diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..80689173 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Each line is a file pattern followed by one or more owners. +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# Default code owner for everything is our aws-crypto-tools group +* @aws/aws-crypto-tools diff --git a/.github/ISSUE_TEMPLATE/amazon-dynamodb-encryption-client-issue.md b/.github/ISSUE_TEMPLATE/amazon-dynamodb-encryption-client-issue.md new file mode 100644 index 00000000..d5b0433d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/amazon-dynamodb-encryption-client-issue.md @@ -0,0 +1,26 @@ +--- +name: Amazon DynamoDB Encryption Client Issue +about: Amazon DynamoDB Encryption Client Issue +title: '' +labels: '' +assignees: '' + +--- + +### Security issue notifications + +If you discover a potential security issue in the Amazon DynamoDB Encryption Client we ask that you notify AWS Security via our [vulnerability reporting page](https://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. + +### Problem: + +A short description of what the problem is and why we need to fix it. Add reproduction steps if necessary. + +### Solution: + +A description of the possible solution in terms of DynamoDB Encryption Client architecture. + +### Out of scope: + +Is there anything the solution will intentionally NOT address? + +[//]: # (NOTE: If you believe this might be a security issue, please email aws-security@amazon.com instead of creating a GitHub issue. For more details, see the AWS Vulnerability Reporting Guide: https://aws.amazon.com/security/vulnerability-reporting/ ) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5cd8ea5c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + # master + - package-ecosystem: "pip" + directory: "/dev_requirements" + schedule: + interval: "daily" + + # Github Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 746f015e..08cda289 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -17,7 +17,6 @@ jobs: category: # Disabled pending completion of integration # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 -# - mypy-py2 # - mypy-py3 - bandit - doc8 @@ -31,13 +30,13 @@ jobs: - pylint-examples - black-check steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.8 - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: check env: TOXENV: ${{ matrix.category }} diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b94c4f9e..fb308d0b 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -9,46 +9,6 @@ on: - cron: '0 0 * * *' jobs: - # Hypothesis no longer supports Python 2 and - # there is a bug that appears with our slow tests - # only on Python 2. - # Until we also drop Python 2 support, - # the workaround is just that we don't run the slow tests - # on Python 2. - py2-tests: - runs-on: ${{ matrix.platform.os }} - strategy: - fail-fast: true - matrix: - platform: - - os: ubuntu-latest - architecture: x64 - - os: windows-latest - architecture: x64 - # x86 builds are only meaningful for Windows - - os: windows-latest - architecture: x86 - - os: macos-latest - architecture: x64 - category: - - local-fast - # These require credentials. - # Enable them once we sort how to provide them. - # - integ-fast - # - examples - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: 2.7 - architecture: ${{ matrix.platform.architecture }} - - run: | - python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt - - name: run test - env: - TOXENV: ${{ matrix.category }} - run: tox -- -vv tests: runs-on: ${{ matrix.platform.os }} strategy: @@ -62,14 +22,15 @@ jobs: # x86 builds are only meaningful for Windows - os: windows-latest architecture: x86 - - os: macos-latest + - os: macos-13 architecture: x64 python: - - 3.5 - - 3.6 - - 3.7 - 3.8 - - 3.x + - 3.9 + - "3.10" + - "3.11" + - "3.12" +# - 3.x 3.13 does not have 'pipes' and maybe other necessary things category: - local-slow # These require credentials. @@ -77,19 +38,20 @@ jobs: # - integ-slow # - examples steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.platform.architecture }} - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: TOXENV: ${{ matrix.category }} run: tox -- -vv - upstream-py3: + + upstream-py311: runs-on: ubuntu-latest strategy: fail-fast: true @@ -97,34 +59,15 @@ jobs: category: - nocmk - sourcebuildcheck - - test-upstream-requirements-py37 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - run: | - python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt - - name: run test - env: - TOXENV: ${{ matrix.category }} - run: tox -- -vv - upstream-py2: - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - category: - - test-upstream-requirements-py27 + - test-upstream-requirements-py311 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: 2.7 + python-version: "3.11" - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: TOXENV: ${{ matrix.category }} diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml new file mode 100644 index 00000000..e3776d39 --- /dev/null +++ b/.github/workflows/repo-sync.yml @@ -0,0 +1,25 @@ +name: Repo Sync + +on: + workflow_dispatch: # allows triggering this manually through the Actions UI + +jobs: + repo-sync: + name: Repo Sync + environment: repo-sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: repo-sync/github-sync@v2 + name: Sync repo to branch + with: + source_repo: ${{ secrets.SOURCE_REPO }} + source_branch: master + destination_branch: ${{ secrets.INTERMEDIATE_BRANCH }} + github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: repo-sync/pull-request@v2 + name: Create pull request + with: + source_branch: ${{ secrets.INTERMEDIATE_BRANCH }} + destination_branch: master + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..82c9c983 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,27 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.8" + +# Build documentation in the doc/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +# Don't need to build documentation for test vectors or any other +# sub modules +submodules: + exclude: all + +python: + install: + - requirements: dev_requirements/doc-requirements.txt + - method: pip + path: . diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d10b71e9..921613e8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,92 @@ Changelog ********* +3.3.0 -- 2024-08-05 +=================== + +Deprecation +----------- +- The AWS DynamoDB Encryption Client for Python no longer supports Python 3.7 as of version 3.3 + - Only Python 3.8+ is supported. +- We no longer support OpenSSL 1.0.1 or 1.0.2, as per `cryptography documentation `_. + +Feature +----------- +* Warn on Deprecated Python 3.7 usage +* Add Python 3.11 to CI +* Add Python 3.12 to CI + +Maintenance +----------- +* Update requirements for boto3 (>=1.10.0) and cryptography (>=3.4.6) + +3.2.0 -- 2021-12-19 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 3.6 +as of version 3.2; only Python 3.7+ is supported. + +Feature +----------- +* Warn on Deprecated Python 3.6 usage + +3.1.0 -- 2021-11-10 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 3.5 +as of version 3.1; only Python 3.6+ is supported. Customers using +Python 3.5 can still use the 2.x line of the AWS DynamoDB Encryption Client for Python, +which will continue to receive security updates, in accordance +with our `Support Policy `__. + +Feature +----------- +* Warn on Deprecated Python usage + `#368 `_ +* Add Python 3.10 to CI +* Remove Python 3.5 from testing + + +3.0.0 -- 2021-07-15 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 2 or Python 3.4 +as of major version 3.x; only Python 3.5+ is supported. Customers using Python 2 +or Python 3.4 can still use the 2.x line of the DynamoDB Encryption Client, +which will continue to receive security updates for the next 12 months, in accordance +with our `Support Policy `__. + + +2.1.0 -- 2021-07-15 +=================== + +Deprecation Announcement +------------------------ +The AWS DynamoDB Encryption Client for Python is discontinuing support for Python 2. +Future major versions of this library will drop support for Python 2 and begin to +adopt changes that are known to break Python 2. + +Support for Python 3.4 will be removed at the same time. Moving forward, we will +support Python 3.5+. + +Security updates will still be available for the DynamoDB Encryption Client 2.x +line for the next 12 months, in accordance with our `Support Policy `__. + + +2.0.0 -- 2021-02-04 +=================== + +Breaking Changes +---------------- +Removes MostRecentProvider. MostRecentProvider is replaced by CachingMostRecentProvider as of 1.3.0. + + 1.3.0 -- 2021-02-04 =================== Adds the CachingMostRecentProvider and deprecates MostRecentProvider. @@ -15,7 +101,7 @@ CachingMostRecentProvider replaces MostRecentProvider and provides a cache entry TTL to reauthorize the key with the key provider. MostRecentProvider is now deprecated, and is removed in 2.0.0. See -https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/most-recent-provider.html +https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/most-recent-provider.html#mrp-versions for more details. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7323620c..b6353e00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,10 @@ Please read through this document before submitting any issues or pull requests information to effectively respond to your bug report or contribution. +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + ## Reporting Bugs/Feature Requests We welcome you to use the GitHub issue tracker to report bugs or suggest features. @@ -50,10 +54,6 @@ For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of opensource-codeofconduct@amazon.com with any additional questions or comments. -## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. - - ## Licensing See the [LICENSE](https://github.com/aws/aws-dynamodb-encryption-python/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/README.rst b/README.rst index d78d3179..438a10ee 100644 --- a/README.rst +++ b/README.rst @@ -38,6 +38,8 @@ You can find our source on `GitHub`_. `Security issue notifications`_ +See `Support Policy`_ for details on the current support status of all major versions of this library. + *************** Getting Started *************** @@ -45,7 +47,8 @@ Getting Started Required Prerequisites ====================== -* Python 2.7 or 3.4+ +* Python 3.8+ + Installation ============ @@ -174,10 +177,10 @@ of the one that the client would normally construct for you. ... ) # this uses my_special_crypto_config -.. _Amazon DynamoDB Encryption Client: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/ +.. _Amazon DynamoDB Encryption Client: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/legacy-dynamodb-encryption-client.html .. _Amazon DynamoDB: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Introduction.html -.. _primary documents: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/ -.. _Concepts Guide: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/concepts.html +.. _primary documents: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/legacy-dynamodb-encryption-client.html +.. _Concepts Guide: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/DDBEC-legacy-concepts.html .. _Amazon DynamoDB Encryption Client for Java: https://github.com/aws/aws-dynamodb-encryption-java/ .. _Amazon DynamoDB Encryption Client for Python: https://github.com/aws/aws-dynamodb-encryption-python/ .. _DynamoDB Stream: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html @@ -192,4 +195,5 @@ of the one that the client would normally construct for you. .. _CryptoConfig: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/encrypted/config.html .. _decrypt_dynamodb_item: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/encrypted/item.html#dynamodb_encryption_sdk.encrypted.item.decrypt_dynamodb_item .. _transformation functions: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/tools/transform.html -.. _Security issue notifications: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/CONTRIBUTING.md#user-content-security-issue-notifications +.. _Security issue notifications: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/CONTRIBUTING.md +.. _Support Policy: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/SUPPORT_POLICY.rst diff --git a/SUPPORT_POLICY.rst b/SUPPORT_POLICY.rst new file mode 100644 index 00000000..3fe938f3 --- /dev/null +++ b/SUPPORT_POLICY.rst @@ -0,0 +1,37 @@ +Overview +======== +This page describes the support policy for the AWS DynamoDB Encryption Client. We regularly provide the AWS DynamoDB Encryption Client with updates that may contain support for new or updated APIs, new features, enhancements, bug fixes, security patches, or documentation updates. Updates may also address changes with dependencies, language runtimes, and operating systems. + +We recommend users to stay up-to-date with DynamoDB Encryption Client releases to keep up with the latest features, security updates, and underlying dependencies. Continued use of an unsupported SDK version is not recommended and is done at the user’s discretion. + + +Major Version Lifecycle +======================== +The AWS DynamoDB Encryption Client follows the same major version lifecycle as the AWS SDK. For details on this lifecycle, see `AWS SDKs and Tools Maintenance Policy`_. + +Version Support Matrix +====================== +This table describes the current support status of each major version of the AWS DynamoDB Encryption Client for Python. It also shows the next status each major version will transition to, and the date at which that transition will happen. + +.. list-table:: + :widths: 30 50 50 50 + :header-rows: 1 + + * - Major version + - Current status + - Next status + - Next status date + * - 1.x + - End of Support + - + - + * - 2.x + - End of Support + - + - + * - 3.x + - Generally Available + - Maintenance + - 2024-08-05 + +.. _AWS SDKs and Tools Maintenance Policy: https://docs.aws.amazon.com/sdkref/latest/guide/maint-policy.html#version-life-cycle diff --git a/buildspec.yml b/buildspec.yml index 7bef08e4..b53801b5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -3,13 +3,25 @@ version: 0.2 batch: fast-fail: false build-list: - - identifier: python2_7 - buildspec: codebuild/python2.7.yml - - identifier: python3_5 - buildspec: codebuild/python3.5.yml - - identifier: python3_6 - buildspec: codebuild/python3.6.yml - - identifier: python3_7 - buildspec: codebuild/python3.7.yml - identifier: python3_8 buildspec: codebuild/python3.8.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: python3_9 + buildspec: codebuild/python3.9.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: python3_10 + buildspec: codebuild/python3.10.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: python3_11 + buildspec: codebuild/python3.11.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: python3_12 + buildspec: codebuild/python3.12.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: code_coverage + buildspec: codebuild/coverage/coverage.yml diff --git a/cfn/CB.yml b/cfn/CB.yml new file mode 100644 index 00000000..30d5966b --- /dev/null +++ b/cfn/CB.yml @@ -0,0 +1,364 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: "Template to build a CodeBuild Project, assumes that GitHub credentials are already set up." +Parameters: + ProjectName: + Type: String + Description: The name of the CodeBuild Project + ProjectDescription: + Type: String + Description: The description for the CodeBuild Project + SourceLocation: + Type: String + Description: The https GitHub URL for the project + NumberOfBuildsInBatch: + Type: Number + MaxValue: 100 + MinValue: 1 + Default: 4 + Description: The number of builds you expect to run in a batch + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - + Label: + default: "Crypto Tools CodeBuild Project Template" + Parameters: + - ProjectName + - ProjectDescription + - SourceLocation + +Resources: + CodeBuildProject: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Ref ProjectName + Description: !Ref ProjectDescription + Source: + Location: !Ref SourceLocation + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Triggers: + BuildType: BUILD_BATCH + Webhook: True + FilterGroups: + - - Type: EVENT + Pattern: PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PUSH,PULL_REQUEST_REOPENED + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildCIServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildCIServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildProjectTestRelease: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Sub "${ProjectName}-test-release" + Description: !Sub "CodeBuild project for ${ProjectName} to release to test PyPi." + Source: + Location: !Ref SourceLocation + BuildSpec: "codebuild/release/test-release.yml" + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildProjectProdRelease: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Sub "${ProjectName}-prod-release" + Description: !Sub "CodeBuild project for ${ProjectName} to release to prod PyPi." + Source: + Location: !Ref SourceLocation + BuildSpec: "codebuild/release/prod-release.yml" + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildServiceRole: + Type: "AWS::IAM::Role" + Properties: + Path: "/service-role/" + RoleName: !Sub "codebuild-${ProjectName}-service-role" + AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + MaxSessionDuration: 3600 + ManagedPolicyArns: + - !Ref CryptoToolsKMS + - !Ref CodeBuildBatchPolicy + - !Ref CodeBuildBasePolicy + - !Ref SecretsManagerPolicy + - !Ref DDBPolicy + - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" + + CodeBuildCIServiceRole: + Type: "AWS::IAM::Role" + Properties: + Path: "/service-role/" + RoleName: !Sub "codebuild-${ProjectName}-CI-service-role" + AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + MaxSessionDuration: 3600 + ManagedPolicyArns: + - !Ref CryptoToolsKMS + - !Ref CodeBuildCIBatchPolicy + - !Ref CodeBuildBasePolicy + - !Ref DDBPolicy + - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" + + CodeBuildBatchPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBuildBatchPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}", + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}-test-release", + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}-prod-release" + ], + "Action": [ + "codebuild:StartBuild", + "codebuild:StopBuild", + "codebuild:RetryBuild" + ] + } + ] + } + + CodeBuildCIBatchPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBuildBatchPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-CI-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}" + ], + "Action": [ + "codebuild:StartBuild", + "codebuild:StopBuild", + "codebuild:RetryBuild" + ] + } + ] + } + + CodeBuildBasePolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBasePolicy-${ProjectName}-${AWS::Region}" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}:*", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-test-release", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-test-release:*", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-prod-release", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-prod-release:*" + ], + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + }, + { + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::codepipeline-${AWS::Region}-*" + ], + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + }, + { + "Effect": "Allow", + "Action": [ + "codebuild:CreateReportGroup", + "codebuild:CreateReport", + "codebuild:UpdateReport", + "codebuild:BatchPutTestCases", + "codebuild:BatchPutCodeCoverages" + ], + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:report-group/${ProjectName}-*" + ] + } + ] + } + + SecretsManagerPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CryptoTools-SecretsManager-${ProjectName}-release" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:secretsmanager:us-west-2:587316601012:secret:TestPyPiAPIToken-uERFjs", + "arn:aws:secretsmanager:us-west-2:587316601012:secret:PyPiAPIToken-nu1Gu6" + ], + "Action": "secretsmanager:GetSecretValue" + } + ] + } + + DDBPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CryptoTools-DynamoDB-${ProjectName}-CI" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb:us-east-1:587316601012:table/ddbec-mrk-testing", + "arn:aws:dynamodb:us-west-2:587316601012:table/ddbec-mrk-testing" + ], + "Action": "*" + } + ] + } + + # There exist public AWS KMS CMKs that are used for testing + # Take care with these CMKs they are **ONLY** for testing!!! + CryptoToolsKMS: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CrypotToolsKMSPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:kms:*:658956600833:key/*", + "arn:aws:kms:*:658956600833:alias/*" + ], + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } + ] + } diff --git a/ci-requirements.txt b/ci-requirements.txt deleted file mode 100644 index 053148f8..00000000 --- a/ci-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tox diff --git a/codebuild/coverage/coverage.yml b/codebuild/coverage/coverage.yml new file mode 100644 index 00000000..51d8b0a6 --- /dev/null +++ b/codebuild/coverage/coverage.yml @@ -0,0 +1,14 @@ +version: 0.2 + +env: + variables: + TOXENV: "coverage" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/python2.7.yml b/codebuild/python3.10.yml similarity index 83% rename from codebuild/python2.7.yml rename to codebuild/python3.10.yml index fd688d77..ad76049f 100644 --- a/codebuild/python2.7.yml +++ b/codebuild/python3.10.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py27-integ-slow" + TOXENV: "py310-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.10 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.6.yml b/codebuild/python3.11.yml similarity index 83% rename from codebuild/python3.6.yml rename to codebuild/python3.11.yml index 602dc113..b21cf15a 100644 --- a/codebuild/python3.6.yml +++ b/codebuild/python3.11.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py36-integ-slow" + TOXENV: "py311-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.11 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.12.yml b/codebuild/python3.12.yml new file mode 100644 index 00000000..cf9e09ef --- /dev/null +++ b/codebuild/python3.12.yml @@ -0,0 +1,23 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-integ-slow" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox diff --git a/codebuild/python3.5.yml b/codebuild/python3.5.yml deleted file mode 100644 index f2b1dbcd..00000000 --- a/codebuild/python3.5.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py35-integ-slow" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - -phases: - install: - runtime-versions: - python: latest - build: - commands: - # The specific versions are manually installed - # because they are not installed - # by default in CodeBuild containers. - # `pyenv` does not have - # a nice way to just install - # the latest patch version. - # I have selected the current latest patch - # rather than try - # and manage a one-liner or script. - # Testing every minor version - # is too extreme at this time. - # The choice of versions should be reviewed. - - pyenv install 3.5.9 - - pyenv local 3.5.9 - - pip install tox tox-pyenv - - tox diff --git a/codebuild/python3.7.yml b/codebuild/python3.7.yml index 1ac0daa6..6a51426a 100644 --- a/codebuild/python3.7.yml +++ b/codebuild/python3.7.yml @@ -11,22 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.7 build: commands: - # The specific versions are manually installed - # because they are not installed - # by default in CodeBuild containers. - # `pyenv` does not have - # a nice way to just install - # the latest patch version. - # I have selected the current latest patch - # rather than try - # and manage a one-liner or script. - # Testing every minor version - # is too extreme at this time. - # The choice of versions should be reviewed. - - pyenv install 3.7.9 - - pyenv local 3.7.9 - - pip install tox tox-pyenv + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.8.yml b/codebuild/python3.8.yml index 1c1524c8..478a3bfc 100644 --- a/codebuild/python3.8.yml +++ b/codebuild/python3.8.yml @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.8 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.9.yml b/codebuild/python3.9.yml new file mode 100644 index 00000000..f572e2a9 --- /dev/null +++ b/codebuild/python3.9.yml @@ -0,0 +1,18 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-integ-slow" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/release/prod-release.yml b/codebuild/release/prod-release.yml new file mode 100644 index 00000000..7f55b526 --- /dev/null +++ b/codebuild/release/prod-release.yml @@ -0,0 +1,41 @@ +version: 0.2 + +env: + variables: + BRANCH: "master" + secrets-manager: + TWINE_USERNAME: PyPiAPIToken:username + TWINE_PASSWORD: PyPiAPIToken:password + +phases: + install: + commands: + - pip install "tox < 4.0" + - pip install --upgrade pip + runtime-versions: + python: latest + pre_build: + commands: + - git checkout $COMMIT_ID + - FOUND_VERSION=$(sed -n 's/__version__ = "\(.*\)"/\1/p' src/dynamodb_encryption_sdk/identifiers.py) + - | + if expr ${FOUND_VERSION} != ${VERSION}; then + echo "identifiers.py version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping" + exit 1; + fi + build: + commands: + - tox -e park + - tox -e release + +batch: + fast-fail: true + build-graph: + - identifier: release_to_prod + - identifier: validate_prod_release + depend-on: + - release_to_prod + buildspec: codebuild/release/validate.yml + env: + variables: + PIP_INDEX_URL: https://pypi.python.org/simple/ diff --git a/codebuild/release/test-release.yml b/codebuild/release/test-release.yml new file mode 100644 index 00000000..03dc4d95 --- /dev/null +++ b/codebuild/release/test-release.yml @@ -0,0 +1,43 @@ +version: 0.2 + +env: + variables: + BRANCH: "master" + secrets-manager: + TWINE_USERNAME: TestPyPiAPIToken:username + TWINE_PASSWORD: TestPyPiAPIToken:password + +phases: + install: + commands: + - pip install "tox < 4.0" + - pip install --upgrade pip + runtime-versions: + python: latest + pre_build: + commands: + - git checkout $COMMIT_ID + - FOUND_VERSION=$(sed -n 's/__version__ = "\(.*\)"/\1/p' src/dynamodb_encryption_sdk/identifiers.py) + - | + if expr ${FOUND_VERSION} != ${VERSION}; then + echo "identifiers.py version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping" + exit 1; + fi + build: + commands: + - tox -e park + - tox -e test-release + + +batch: + fast-fail: true + build-graph: + - identifier: release_to_staging + - identifier: validate_staging_release + depend-on: + - release_to_staging + buildspec: codebuild/release/validate.yml + env: + variables: + PIP_INDEX_URL: https://test.pypi.org/simple/ + PIP_EXTRA_INDEX_URL: https://pypi.python.org/simple/ diff --git a/codebuild/release/validate.yml b/codebuild/release/validate.yml new file mode 100644 index 00000000..f710aa5a --- /dev/null +++ b/codebuild/release/validate.yml @@ -0,0 +1,43 @@ +version: 0.2 + +env: + variables: + BRANCH: "master" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:eu-west-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME: ddbec-release-validation + + +phases: + install: + commands: + - pip install "tox < 4.0" + runtime-versions: + python: latest + pre_build: + commands: + - cd examples + - sed -i "s/dynamodb-encryption-sdk/dynamodb-encryption-sdk==$VERSION/" test/requirements.txt + build: + commands: + - NUM_RETRIES=3 + - | + while [ $NUM_RETRIES -gt 0 ] + do + tox -re py3-examples + if [ $? -eq 0 ]; then + break + fi + NUM_RETRIES=$((NUM_RETRIES-1)) + if [ $NUM_RETRIES -eq 0 ]; then + echo "All validation attempts failed, stopping" + exit 1; + else + echo "Validation failed, retrying in 60 seconds; will retry $NUM_RETRIES more times" && sleep 60 + fi + done + diff --git a/dev_requirements/ci-requirements.txt b/dev_requirements/ci-requirements.txt new file mode 100644 index 00000000..b673eb36 --- /dev/null +++ b/dev_requirements/ci-requirements.txt @@ -0,0 +1,2 @@ +setuptools +tox==3.24.5 diff --git a/dev_requirements/doc-requirements.txt b/dev_requirements/doc-requirements.txt new file mode 100644 index 00000000..9364148e --- /dev/null +++ b/dev_requirements/doc-requirements.txt @@ -0,0 +1,2 @@ +sphinx==4.4.0 +sphinx_rtd_theme==1.0.0 diff --git a/dev_requirements/linter-requirements.txt b/dev_requirements/linter-requirements.txt new file mode 100644 index 00000000..1f8d8c16 --- /dev/null +++ b/dev_requirements/linter-requirements.txt @@ -0,0 +1,16 @@ +bandit==1.7.2 +black==22.3.0 +doc8==0.10.1 +flake8==4.0.1 +flake8-docstrings==1.7.0 +flake8-isort==4.1.1 +# https://github.com/JBKahn/flake8-print/pull/30 +flake8-print==5.0.0 +isort==5.12.0 +pylint==2.12.2 +pyflakes==2.4.0 +# https://github.com/PyCQA/pydocstyle/issues/375 +pydocstyle==3.0.0 +readme_renderer==34.0 +seed-isort-config==2.2.0 +vulture==2.3 diff --git a/dev_requirements/release-requirements.txt b/dev_requirements/release-requirements.txt new file mode 100644 index 00000000..fa2ef83b --- /dev/null +++ b/dev_requirements/release-requirements.txt @@ -0,0 +1,6 @@ +pypi-parker==0.1.2 +setuptools==66.1.1 +twine==3.8.0 +wheel==0.38.4 +#This is required for twine < 4.0 +packaging \ No newline at end of file diff --git a/dev_requirements/test-requirements.txt b/dev_requirements/test-requirements.txt new file mode 100644 index 00000000..c73de305 --- /dev/null +++ b/dev_requirements/test-requirements.txt @@ -0,0 +1,9 @@ +hypothesis==6.31.6 +mock==4.0.3 +moto==3.0.2 +pytest==7.2.1 +pytest-cov==3.0.0 +pytest-mock==3.10.0 +pytest-xdist==3.2.0 +boto3==1.28.38 +botocore==1.31.38 diff --git a/doc/_static/.gitignore b/doc/_static/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/doc/conf.py b/doc/conf.py index 576f77c5..4e87cadd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,7 +11,7 @@ def read(*args): """Reads complete file contents.""" - return io.open(os.path.join(HERE, *args), encoding="utf-8").read() + return io.open(os.path.join(HERE, *args), encoding="utf-8").read() # pylint: disable=consider-using-with def get_release(): @@ -29,7 +29,7 @@ def get_version(): return _release -project = u"dynamodb-encryption-sdk-python" +project = "dynamodb-encryption-sdk-python" version = get_version() release = get_release() @@ -53,7 +53,7 @@ def get_version(): source_suffix = ".rst" # The suffix of source filenames. master_doc = "index" # The master toctree document. -copyright = u"%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin +copyright = "%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin # List of directories, relative to source directory, that shouldn't be searched # for source files. @@ -62,7 +62,7 @@ def get_version(): pygments_style = "sphinx" autoclass_content = "both" -autodoc_default_flags = ["show-inheritance", "members"] +autodoc_default_options = {"members": True, "show-inheritance": True} autodoc_member_order = "bysource" html_theme = "sphinx_rtd_theme" diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 29e31945..00000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sphinx>=1.3.0 -sphinx_rtd_theme \ No newline at end of file diff --git a/examples/README.rst b/examples/README.rst new file mode 100644 index 00000000..ea206ff0 --- /dev/null +++ b/examples/README.rst @@ -0,0 +1,54 @@ +######################################### +AWS DynamoDB Encryption Client Examples +######################################### + +This section features examples that show you +how to use the AWS DynamoDB Encryption Client. +We demonstrate how to use the encryption and decryption APIs +and how to set up some common configuration patterns. + +APIs +==== + +The AWS DynamoDB Encryption Client provides four high-level APIs: `EncryptedClient`, `EncryptedItem`, +`EncryptedResource`, and `EncryptedTable`. + +You can find examples that demonstrate these APIs +in the `examples/src/dynamodb_encryption_sdk_examples <./src/dynamodb_encryption_sdk_examples>`_ directory. +Each of these examples uses AWS KMS as the materials provider. + +* `How to use the EncryptedClient API <./src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_client.py>`_ +* `How to use the EncryptedItem API <./src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_item.py>`_ +* `How to use the EncryptedResource API <./src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_resource.py>`_ +* `How to use the EncryptedTable API <./src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_table.py>`_ + +Material Providers +================== + +To use the encryption and decryption APIs, you need to describe how you want the library to protect your data keys. +You can do this by configuring material providers. AWS KMS is the most common material provider used with the AWS DynamoDB Encryption +SDK, and each of the API examples above uses AWS KMS. This section describes the other providers that come bundled +with this library. + +* `How to use the CachingMostRecentProvider <./src/dynamodb_encryption_sdk_examples/most_recent_provider_encrypted_table.py>`_ +* `How to use raw symmetric wrapping keys <./src/dynamodb_encryption_sdk_examples/wrapped_symmetric_encrypted_table.py>`_ +* `How to use raw asymmetric wrapping keys <./src/dynamodb_encryption_sdk_examples/wrapped_rsa_encrypted_table.py>`_ + +For more details on the different type of material providers, see `How to choose a cryptographic materials provider `_. + +Running the examples +==================== + +In order to run these examples, these things must be configured: + +#. Ensure that AWS credentials are available in one of the `automatically discoverable credential locations`_. +#. The ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID`` environment variable + must be set to a valid `AWS KMS CMK ARN`_ that can be used by the available credentials. +#. The ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID`` and ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2`` environment variables + must be set to two related AWS KMS Multi-Region key ids in different regions. +#. The ``DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME`` environment variable must be set to a valid + DynamoDB table name, in the default region, to which the discoverable credentials have + read, write, and describe permissions. + +.. _automatically discoverable credential locations: http://boto3.readthedocs.io/en/latest/guide/configuration.html +.. _AWS KMS CMK ARN: http://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html diff --git a/examples/requirements.txt b/examples/requirements.txt new file mode 100644 index 00000000..138d8645 --- /dev/null +++ b/examples/requirements.txt @@ -0,0 +1 @@ +dynamodb-encryption-sdk diff --git a/examples/setup.py b/examples/setup.py new file mode 100644 index 00000000..b856ef53 --- /dev/null +++ b/examples/setup.py @@ -0,0 +1,60 @@ +"""DynamoDB Encryption Client for Python examples.""" +import io +import os +import re + +from setuptools import find_packages, setup + +VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") +HERE = os.path.abspath(os.path.dirname(__file__)) + + +def read(*args): + """Reads complete file contents.""" + return io.open(os.path.join(HERE, *args), encoding="utf-8").read() + + +def get_version(): + """Reads the version from this module.""" + init = read("src", "dynamodb_encryption_sdk_examples", "__init__.py") + return VERSION_RE.search(init).group(1) + + +def get_requirements(): + """Reads the requirements file.""" + requirements = read("requirements.txt") + return requirements.strip().splitlines() + + +setup( + name="dynamodb-encryption-sdk-examples", + version=get_version(), + packages=find_packages("src"), + package_dir={"": "src"}, + url="https://github.com/aws/aws-dynamodb-encryption-python", + author="Amazon Web Services", + author_email="aws-cryptools@amazon.com", + maintainer="Amazon Web Services", + description="DynamoDB Encryption Client for Python examples", + long_description=read("README.rst"), + keywords="dynamodb-encryption-sdk aws kms encryption dynamodb", + data_files=["requirements.txt"], + license="Apache License 2.0", + install_requires=get_requirements(), + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Security", + "Topic :: Security :: Cryptography", + ], +) diff --git a/examples/src/__init__.py b/examples/src/dynamodb_encryption_sdk_examples/__init__.py similarity index 96% rename from examples/src/__init__.py rename to examples/src/dynamodb_encryption_sdk_examples/__init__.py index b08a227c..887f4840 100644 --- a/examples/src/__init__.py +++ b/examples/src/dynamodb_encryption_sdk_examples/__init__.py @@ -11,3 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Stub module indicator to make linter configuration simpler.""" +__version__ = "1.0.0" diff --git a/examples/src/aws_kms_encrypted_client.py b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_client.py similarity index 97% rename from examples/src/aws_kms_encrypted_client.py rename to examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_client.py index 87af4f91..a190b09d 100644 --- a/examples/src/aws_kms_encrypted_client.py +++ b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_client.py @@ -54,7 +54,7 @@ def encrypt_item(table_name, aws_cmk_id): # Get the encrypted item using the standard client. encrypted_item = client.get_item(TableName=table_name, Key=index_key)["Item"] - # Get the item using the encrypted client, transparently decyrpting it. + # Get the item using the encrypted client, transparently decrypting it. decrypted_item = encrypted_client.get_item(TableName=table_name, Key=index_key)["Item"] # Verify that all of the attributes are different in the encrypted item @@ -124,7 +124,7 @@ def encrypt_batch_items(table_name, aws_cmk_id): def _select_index_from_item(item): """Find the index keys that match this item.""" for index in index_keys: - if all([item[key] == value for key, value in index.items()]): + if all(item[key] == value for key, value in index.items()): return index raise Exception("Index key not found in item.") @@ -132,7 +132,7 @@ def _select_index_from_item(item): def _select_item_from_index(index, all_items): """Find the item that matches these index keys.""" for item in all_items: - if all([item[key] == value for key, value in index.items()]): + if all(item[key] == value for key, value in index.items()): return item raise Exception("Index key not found in item.") diff --git a/examples/src/aws_kms_encrypted_item.py b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_item.py similarity index 100% rename from examples/src/aws_kms_encrypted_item.py rename to examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_item.py diff --git a/examples/src/aws_kms_encrypted_resource.py b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_resource.py similarity index 97% rename from examples/src/aws_kms_encrypted_resource.py rename to examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_resource.py index dabcf311..5a8d3907 100644 --- a/examples/src/aws_kms_encrypted_resource.py +++ b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_resource.py @@ -76,7 +76,7 @@ def encrypt_batch_items(table_name, aws_cmk_id): def _select_index_from_item(item): """Find the index keys that match this item.""" for index in index_keys: - if all([item[key] == value for key, value in index.items()]): + if all(item[key] == value for key, value in index.items()): return index raise Exception("Index key not found in item.") @@ -84,7 +84,7 @@ def _select_index_from_item(item): def _select_item_from_index(index, all_items): """Find the item that matches these index keys.""" for item in all_items: - if all([item[key] == value for key, value in index.items()]): + if all(item[key] == value for key, value in index.items()): return item raise Exception("Index key not found in item.") diff --git a/examples/src/aws_kms_encrypted_table.py b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_table.py similarity index 100% rename from examples/src/aws_kms_encrypted_table.py rename to examples/src/dynamodb_encryption_sdk_examples/aws_kms_encrypted_table.py diff --git a/examples/src/dynamodb_encryption_sdk_examples/aws_kms_multi_region_key.py b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_multi_region_key.py new file mode 100644 index 00000000..5c32b501 --- /dev/null +++ b/examples/src/dynamodb_encryption_sdk_examples/aws_kms_multi_region_key.py @@ -0,0 +1,78 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example showing use of AWS KMS CMP with a DynamoDB Global table and an AWS Multi-Region Key.""" + +import time + +import boto3 + +from dynamodb_encryption_sdk.encrypted.client import EncryptedClient +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions + +SECOND_REGION = "eu-west-1" + + +def encrypt_item(table_name, cmk_mrk_arn_first_region, cmk_mrk_arn_second_region): + """Demonstrate use of Multi-Region Keys with DynamoDB Encryption Client. + + This example encrypts an item with a Multi-Region Key in one region and decrypts it in another region. It + assumes that you have a Dynamo DB Global table in two regions, as well as a KMS + Multi-Region Key replicated to these regions. + """ + index_key = {"partition_attribute": {"S": "is this"}, "sort_attribute": {"N": "55"}} + plaintext_item = { + "example": {"S": "data"}, + "some numbers": {"N": "99"}, + "and some binary": {"B": b"\x00\x01\x02"}, + "leave me": {"S": "alone"}, # We want to ignore this attribute + } + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_item.keys()) + encrypted_attributes.remove("leave me") + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_key.keys()) + unencrypted_attributes.add("leave me") + # Add the index pairs to the item. + plaintext_item.update(index_key) + + # Create attribute actions that tells the encrypted client to encrypt all attributes except one. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, attribute_actions={"leave me": CryptoAction.DO_NOTHING} + ) + + # Create a DDB client and KMS crypto materials provider in the first region using the specified AWS KMS key. + split_arn = cmk_mrk_arn_first_region.split(":") + encryption_region = split_arn[3] + ddb_client = boto3.client("dynamodb", region_name=encryption_region) + encryption_cmp = AwsKmsCryptographicMaterialsProvider(key_id=cmk_mrk_arn_first_region) + # Use these objects to create an encrypted client. + encryption_client = EncryptedClient(client=ddb_client, materials_provider=encryption_cmp, attribute_actions=actions) + + # Put the item to the table, using the encrypted client to transparently encrypt it. + encryption_client.put_item(TableName=table_name, Item=plaintext_item) + + # Create a DDB client and KMS crypto materials provider in the second region + split_arn = cmk_mrk_arn_second_region.split(":") + decryption_region = split_arn[3] + decryption_cmp = AwsKmsCryptographicMaterialsProvider(key_id=cmk_mrk_arn_second_region) + ddb_client = boto3.client("dynamodb", region_name=decryption_region) + # Use these objects to create an encrypted client. + decryption_client = EncryptedClient(client=ddb_client, materials_provider=decryption_cmp, attribute_actions=actions) + + # DDB Global Table replication takes some time. Sleep for a moment to give the item a chance to replicate to the + # second region + time.sleep(1) + + # Get the item from the second region, transparently decrypting it. This allows you to avoid a cross-region KMS + # call to the first region if your application is running in the second region + decrypted_item = decryption_client.get_item(TableName=table_name, Key=index_key)["Item"] + + # Verify that the decryption successfully retrieved the original plaintext + for name in encrypted_attributes: + assert plaintext_item[name] == decrypted_item[name] + + # Clean up the item + encryption_client.delete_item(TableName=table_name, Key=index_key) diff --git a/examples/src/most_recent_provider_encrypted_table.py b/examples/src/dynamodb_encryption_sdk_examples/most_recent_provider_encrypted_table.py similarity index 100% rename from examples/src/most_recent_provider_encrypted_table.py rename to examples/src/dynamodb_encryption_sdk_examples/most_recent_provider_encrypted_table.py diff --git a/examples/src/wrapped_rsa_encrypted_table.py b/examples/src/dynamodb_encryption_sdk_examples/wrapped_rsa_encrypted_table.py similarity index 100% rename from examples/src/wrapped_rsa_encrypted_table.py rename to examples/src/dynamodb_encryption_sdk_examples/wrapped_rsa_encrypted_table.py diff --git a/examples/src/wrapped_symmetric_encrypted_table.py b/examples/src/dynamodb_encryption_sdk_examples/wrapped_symmetric_encrypted_table.py similarity index 100% rename from examples/src/wrapped_symmetric_encrypted_table.py rename to examples/src/dynamodb_encryption_sdk_examples/wrapped_symmetric_encrypted_table.py diff --git a/examples/src/pylintrc b/examples/src/pylintrc index 5ea9fbcc..2a3a443a 100644 --- a/examples/src/pylintrc +++ b/examples/src/pylintrc @@ -3,6 +3,7 @@ disable = duplicate-code, # these examples often feature similar code too-many-locals, # for these examples, we prioritize keeping everything together for simple readability + consider-using-f-string, # Not supported in Python 3.5 [BASIC] # Allow function names up to 50 characters diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 252132e7..89ba1bba 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -1,8 +1,25 @@ -"""Helper utilities for use while testing examples.""" +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Helper utilities for use while testing examples. + +isort:skip_file +""" import os import sys os.environ["AWS_ENCRYPTION_SDK_EXAMPLES_TESTING"] = "yes" sys.path.extend([os.sep.join([os.path.dirname(__file__), "..", "..", "test", "integration"])]) -from integration_test_utils import cmk_arn, ddb_table_name # noqa pylint: disable=unused-import +# fmt: off +from integration_test_utils import cmk_arn, cmk_mrk_arn, ddb_table_name, second_cmk_mrk_arn # noqa pylint: disable=unused-import +# fmt: on diff --git a/examples/test/pylintrc b/examples/test/pylintrc index f4dfcfe6..f9671d06 100644 --- a/examples/test/pylintrc +++ b/examples/test/pylintrc @@ -10,6 +10,7 @@ disable = # pylint does not recognize this duplicate-code, # tests for similar things tend to be similar redefined-outer-name, # raises false positives with fixtures + consider-using-f-string, # Not supported in Python 3.5 [DESIGN] max-args = 10 diff --git a/examples/test/requirements.txt b/examples/test/requirements.txt new file mode 100644 index 00000000..882624b6 --- /dev/null +++ b/examples/test/requirements.txt @@ -0,0 +1,2 @@ +dynamodb-encryption-sdk +pytest diff --git a/examples/test/test_aws_kms_encrypted_examples.py b/examples/test/test_aws_kms_encrypted_examples.py index f7df24b3..a815683a 100644 --- a/examples/test/test_aws_kms_encrypted_examples.py +++ b/examples/test/test_aws_kms_encrypted_examples.py @@ -12,9 +12,20 @@ # language governing permissions and limitations under the License. """Test ``aws_kms_encrypted_*`` examples.""" import pytest - -from ..src import aws_kms_encrypted_client, aws_kms_encrypted_item, aws_kms_encrypted_resource, aws_kms_encrypted_table -from .examples_test_utils import cmk_arn, ddb_table_name # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk_examples import ( + aws_kms_encrypted_client, + aws_kms_encrypted_item, + aws_kms_encrypted_resource, + aws_kms_encrypted_table, + aws_kms_multi_region_key, +) + +from .examples_test_utils import ( # noqa pylint: disable=unused-import + cmk_arn, + cmk_mrk_arn, + ddb_table_name, + second_cmk_mrk_arn, +) pytestmark = [pytest.mark.examples] @@ -37,3 +48,7 @@ def test_aws_kms_encrypted_item(ddb_table_name, cmk_arn): def test_aws_kms_encrypted_resource(ddb_table_name, cmk_arn): aws_kms_encrypted_resource.encrypt_batch_items(ddb_table_name, cmk_arn) + + +def test_aws_kms_mrk_client(ddb_table_name, cmk_mrk_arn, second_cmk_mrk_arn): + aws_kms_multi_region_key.encrypt_item(ddb_table_name, cmk_mrk_arn, second_cmk_mrk_arn) diff --git a/examples/test/test_most_recent_provider_encrypted_examples.py b/examples/test/test_most_recent_provider_encrypted_examples.py index 8894001c..1821bf32 100644 --- a/examples/test/test_most_recent_provider_encrypted_examples.py +++ b/examples/test/test_most_recent_provider_encrypted_examples.py @@ -15,10 +15,10 @@ import boto3 import pytest +from dynamodb_encryption_sdk_examples import most_recent_provider_encrypted_table from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore -from ..src import most_recent_provider_encrypted_table from .examples_test_utils import cmk_arn, ddb_table_name # noqa pylint: disable=unused-import pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_wrapped_encrypted_examples.py b/examples/test/test_wrapped_encrypted_examples.py index f6bb7aa2..1c06e813 100644 --- a/examples/test/test_wrapped_encrypted_examples.py +++ b/examples/test/test_wrapped_encrypted_examples.py @@ -12,10 +12,10 @@ # language governing permissions and limitations under the License. """Test ``wrapped_*_encrypted_*`` examples.""" import pytest +from dynamodb_encryption_sdk_examples import wrapped_rsa_encrypted_table, wrapped_symmetric_encrypted_table from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey -from ..src import wrapped_rsa_encrypted_table, wrapped_symmetric_encrypted_table from .examples_test_utils import ddb_table_name # noqa pylint: disable=unused-import pytestmark = [pytest.mark.examples] diff --git a/examples/tox.ini b/examples/tox.ini new file mode 100644 index 00000000..e4b39b37 --- /dev/null +++ b/examples/tox.ini @@ -0,0 +1,31 @@ +# Basic environments for running examples against various versions of Python + +[tox] +envlist = + py{3,37,38,39}-examples + +[testenv:base-command] +commands = python -m pytest --basetemp={envtmpdir} -l {posargs} + +[testenv] +passenv = + # Identifies AWS KMS key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ + # Identifes AWS KMS Multi-Region key ids to use in examples \ + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID \ + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ + # DynamoDB Table to use in integration tests + DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME \ + # Pass through AWS credentials + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + # AWS Role access in CodeBuild is via the container URI + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + # Pass through the default AWS region + AWS_DEFAULT_REGION + +sitepackages = False +deps = -rtest/requirements.txt +# 'download' forces tox to always upgrade pip to the latest +download = true +commands = + examples: {[testenv:base-command]commands} test/ -m "examples" diff --git a/requirements.txt b/requirements.txt index b10f60f4..34c7e6a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -boto3>=1.4.4 -cryptography>=1.8.1 +boto3>=1.10.0 +cryptography>=3.4.6 attrs>=17.4.0 -enum34; python_version < '3.4' \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a4080d60..0b64cb5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,7 @@ branch = True [coverage:report] show_missing = True +fail_under = 90 [mypy] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 2dc604cd..e64e7d4e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def read(*args): """Reads complete file contents.""" - return io.open(os.path.join(HERE, *args), encoding="utf-8").read() + return io.open(os.path.join(HERE, *args), encoding="utf-8").read() # pylint: disable=consider-using-with def get_version(): @@ -47,13 +47,12 @@ def get_requirements(): "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/dynamodb_encryption_sdk/__init__.py b/src/dynamodb_encryption_sdk/__init__.py index 7b5dba80..d1536792 100644 --- a/src/dynamodb_encryption_sdk/__init__.py +++ b/src/dynamodb_encryption_sdk/__init__.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """DynamoDB Encryption Client.""" +from dynamodb_encryption_sdk.compatability import _warn_deprecated_python from dynamodb_encryption_sdk.encrypted.client import EncryptedClient from dynamodb_encryption_sdk.encrypted.item import ( decrypt_dynamodb_item, @@ -22,6 +23,8 @@ from dynamodb_encryption_sdk.encrypted.table import EncryptedTable from dynamodb_encryption_sdk.identifiers import __version__ +_warn_deprecated_python() + __all__ = ( "decrypt_dynamodb_item", "decrypt_python_item", diff --git a/src/dynamodb_encryption_sdk/compatability.py b/src/dynamodb_encryption_sdk/compatability.py new file mode 100644 index 00000000..ccd7be9f --- /dev/null +++ b/src/dynamodb_encryption_sdk/compatability.py @@ -0,0 +1,41 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Contains logic for checking the Python Version""" +import sys +import warnings + +DEPRECATION_DATE_MAP = {"1.x": "2022-07-08", "2.x": "2022-07-15"} + + +def _warn_deprecated_python(): + """Template for deprecation of Python warning.""" + deprecated_versions = { + (2, 7): {"date": DEPRECATION_DATE_MAP["2.x"]}, + (3, 4): {"date": DEPRECATION_DATE_MAP["2.x"]}, + (3, 5): {"date": "2021-11-10"}, + (3, 6): {"date": "2021-12-19"}, + (3, 7): {"date": "2024-03-04"}, + } + py_version = (sys.version_info.major, sys.version_info.minor) + minimum_version = (3, 8) + + if py_version in deprecated_versions: + params = deprecated_versions[py_version] + warning = ( + "aws-dynamodb-encryption will no longer support Python {}.{} " + "starting {}. To continue receiving service updates, " + "bug fixes, and security updates please upgrade to Python {}.{} or " + "later. For more information, see SUPPORT_POLICY.rst: " + "https://github.com/aws/aws-dynamodb-encryption-python/blob/master/SUPPORT_POLICY.rst" + ).format(py_version[0], py_version[1], params["date"], minimum_version[0], minimum_version[1]) + warnings.warn(warning, DeprecationWarning) diff --git a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py index b41caeee..ac0aa734 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py @@ -12,18 +12,12 @@ # language governing permissions and limitations under the License. """Delegated keys.""" import abc +from typing import Dict, Optional, Text import six from dynamodb_encryption_sdk.identifiers import EncryptionKeyType # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("DelegatedKey",) @@ -45,7 +39,8 @@ class DelegatedKey(object): a :class:`NotImplementedError` detailing this. """ - @abc.abstractproperty + @property + @abc.abstractmethod def algorithm(self): # type: () -> Text """Text description of algorithm used by this delegated key.""" diff --git a/src/dynamodb_encryption_sdk/delegated_keys/jce.py b/src/dynamodb_encryption_sdk/delegated_keys/jce.py index 4edc6b2c..c2be9b5c 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/jce.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/jce.py @@ -15,6 +15,7 @@ import logging import os +from typing import Dict, Optional, Text import attr import six @@ -28,13 +29,6 @@ from . import DelegatedKey -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("JceNameLocalDelegatedKey",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/encrypted/__init__.py b/src/dynamodb_encryption_sdk/encrypted/__init__.py index e3e89ec1..d03f3f4a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/__init__.py +++ b/src/dynamodb_encryption_sdk/encrypted/__init__.py @@ -21,13 +21,6 @@ from dynamodb_encryption_sdk.materials import CryptographicMaterials # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("CryptoConfig",) diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py index bd8f2c58..e13533f3 100644 --- a/src/dynamodb_encryption_sdk/encrypted/client.py +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Any, Callable, Dict, Iterator, Optional import attr import botocore @@ -34,13 +35,6 @@ from .item import decrypt_dynamodb_item, decrypt_python_item, encrypt_dynamodb_item, encrypt_python_item -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict, Iterator, Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedClient", "EncryptedPaginator") @@ -134,7 +128,7 @@ class EncryptedClient(object): This class provides a superset of the boto3 DynamoDB client API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#client + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#client If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, ``query``, ``scan``, ``batch_write_item``, and ``batch_get_item`` methods will also diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index b491a34a..33c109cf 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -11,14 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Top-level functions for encrypting and decrypting DynamoDB items.""" -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - from dynamodb_encryption_sdk.exceptions import DecryptionError, EncryptionError from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.crypto.authentication import sign_item, verify_item_signature from dynamodb_encryption_sdk.internal.crypto.encryption import decrypt_attribute, encrypt_attribute from dynamodb_encryption_sdk.internal.formatting.material_description import ( diff --git a/src/dynamodb_encryption_sdk/encrypted/resource.py b/src/dynamodb_encryption_sdk/encrypted/resource.py index b5b71f8b..f040ea7a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/resource.py +++ b/src/dynamodb_encryption_sdk/encrypted/resource.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Optional import attr from boto3.resources.base import ServiceResource @@ -29,13 +30,6 @@ from .item import decrypt_python_item, encrypt_python_item from .table import EncryptedTable -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedResource", "EncryptedTablesCollectionManager") @@ -44,7 +38,7 @@ class EncryptedTablesCollectionManager(object): # pylint: disable=too-few-public-methods,too-many-instance-attributes """Tables collection manager that provides :class:`EncryptedTable` objects. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html :param collection: Pre-configured boto3 DynamoDB table collection manager :type collection: boto3.resources.collection.CollectionManager @@ -137,7 +131,7 @@ class EncryptedResource(object): This class provides a superset of the boto3 DynamoDB service resource API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#service-resource + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/index.html If you want to provide per-request cryptographic details, the ``batch_write_item`` and ``batch_get_item`` methods will also accept a ``crypto_config`` parameter, defining @@ -217,7 +211,7 @@ def Table(self, name, **kwargs): If any of the optional configuration values are not provided, the corresponding values for this ``EncryptedResource`` will be used. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.Table + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table :param name: The table name. :param CryptographicMaterialsProvider materials_provider: Cryptographic materials diff --git a/src/dynamodb_encryption_sdk/encrypted/table.py b/src/dynamodb_encryption_sdk/encrypted/table.py index 128cb896..98386b81 100644 --- a/src/dynamodb_encryption_sdk/encrypted/table.py +++ b/src/dynamodb_encryption_sdk/encrypted/table.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Optional import attr from boto3.dynamodb.table import BatchWriter @@ -30,13 +31,6 @@ from .client import EncryptedClient from .item import decrypt_python_item, encrypt_python_item -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedTable",) @@ -60,7 +54,7 @@ class EncryptedTable(object): This class provides a superset of the boto3 DynamoDB Table API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#table + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, ``query``, and ``scan`` methods will also accept a ``crypto_config`` parameter, defining @@ -158,7 +152,7 @@ def update_item(self, **kwargs): def batch_writer(self, overwrite_by_pkeys=None): """Create a batch writer object. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Table.batch_writer + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/batch_writer.html :type overwrite_by_pkeys: list(string) :param overwrite_by_pkeys: De-duplicate request items in buffer if match new request diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index f4edd670..5c63f095 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -14,7 +14,7 @@ from enum import Enum __all__ = ("LOGGER_NAME", "CryptoAction", "EncryptionKeyType", "KeyEncodingType") -__version__ = "1.3.0" +__version__ = "3.3.0" LOGGER_NAME = "dynamodb_encryption_sdk" USER_AGENT_SUFFIX = "DynamodbEncryptionSdkPython/{}".format(__version__) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index d5247688..622e03b5 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -16,25 +16,19 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ +from typing import Text + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.encrypted import CryptoConfig # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, SignatureValues, Tag from dynamodb_encryption_sdk.structures import AttributeActions # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Text # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("sign_item", "verify_item_signature") diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 3737d520..e8b72749 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -16,15 +16,11 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Text # noqa pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Text from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.deserialize.attribute import deserialize_attribute from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import Tag diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index b2244ac7..0d1b08e8 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -18,6 +18,7 @@ """ import abc import logging +from typing import Any, Callable, Text import attr import six @@ -32,12 +33,6 @@ from .primitives import load_rsa_key -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - __all__ = ("JavaAuthenticator", "JavaMac", "JavaSignature", "JAVA_AUTHENTICATOR") _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 564bed80..2d6f667c 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -19,6 +19,7 @@ import abc import logging import os +from typing import Any, Callable, Text import attr import six @@ -38,13 +39,6 @@ from dynamodb_encryption_sdk.internal.identifiers import MinimumKeySizes from dynamodb_encryption_sdk.internal.validators import callable_validator -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ( "JavaPadding", "SimplePadding", diff --git a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py index 0509a59b..01b4becb 100644 --- a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py +++ b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py @@ -5,24 +5,20 @@ namespace staying consistent. Directly reference at your own risk. """ # constant naming for types so pylint: disable=invalid-name -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, AnyStr, ByteString, Dict, List, Text +from typing import Any, AnyStr, ByteString, Dict, List, Text - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - ATTRIBUTE = Dict[Text, Any] # narrow this down - ITEM = Dict[Text, ATTRIBUTE] - RAW_ATTRIBUTE = ITEM - NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} - BOOLEAN = bool - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - NUMBER = int # This misses long on Python 2...figure out something for this - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - STRING = AnyStr # can be unicode but should not be bytes - BINARY = ByteString - BINARY_ATTRIBUTE = Dict[Text, BINARY] - SET = List # DynamoDB TypeSerializer converts sets into lists - MAP = RAW_ATTRIBUTE - LIST = List[RAW_ATTRIBUTE] -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +ATTRIBUTE = Dict[Text, Any] # narrow this down +ITEM = Dict[Text, ATTRIBUTE] +RAW_ATTRIBUTE = ITEM +NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} +BOOLEAN = bool +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +NUMBER = int # This misses long on Python 2...figure out something for this +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +STRING = AnyStr # can be unicode but should not be bytes +BINARY = ByteString +BINARY_ATTRIBUTE = Dict[Text, BINARY] +SET = List # DynamoDB TypeSerializer converts sets into lists +MAP = RAW_ATTRIBUTE +LIST = List[RAW_ATTRIBUTE] diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index 41058916..164ad303 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -21,24 +21,17 @@ import logging import struct from decimal import Decimal +from typing import Callable, Dict, List, Text, Union from boto3.dynamodb.types import Binary from dynamodb_encryption_sdk.exceptions import DeserializationError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.deserialize import decode_byte, decode_length, decode_tag, decode_value from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_str -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Callable, Dict, List, Text, Union # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("deserialize_attribute",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index 1f6af4a2..4657a34c 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -19,24 +19,17 @@ import io import logging import struct +from typing import Dict, Text from dynamodb_encryption_sdk.exceptions import InvalidMaterialDescriptionError, InvalidMaterialDescriptionVersionError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.identifiers import Tag from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str from .deserialize import decode_value, unpack_value from .serialize import encode_value -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("serialize", "deserialize") _LOGGER = logging.getLogger(LOGGER_NAME) _MATERIAL_DESCRIPTION_VERSION = b"\00" * 4 diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py index 1c7f7ee2..07caf22b 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py @@ -17,12 +17,7 @@ namespace staying consistent. Directly reference at your own risk. """ import struct - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Sized # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Sized __all__ = ("encode_length", "encode_value") diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index 1ca416a1..49a0097c 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -18,24 +18,17 @@ """ import io import logging +from typing import Callable from boto3.dynamodb.types import DYNAMODB_CONTEXT, Binary from dynamodb_encryption_sdk.exceptions import SerializationError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.serialize import encode_length, encode_value from dynamodb_encryption_sdk.internal.identifiers import Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_bytes -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Callable # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("serialize_attribute",) _LOGGER = logging.getLogger(LOGGER_NAME) _RESERVED = b"\x00" diff --git a/src/dynamodb_encryption_sdk/internal/identifiers.py b/src/dynamodb_encryption_sdk/internal/identifiers.py index 94d7bd41..facc1266 100644 --- a/src/dynamodb_encryption_sdk/internal/identifiers.py +++ b/src/dynamodb_encryption_sdk/internal/identifiers.py @@ -17,12 +17,7 @@ namespace staying consistent. Directly reference at your own risk. """ from enum import Enum - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Optional, Text __all__ = ( "ReservedAttributes", diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 988576b2..cdb6266c 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -18,6 +18,7 @@ """ import copy from functools import partial +from typing import Any, Callable, Dict, Iterable, Text import attr import botocore.client @@ -28,12 +29,6 @@ from dynamodb_encryption_sdk.structures import CryptoAction, EncryptionContext, TableInfo from dynamodb_encryption_sdk.transform import dict_to_ddb -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Bool, Callable, Dict, Iterable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - __all__ = ( "TableInfoCache", "crypto_config_from_kwargs", @@ -198,7 +193,7 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg :param callable decrypt_method: Method to use to decrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -220,7 +215,7 @@ def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs :param callable decrypt_method: Method to use to decrypt item :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -244,7 +239,7 @@ def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, ** :param callable decrypt_method: Method to use to decrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -276,7 +271,7 @@ def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwarg :param callable encrypt_method: Method to use to encrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable write_method: Method that writes to the table - :param **kwargs: Keyword arguments to pass to ``write_method`` + :param ``**kwargs``: Keyword arguments to pass to ``write_method`` :return: DynamoDB response :rtype: dict """ @@ -297,7 +292,7 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, :param callable encrypt_method: Method to use to encrypt items :param callable crypto_config_method: Method that accepts a table name string and provides a :class:`CryptoConfig` :param callable write_method: Method that writes to the table - :param **kwargs: Keyword arguments to pass to ``write_method`` + :param ``**kwargs``: Keyword arguments to pass to ``write_method`` :return: DynamoDB response :rtype: dict """ @@ -366,7 +361,7 @@ def _process_batch_write_response(request, response, table_crypto_config): def _item_keys_match(crypto_config, item1, item2): - # type: (CryptoConfig, Dict, Dict) -> Bool + # type: (CryptoConfig, Dict, Dict) -> bool """Determines whether the values in the primary and sort keys (if they exist) are the same :param CryptoConfig crypto_config: CryptoConfig used in encrypting the given items @@ -387,7 +382,7 @@ def _item_keys_match(crypto_config, item1, item2): def _item_attributes_match(crypto_config, plaintext_item, encrypted_item): - # type: (CryptoConfig, Dict, Dict) -> Bool + # type: (CryptoConfig, Dict, Dict) -> bool """Determines whether the unencrypted values in the plaintext items attributes are the same as those in the encrypted item. Essentially this uses brute force to cover when we don't know the primary and sort index attribute names, since they can't be encrypted. diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index ea7a55f2..73212999 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -16,6 +16,7 @@ import base64 import logging from enum import Enum +from typing import Dict, Optional, Text, Tuple import attr import boto3 @@ -28,6 +29,7 @@ from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey from dynamodb_encryption_sdk.exceptions import UnknownRegionError, UnwrappingError, WrappingError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME, USER_AGENT_SUFFIX, EncryptionKeyType, KeyEncodingType +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, MaterialDescriptionKeys from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str from dynamodb_encryption_sdk.internal.validators import dictionary_validator, iterable_validator @@ -36,15 +38,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("AwsKmsCryptographicMaterialsProvider",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 55b6376a..8a003f17 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -13,10 +13,10 @@ """Cryptographic materials provider that uses a provider store to obtain cryptographic materials.""" import logging import time -import warnings from collections import OrderedDict from enum import Enum from threading import Lock, RLock +from typing import Any, Text import attr import six @@ -29,17 +29,7 @@ from . import CryptographicMaterialsProvider from .store import ProviderStore -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - -__all__ = ( - "MostRecentProvider", - "CachingMostRecentProvider", -) +__all__ = ("CachingMostRecentProvider",) _LOGGER = logging.getLogger(LOGGER_NAME) #: Grace period during which we will return the latest local materials. This allows multiple #: threads to be using this same provider without risking lock contention or many threads @@ -135,10 +125,12 @@ def evict(self, name): @attr.s(init=False) -class MostRecentProvider(CryptographicMaterialsProvider): +@attr.s(init=False) +class CachingMostRecentProvider(CryptographicMaterialsProvider): # pylint: disable=too-many-instance-attributes """Cryptographic materials provider that uses a provider store to obtain cryptography - materials. + materials. Materials obtained from the store are cached for a user-defined amount of time, + then removed from the cache and re-retrieved from the store. When encrypting, the most recent provider that the provider store knows about will always be used. @@ -160,7 +152,6 @@ def __init__(self, provider_store, material_name, version_ttl, cache_size=1000): # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 - warnings.warn("MostRecentProvider is deprecated, use CachingMostRecentProvider instead.", DeprecationWarning) self._provider_store = provider_store self._material_name = material_name self._version_ttl = version_ttl @@ -185,15 +176,26 @@ def decryption_materials(self, encryption_context): :param EncryptionContext encryption_context: Encryption context for request :raises AttributeError: if no decryption materials are available """ + provider = None + version = self._provider_store.version_from_material_description(encryption_context.material_description) - try: - _LOGGER.debug("Looking in cache for decryption materials provider version %d", version) - _, provider = self._cache.get(version) - except KeyError: - _LOGGER.debug("Decryption materials provider not found in cache") + + ttl_action = self._ttl_action(version, _DECRYPT_ACTION) + + if ttl_action is TtlActions.EXPIRED: + self._cache.evict(self._version) + + _LOGGER.debug('TTL Action "%s" when getting decryption materials', ttl_action.name) + if ttl_action is TtlActions.LIVE: try: - provider = self._provider_store.provider(self._material_name, version) - self._cache.put(version, (time.time(), provider)) + _LOGGER.debug("Looking in cache for encryption materials provider version %d", version) + _, provider = self._cache.get(version) + except KeyError: + _LOGGER.debug("Decryption materials provider not found in cache") + + if provider is None: + try: + provider = self._get_provider_with_grace_period(version, ttl_action) except InvalidVersionError: _LOGGER.exception("Unable to get decryption materials from provider store.") raise AttributeError("No decryption materials available") @@ -272,7 +274,7 @@ def _get_provider_with_grace_period(self, version, ttl_action): :raises AttributeError: if provider could not locate version """ blocking_wait = bool(ttl_action is TtlActions.EXPIRED) - acquired = self._lock.acquire(blocking_wait) + acquired = self._lock.acquire(blocking_wait) # pylint: disable=consider-using-with if not acquired: # We failed to acquire the lock. # If blocking, we will never reach this point. @@ -310,7 +312,7 @@ def _get_most_recent_version(self, ttl_action): :rtype: CryptographicMaterialsProvider """ blocking_wait = bool(ttl_action is TtlActions.EXPIRED) - acquired = self._lock.acquire(blocking_wait) + acquired = self._lock.acquire(blocking_wait) # pylint: disable=consider-using-with if not acquired: # We failed to acquire the lock. @@ -385,52 +387,8 @@ def encryption_materials(self, encryption_context): def refresh(self): # type: () -> None """Clear all local caches for this provider.""" - _LOGGER.debug("Refreshing MostRecentProvider instance.") + _LOGGER.debug("Refreshing CachingMostRecentProvider instance.") with self._lock: self._cache.clear() self._version = None # type: int # pylint: disable=attribute-defined-outside-init self._last_updated = None # type: float # pylint: disable=attribute-defined-outside-init - - -@attr.s(init=False) -class CachingMostRecentProvider(MostRecentProvider): - """Cryptographic materials provider that uses a provider store to obtain cryptography - materials. Materials obtained from the store are cached for a user-defined amount of time, - then removed from the cache and re-retrieved from the store. - - When encrypting, the most recent provider that the provider store knows about will always - be used. - """ - - def decryption_materials(self, encryption_context): - # type: (EncryptionContext) -> CryptographicMaterials - """Return decryption materials. - - :param EncryptionContext encryption_context: Encryption context for request - :raises AttributeError: if no decryption materials are available - """ - provider = None - - version = self._provider_store.version_from_material_description(encryption_context.material_description) - - ttl_action = self._ttl_action(version, _DECRYPT_ACTION) - - if ttl_action is TtlActions.EXPIRED: - self._cache.evict(self._version) - - _LOGGER.debug('TTL Action "%s" when getting decryption materials', ttl_action.name) - if ttl_action is TtlActions.LIVE: - try: - _LOGGER.debug("Looking in cache for encryption materials provider version %d", version) - _, provider = self._cache.get(version) - except KeyError: - _LOGGER.debug("Decryption materials provider not found in cache") - - if provider is None: - try: - provider = self._get_provider_with_grace_period(version, ttl_action) - except InvalidVersionError: - _LOGGER.exception("Unable to get decryption materials from provider store.") - raise AttributeError("No decryption materials available") - - return provider.decryption_materials(encryption_context) diff --git a/src/dynamodb_encryption_sdk/material_providers/static.py b/src/dynamodb_encryption_sdk/material_providers/static.py index 966002cb..77af8478 100644 --- a/src/dynamodb_encryption_sdk/material_providers/static.py +++ b/src/dynamodb_encryption_sdk/material_providers/static.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials provider for use with pre-configured encryption and decryption materials.""" +from typing import Optional + import attr from dynamodb_encryption_sdk.materials import CryptographicMaterials # noqa pylint: disable=unused-import @@ -19,13 +21,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("StaticCryptographicMaterialsProvider",) diff --git a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py index e03b57b9..1948c388 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """Cryptographic materials provider stores.""" import abc +from typing import Optional, Text import six @@ -20,13 +21,6 @@ CryptographicMaterialsProvider, ) -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("ProviderStore",) diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index 46d7410f..da545ba4 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -13,6 +13,7 @@ """Meta cryptographic provider store.""" import logging from enum import Enum +from typing import Dict, Optional, Text, Tuple import attr import botocore @@ -29,13 +30,6 @@ from . import ProviderStore -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("MetaStore",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/material_providers/wrapped.py b/src/dynamodb_encryption_sdk/material_providers/wrapped.py index 13f6a346..416156c7 100644 --- a/src/dynamodb_encryption_sdk/material_providers/wrapped.py +++ b/src/dynamodb_encryption_sdk/material_providers/wrapped.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials provider to use ephemeral content encryption keys wrapped by delegated keys.""" +from typing import Dict, Optional, Text + import attr import six @@ -22,19 +24,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("WrappedCryptographicMaterialsProvider",) diff --git a/src/dynamodb_encryption_sdk/materials/__init__.py b/src/dynamodb_encryption_sdk/materials/__init__.py index 66797ec6..3b9788d0 100644 --- a/src/dynamodb_encryption_sdk/materials/__init__.py +++ b/src/dynamodb_encryption_sdk/materials/__init__.py @@ -12,20 +12,12 @@ # language governing permissions and limitations under the License. """Cryptographic materials are containers that provide delegated keys for cryptographic operations.""" import abc +from typing import Dict, Text import six from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text # noqa pylint: disable=unused-import - - from mypy_extensions import NoReturn # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("CryptographicMaterials", "EncryptionMaterials", "DecryptionMaterials") @@ -33,7 +25,8 @@ class CryptographicMaterials(object): """Base class for all cryptographic materials.""" - @abc.abstractproperty + @property + @abc.abstractmethod def material_description(self): # type: () -> Dict[Text, Text] """Material description to use with these cryptographic materials. @@ -42,7 +35,8 @@ def material_description(self): :rtype: dict """ - @abc.abstractproperty + @property + @abc.abstractmethod def encryption_key(self): # type: () -> DelegatedKey """Delegated key used for encrypting attributes. @@ -51,7 +45,8 @@ def encryption_key(self): :rtype: DelegatedKey """ - @abc.abstractproperty + @property + @abc.abstractmethod def decryption_key(self): # type: () -> DelegatedKey """Delegated key used for decrypting attributes. @@ -60,7 +55,8 @@ def decryption_key(self): :rtype: DelegatedKey """ - @abc.abstractproperty + @property + @abc.abstractmethod def signing_key(self): # type: () -> DelegatedKey """Delegated key used for calculating digital signatures. @@ -69,7 +65,8 @@ def signing_key(self): :rtype: DelegatedKey """ - @abc.abstractproperty + @property + @abc.abstractmethod def verification_key(self): # type: () -> DelegatedKey """Delegated key used for verifying digital signatures. @@ -84,7 +81,6 @@ class EncryptionMaterials(CryptographicMaterials): @property def decryption_key(self): - # type: () -> NoReturn """Encryption materials do not provide decryption keys. :raises NotImplementedError: because encryption materials do not contain decryption keys @@ -93,7 +89,6 @@ def decryption_key(self): @property def verification_key(self): - # type: () -> NoReturn """Encryption materials do not provide verification keys. :raises NotImplementedError: because encryption materials do not contain verification keys @@ -106,7 +101,6 @@ class DecryptionMaterials(CryptographicMaterials): @property def encryption_key(self): - # type: () -> NoReturn """Decryption materials do not provide encryption keys. :raises NotImplementedError: because decryption materials do not contain encryption keys @@ -115,7 +109,6 @@ def encryption_key(self): @property def signing_key(self): - # type: () -> NoReturn """Decryption materials do not provide signing keys. :raises NotImplementedError: because decryption materials do not contain signing keys diff --git a/src/dynamodb_encryption_sdk/materials/raw.py b/src/dynamodb_encryption_sdk/materials/raw.py index 7c2e85e4..d2587339 100644 --- a/src/dynamodb_encryption_sdk/materials/raw.py +++ b/src/dynamodb_encryption_sdk/materials/raw.py @@ -23,6 +23,7 @@ that you use wrapped cryptographic materials instead. """ import copy +from typing import Dict, Optional, Text import attr import six @@ -31,13 +32,6 @@ from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("RawEncryptionMaterials", "RawDecryptionMaterials") diff --git a/src/dynamodb_encryption_sdk/materials/wrapped.py b/src/dynamodb_encryption_sdk/materials/wrapped.py index f85924ea..74d2784c 100644 --- a/src/dynamodb_encryption_sdk/materials/wrapped.py +++ b/src/dynamodb_encryption_sdk/materials/wrapped.py @@ -13,6 +13,7 @@ """Cryptographic materials to use ephemeral content encryption keys wrapped by delegated keys.""" import base64 import copy +from typing import Dict, Optional, Text import attr import six @@ -25,13 +26,6 @@ from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import CryptographicMaterials -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("WrappedCryptographicMaterials",) _DEFAULT_CONTENT_ENCRYPTION_ALGORITHM = "AES/256" _WRAPPING_TRANSFORMATION = {"AES": "AESWrap", "RSA": "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"} diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index 61b329c6..16976b98 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """Common structures used by the DynamoDB Encryption Client.""" import copy +from typing import Dict, Iterable, List, Optional, Set, Text import attr import six @@ -22,13 +23,6 @@ from .identifiers import CryptoAction -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Iterable, List, Optional, Set, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptionContext", "AttributeActions", "TableIndex", "TableInfo") @@ -176,7 +170,7 @@ def set_index_keys(self, *keys): SIGN_ONLY -> SIGN_ONLY ENCRYPT_AND_SIGN -> SIGN_ONLY - :param str *keys: Attribute names to treat as indexed + :param str ``*keys``: Attribute names to treat as indexed :raises InvalidArgumentError: if a custom action was previously set for any specified attributes """ diff --git a/src/dynamodb_encryption_sdk/transform.py b/src/dynamodb_encryption_sdk/transform.py index 347024d3..d79b8504 100644 --- a/src/dynamodb_encryption_sdk/transform.py +++ b/src/dynamodb_encryption_sdk/transform.py @@ -11,11 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Helper tools for translating between native and DynamoDB items.""" -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Dict # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Any, Dict from boto3.dynamodb.types import TypeDeserializer, TypeSerializer diff --git a/src/pylintrc b/src/pylintrc index bc0406f6..399920a7 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -8,6 +8,7 @@ disable = useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string, [BASIC] # Allow function names up to 50 characters diff --git a/test/README.rst b/test/README.rst index 42cace9f..747522bb 100644 --- a/test/README.rst +++ b/test/README.rst @@ -8,6 +8,8 @@ In order to run these integration tests successfully, these things which must be `automatically discoverable credential locations`_. #. The ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID`` environment variable must be set to a valid `AWS KMS CMK ARN`_ that can be used by the available credentials. +#. The ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID`` and ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2`` environment variables + must be set to two related AWS KMS Multi-Region key ids in different regions. #. The ``DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME`` environment variable must be set to a valid DynamoDB table name, in the default region, to which the discoverable credentials have read, write, and describe permissions. @@ -20,16 +22,12 @@ Updating Upstream Requirements The purpose of the upstream requirements files is to provide a stable list of packages for dependencies to run downstream tests of the DynamoDB Encryption -Client. In order to update the upstream requirements in `upstream-requirements-py37.txt` -and `upstream-requirements-py27.txt`, run these commands: +Client. In order to update the upstream requirements in `upstream-requirements-py37.txt`, +run these commands:: - .. code:: + $ tox -e freeze-upstream-requirements-py37 - $ tox -e freeze-upstream-requirements-py27 - $ tox -e freeze-upstream-requirements-py37 +Test them using:: -Test them using: + $ tox -e test-upstream-requirements-py37 - .. code:: - $ tox -e test-upstream-requirements-py27 - $ tox -e test-upstream-requirements-py37 diff --git a/test/acceptance/acceptance_test_generators.py b/test/acceptance/acceptance_test_generators.py index 9ba01174..1c513bd3 100644 --- a/test/acceptance/acceptance_test_generators.py +++ b/test/acceptance/acceptance_test_generators.py @@ -43,7 +43,7 @@ def load_scenarios(online): into a shared method. """ # pylint: disable=too-many-locals - with open(_SCENARIO_FILE) as f: + with open(_SCENARIO_FILE, encoding="utf-8") as f: scenarios = json.load(f) keys_file = _filename_from_uri(scenarios["keys"]) keys = _load_keys(keys_file) @@ -128,7 +128,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info): if table: table.delete() - with open(ciphertext_file, "w") as outfile: + with open(ciphertext_file, "w", encoding="utf-8") as outfile: json.dump(data_table_output, outfile, indent=4) if metatable: @@ -137,7 +137,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info): metastore_output[metastore_info["table_name"]].append(ddb_to_json(wrapping_key)) metastore_ciphertext_file = _filename_from_uri(metastore_info["ciphertext"]) - with open(metastore_ciphertext_file, "w") as outfile: + with open(metastore_ciphertext_file, "w", encoding="utf-8") as outfile: json.dump(metastore_output, outfile, indent=4) metatable.delete() diff --git a/test/acceptance/acceptance_test_utils.py b/test/acceptance/acceptance_test_utils.py index c4f06b46..a7fd4c03 100644 --- a/test/acceptance/acceptance_test_utils.py +++ b/test/acceptance/acceptance_test_utils.py @@ -61,7 +61,7 @@ def _decode_item(item): def _build_plaintext_items(plaintext_file, version): # pylint: disable=too-many-locals - with open(plaintext_file) as f: + with open(plaintext_file, encoding="utf-8") as f: plaintext_data = json.load(f) actions = {} @@ -92,7 +92,7 @@ def _build_plaintext_items(plaintext_file, version): def _load_ciphertext_items(ciphertext_file): - with open(ciphertext_file) as f: + with open(ciphertext_file, encoding="utf-8") as f: ciphertexts = json.load(f) for _table, items in ciphertexts.items(): @@ -103,7 +103,7 @@ def _load_ciphertext_items(ciphertext_file): def _load_keys(keys_file): - with open(keys_file) as f: + with open(keys_file, encoding="utf-8") as f: return json.load(f) @@ -165,7 +165,7 @@ def _meta_table_prep(table_name, items_filename): table = boto3.resource("dynamodb", region_name="us-west-2").Table(table_name) table.wait_until_exists() try: - with open(_filename_from_uri(items_filename)) as f: + with open(_filename_from_uri(items_filename), encoding="utf-8") as f: table_data = json.load(f) request_items = {} @@ -255,7 +255,7 @@ def _expand_items(ciphertext_items, plaintext_items): def load_scenarios(online): # pylint: disable=too-many-locals - with open(_SCENARIO_FILE) as f: + with open(_SCENARIO_FILE, encoding="utf-8") as f: scenarios = json.load(f) keys_file = _filename_from_uri(scenarios["keys"]) keys = _load_keys(keys_file) diff --git a/test/freeze-upstream-requirements.sh b/test/freeze-upstream-requirements.sh index 293ae16d..2be3824d 100755 --- a/test/freeze-upstream-requirements.sh +++ b/test/freeze-upstream-requirements.sh @@ -6,5 +6,5 @@ if [ -z ${1} ]; then fi pip install -r requirements.txt -pip install -r test/requirements.txt +pip install -r dev_requirements/test-requirements.txt pip freeze > ${1} diff --git a/test/functional/functional_test_vector_generators.py b/test/functional/functional_test_vector_generators.py index 02906f35..9e711ad2 100644 --- a/test/functional/functional_test_vector_generators.py +++ b/test/functional/functional_test_vector_generators.py @@ -104,14 +104,14 @@ def _decode_complex_value(_value): def attribute_test_vectors(mode): filepath = _ATTRIBUTE_TEST_VECTOR_FILE_TEMPLATE.format(mode=mode) - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: yield (decode_value(vector["attribute"]), base64.b64decode(codecs.encode(vector["serialized"], "utf-8"))) def material_description_test_vectors(): - with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE) as f: + with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: yield (vector["material_description"], decode_value({"B": codecs.encode(vector["serialized"], "utf-8")})) @@ -125,7 +125,7 @@ def material_description_test_vectors(): def string_to_sign_test_vectors(): - with open(_STRING_TO_SIGN_TEST_VECTORS_FILE) as f: + with open(_STRING_TO_SIGN_TEST_VECTORS_FILE, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: item = {key: decode_value(value["value"]) for key, value in vector["item"].items()} diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index 6a39d4cf..059e14b6 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -23,6 +23,10 @@ hypothesis.HealthCheck.too_slow, hypothesis.HealthCheck.data_too_large, hypothesis.HealthCheck.large_base_example, + # Hypothesis requires that we acknowledge that the example_table fixure + # is not reset between examples generated by hypothesis.given. + # This is the desired behavior for example_table, so supress this check + hypothesis.HealthCheck.function_scoped_fixture, ), deadline=None, ) diff --git a/test/functional/internal/test_str_ops.py b/test/functional/internal/test_str_ops.py index 704e3e3f..1d9f7443 100644 --- a/test/functional/internal/test_str_ops.py +++ b/test/functional/internal/test_str_ops.py @@ -26,8 +26,8 @@ ( ("asdf", "asdf"), (b"asdf", "asdf"), - (codecs.encode(u"Предисловие", "utf-8"), u"Предисловие"), - (u"Предисловие", u"Предисловие"), + (codecs.encode("Предисловие", "utf-8"), "Предисловие"), + ("Предисловие", "Предисловие"), ), ) def test_to_str(data, expected_output): @@ -41,8 +41,8 @@ def test_to_str(data, expected_output): ("asdf", b"asdf"), (b"asdf", b"asdf"), (b"\x3a\x00\x99", b"\x3a\x00\x99"), - (u"Предисловие", codecs.encode(u"Предисловие", "utf-8")), - (codecs.encode(u"Предисловие", "utf-8"), codecs.encode(u"Предисловие", "utf-8")), + ("Предисловие", codecs.encode("Предисловие", "utf-8")), + (codecs.encode("Предисловие", "utf-8"), codecs.encode("Предисловие", "utf-8")), ), ) def test_to_bytes(data, expected_output): diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index 3e46abe2..778e26af 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -12,7 +12,6 @@ # language governing permissions and limitations under the License. """Functional tests for ``dynamodb_encryption_sdk.material_providers.most_recent``.""" import time -import warnings from collections import defaultdict import pytest @@ -20,11 +19,7 @@ from dynamodb_encryption_sdk.exceptions import NoKnownVersionError from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider -from dynamodb_encryption_sdk.material_providers.most_recent import ( - CachingMostRecentProvider, - MostRecentProvider, - TtlActions, -) +from dynamodb_encryption_sdk.material_providers.most_recent import CachingMostRecentProvider, TtlActions from dynamodb_encryption_sdk.material_providers.store import ProviderStore from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import @@ -76,12 +71,11 @@ def version_from_material_description(self, material_description): return material_description -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_constructor(provider_class): +def test_constructor(): """Tests that when the cache is expired on encrypt, we evict the entry from the cache.""" store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=1.0, cache_size=42) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=1.0, cache_size=42) assert provider._provider_store == store assert provider._material_name == name @@ -277,10 +271,9 @@ def test_get_most_recent_version_grace_period_lock_not_acquired(): assert store.provider_calls == expected_calls -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_failed_lock_acquisition(provider_class): +def test_failed_lock_acquisition(): store = MagicMock(__class__=ProviderStore) - provider = provider_class(provider_store=store, material_name="my material", version_ttl=10.0) + provider = CachingMostRecentProvider(provider_store=store, material_name="my material", version_ttl=10.0) provider._version = 9 provider._cache.put(provider._version, (time.time(), sentinel.nine)) @@ -291,11 +284,10 @@ def test_failed_lock_acquisition(provider_class): assert not store.mock_calls -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_encryption_materials_cache_use(provider_class): +def test_encryption_materials_cache_use(): store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=10.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=10.0) test1 = provider.encryption_materials(sentinel.encryption_context_1) assert test1 is sentinel.material_0_encryption @@ -320,11 +312,10 @@ def test_encryption_materials_cache_use(provider_class): assert store.provider_calls == expected_calls -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_encryption_materials_cache_expired(provider_class): +def test_encryption_materials_cache_expired(): store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=0.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) test1 = provider.encryption_materials(sentinel.encryption_context_1) assert test1 is sentinel.material_0_encryption @@ -354,12 +345,11 @@ def test_encryption_materials_cache_expired(provider_class): assert store.provider_calls == expected_calls -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_encryption_materials_cache_expired_cache_removed(provider_class): +def test_encryption_materials_cache_expired_cache_removed(): """Tests that when the cache is expired on encrypt, we evict the entry from the cache.""" store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=0.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) provider._cache = MagicMock() provider._cache.get.return_value = (0.0, MagicMock()) @@ -379,8 +369,7 @@ def test_decryption_materials_cache_expired_cache_removed(): provider._cache.evict.assert_called_once() -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_encryption_materials_cache_in_grace_period_acquire_lock(provider_class): +def test_encryption_materials_cache_in_grace_period_acquire_lock(): """Test encryption grace period behavior. When the TTL is GRACE_PERIOD and we successfully acquire the lock for retrieving new materials, @@ -388,7 +377,7 @@ def test_encryption_materials_cache_in_grace_period_acquire_lock(provider_class) """ store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=0.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) provider._grace_period = 10.0 test1 = provider.encryption_materials(sentinel.encryption_context_1) @@ -422,8 +411,7 @@ def test_encryption_materials_cache_in_grace_period_acquire_lock(provider_class) assert store.provider_calls == expected_calls -@pytest.mark.parametrize("provider_class", (MostRecentProvider, CachingMostRecentProvider)) -def test_encryption_materials_cache_in_grace_period_fail_to_acquire_lock(provider_class): +def test_encryption_materials_cache_in_grace_period_fail_to_acquire_lock(): """Test encryption grace period behavior. When the TTL is GRACE_PERIOD and we fail to acquire the lock for retrieving new materials, @@ -431,7 +419,7 @@ def test_encryption_materials_cache_in_grace_period_fail_to_acquire_lock(provide """ store = MockProviderStore() name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=0.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) provider._grace_period = 10.0 test1 = provider.encryption_materials(sentinel.encryption_context_1) @@ -463,43 +451,10 @@ def test_encryption_materials_cache_in_grace_period_fail_to_acquire_lock(provide assert store.provider_calls == expected_calls -@pytest.mark.parametrize("provider_class", (CachingMostRecentProvider, CachingMostRecentProvider)) -def test_decryption_materials_cache_use(provider_class): - store = MockProviderStore() - name = "material" - provider = provider_class(provider_store=store, material_name=name, version_ttl=10.0) - - context = MagicMock(material_description=0) - - test1 = provider.decryption_materials(context) - assert test1 is sentinel.material_0_decryption - - assert len(provider._cache._cache) == 1 - - expected_calls = [("version_from_material_description", 0), ("get_or_create_provider", name, 0)] - - assert store.provider_calls == expected_calls - - test2 = provider.decryption_materials(context) - assert test2 is sentinel.material_0_decryption - - assert len(provider._cache._cache) == 1 - - expected_calls.append(("version_from_material_description", 0)) - - assert store.provider_calls == expected_calls - - -def test_most_recent_provider_decryption_materials_cache_expired(): - """Test decryption expiration behavior for MostRecentProvider. - - When using a MostRecentProvider and the cache is expired on decryption, we do not retrieve new - materials from the provider store. Note that this test only runs for MostRecentProvider, to ensure that our legacy - behavior has not changed. - """ +def test_decryption_materials_cache_use(): store = MockProviderStore() name = "material" - provider = MostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) + provider = CachingMostRecentProvider(provider_store=store, material_name=name, version_ttl=10.0) context = MagicMock(material_description=0) @@ -517,7 +472,6 @@ def test_most_recent_provider_decryption_materials_cache_expired(): assert len(provider._cache._cache) == 1 - # The MostRecentProvider does not use TTLs on decryption, so we should not see a new call to the provider store expected_calls.append(("version_from_material_description", 0)) assert store.provider_calls == expected_calls @@ -629,13 +583,3 @@ def test_caching_provider_decryption_materials_cache_in_grace_period_fail_to_acq def test_cache_use_encrypt(mock_metastore, example_table, caplog): check_metastore_cache_use_encrypt(mock_metastore, TEST_TABLE_NAME, caplog) - - -def test_most_recent_provider_deprecated(): - warnings.simplefilter("error") - - with pytest.raises(DeprecationWarning) as excinfo: - store = MockProviderStore() - name = "material" - MostRecentProvider(provider_store=store, material_name=name, version_ttl=0.0) - excinfo.match("MostRecentProvider is deprecated, use CachingMostRecentProvider instead") diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index 71106ef7..006d5319 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -27,16 +27,18 @@ raise AWS_KMS_KEY_ID = "AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID" +AWS_KMS_MRK_KEY_ID = "AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID" +AWS_KMS_MRK_KEY_ID_2 = "AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2" DDB_TABLE_NAME = "DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME" -def cmk_arn_value(): +def cmk_arn_value(env_variable=AWS_KMS_KEY_ID): """Retrieve the target CMK ARN from environment variable.""" - arn = os.environ.get(AWS_KMS_KEY_ID, None) + arn = os.environ.get(env_variable, None) if arn is None: raise ValueError( 'Environment variable "{}" must be set to a valid KMS CMK ARN for integration tests to run'.format( - AWS_KMS_KEY_ID + env_variable ) ) if arn.startswith("arn:") and ":alias/" not in arn: @@ -47,7 +49,19 @@ def cmk_arn_value(): @pytest.fixture def cmk_arn(): """As of Pytest 4.0.0, fixtures cannot be called directly.""" - return cmk_arn_value() + return cmk_arn_value(AWS_KMS_KEY_ID) + + +@pytest.fixture +def cmk_mrk_arn(): + """As of Pytest 4.0.0, fixtures cannot be called directly.""" + return cmk_arn_value(AWS_KMS_MRK_KEY_ID) + + +@pytest.fixture +def second_cmk_mrk_arn(): + """As of Pytest 4.0.0, fixtures cannot be called directly.""" + return cmk_arn_value(AWS_KMS_MRK_KEY_ID_2) def _build_kms_cmp(require_attributes): diff --git a/test/pylintrc b/test/pylintrc index ce2bba60..f63b3263 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -10,10 +10,12 @@ disable = protected-access, # raised when calling _ methods redefined-outer-name, # raised when using pytest-mock unused-argument, # raised when patches and fixtures are needed but not called + no-self-use, # raised on Classes in tests used for logically grouping tests # All below are disabled because we need to support Python 2 useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string, [DESIGN] max-args = 10 diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index 24ace5ac..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -hypothesis>=5.0.0,<6.0.0;python_version>='3' -hypothesis==4.57.1;python_version=='2.7' -mock -moto>=1.3.8 -pytest>=3.4.0 -pytest-cov -pytest-mock -pytest-xdist -boto3 -botocore diff --git a/test/source-build-check.sh b/test/source-build-check.sh index 22e31a83..1d6986a1 100755 --- a/test/source-build-check.sh +++ b/test/source-build-check.sh @@ -26,7 +26,7 @@ EXTRACTEDDIR=$(ls | tail -1) cd ${EXTRACTEDDIR} echo "Installing requirements from extracted source build." -pip install -r test/requirements.txt +pip install -r dev_requirements/test-requirements.txt pip install -e . echo "Running tests from extracted source build." diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index edcd301d..7fdc9f83 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -220,36 +220,32 @@ def test_loaded_key_infos(): assert cmp._regional_clients == {} -@pytest.mark.parametrize( - "kwargs", - [ - pytest.param(val, id=str(val)) - for val in all_possible_combinations_kwargs( - dict(), - dict(botocore_session=botocore.session.Session()), - dict(grant_tokens=("sdvoaweih", "auwshefiouawh")), - dict(material_description={"asoiufeoia": "soajfijewi"}), - dict( - regional_clients={ - "my-region-1": boto3.session.Session().client( - "kms", region_name="not-a-real-region", endpoint_url="https://not-a-real-url" - ) - } - ), - ) - ], -) -def test_kms_cmp_values_set(kwargs): - cmp = AwsKmsCryptographicMaterialsProvider(key_id="example_key_id", **kwargs) +def test_kms_cmp_values_set(): + # These aren't parametrized to avoid issues with pytest-xdist test mismatches + # due to different session objects per process + for kwargs in all_possible_combinations_kwargs( + {}, + dict(botocore_session=botocore.session.Session()), + dict(grant_tokens=("sdvoaweih", "auwshefiouawh")), + dict(material_description={"asoiufeoia": "soajfijewi"}), + dict( + regional_clients={ + "my-region-1": boto3.session.Session().client( + "kms", region_name="not-a-real-region", endpoint_url="https://not-a-real-url" + ) + } + ), + ): + cmp = AwsKmsCryptographicMaterialsProvider(key_id="example_key_id", **kwargs) - assert cmp._key_id == "example_key_id" + assert cmp._key_id == "example_key_id" - if "botocore_session" in kwargs: - assert cmp._botocore_session == kwargs["botocore_session"] + if "botocore_session" in kwargs: + assert cmp._botocore_session == kwargs["botocore_session"] - assert cmp._grant_tokens == kwargs.get("grant_tokens", ()) - assert cmp._material_description == kwargs.get("material_description", {}) - assert cmp._regional_clients == kwargs.get("regional_clients", {}) + assert cmp._grant_tokens == kwargs.get("grant_tokens", ()) + assert cmp._material_description == kwargs.get("material_description", {}) + assert cmp._regional_clients == kwargs.get("regional_clients", {}) def test_add_regional_client_known_region(default_kms_cmp, patch_boto3_session): diff --git a/test/unit/test_compatability.py b/test/unit/test_compatability.py new file mode 100644 index 00000000..314017e9 --- /dev/null +++ b/test/unit/test_compatability.py @@ -0,0 +1,37 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite for dynamodb_encryption_sdk.compatability.""" +import sys + +import mock +import pytest + +from dynamodb_encryption_sdk.compatability import _warn_deprecated_python + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +class TestWarnDeprecatedPython: + def test_happy_version(self, recwarn): + with mock.patch.object(sys, "version_info") as v_info: + v_info.major = 3 + v_info.minor = 8 + _warn_deprecated_python() + assert len(recwarn) == 0 + + def test_below_warn(self): + with mock.patch.object(sys, "version_info") as v_info: + v_info.major = 2 + v_info.minor = 7 + with pytest.warns(DeprecationWarning): + _warn_deprecated_python() diff --git a/test/upstream-requirements-py27.txt b/test/upstream-requirements-py27.txt deleted file mode 100644 index 21acbcd8..00000000 --- a/test/upstream-requirements-py27.txt +++ /dev/null @@ -1,78 +0,0 @@ -apipkg==1.5 -asn1crypto==1.0.1 -atomicwrites==1.3.0 -attrs==19.2.0 -aws-sam-translator==1.15.0 -aws-xray-sdk==2.4.2 -backports.ssl-match-hostname==3.7.0.1 -backports.tempfile==1.0 -backports.weakref==1.0.post1 -boto==2.49.0 -boto3==1.9.246 -botocore==1.12.246 -certifi==2019.9.11 -cffi==1.12.3 -cfn-lint==0.24.4 -chardet==3.0.4 -configparser==4.0.2 -contextlib2==0.6.0.post1 -cookies==2.2.1 -coverage==4.5.4 -cryptography==3.2 -DateTime==4.3 -docker==4.1.0 -docutils==0.15.2 -ecdsa==0.13.3 -enum34==1.1.6 -execnet==1.7.1 -funcsigs==1.0.2 -functools32==3.2.3.post2 -future==0.18.0 -futures==3.3.0 -hypothesis==4.40.0 -idna==2.8 -importlib-metadata==0.23 -ipaddress==1.0.22 -Jinja2==2.10.3 -jmespath==0.9.4 -jsondiff==1.1.2 -jsonpatch==1.24 -jsonpickle==1.2 -jsonpointer==2.0 -jsonschema==3.1.1 -MarkupSafe==1.1.1 -mock==3.0.5 -more-itertools==5.0.0 -moto==1.3.13 -packaging==19.2 -pathlib2==2.3.5 -pluggy==0.13.0 -py==1.8.0 -pyasn1==0.4.7 -pycparser==2.19 -pyparsing==2.4.2 -pyrsistent==0.15.4 -pytest==4.6.5 -pytest-cov==2.8.1 -pytest-forked==1.0.2 -pytest-mock==1.11.1 -pytest-xdist==1.30.0 -python-dateutil==2.8.0 -python-jose==3.0.1 -pytz==2019.3 -PyYAML==5.1.2 -requests==2.22.0 -responses==0.10.6 -rsa==4.0 -s3transfer==0.2.1 -scandir==1.10.0 -six==1.12.0 -sshpubkeys==3.1.0 -urllib3==1.25.6 -wcwidth==0.1.7 -websocket-client==0.56.0 -Werkzeug==0.16.0 -wrapt==1.11.2 -xmltodict==0.12.0 -zipp==0.6.0 -zope.interface==4.6.0 diff --git a/test/upstream-requirements-py311.txt b/test/upstream-requirements-py311.txt new file mode 100644 index 00000000..1c3051be --- /dev/null +++ b/test/upstream-requirements-py311.txt @@ -0,0 +1,38 @@ +attrs==22.2.0 +boto3==1.20.51 +botocore==1.23.51 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.0.1 +coverage==7.1.0 +cryptography==42.0.4 +execnet==1.9.0 +hypothesis==6.31.6 +idna==3.4 +iniconfig==2.0.0 +Jinja2==3.1.3 +jmespath==0.10.0 +MarkupSafe==2.1.2 +mock==4.0.3 +moto==3.0.2 +packaging==23.0 +pluggy==1.5.0 +py==1.11.0 +pycparser==2.21 +pytest==8.2.0 +pytest-cov==3.0.0 +pytest-forked==1.6.0 +pytest-mock==3.10.0 +pytest-xdist==3.2.0 +python-dateutil==2.8.2 +pytz==2022.7.1 +requests==2.31.0 +responses==0.22.0 +s3transfer==0.5.2 +six==1.16.0 +sortedcontainers==2.4.0 +toml==0.10.2 +types-toml==0.10.8.5 +urllib3==1.26.18 +Werkzeug==3.0.3 +xmltodict==0.13.0 diff --git a/test/upstream-requirements-py37.txt b/test/upstream-requirements-py37.txt deleted file mode 100644 index 6b0de40c..00000000 --- a/test/upstream-requirements-py37.txt +++ /dev/null @@ -1,65 +0,0 @@ -apipkg==1.5 -asn1crypto==1.0.1 -atomicwrites==1.3.0 -attrs==19.2.0 -aws-sam-translator==1.15.0 -aws-xray-sdk==2.4.2 -boto==2.49.0 -boto3==1.9.246 -botocore==1.12.246 -certifi==2019.9.11 -cffi==1.12.3 -cfn-lint==0.24.4 -chardet==3.0.4 -coverage==4.5.4 -cryptography==3.2 -DateTime==4.3 -docker==4.1.0 -docutils==0.15.2 -ecdsa==0.13.3 -execnet==1.7.1 -future==0.18.0 -hypothesis==4.40.0 -idna==2.8 -importlib-metadata==0.23 -Jinja2==2.10.3 -jmespath==0.9.4 -jsondiff==1.1.2 -jsonpatch==1.24 -jsonpickle==1.2 -jsonpointer==2.0 -jsonschema==3.1.1 -MarkupSafe==1.1.1 -mock==3.0.5 -more-itertools==7.2.0 -moto==1.3.13 -packaging==19.2 -pluggy==0.13.0 -py==1.8.0 -pyasn1==0.4.7 -pycparser==2.19 -pyparsing==2.4.2 -pyrsistent==0.15.4 -pytest==5.2.1 -pytest-cov==2.8.1 -pytest-forked==1.0.2 -pytest-mock==1.11.1 -pytest-xdist==1.30.0 -python-dateutil==2.8.0 -python-jose==3.0.1 -pytz==2019.3 -PyYAML==5.1.2 -requests==2.22.0 -responses==0.10.6 -rsa==4.0 -s3transfer==0.2.1 -six==1.12.0 -sshpubkeys==3.1.0 -urllib3==1.25.6 -wcwidth==0.1.7 -websocket-client==0.56.0 -Werkzeug==0.16.0 -wrapt==1.11.2 -xmltodict==0.12.0 -zipp==0.6.0 -zope.interface==4.6.0 diff --git a/test/upstream.md b/test/upstream.md new file mode 100644 index 00000000..aeb3ed63 --- /dev/null +++ b/test/upstream.md @@ -0,0 +1,7 @@ +AWS Crypto Tools maintains `test/upstream-requirements-py.txt` in our Python products such that +our Cryptographic Primitive Provider for Python ([pyca/cryptography](https://github.com/pyca/cryptography)) +may execute downstream tests against AWS Crypto Tools Python products. +These files allow pyca to install and test the Crypto Tools products. +Additionally, Crypto Tools should maintain a test configuration that can be completed without using any AWS resources. +If Crypto Tools needs to contact pyca about this expectation, +they should cut a issue to the pyca/cryptography repo. diff --git a/tox.ini b/tox.ini index 33f1aff9..9024f22b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] envlist = - py{27,35,36,37,38}-{local,integ,ddb,examples}-fast, + py{38,39,310,311,312}-{local,integ,ddb,examples}-fast, nocmk, sourcebuildcheck, docs, bandit, doc8, readme, flake8{,-tests,-examples}, pylint{,-tests,-examples}, vulture, - test-upstream-requirements-py{2,3}7 + test-upstream-requirements-py3{11} # Additional environments: # @@ -35,25 +35,34 @@ envlist = # test-release :: Builds dist files and uploads to testpypi pypirc profile. # release :: Builds dist files and uploads to pypi pypirc profile. +# Reporting environments: +# +# coverage :: Runs code coverage, failing the build if coverage is below the configured threshold + [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov dynamodb_encryption_sdk {posargs} +commands = pytest -n auto --basetemp={envtmpdir} -l {posargs} [testenv] passenv = # Identifies AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID + # Identifies AWS KMS MRK key ids to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 # DynamoDB Table to use in integration tests - DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME \ + DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME # Pass through AWS credentials - AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN # AWS Role access in CodeBuild is via the contaner URI - AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI # Pass through AWS profile name (useful for local testing) - AWS_PROFILE \ + AWS_PROFILE # Pass through the default AWS region (used for integration tests) AWS_DEFAULT_REGION sitepackages = False -deps = -rtest/requirements.txt +deps = -rdev_requirements/test-requirements.txt # 'download' forces tox to always upgrade pip to the latest download = true commands = @@ -84,7 +93,11 @@ commands = # Do not select any specific markers manual: {[testenv:base-command]commands} # Only run examples tests - examples: {[testenv:base-command]commands} examples/test/ -m "examples" + examples: {[testenv:base-command]commands} examples/test -m "examples" + +# Run code coverage on the unit tests +[testenv:coverage] +commands = {[testenv:base-command]commands} --cov dynamodb_encryption_sdk test/ -m "local and not slow and not veryslow and not nope" # Verify that local tests work without environment variables present [testenv:nocmk] @@ -95,8 +108,8 @@ sitepackages = False passenv = setenv = ######################################################### -deps = -rtest/requirements.txt -commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" +deps = -rdev_requirements/test-requirements.txt +commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" --ignore=examples # Collect requirements for use in upstream tests [testenv:freeze-upstream-requirements-base] @@ -106,45 +119,27 @@ recreate = True deps = commands = {toxinidir}/test/freeze-upstream-requirements.sh -# Freeze for Python 2.7 -[testenv:freeze-upstream-requirements-py27] -basepython = python2.7 +# Freeze for Python 3.11 +[testenv:freeze-upstream-requirements-py311] +basepython = python3.11 sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} recreate = {[testenv:freeze-upstream-requirements-base]recreate} deps = {[testenv:freeze-upstream-requirements-base]deps} -commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py27.txt - -# Freeze for Python 3.7 -[testenv:freeze-upstream-requirements-py37] -basepython = python3.7 -sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} -skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} -recreate = {[testenv:freeze-upstream-requirements-base]recreate} -deps = {[testenv:freeze-upstream-requirements-base]deps} -commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py37.txt +commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py311.txt # Test frozen upstream requirements [testenv:test-upstream-requirements-base] sitepackages = False recreate = True passenv = -commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" - -# Test frozen upstream requirements for Python 2.7 -[testenv:test-upstream-requirements-py27] -basepython = python2.7 -passenv = -deps = -rtest/upstream-requirements-py27.txt -sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} -recreate = {[testenv:test-upstream-requirements-base]recreate} -commands = {[testenv:test-upstream-requirements-base]commands} +commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" --ignore=examples -# Test frozen upstream requirements for Python 3.7 -[testenv:test-upstream-requirements-py37] -basepython = python3.7 +# Test frozen upstream requirements for Python 3.11 +[testenv:test-upstream-requirements-py311] +basepython = python3.11 passenv = -deps = -rtest/upstream-requirements-py37.txt +deps = -rtest/upstream-requirements-py311.txt sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} @@ -156,6 +151,7 @@ sitepackages = False recreate = True deps = {[testenv:build]deps} + -rdev_requirements/test-requirements.txt commands = {[testenv:build]commands} {toxinidir}/test/source-build-check.sh {envtmpdir} {toxinidir}/dist @@ -190,28 +186,10 @@ commands = {posargs} {[testenv:mypy-coverage]commands} -[testenv:mypy-py2] -basepython = python2.7 -deps = {[testenv:mypy-common]deps} -commands = - python -m mypy \ - --py2 \ - --linecoverage-report build \ - src/dynamodb_encryption_sdk/ \ - {posargs} - {[testenv:mypy-coverage]commands} - # Linters [testenv:flake8] basepython = python3 -deps = - flake8 - flake8-docstrings - flake8-isort - # https://github.com/PyCQA/pydocstyle/issues/375 - pydocstyle<4.0.0 - # https://github.com/JBKahn/flake8-print/pull/30 - flake8-print>=3.1.0 +deps = -rdev_requirements/linter-requirements.txt commands = flake8 \ src/dynamodb_encryption_sdk/ \ @@ -220,7 +198,7 @@ commands = [testenv:flake8-tests] basepython = {[testenv:flake8]basepython} -deps = {[testenv:flake8]deps} +deps = -rdev_requirements/linter-requirements.txt commands = flake8 \ # Ignore F811 redefinition errors in tests (breaks with pytest-mock use) @@ -237,21 +215,20 @@ commands = flake8 \ # Ignore C901 complexity requirements (examples optimize for straightforward readability) --ignore C901 \ - examples/src/ + examples/src/dynamodb_encryption_sdk_examples/ flake8 \ # Ignore F811 redefinition errors in tests (breaks with fixture use) # Ignore D103 docstring requirements for tests --ignore F811,D103 \ # Our path munging confuses isort, so disable flake8-isort checks on that file - --per-file-ignores="examples/test/examples_test_utils.py:I003,I004" \ + --per-file-ignores="examples/test/examples_test_utils.py:I003,I004,I005,examples/test/test_aws_kms_encrypted_examples.py:I005" \ examples/test/ [testenv:pylint] basepython = python3 deps = {[testenv]deps} - pyflakes - pylint + -rdev_requirements/linter-requirements.txt commands = pylint \ --rcfile=src/pylintrc \ @@ -274,51 +251,48 @@ commands = basepython = {[testenv:pylint]basepython} deps = {[testenv:pylint]deps} commands = - pylint --rcfile=examples/src/pylintrc examples/src/ + pylint --rcfile=examples/src/pylintrc examples/src/dynamodb_encryption_sdk_examples pylint --rcfile=examples/test/pylintrc examples/test/ [testenv:blacken-src] basepython = python3 -deps = - black +deps = -rdev_requirements/linter-requirements.txt commands = black --line-length 120 \ src/dynamodb_encryption_sdk/ \ setup.py \ doc/conf.py \ test/ \ - examples/ \ + examples/src \ + examples/test \ {posargs} [testenv:blacken] basepython = python3 -deps = - {[testenv:blacken-src]deps} +deps = {[testenv:blacken-src]deps} commands = {[testenv:blacken-src]commands} [testenv:black-check] basepython = python3 -deps = - {[testenv:blacken]deps} +deps = {[testenv:blacken]deps} commands = {[testenv:blacken-src]commands} --diff [testenv:isort-seed] basepython = python3 -deps = seed-isort-config +deps = -rdev_requirements/linter-requirements.txt commands = seed-isort-config [testenv:isort] basepython = python3 -# We need >=5.0.0 because -# several configuration settings changed with 5.0.0 -deps = isort>=5.0.0 +deps = -rdev_requirements/linter-requirements.txt commands = isort \ src \ test \ - examples/ \ + examples/src \ + examples/test \ doc \ setup.py \ --skip examples/test/examples_test_utils.py \ @@ -332,8 +306,8 @@ commands = {[testenv:isort]commands} -c [testenv:autoformat] basepython = python3 deps = - {[testenv:isort]deps} {[testenv:blacken]deps} + {[testenv:isort]deps} commands = {[testenv:isort]commands} {[testenv:blacken]commands} @@ -353,27 +327,27 @@ commands = basepython = python3 whitelist_externals = {[testenv:resetdocs]whitelist_externals} deps = - sphinx - doc8 -commands = + -rdev_requirements/doc-requirements.txt + -rdev_requirements/linter-requirements.txt +commands = {[testenv:resetdocs]commands} doc8 doc/index.rst doc/lib/ README.rst CHANGELOG.rst - + [testenv:readme] basepython = python3 -deps = readme_renderer +deps = -rdev_requirements/linter-requirements.txt commands = python setup.py check -r -s [testenv:bandit] basepython = python3 -deps = bandit +deps = -rdev_requirements/linter-requirements.txt commands = bandit -r src/dynamodb_encryption_sdk/ # Prone to false positives: only run independently [testenv:vulture] basepython = python3 -deps = vulture +deps = -rdev_requirements/linter-requirements.txt commands = vulture src/dynamodb_encryption_sdk/ [testenv:linters] @@ -403,7 +377,7 @@ commands = # Documentation [testenv:docs] basepython = python3 -deps = -rdoc/requirements.txt +deps = -rdev_requirements/doc-requirements.txt commands = sphinx-build -E -c doc/ -b html doc/ doc/build/html sphinx-build -E -c doc/ -b linkcheck doc/ doc/build/html @@ -420,27 +394,21 @@ commands = [testenv:park] basepython = python3 skip_install = true -deps = - pypi-parker - setuptools +deps = -rdev_requirements/release-requirements.txt commands = python setup.py park # Release tooling [testenv:build] basepython = python3 skip_install = true -deps = - wheel - setuptools +deps = -rdev_requirements/release-requirements.txt commands = python setup.py sdist bdist_wheel [testenv:release-base] basepython = python3 skip_install = true -deps = - {[testenv:build]deps} - twine +deps = -rdev_requirements/release-requirements.txt passenv = # Intentionally omit TWINE_REPOSITORY_URL from the passenv list, # as this overrides other ways of setting the repository and could