diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..80689173 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Each line is a file pattern followed by one or more owners. +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# Default code owner for everything is our aws-crypto-tools group +* @aws/aws-crypto-tools diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5cd8ea5c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + # master + - package-ecosystem: "pip" + directory: "/dev_requirements" + schedule: + interval: "daily" + + # Github Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 746f015e..08cda289 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -17,7 +17,6 @@ jobs: category: # Disabled pending completion of integration # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 -# - mypy-py2 # - mypy-py3 - bandit - doc8 @@ -31,13 +30,13 @@ jobs: - pylint-examples - black-check steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.8 - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: check env: TOXENV: ${{ matrix.category }} diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b8e72507..fb308d0b 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -9,46 +9,6 @@ on: - cron: '0 0 * * *' jobs: - # Hypothesis no longer supports Python 2 and - # there is a bug that appears with our slow tests - # only on Python 2. - # Until we also drop Python 2 support, - # the workaround is just that we don't run the slow tests - # on Python 2. - py2-tests: - runs-on: ${{ matrix.platform.os }} - strategy: - fail-fast: true - matrix: - platform: - - os: ubuntu-latest - architecture: x64 - - os: windows-latest - architecture: x64 - # x86 builds are only meaningful for Windows - - os: windows-latest - architecture: x86 - - os: macos-latest - architecture: x64 - category: - - local-fast - # These require credentials. - # Enable them once we sort how to provide them. - # - integ-fast - # - examples - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: 2.7 - architecture: ${{ matrix.platform.architecture }} - - run: | - python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt - - name: run test - env: - TOXENV: ${{ matrix.category }} - run: tox -- -vv tests: runs-on: ${{ matrix.platform.os }} strategy: @@ -62,15 +22,15 @@ jobs: # x86 builds are only meaningful for Windows - os: windows-latest architecture: x86 - - os: macos-latest + - os: macos-13 architecture: x64 python: - - 3.5 - - 3.6 - - 3.7 - 3.8 - 3.9 - - 3.x + - "3.10" + - "3.11" + - "3.12" +# - 3.x 3.13 does not have 'pipes' and maybe other necessary things category: - local-slow # These require credentials. @@ -78,19 +38,20 @@ jobs: # - integ-slow # - examples steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.platform.architecture }} - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: TOXENV: ${{ matrix.category }} run: tox -- -vv - upstream-py3: + + upstream-py311: runs-on: ubuntu-latest strategy: fail-fast: true @@ -98,34 +59,15 @@ jobs: category: - nocmk - sourcebuildcheck - - test-upstream-requirements-py37 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - run: | - python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt - - name: run test - env: - TOXENV: ${{ matrix.category }} - run: tox -- -vv - upstream-py2: - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - category: - - test-upstream-requirements-py27 + - test-upstream-requirements-py311 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: 2.7 + python-version: "3.11" - run: | python -m pip install --upgrade pip - pip install --upgrade -r ci-requirements.txt + pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: TOXENV: ${{ matrix.category }} diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index b7605354..e3776d39 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -9,7 +9,7 @@ jobs: environment: repo-sync runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: repo-sync/github-sync@v2 name: Sync repo to branch with: diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..82c9c983 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,27 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.8" + +# Build documentation in the doc/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +# Don't need to build documentation for test vectors or any other +# sub modules +submodules: + exclude: all + +python: + install: + - requirements: dev_requirements/doc-requirements.txt + - method: pip + path: . diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd602e43..921613e8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,68 @@ Changelog ********* +3.3.0 -- 2024-08-05 +=================== + +Deprecation +----------- +- The AWS DynamoDB Encryption Client for Python no longer supports Python 3.7 as of version 3.3 + - Only Python 3.8+ is supported. +- We no longer support OpenSSL 1.0.1 or 1.0.2, as per `cryptography documentation `_. + +Feature +----------- +* Warn on Deprecated Python 3.7 usage +* Add Python 3.11 to CI +* Add Python 3.12 to CI + +Maintenance +----------- +* Update requirements for boto3 (>=1.10.0) and cryptography (>=3.4.6) + +3.2.0 -- 2021-12-19 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 3.6 +as of version 3.2; only Python 3.7+ is supported. + +Feature +----------- +* Warn on Deprecated Python 3.6 usage + +3.1.0 -- 2021-11-10 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 3.5 +as of version 3.1; only Python 3.6+ is supported. Customers using +Python 3.5 can still use the 2.x line of the AWS DynamoDB Encryption Client for Python, +which will continue to receive security updates, in accordance +with our `Support Policy `__. + +Feature +----------- +* Warn on Deprecated Python usage + `#368 `_ +* Add Python 3.10 to CI +* Remove Python 3.5 from testing + + +3.0.0 -- 2021-07-15 +=================== + +Deprecation +----------- +The AWS DynamoDB Encryption Client for Python no longer supports Python 2 or Python 3.4 +as of major version 3.x; only Python 3.5+ is supported. Customers using Python 2 +or Python 3.4 can still use the 2.x line of the DynamoDB Encryption Client, +which will continue to receive security updates for the next 12 months, in accordance +with our `Support Policy `__. + + 2.1.0 -- 2021-07-15 =================== @@ -39,7 +101,7 @@ CachingMostRecentProvider replaces MostRecentProvider and provides a cache entry TTL to reauthorize the key with the key provider. MostRecentProvider is now deprecated, and is removed in 2.0.0. See -https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/most-recent-provider.html +https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/most-recent-provider.html#mrp-versions for more details. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7323620c..b6353e00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,10 @@ Please read through this document before submitting any issues or pull requests information to effectively respond to your bug report or contribution. +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + ## Reporting Bugs/Feature Requests We welcome you to use the GitHub issue tracker to report bugs or suggest features. @@ -50,10 +54,6 @@ For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of opensource-codeofconduct@amazon.com with any additional questions or comments. -## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. - - ## Licensing See the [LICENSE](https://github.com/aws/aws-dynamodb-encryption-python/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/README.rst b/README.rst index 42b8ce42..438a10ee 100644 --- a/README.rst +++ b/README.rst @@ -47,12 +47,7 @@ Getting Started Required Prerequisites ====================== -* Python 2.7 or 3.4+ - - **NOTE: 2.x is the last major version of this library that will - support Python 2. Future major versions will begin to adopt changes - known to break Python 2. Python 3.4 support will also be removed - in future major versions; Python 3.5+ will be required.** +* Python 3.8+ Installation @@ -182,10 +177,10 @@ of the one that the client would normally construct for you. ... ) # this uses my_special_crypto_config -.. _Amazon DynamoDB Encryption Client: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/ +.. _Amazon DynamoDB Encryption Client: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/legacy-dynamodb-encryption-client.html .. _Amazon DynamoDB: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Introduction.html -.. _primary documents: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/ -.. _Concepts Guide: https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/concepts.html +.. _primary documents: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/legacy-dynamodb-encryption-client.html +.. _Concepts Guide: https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/DDBEC-legacy-concepts.html .. _Amazon DynamoDB Encryption Client for Java: https://github.com/aws/aws-dynamodb-encryption-java/ .. _Amazon DynamoDB Encryption Client for Python: https://github.com/aws/aws-dynamodb-encryption-python/ .. _DynamoDB Stream: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html @@ -200,5 +195,5 @@ of the one that the client would normally construct for you. .. _CryptoConfig: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/encrypted/config.html .. _decrypt_dynamodb_item: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/encrypted/item.html#dynamodb_encryption_sdk.encrypted.item.decrypt_dynamodb_item .. _transformation functions: https://aws-dynamodb-encryption-python.readthedocs.io/en/latest/lib/tools/transform.html -.. _Security issue notifications: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/CONTRIBUTING.md#user-content-security-issue-notifications +.. _Security issue notifications: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/CONTRIBUTING.md .. _Support Policy: https://github.com/aws/aws-dynamodb-encryption-python/blob/master/SUPPORT_POLICY.rst diff --git a/SUPPORT_POLICY.rst b/SUPPORT_POLICY.rst index 26667e42..3fe938f3 100644 --- a/SUPPORT_POLICY.rst +++ b/SUPPORT_POLICY.rst @@ -22,16 +22,16 @@ This table describes the current support status of each major version of the AWS - Next status - Next status date * - 1.x - - Maintenance - End of Support - - 2022-07-08 + - + - * - 2.x - - Generally Available - - Maintenance - - 2021-07-13 + - End of Support + - + - * - 3.x - - - Generally Available - - 2021-07-13 + - Maintenance + - 2024-08-05 .. _AWS SDKs and Tools Maintenance Policy: https://docs.aws.amazon.com/sdkref/latest/guide/maint-policy.html#version-life-cycle diff --git a/buildspec.yml b/buildspec.yml index 82f32b41..b53801b5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -3,18 +3,25 @@ version: 0.2 batch: fast-fail: false build-list: - - identifier: python2_7 - buildspec: codebuild/python2.7.yml - - identifier: python3_5 - buildspec: codebuild/python3.5.yml - - identifier: python3_6 - buildspec: codebuild/python3.6.yml - - identifier: python3_7 - buildspec: codebuild/python3.7.yml - identifier: python3_8 buildspec: codebuild/python3.8.yml + env: + image: aws/codebuild/standard:5.0 - identifier: python3_9 buildspec: codebuild/python3.9.yml - + env: + image: aws/codebuild/standard:5.0 + - identifier: python3_10 + buildspec: codebuild/python3.10.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: python3_11 + buildspec: codebuild/python3.11.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: python3_12 + buildspec: codebuild/python3.12.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml diff --git a/cfn/CB.yml b/cfn/CB.yml new file mode 100644 index 00000000..30d5966b --- /dev/null +++ b/cfn/CB.yml @@ -0,0 +1,364 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: "Template to build a CodeBuild Project, assumes that GitHub credentials are already set up." +Parameters: + ProjectName: + Type: String + Description: The name of the CodeBuild Project + ProjectDescription: + Type: String + Description: The description for the CodeBuild Project + SourceLocation: + Type: String + Description: The https GitHub URL for the project + NumberOfBuildsInBatch: + Type: Number + MaxValue: 100 + MinValue: 1 + Default: 4 + Description: The number of builds you expect to run in a batch + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - + Label: + default: "Crypto Tools CodeBuild Project Template" + Parameters: + - ProjectName + - ProjectDescription + - SourceLocation + +Resources: + CodeBuildProject: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Ref ProjectName + Description: !Ref ProjectDescription + Source: + Location: !Ref SourceLocation + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Triggers: + BuildType: BUILD_BATCH + Webhook: True + FilterGroups: + - - Type: EVENT + Pattern: PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PUSH,PULL_REQUEST_REOPENED + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildCIServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildCIServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildProjectTestRelease: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Sub "${ProjectName}-test-release" + Description: !Sub "CodeBuild project for ${ProjectName} to release to test PyPi." + Source: + Location: !Ref SourceLocation + BuildSpec: "codebuild/release/test-release.yml" + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildProjectProdRelease: + Type: "AWS::CodeBuild::Project" + Properties: + Name: !Sub "${ProjectName}-prod-release" + Description: !Sub "CodeBuild project for ${ProjectName} to release to prod PyPi." + Source: + Location: !Ref SourceLocation + BuildSpec: "codebuild/release/prod-release.yml" + GitCloneDepth: 1 + GitSubmodulesConfig: + FetchSubmodules: false + InsecureSsl: false + ReportBuildStatus: false + Type: "GITHUB" + Artifacts: + Type: "NO_ARTIFACTS" + Cache: + Type: "NO_CACHE" + Environment: + ComputeType: "BUILD_GENERAL1_SMALL" + Image: "aws/codebuild/standard:3.0" + ImagePullCredentialsType: "CODEBUILD" + PrivilegedMode: false + Type: "LINUX_CONTAINER" + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + TimeoutInMinutes: 60 + QueuedTimeoutInMinutes: 480 + EncryptionKey: !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/aws/s3" + BadgeEnabled: false + BuildBatchConfig: + ServiceRole: !GetAtt CodeBuildServiceRole.Arn + Restrictions: + MaximumBuildsAllowed: !Ref NumberOfBuildsInBatch + ComputeTypesAllowed: + - BUILD_GENERAL1_SMALL + - BUILD_GENERAL1_MEDIUM + TimeoutInMins: 480 + LogsConfig: + CloudWatchLogs: + Status: "ENABLED" + S3Logs: + Status: "DISABLED" + EncryptionDisabled: false + + CodeBuildServiceRole: + Type: "AWS::IAM::Role" + Properties: + Path: "/service-role/" + RoleName: !Sub "codebuild-${ProjectName}-service-role" + AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + MaxSessionDuration: 3600 + ManagedPolicyArns: + - !Ref CryptoToolsKMS + - !Ref CodeBuildBatchPolicy + - !Ref CodeBuildBasePolicy + - !Ref SecretsManagerPolicy + - !Ref DDBPolicy + - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" + + CodeBuildCIServiceRole: + Type: "AWS::IAM::Role" + Properties: + Path: "/service-role/" + RoleName: !Sub "codebuild-${ProjectName}-CI-service-role" + AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + MaxSessionDuration: 3600 + ManagedPolicyArns: + - !Ref CryptoToolsKMS + - !Ref CodeBuildCIBatchPolicy + - !Ref CodeBuildBasePolicy + - !Ref DDBPolicy + - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" + + CodeBuildBatchPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBuildBatchPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}", + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}-test-release", + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}-prod-release" + ], + "Action": [ + "codebuild:StartBuild", + "codebuild:StopBuild", + "codebuild:RetryBuild" + ] + } + ] + } + + CodeBuildCIBatchPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBuildBatchPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-CI-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${ProjectName}" + ], + "Action": [ + "codebuild:StartBuild", + "codebuild:StopBuild", + "codebuild:RetryBuild" + ] + } + ] + } + + CodeBuildBasePolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CodeBuildBasePolicy-${ProjectName}-${AWS::Region}" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}:*", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-test-release", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-test-release:*", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-prod-release", + "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${ProjectName}-prod-release:*" + ], + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + }, + { + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::codepipeline-${AWS::Region}-*" + ], + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + }, + { + "Effect": "Allow", + "Action": [ + "codebuild:CreateReportGroup", + "codebuild:CreateReport", + "codebuild:UpdateReport", + "codebuild:BatchPutTestCases", + "codebuild:BatchPutCodeCoverages" + ], + "Resource": [ + "arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:report-group/${ProjectName}-*" + ] + } + ] + } + + SecretsManagerPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CryptoTools-SecretsManager-${ProjectName}-release" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:secretsmanager:us-west-2:587316601012:secret:TestPyPiAPIToken-uERFjs", + "arn:aws:secretsmanager:us-west-2:587316601012:secret:PyPiAPIToken-nu1Gu6" + ], + "Action": "secretsmanager:GetSecretValue" + } + ] + } + + DDBPolicy: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CryptoTools-DynamoDB-${ProjectName}-CI" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb:us-east-1:587316601012:table/ddbec-mrk-testing", + "arn:aws:dynamodb:us-west-2:587316601012:table/ddbec-mrk-testing" + ], + "Action": "*" + } + ] + } + + # There exist public AWS KMS CMKs that are used for testing + # Take care with these CMKs they are **ONLY** for testing!!! + CryptoToolsKMS: + Type: "AWS::IAM::ManagedPolicy" + Properties: + ManagedPolicyName: !Sub "CrypotToolsKMSPolicy-${ProjectName}-${AWS::Region}-codebuild-${ProjectName}-service-role" + Path: "/service-role/" + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Resource": [ + "arn:aws:kms:*:658956600833:key/*", + "arn:aws:kms:*:658956600833:alias/*" + ], + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } + ] + } diff --git a/ci-requirements.txt b/ci-requirements.txt deleted file mode 100644 index 053148f8..00000000 --- a/ci-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tox diff --git a/codebuild/coverage/coverage.yml b/codebuild/coverage/coverage.yml index f82a3a98..51d8b0a6 100644 --- a/codebuild/coverage/coverage.yml +++ b/codebuild/coverage/coverage.yml @@ -10,5 +10,5 @@ phases: python: latest build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python2.7.yml b/codebuild/python3.10.yml similarity index 83% rename from codebuild/python2.7.yml rename to codebuild/python3.10.yml index fd688d77..ad76049f 100644 --- a/codebuild/python2.7.yml +++ b/codebuild/python3.10.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py27-integ-slow" + TOXENV: "py310-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.10 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.6.yml b/codebuild/python3.11.yml similarity index 83% rename from codebuild/python3.6.yml rename to codebuild/python3.11.yml index 602dc113..b21cf15a 100644 --- a/codebuild/python3.6.yml +++ b/codebuild/python3.11.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py36-integ-slow" + TOXENV: "py311-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.11 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.12.yml b/codebuild/python3.12.yml new file mode 100644 index 00000000..cf9e09ef --- /dev/null +++ b/codebuild/python3.12.yml @@ -0,0 +1,23 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-integ-slow" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox diff --git a/codebuild/python3.5.yml b/codebuild/python3.5.yml deleted file mode 100644 index f2b1dbcd..00000000 --- a/codebuild/python3.5.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py35-integ-slow" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - -phases: - install: - runtime-versions: - python: latest - build: - commands: - # The specific versions are manually installed - # because they are not installed - # by default in CodeBuild containers. - # `pyenv` does not have - # a nice way to just install - # the latest patch version. - # I have selected the current latest patch - # rather than try - # and manage a one-liner or script. - # Testing every minor version - # is too extreme at this time. - # The choice of versions should be reviewed. - - pyenv install 3.5.9 - - pyenv local 3.5.9 - - pip install tox tox-pyenv - - tox diff --git a/codebuild/python3.7.yml b/codebuild/python3.7.yml index 1ac0daa6..6a51426a 100644 --- a/codebuild/python3.7.yml +++ b/codebuild/python3.7.yml @@ -11,22 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.7 build: commands: - # The specific versions are manually installed - # because they are not installed - # by default in CodeBuild containers. - # `pyenv` does not have - # a nice way to just install - # the latest patch version. - # I have selected the current latest patch - # rather than try - # and manage a one-liner or script. - # Testing every minor version - # is too extreme at this time. - # The choice of versions should be reviewed. - - pyenv install 3.7.9 - - pyenv local 3.7.9 - - pip install tox tox-pyenv + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.8.yml b/codebuild/python3.8.yml index 1c1524c8..478a3bfc 100644 --- a/codebuild/python3.8.yml +++ b/codebuild/python3.8.yml @@ -11,8 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.8 build: commands: - - pip install tox + - pip install "tox < 4.0" - tox diff --git a/codebuild/python3.9.yml b/codebuild/python3.9.yml index 62868c80..f572e2a9 100644 --- a/codebuild/python3.9.yml +++ b/codebuild/python3.9.yml @@ -11,10 +11,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.9 build: commands: - - pyenv install 3.9.0 - - pyenv local 3.9.0 - - pip install tox tox-pyenv + - pip install "tox < 4.0" - tox diff --git a/codebuild/release/prod-release.yml b/codebuild/release/prod-release.yml index c729c96c..7f55b526 100644 --- a/codebuild/release/prod-release.yml +++ b/codebuild/release/prod-release.yml @@ -4,13 +4,13 @@ env: variables: BRANCH: "master" secrets-manager: - TWINE_USERNAME: PyPiAdmin:username - TWINE_PASSWORD: PyPiAdmin:password + TWINE_USERNAME: PyPiAPIToken:username + TWINE_PASSWORD: PyPiAPIToken:password phases: install: commands: - - pip install tox + - pip install "tox < 4.0" - pip install --upgrade pip runtime-versions: python: latest diff --git a/codebuild/release/test-release.yml b/codebuild/release/test-release.yml index 1dc9feae..03dc4d95 100644 --- a/codebuild/release/test-release.yml +++ b/codebuild/release/test-release.yml @@ -4,13 +4,13 @@ env: variables: BRANCH: "master" secrets-manager: - TWINE_USERNAME: TestPyPiCryptoTools:username - TWINE_PASSWORD: TestPyPiCryptoTools:password + TWINE_USERNAME: TestPyPiAPIToken:username + TWINE_PASSWORD: TestPyPiAPIToken:password phases: install: commands: - - pip install tox + - pip install "tox < 4.0" - pip install --upgrade pip runtime-versions: python: latest diff --git a/codebuild/release/validate.yml b/codebuild/release/validate.yml index 9c242630..f710aa5a 100644 --- a/codebuild/release/validate.yml +++ b/codebuild/release/validate.yml @@ -5,13 +5,17 @@ env: BRANCH: "master" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:eu-west-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME: ddbec-release-validation phases: install: commands: - - pip install tox + - pip install "tox < 4.0" runtime-versions: python: latest pre_build: @@ -24,7 +28,7 @@ phases: - | while [ $NUM_RETRIES -gt 0 ] do - tox -re py38-examples + tox -re py3-examples if [ $? -eq 0 ]; then break fi diff --git a/dev_requirements/ci-requirements.txt b/dev_requirements/ci-requirements.txt new file mode 100644 index 00000000..b673eb36 --- /dev/null +++ b/dev_requirements/ci-requirements.txt @@ -0,0 +1,2 @@ +setuptools +tox==3.24.5 diff --git a/dev_requirements/doc-requirements.txt b/dev_requirements/doc-requirements.txt new file mode 100644 index 00000000..9364148e --- /dev/null +++ b/dev_requirements/doc-requirements.txt @@ -0,0 +1,2 @@ +sphinx==4.4.0 +sphinx_rtd_theme==1.0.0 diff --git a/dev_requirements/linter-requirements.txt b/dev_requirements/linter-requirements.txt new file mode 100644 index 00000000..1f8d8c16 --- /dev/null +++ b/dev_requirements/linter-requirements.txt @@ -0,0 +1,16 @@ +bandit==1.7.2 +black==22.3.0 +doc8==0.10.1 +flake8==4.0.1 +flake8-docstrings==1.7.0 +flake8-isort==4.1.1 +# https://github.com/JBKahn/flake8-print/pull/30 +flake8-print==5.0.0 +isort==5.12.0 +pylint==2.12.2 +pyflakes==2.4.0 +# https://github.com/PyCQA/pydocstyle/issues/375 +pydocstyle==3.0.0 +readme_renderer==34.0 +seed-isort-config==2.2.0 +vulture==2.3 diff --git a/dev_requirements/release-requirements.txt b/dev_requirements/release-requirements.txt new file mode 100644 index 00000000..fa2ef83b --- /dev/null +++ b/dev_requirements/release-requirements.txt @@ -0,0 +1,6 @@ +pypi-parker==0.1.2 +setuptools==66.1.1 +twine==3.8.0 +wheel==0.38.4 +#This is required for twine < 4.0 +packaging \ No newline at end of file diff --git a/dev_requirements/test-requirements.txt b/dev_requirements/test-requirements.txt new file mode 100644 index 00000000..c73de305 --- /dev/null +++ b/dev_requirements/test-requirements.txt @@ -0,0 +1,9 @@ +hypothesis==6.31.6 +mock==4.0.3 +moto==3.0.2 +pytest==7.2.1 +pytest-cov==3.0.0 +pytest-mock==3.10.0 +pytest-xdist==3.2.0 +boto3==1.28.38 +botocore==1.31.38 diff --git a/doc/_static/.gitignore b/doc/_static/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/doc/conf.py b/doc/conf.py index 9c0b817c..4e87cadd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -29,7 +29,7 @@ def get_version(): return _release -project = u"dynamodb-encryption-sdk-python" +project = "dynamodb-encryption-sdk-python" version = get_version() release = get_release() @@ -53,7 +53,7 @@ def get_version(): source_suffix = ".rst" # The suffix of source filenames. master_doc = "index" # The master toctree document. -copyright = u"%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin +copyright = "%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin # List of directories, relative to source directory, that shouldn't be searched # for source files. @@ -62,7 +62,7 @@ def get_version(): pygments_style = "sphinx" autoclass_content = "both" -autodoc_default_flags = ["show-inheritance", "members"] +autodoc_default_options = {"members": True, "show-inheritance": True} autodoc_member_order = "bysource" html_theme = "sphinx_rtd_theme" diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 29e31945..00000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sphinx>=1.3.0 -sphinx_rtd_theme \ No newline at end of file diff --git a/examples/README.rst b/examples/README.rst index f636a8a0..ea206ff0 100644 --- a/examples/README.rst +++ b/examples/README.rst @@ -34,7 +34,7 @@ with this library. * `How to use raw symmetric wrapping keys <./src/dynamodb_encryption_sdk_examples/wrapped_symmetric_encrypted_table.py>`_ * `How to use raw asymmetric wrapping keys <./src/dynamodb_encryption_sdk_examples/wrapped_rsa_encrypted_table.py>`_ -For more details on the different type of material providers, see `How to choose a cryptographic materials provider `_. +For more details on the different type of material providers, see `How to choose a cryptographic materials provider `_. Running the examples ==================== diff --git a/examples/setup.py b/examples/setup.py index 8f1fa53d..b856ef53 100644 --- a/examples/setup.py +++ b/examples/setup.py @@ -36,7 +36,7 @@ def get_requirements(): author_email="aws-cryptools@amazon.com", maintainer="Amazon Web Services", description="DynamoDB Encryption Client for Python examples", - long_description=read("README.md"), + long_description=read("README.rst"), keywords="dynamodb-encryption-sdk aws kms encryption dynamodb", data_files=["requirements.txt"], license="Apache License 2.0", @@ -50,8 +50,6 @@ def get_requirements(): "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/examples/src/pylintrc b/examples/src/pylintrc index 5ea9fbcc..2a3a443a 100644 --- a/examples/src/pylintrc +++ b/examples/src/pylintrc @@ -3,6 +3,7 @@ disable = duplicate-code, # these examples often feature similar code too-many-locals, # for these examples, we prioritize keeping everything together for simple readability + consider-using-f-string, # Not supported in Python 3.5 [BASIC] # Allow function names up to 50 characters diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 889b1290..89ba1bba 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -1,8 +1,25 @@ -"""Helper utilities for use while testing examples.""" +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Helper utilities for use while testing examples. + +isort:skip_file +""" import os import sys os.environ["AWS_ENCRYPTION_SDK_EXAMPLES_TESTING"] = "yes" sys.path.extend([os.sep.join([os.path.dirname(__file__), "..", "..", "test", "integration"])]) +# fmt: off from integration_test_utils import cmk_arn, cmk_mrk_arn, ddb_table_name, second_cmk_mrk_arn # noqa pylint: disable=unused-import +# fmt: on diff --git a/examples/test/pylintrc b/examples/test/pylintrc index f4dfcfe6..f9671d06 100644 --- a/examples/test/pylintrc +++ b/examples/test/pylintrc @@ -10,6 +10,7 @@ disable = # pylint does not recognize this duplicate-code, # tests for similar things tend to be similar redefined-outer-name, # raises false positives with fixtures + consider-using-f-string, # Not supported in Python 3.5 [DESIGN] max-args = 10 diff --git a/examples/test/test_aws_kms_encrypted_examples.py b/examples/test/test_aws_kms_encrypted_examples.py index 3c0e8a36..a815683a 100644 --- a/examples/test/test_aws_kms_encrypted_examples.py +++ b/examples/test/test_aws_kms_encrypted_examples.py @@ -20,7 +20,12 @@ aws_kms_multi_region_key, ) -from .examples_test_utils import cmk_arn, cmk_mrk_arn, ddb_table_name, second_cmk_mrk_arn # noqa pylint: disable=unused-import +from .examples_test_utils import ( # noqa pylint: disable=unused-import + cmk_arn, + cmk_mrk_arn, + ddb_table_name, + second_cmk_mrk_arn, +) pytestmark = [pytest.mark.examples] diff --git a/examples/tox.ini b/examples/tox.ini index 67fa6318..e4b39b37 100644 --- a/examples/tox.ini +++ b/examples/tox.ini @@ -2,7 +2,7 @@ [tox] envlist = - py{27,35,36,37,38,39}-examples + py{3,37,38,39}-examples [testenv:base-command] commands = python -m pytest --basetemp={envtmpdir} -l {posargs} diff --git a/requirements.txt b/requirements.txt index b10f60f4..34c7e6a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -boto3>=1.4.4 -cryptography>=1.8.1 +boto3>=1.10.0 +cryptography>=3.4.6 attrs>=17.4.0 -enum34; python_version < '3.4' \ No newline at end of file diff --git a/setup.py b/setup.py index 8b50e8dc..e64e7d4e 100644 --- a/setup.py +++ b/setup.py @@ -47,15 +47,12 @@ def get_requirements(): "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/dynamodb_encryption_sdk/__init__.py b/src/dynamodb_encryption_sdk/__init__.py index 7b5dba80..d1536792 100644 --- a/src/dynamodb_encryption_sdk/__init__.py +++ b/src/dynamodb_encryption_sdk/__init__.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """DynamoDB Encryption Client.""" +from dynamodb_encryption_sdk.compatability import _warn_deprecated_python from dynamodb_encryption_sdk.encrypted.client import EncryptedClient from dynamodb_encryption_sdk.encrypted.item import ( decrypt_dynamodb_item, @@ -22,6 +23,8 @@ from dynamodb_encryption_sdk.encrypted.table import EncryptedTable from dynamodb_encryption_sdk.identifiers import __version__ +_warn_deprecated_python() + __all__ = ( "decrypt_dynamodb_item", "decrypt_python_item", diff --git a/src/dynamodb_encryption_sdk/compatability.py b/src/dynamodb_encryption_sdk/compatability.py new file mode 100644 index 00000000..ccd7be9f --- /dev/null +++ b/src/dynamodb_encryption_sdk/compatability.py @@ -0,0 +1,41 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Contains logic for checking the Python Version""" +import sys +import warnings + +DEPRECATION_DATE_MAP = {"1.x": "2022-07-08", "2.x": "2022-07-15"} + + +def _warn_deprecated_python(): + """Template for deprecation of Python warning.""" + deprecated_versions = { + (2, 7): {"date": DEPRECATION_DATE_MAP["2.x"]}, + (3, 4): {"date": DEPRECATION_DATE_MAP["2.x"]}, + (3, 5): {"date": "2021-11-10"}, + (3, 6): {"date": "2021-12-19"}, + (3, 7): {"date": "2024-03-04"}, + } + py_version = (sys.version_info.major, sys.version_info.minor) + minimum_version = (3, 8) + + if py_version in deprecated_versions: + params = deprecated_versions[py_version] + warning = ( + "aws-dynamodb-encryption will no longer support Python {}.{} " + "starting {}. To continue receiving service updates, " + "bug fixes, and security updates please upgrade to Python {}.{} or " + "later. For more information, see SUPPORT_POLICY.rst: " + "https://github.com/aws/aws-dynamodb-encryption-python/blob/master/SUPPORT_POLICY.rst" + ).format(py_version[0], py_version[1], params["date"], minimum_version[0], minimum_version[1]) + warnings.warn(warning, DeprecationWarning) diff --git a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py index d301543a..ac0aa734 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py @@ -12,18 +12,12 @@ # language governing permissions and limitations under the License. """Delegated keys.""" import abc +from typing import Dict, Optional, Text import six from dynamodb_encryption_sdk.identifiers import EncryptionKeyType # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("DelegatedKey",) diff --git a/src/dynamodb_encryption_sdk/delegated_keys/jce.py b/src/dynamodb_encryption_sdk/delegated_keys/jce.py index 4edc6b2c..c2be9b5c 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/jce.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/jce.py @@ -15,6 +15,7 @@ import logging import os +from typing import Dict, Optional, Text import attr import six @@ -28,13 +29,6 @@ from . import DelegatedKey -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("JceNameLocalDelegatedKey",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/encrypted/__init__.py b/src/dynamodb_encryption_sdk/encrypted/__init__.py index e3e89ec1..d03f3f4a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/__init__.py +++ b/src/dynamodb_encryption_sdk/encrypted/__init__.py @@ -21,13 +21,6 @@ from dynamodb_encryption_sdk.materials import CryptographicMaterials # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("CryptoConfig",) diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py index bd8f2c58..e13533f3 100644 --- a/src/dynamodb_encryption_sdk/encrypted/client.py +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Any, Callable, Dict, Iterator, Optional import attr import botocore @@ -34,13 +35,6 @@ from .item import decrypt_dynamodb_item, decrypt_python_item, encrypt_dynamodb_item, encrypt_python_item -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict, Iterator, Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedClient", "EncryptedPaginator") @@ -134,7 +128,7 @@ class EncryptedClient(object): This class provides a superset of the boto3 DynamoDB client API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#client + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#client If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, ``query``, ``scan``, ``batch_write_item``, and ``batch_get_item`` methods will also diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index b491a34a..33c109cf 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -11,14 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Top-level functions for encrypting and decrypting DynamoDB items.""" -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - from dynamodb_encryption_sdk.exceptions import DecryptionError, EncryptionError from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.crypto.authentication import sign_item, verify_item_signature from dynamodb_encryption_sdk.internal.crypto.encryption import decrypt_attribute, encrypt_attribute from dynamodb_encryption_sdk.internal.formatting.material_description import ( diff --git a/src/dynamodb_encryption_sdk/encrypted/resource.py b/src/dynamodb_encryption_sdk/encrypted/resource.py index b5b71f8b..f040ea7a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/resource.py +++ b/src/dynamodb_encryption_sdk/encrypted/resource.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Optional import attr from boto3.resources.base import ServiceResource @@ -29,13 +30,6 @@ from .item import decrypt_python_item, encrypt_python_item from .table import EncryptedTable -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedResource", "EncryptedTablesCollectionManager") @@ -44,7 +38,7 @@ class EncryptedTablesCollectionManager(object): # pylint: disable=too-few-public-methods,too-many-instance-attributes """Tables collection manager that provides :class:`EncryptedTable` objects. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html :param collection: Pre-configured boto3 DynamoDB table collection manager :type collection: boto3.resources.collection.CollectionManager @@ -137,7 +131,7 @@ class EncryptedResource(object): This class provides a superset of the boto3 DynamoDB service resource API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#service-resource + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/index.html If you want to provide per-request cryptographic details, the ``batch_write_item`` and ``batch_get_item`` methods will also accept a ``crypto_config`` parameter, defining @@ -217,7 +211,7 @@ def Table(self, name, **kwargs): If any of the optional configuration values are not provided, the corresponding values for this ``EncryptedResource`` will be used. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.Table + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table :param name: The table name. :param CryptographicMaterialsProvider materials_provider: Cryptographic materials diff --git a/src/dynamodb_encryption_sdk/encrypted/table.py b/src/dynamodb_encryption_sdk/encrypted/table.py index 128cb896..98386b81 100644 --- a/src/dynamodb_encryption_sdk/encrypted/table.py +++ b/src/dynamodb_encryption_sdk/encrypted/table.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """High-level helper class to provide a familiar interface to encrypted tables.""" from functools import partial +from typing import Optional import attr from boto3.dynamodb.table import BatchWriter @@ -30,13 +31,6 @@ from .client import EncryptedClient from .item import decrypt_python_item, encrypt_python_item -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptedTable",) @@ -60,7 +54,7 @@ class EncryptedTable(object): This class provides a superset of the boto3 DynamoDB Table API, so should work as a drop-in replacement once configured. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#table + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, ``query``, and ``scan`` methods will also accept a ``crypto_config`` parameter, defining @@ -158,7 +152,7 @@ def update_item(self, **kwargs): def batch_writer(self, overwrite_by_pkeys=None): """Create a batch writer object. - https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Table.batch_writer + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/batch_writer.html :type overwrite_by_pkeys: list(string) :param overwrite_by_pkeys: De-duplicate request items in buffer if match new request diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index e7a0fcfd..5c63f095 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -14,7 +14,7 @@ from enum import Enum __all__ = ("LOGGER_NAME", "CryptoAction", "EncryptionKeyType", "KeyEncodingType") -__version__ = "2.1.0" +__version__ = "3.3.0" LOGGER_NAME = "dynamodb_encryption_sdk" USER_AGENT_SUFFIX = "DynamodbEncryptionSdkPython/{}".format(__version__) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index d5247688..622e03b5 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -16,25 +16,19 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ +from typing import Text + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.encrypted import CryptoConfig # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, SignatureValues, Tag from dynamodb_encryption_sdk.structures import AttributeActions # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Text # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("sign_item", "verify_item_signature") diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 3737d520..e8b72749 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -16,15 +16,11 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Text # noqa pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Text from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.deserialize.attribute import deserialize_attribute from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import Tag diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index b2244ac7..0d1b08e8 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -18,6 +18,7 @@ """ import abc import logging +from typing import Any, Callable, Text import attr import six @@ -32,12 +33,6 @@ from .primitives import load_rsa_key -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - __all__ = ("JavaAuthenticator", "JavaMac", "JavaSignature", "JAVA_AUTHENTICATOR") _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 564bed80..2d6f667c 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -19,6 +19,7 @@ import abc import logging import os +from typing import Any, Callable, Text import attr import six @@ -38,13 +39,6 @@ from dynamodb_encryption_sdk.internal.identifiers import MinimumKeySizes from dynamodb_encryption_sdk.internal.validators import callable_validator -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ( "JavaPadding", "SimplePadding", diff --git a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py index 0509a59b..01b4becb 100644 --- a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py +++ b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py @@ -5,24 +5,20 @@ namespace staying consistent. Directly reference at your own risk. """ # constant naming for types so pylint: disable=invalid-name -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, AnyStr, ByteString, Dict, List, Text +from typing import Any, AnyStr, ByteString, Dict, List, Text - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - ATTRIBUTE = Dict[Text, Any] # narrow this down - ITEM = Dict[Text, ATTRIBUTE] - RAW_ATTRIBUTE = ITEM - NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} - BOOLEAN = bool - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - NUMBER = int # This misses long on Python 2...figure out something for this - # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 - STRING = AnyStr # can be unicode but should not be bytes - BINARY = ByteString - BINARY_ATTRIBUTE = Dict[Text, BINARY] - SET = List # DynamoDB TypeSerializer converts sets into lists - MAP = RAW_ATTRIBUTE - LIST = List[RAW_ATTRIBUTE] -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +ATTRIBUTE = Dict[Text, Any] # narrow this down +ITEM = Dict[Text, ATTRIBUTE] +RAW_ATTRIBUTE = ITEM +NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} +BOOLEAN = bool +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +NUMBER = int # This misses long on Python 2...figure out something for this +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +STRING = AnyStr # can be unicode but should not be bytes +BINARY = ByteString +BINARY_ATTRIBUTE = Dict[Text, BINARY] +SET = List # DynamoDB TypeSerializer converts sets into lists +MAP = RAW_ATTRIBUTE +LIST = List[RAW_ATTRIBUTE] diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index 41058916..164ad303 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -21,24 +21,17 @@ import logging import struct from decimal import Decimal +from typing import Callable, Dict, List, Text, Union from boto3.dynamodb.types import Binary from dynamodb_encryption_sdk.exceptions import DeserializationError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.deserialize import decode_byte, decode_length, decode_tag, decode_value from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_str -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Callable, Dict, List, Text, Union # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("deserialize_attribute",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index 1f6af4a2..4657a34c 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -19,24 +19,17 @@ import io import logging import struct +from typing import Dict, Text from dynamodb_encryption_sdk.exceptions import InvalidMaterialDescriptionError, InvalidMaterialDescriptionVersionError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.identifiers import Tag from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str from .deserialize import decode_value, unpack_value from .serialize import encode_value -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("serialize", "deserialize") _LOGGER = logging.getLogger(LOGGER_NAME) _MATERIAL_DESCRIPTION_VERSION = b"\00" * 4 diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py index 1c7f7ee2..07caf22b 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py @@ -17,12 +17,7 @@ namespace staying consistent. Directly reference at your own risk. """ import struct - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Sized # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Sized __all__ = ("encode_length", "encode_value") diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index 1ca416a1..49a0097c 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -18,24 +18,17 @@ """ import io import logging +from typing import Callable from boto3.dynamodb.types import DYNAMODB_CONTEXT, Binary from dynamodb_encryption_sdk.exceptions import SerializationError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.formatting.serialize import encode_length, encode_value from dynamodb_encryption_sdk.internal.identifiers import Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_bytes -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Callable # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("serialize_attribute",) _LOGGER = logging.getLogger(LOGGER_NAME) _RESERVED = b"\x00" diff --git a/src/dynamodb_encryption_sdk/internal/identifiers.py b/src/dynamodb_encryption_sdk/internal/identifiers.py index 94d7bd41..facc1266 100644 --- a/src/dynamodb_encryption_sdk/internal/identifiers.py +++ b/src/dynamodb_encryption_sdk/internal/identifiers.py @@ -17,12 +17,7 @@ namespace staying consistent. Directly reference at your own risk. """ from enum import Enum - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Optional, Text __all__ = ( "ReservedAttributes", diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 988576b2..cdb6266c 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -18,6 +18,7 @@ """ import copy from functools import partial +from typing import Any, Callable, Dict, Iterable, Text import attr import botocore.client @@ -28,12 +29,6 @@ from dynamodb_encryption_sdk.structures import CryptoAction, EncryptionContext, TableInfo from dynamodb_encryption_sdk.transform import dict_to_ddb -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Bool, Callable, Dict, Iterable, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - __all__ = ( "TableInfoCache", "crypto_config_from_kwargs", @@ -198,7 +193,7 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg :param callable decrypt_method: Method to use to decrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -220,7 +215,7 @@ def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs :param callable decrypt_method: Method to use to decrypt item :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -244,7 +239,7 @@ def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, ** :param callable decrypt_method: Method to use to decrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable read_method: Method that reads from the table - :param **kwargs: Keyword arguments to pass to ``read_method`` + :param ``**kwargs``: Keyword arguments to pass to ``read_method`` :return: DynamoDB response :rtype: dict """ @@ -276,7 +271,7 @@ def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwarg :param callable encrypt_method: Method to use to encrypt items :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` :param callable write_method: Method that writes to the table - :param **kwargs: Keyword arguments to pass to ``write_method`` + :param ``**kwargs``: Keyword arguments to pass to ``write_method`` :return: DynamoDB response :rtype: dict """ @@ -297,7 +292,7 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, :param callable encrypt_method: Method to use to encrypt items :param callable crypto_config_method: Method that accepts a table name string and provides a :class:`CryptoConfig` :param callable write_method: Method that writes to the table - :param **kwargs: Keyword arguments to pass to ``write_method`` + :param ``**kwargs``: Keyword arguments to pass to ``write_method`` :return: DynamoDB response :rtype: dict """ @@ -366,7 +361,7 @@ def _process_batch_write_response(request, response, table_crypto_config): def _item_keys_match(crypto_config, item1, item2): - # type: (CryptoConfig, Dict, Dict) -> Bool + # type: (CryptoConfig, Dict, Dict) -> bool """Determines whether the values in the primary and sort keys (if they exist) are the same :param CryptoConfig crypto_config: CryptoConfig used in encrypting the given items @@ -387,7 +382,7 @@ def _item_keys_match(crypto_config, item1, item2): def _item_attributes_match(crypto_config, plaintext_item, encrypted_item): - # type: (CryptoConfig, Dict, Dict) -> Bool + # type: (CryptoConfig, Dict, Dict) -> bool """Determines whether the unencrypted values in the plaintext items attributes are the same as those in the encrypted item. Essentially this uses brute force to cover when we don't know the primary and sort index attribute names, since they can't be encrypted. diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index ea7a55f2..73212999 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -16,6 +16,7 @@ import base64 import logging from enum import Enum +from typing import Dict, Optional, Text, Tuple import attr import boto3 @@ -28,6 +29,7 @@ from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey from dynamodb_encryption_sdk.exceptions import UnknownRegionError, UnwrappingError, WrappingError from dynamodb_encryption_sdk.identifiers import LOGGER_NAME, USER_AGENT_SUFFIX, EncryptionKeyType, KeyEncodingType +from dynamodb_encryption_sdk.internal import dynamodb_types from dynamodb_encryption_sdk.internal.identifiers import TEXT_ENCODING, MaterialDescriptionKeys from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str from dynamodb_encryption_sdk.internal.validators import dictionary_validator, iterable_validator @@ -36,15 +38,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import - - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("AwsKmsCryptographicMaterialsProvider",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 349163e0..8a003f17 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -16,6 +16,7 @@ from collections import OrderedDict from enum import Enum from threading import Lock, RLock +from typing import Any, Text import attr import six @@ -28,16 +29,7 @@ from . import CryptographicMaterialsProvider from .store import ProviderStore -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - -__all__ = ( - "CachingMostRecentProvider", -) +__all__ = ("CachingMostRecentProvider",) _LOGGER = logging.getLogger(LOGGER_NAME) #: Grace period during which we will return the latest local materials. This allows multiple #: threads to be using this same provider without risking lock contention or many threads diff --git a/src/dynamodb_encryption_sdk/material_providers/static.py b/src/dynamodb_encryption_sdk/material_providers/static.py index 966002cb..77af8478 100644 --- a/src/dynamodb_encryption_sdk/material_providers/static.py +++ b/src/dynamodb_encryption_sdk/material_providers/static.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials provider for use with pre-configured encryption and decryption materials.""" +from typing import Optional + import attr from dynamodb_encryption_sdk.materials import CryptographicMaterials # noqa pylint: disable=unused-import @@ -19,13 +21,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("StaticCryptographicMaterialsProvider",) diff --git a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py index e03b57b9..1948c388 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """Cryptographic materials provider stores.""" import abc +from typing import Optional, Text import six @@ -20,13 +21,6 @@ CryptographicMaterialsProvider, ) -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("ProviderStore",) diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index 46d7410f..da545ba4 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -13,6 +13,7 @@ """Meta cryptographic provider store.""" import logging from enum import Enum +from typing import Dict, Optional, Text, Tuple import attr import botocore @@ -29,13 +30,6 @@ from . import ProviderStore -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("MetaStore",) _LOGGER = logging.getLogger(LOGGER_NAME) diff --git a/src/dynamodb_encryption_sdk/material_providers/wrapped.py b/src/dynamodb_encryption_sdk/material_providers/wrapped.py index 13f6a346..416156c7 100644 --- a/src/dynamodb_encryption_sdk/material_providers/wrapped.py +++ b/src/dynamodb_encryption_sdk/material_providers/wrapped.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials provider to use ephemeral content encryption keys wrapped by delegated keys.""" +from typing import Dict, Optional, Text + import attr import six @@ -22,19 +24,6 @@ from . import CryptographicMaterialsProvider -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Optional # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("WrappedCryptographicMaterialsProvider",) diff --git a/src/dynamodb_encryption_sdk/materials/__init__.py b/src/dynamodb_encryption_sdk/materials/__init__.py index 09c4a470..3b9788d0 100644 --- a/src/dynamodb_encryption_sdk/materials/__init__.py +++ b/src/dynamodb_encryption_sdk/materials/__init__.py @@ -12,20 +12,12 @@ # language governing permissions and limitations under the License. """Cryptographic materials are containers that provide delegated keys for cryptographic operations.""" import abc +from typing import Dict, Text import six from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text # noqa pylint: disable=unused-import - - from mypy_extensions import NoReturn # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("CryptographicMaterials", "EncryptionMaterials", "DecryptionMaterials") @@ -89,7 +81,6 @@ class EncryptionMaterials(CryptographicMaterials): @property def decryption_key(self): - # type: () -> NoReturn """Encryption materials do not provide decryption keys. :raises NotImplementedError: because encryption materials do not contain decryption keys @@ -98,7 +89,6 @@ def decryption_key(self): @property def verification_key(self): - # type: () -> NoReturn """Encryption materials do not provide verification keys. :raises NotImplementedError: because encryption materials do not contain verification keys @@ -111,7 +101,6 @@ class DecryptionMaterials(CryptographicMaterials): @property def encryption_key(self): - # type: () -> NoReturn """Decryption materials do not provide encryption keys. :raises NotImplementedError: because decryption materials do not contain encryption keys @@ -120,7 +109,6 @@ def encryption_key(self): @property def signing_key(self): - # type: () -> NoReturn """Decryption materials do not provide signing keys. :raises NotImplementedError: because decryption materials do not contain signing keys diff --git a/src/dynamodb_encryption_sdk/materials/raw.py b/src/dynamodb_encryption_sdk/materials/raw.py index 7c2e85e4..d2587339 100644 --- a/src/dynamodb_encryption_sdk/materials/raw.py +++ b/src/dynamodb_encryption_sdk/materials/raw.py @@ -23,6 +23,7 @@ that you use wrapped cryptographic materials instead. """ import copy +from typing import Dict, Optional, Text import attr import six @@ -31,13 +32,6 @@ from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("RawEncryptionMaterials", "RawDecryptionMaterials") diff --git a/src/dynamodb_encryption_sdk/materials/wrapped.py b/src/dynamodb_encryption_sdk/materials/wrapped.py index f85924ea..74d2784c 100644 --- a/src/dynamodb_encryption_sdk/materials/wrapped.py +++ b/src/dynamodb_encryption_sdk/materials/wrapped.py @@ -13,6 +13,7 @@ """Cryptographic materials to use ephemeral content encryption keys wrapped by delegated keys.""" import base64 import copy +from typing import Dict, Optional, Text import attr import six @@ -25,13 +26,6 @@ from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import CryptographicMaterials -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Optional, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("WrappedCryptographicMaterials",) _DEFAULT_CONTENT_ENCRYPTION_ALGORITHM = "AES/256" _WRAPPING_TRANSFORMATION = {"AES": "AESWrap", "RSA": "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"} diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index 61b329c6..16976b98 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """Common structures used by the DynamoDB Encryption Client.""" import copy +from typing import Dict, Iterable, List, Optional, Set, Text import attr import six @@ -22,13 +23,6 @@ from .identifiers import CryptoAction -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Iterable, List, Optional, Set, Text # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass - - __all__ = ("EncryptionContext", "AttributeActions", "TableIndex", "TableInfo") @@ -176,7 +170,7 @@ def set_index_keys(self, *keys): SIGN_ONLY -> SIGN_ONLY ENCRYPT_AND_SIGN -> SIGN_ONLY - :param str *keys: Attribute names to treat as indexed + :param str ``*keys``: Attribute names to treat as indexed :raises InvalidArgumentError: if a custom action was previously set for any specified attributes """ diff --git a/src/dynamodb_encryption_sdk/transform.py b/src/dynamodb_encryption_sdk/transform.py index 347024d3..d79b8504 100644 --- a/src/dynamodb_encryption_sdk/transform.py +++ b/src/dynamodb_encryption_sdk/transform.py @@ -11,11 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Helper tools for translating between native and DynamoDB items.""" -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Dict # noqa pylint: disable=unused-import -except ImportError: # pragma: no cover - # We only actually need these imports when running the mypy checks - pass +from typing import Any, Dict from boto3.dynamodb.types import TypeDeserializer, TypeSerializer diff --git a/src/pylintrc b/src/pylintrc index bc0406f6..399920a7 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -8,6 +8,7 @@ disable = useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string, [BASIC] # Allow function names up to 50 characters diff --git a/test/README.rst b/test/README.rst index ead71c42..747522bb 100644 --- a/test/README.rst +++ b/test/README.rst @@ -22,16 +22,12 @@ Updating Upstream Requirements The purpose of the upstream requirements files is to provide a stable list of packages for dependencies to run downstream tests of the DynamoDB Encryption -Client. In order to update the upstream requirements in `upstream-requirements-py37.txt` -and `upstream-requirements-py27.txt`, run these commands: +Client. In order to update the upstream requirements in `upstream-requirements-py37.txt`, +run these commands:: - .. code:: + $ tox -e freeze-upstream-requirements-py37 - $ tox -e freeze-upstream-requirements-py27 - $ tox -e freeze-upstream-requirements-py37 +Test them using:: -Test them using: + $ tox -e test-upstream-requirements-py37 - .. code:: - $ tox -e test-upstream-requirements-py27 - $ tox -e test-upstream-requirements-py37 diff --git a/test/acceptance/acceptance_test_generators.py b/test/acceptance/acceptance_test_generators.py index 9ba01174..1c513bd3 100644 --- a/test/acceptance/acceptance_test_generators.py +++ b/test/acceptance/acceptance_test_generators.py @@ -43,7 +43,7 @@ def load_scenarios(online): into a shared method. """ # pylint: disable=too-many-locals - with open(_SCENARIO_FILE) as f: + with open(_SCENARIO_FILE, encoding="utf-8") as f: scenarios = json.load(f) keys_file = _filename_from_uri(scenarios["keys"]) keys = _load_keys(keys_file) @@ -128,7 +128,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info): if table: table.delete() - with open(ciphertext_file, "w") as outfile: + with open(ciphertext_file, "w", encoding="utf-8") as outfile: json.dump(data_table_output, outfile, indent=4) if metatable: @@ -137,7 +137,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info): metastore_output[metastore_info["table_name"]].append(ddb_to_json(wrapping_key)) metastore_ciphertext_file = _filename_from_uri(metastore_info["ciphertext"]) - with open(metastore_ciphertext_file, "w") as outfile: + with open(metastore_ciphertext_file, "w", encoding="utf-8") as outfile: json.dump(metastore_output, outfile, indent=4) metatable.delete() diff --git a/test/acceptance/acceptance_test_utils.py b/test/acceptance/acceptance_test_utils.py index c4f06b46..a7fd4c03 100644 --- a/test/acceptance/acceptance_test_utils.py +++ b/test/acceptance/acceptance_test_utils.py @@ -61,7 +61,7 @@ def _decode_item(item): def _build_plaintext_items(plaintext_file, version): # pylint: disable=too-many-locals - with open(plaintext_file) as f: + with open(plaintext_file, encoding="utf-8") as f: plaintext_data = json.load(f) actions = {} @@ -92,7 +92,7 @@ def _build_plaintext_items(plaintext_file, version): def _load_ciphertext_items(ciphertext_file): - with open(ciphertext_file) as f: + with open(ciphertext_file, encoding="utf-8") as f: ciphertexts = json.load(f) for _table, items in ciphertexts.items(): @@ -103,7 +103,7 @@ def _load_ciphertext_items(ciphertext_file): def _load_keys(keys_file): - with open(keys_file) as f: + with open(keys_file, encoding="utf-8") as f: return json.load(f) @@ -165,7 +165,7 @@ def _meta_table_prep(table_name, items_filename): table = boto3.resource("dynamodb", region_name="us-west-2").Table(table_name) table.wait_until_exists() try: - with open(_filename_from_uri(items_filename)) as f: + with open(_filename_from_uri(items_filename), encoding="utf-8") as f: table_data = json.load(f) request_items = {} @@ -255,7 +255,7 @@ def _expand_items(ciphertext_items, plaintext_items): def load_scenarios(online): # pylint: disable=too-many-locals - with open(_SCENARIO_FILE) as f: + with open(_SCENARIO_FILE, encoding="utf-8") as f: scenarios = json.load(f) keys_file = _filename_from_uri(scenarios["keys"]) keys = _load_keys(keys_file) diff --git a/test/freeze-upstream-requirements.sh b/test/freeze-upstream-requirements.sh index 293ae16d..2be3824d 100755 --- a/test/freeze-upstream-requirements.sh +++ b/test/freeze-upstream-requirements.sh @@ -6,5 +6,5 @@ if [ -z ${1} ]; then fi pip install -r requirements.txt -pip install -r test/requirements.txt +pip install -r dev_requirements/test-requirements.txt pip freeze > ${1} diff --git a/test/functional/functional_test_vector_generators.py b/test/functional/functional_test_vector_generators.py index 02906f35..9e711ad2 100644 --- a/test/functional/functional_test_vector_generators.py +++ b/test/functional/functional_test_vector_generators.py @@ -104,14 +104,14 @@ def _decode_complex_value(_value): def attribute_test_vectors(mode): filepath = _ATTRIBUTE_TEST_VECTOR_FILE_TEMPLATE.format(mode=mode) - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: yield (decode_value(vector["attribute"]), base64.b64decode(codecs.encode(vector["serialized"], "utf-8"))) def material_description_test_vectors(): - with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE) as f: + with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: yield (vector["material_description"], decode_value({"B": codecs.encode(vector["serialized"], "utf-8")})) @@ -125,7 +125,7 @@ def material_description_test_vectors(): def string_to_sign_test_vectors(): - with open(_STRING_TO_SIGN_TEST_VECTORS_FILE) as f: + with open(_STRING_TO_SIGN_TEST_VECTORS_FILE, encoding="utf-8") as f: vectors = json.load(f) for vector in vectors: item = {key: decode_value(value["value"]) for key, value in vector["item"].items()} diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index 6a39d4cf..059e14b6 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -23,6 +23,10 @@ hypothesis.HealthCheck.too_slow, hypothesis.HealthCheck.data_too_large, hypothesis.HealthCheck.large_base_example, + # Hypothesis requires that we acknowledge that the example_table fixure + # is not reset between examples generated by hypothesis.given. + # This is the desired behavior for example_table, so supress this check + hypothesis.HealthCheck.function_scoped_fixture, ), deadline=None, ) diff --git a/test/functional/internal/test_str_ops.py b/test/functional/internal/test_str_ops.py index 704e3e3f..1d9f7443 100644 --- a/test/functional/internal/test_str_ops.py +++ b/test/functional/internal/test_str_ops.py @@ -26,8 +26,8 @@ ( ("asdf", "asdf"), (b"asdf", "asdf"), - (codecs.encode(u"Предисловие", "utf-8"), u"Предисловие"), - (u"Предисловие", u"Предисловие"), + (codecs.encode("Предисловие", "utf-8"), "Предисловие"), + ("Предисловие", "Предисловие"), ), ) def test_to_str(data, expected_output): @@ -41,8 +41,8 @@ def test_to_str(data, expected_output): ("asdf", b"asdf"), (b"asdf", b"asdf"), (b"\x3a\x00\x99", b"\x3a\x00\x99"), - (u"Предисловие", codecs.encode(u"Предисловие", "utf-8")), - (codecs.encode(u"Предисловие", "utf-8"), codecs.encode(u"Предисловие", "utf-8")), + ("Предисловие", codecs.encode("Предисловие", "utf-8")), + (codecs.encode("Предисловие", "utf-8"), codecs.encode("Предисловие", "utf-8")), ), ) def test_to_bytes(data, expected_output): diff --git a/test/pylintrc b/test/pylintrc index ce2bba60..f63b3263 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -10,10 +10,12 @@ disable = protected-access, # raised when calling _ methods redefined-outer-name, # raised when using pytest-mock unused-argument, # raised when patches and fixtures are needed but not called + no-self-use, # raised on Classes in tests used for logically grouping tests # All below are disabled because we need to support Python 2 useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string, [DESIGN] max-args = 10 diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index 24ace5ac..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -hypothesis>=5.0.0,<6.0.0;python_version>='3' -hypothesis==4.57.1;python_version=='2.7' -mock -moto>=1.3.8 -pytest>=3.4.0 -pytest-cov -pytest-mock -pytest-xdist -boto3 -botocore diff --git a/test/source-build-check.sh b/test/source-build-check.sh index 22e31a83..1d6986a1 100755 --- a/test/source-build-check.sh +++ b/test/source-build-check.sh @@ -26,7 +26,7 @@ EXTRACTEDDIR=$(ls | tail -1) cd ${EXTRACTEDDIR} echo "Installing requirements from extracted source build." -pip install -r test/requirements.txt +pip install -r dev_requirements/test-requirements.txt pip install -e . echo "Running tests from extracted source build." diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index edcd301d..7fdc9f83 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -220,36 +220,32 @@ def test_loaded_key_infos(): assert cmp._regional_clients == {} -@pytest.mark.parametrize( - "kwargs", - [ - pytest.param(val, id=str(val)) - for val in all_possible_combinations_kwargs( - dict(), - dict(botocore_session=botocore.session.Session()), - dict(grant_tokens=("sdvoaweih", "auwshefiouawh")), - dict(material_description={"asoiufeoia": "soajfijewi"}), - dict( - regional_clients={ - "my-region-1": boto3.session.Session().client( - "kms", region_name="not-a-real-region", endpoint_url="https://not-a-real-url" - ) - } - ), - ) - ], -) -def test_kms_cmp_values_set(kwargs): - cmp = AwsKmsCryptographicMaterialsProvider(key_id="example_key_id", **kwargs) +def test_kms_cmp_values_set(): + # These aren't parametrized to avoid issues with pytest-xdist test mismatches + # due to different session objects per process + for kwargs in all_possible_combinations_kwargs( + {}, + dict(botocore_session=botocore.session.Session()), + dict(grant_tokens=("sdvoaweih", "auwshefiouawh")), + dict(material_description={"asoiufeoia": "soajfijewi"}), + dict( + regional_clients={ + "my-region-1": boto3.session.Session().client( + "kms", region_name="not-a-real-region", endpoint_url="https://not-a-real-url" + ) + } + ), + ): + cmp = AwsKmsCryptographicMaterialsProvider(key_id="example_key_id", **kwargs) - assert cmp._key_id == "example_key_id" + assert cmp._key_id == "example_key_id" - if "botocore_session" in kwargs: - assert cmp._botocore_session == kwargs["botocore_session"] + if "botocore_session" in kwargs: + assert cmp._botocore_session == kwargs["botocore_session"] - assert cmp._grant_tokens == kwargs.get("grant_tokens", ()) - assert cmp._material_description == kwargs.get("material_description", {}) - assert cmp._regional_clients == kwargs.get("regional_clients", {}) + assert cmp._grant_tokens == kwargs.get("grant_tokens", ()) + assert cmp._material_description == kwargs.get("material_description", {}) + assert cmp._regional_clients == kwargs.get("regional_clients", {}) def test_add_regional_client_known_region(default_kms_cmp, patch_boto3_session): diff --git a/test/unit/test_compatability.py b/test/unit/test_compatability.py new file mode 100644 index 00000000..314017e9 --- /dev/null +++ b/test/unit/test_compatability.py @@ -0,0 +1,37 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite for dynamodb_encryption_sdk.compatability.""" +import sys + +import mock +import pytest + +from dynamodb_encryption_sdk.compatability import _warn_deprecated_python + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +class TestWarnDeprecatedPython: + def test_happy_version(self, recwarn): + with mock.patch.object(sys, "version_info") as v_info: + v_info.major = 3 + v_info.minor = 8 + _warn_deprecated_python() + assert len(recwarn) == 0 + + def test_below_warn(self): + with mock.patch.object(sys, "version_info") as v_info: + v_info.major = 2 + v_info.minor = 7 + with pytest.warns(DeprecationWarning): + _warn_deprecated_python() diff --git a/test/upstream-requirements-py27.txt b/test/upstream-requirements-py27.txt deleted file mode 100644 index cb3364b1..00000000 --- a/test/upstream-requirements-py27.txt +++ /dev/null @@ -1,78 +0,0 @@ -apipkg==1.5 -asn1crypto==1.0.1 -atomicwrites==1.3.0 -attrs==19.2.0 -aws-sam-translator==1.15.0 -aws-xray-sdk==2.4.2 -backports.ssl-match-hostname==3.7.0.1 -backports.tempfile==1.0 -backports.weakref==1.0.post1 -boto==2.49.0 -boto3==1.9.246 -botocore==1.12.246 -certifi==2019.9.11 -cffi==1.12.3 -cfn-lint==0.24.4 -chardet==3.0.4 -configparser==4.0.2 -contextlib2==0.6.0.post1 -cookies==2.2.1 -coverage==4.5.4 -cryptography==3.3.2 -DateTime==4.3 -docker==4.1.0 -docutils==0.15.2 -ecdsa==0.13.3 -enum34==1.1.6 -execnet==1.7.1 -funcsigs==1.0.2 -functools32==3.2.3.post2 -future==0.18.0 -futures==3.3.0 -hypothesis==4.40.0 -idna==2.8 -importlib-metadata==0.23 -ipaddress==1.0.22 -Jinja2==2.11.3 -jmespath==0.9.4 -jsondiff==1.1.2 -jsonpatch==1.24 -jsonpickle==1.2 -jsonpointer==2.0 -jsonschema==3.1.1 -MarkupSafe==1.1.1 -mock==3.0.5 -more-itertools==5.0.0 -moto==1.3.13 -packaging==19.2 -pathlib2==2.3.5 -pluggy==0.13.0 -py==1.10.0 -pyasn1==0.4.7 -pycparser==2.19 -pyparsing==2.4.2 -pyrsistent==0.15.4 -pytest==4.6.5 -pytest-cov==2.8.1 -pytest-forked==1.0.2 -pytest-mock==1.11.1 -pytest-xdist==1.30.0 -python-dateutil==2.8.0 -python-jose==3.0.1 -pytz==2019.3 -PyYAML==5.4 -requests==2.22.0 -responses==0.10.6 -rsa==4.5 -s3transfer==0.2.1 -scandir==1.10.0 -six==1.12.0 -sshpubkeys==3.1.0 -urllib3==1.25.8 -wcwidth==0.1.7 -websocket-client==0.56.0 -Werkzeug==0.16.0 -wrapt==1.11.2 -xmltodict==0.12.0 -zipp==0.6.0 -zope.interface==4.6.0 diff --git a/test/upstream-requirements-py311.txt b/test/upstream-requirements-py311.txt new file mode 100644 index 00000000..1c3051be --- /dev/null +++ b/test/upstream-requirements-py311.txt @@ -0,0 +1,38 @@ +attrs==22.2.0 +boto3==1.20.51 +botocore==1.23.51 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.0.1 +coverage==7.1.0 +cryptography==42.0.4 +execnet==1.9.0 +hypothesis==6.31.6 +idna==3.4 +iniconfig==2.0.0 +Jinja2==3.1.3 +jmespath==0.10.0 +MarkupSafe==2.1.2 +mock==4.0.3 +moto==3.0.2 +packaging==23.0 +pluggy==1.5.0 +py==1.11.0 +pycparser==2.21 +pytest==8.2.0 +pytest-cov==3.0.0 +pytest-forked==1.6.0 +pytest-mock==3.10.0 +pytest-xdist==3.2.0 +python-dateutil==2.8.2 +pytz==2022.7.1 +requests==2.31.0 +responses==0.22.0 +s3transfer==0.5.2 +six==1.16.0 +sortedcontainers==2.4.0 +toml==0.10.2 +types-toml==0.10.8.5 +urllib3==1.26.18 +Werkzeug==3.0.3 +xmltodict==0.13.0 diff --git a/test/upstream-requirements-py37.txt b/test/upstream-requirements-py37.txt deleted file mode 100644 index fe00514f..00000000 --- a/test/upstream-requirements-py37.txt +++ /dev/null @@ -1,65 +0,0 @@ -apipkg==1.5 -asn1crypto==1.0.1 -atomicwrites==1.3.0 -attrs==19.2.0 -aws-sam-translator==1.15.0 -aws-xray-sdk==2.4.2 -boto==2.49.0 -boto3==1.9.246 -botocore==1.12.246 -certifi==2019.9.11 -cffi==1.12.3 -cfn-lint==0.24.4 -chardet==3.0.4 -coverage==4.5.4 -cryptography==3.3.2 -DateTime==4.3 -docker==4.1.0 -docutils==0.15.2 -ecdsa==0.13.3 -execnet==1.7.1 -future==0.18.0 -hypothesis==4.40.0 -idna==2.8 -importlib-metadata==0.23 -Jinja2==2.11.3 -jmespath==0.9.4 -jsondiff==1.1.2 -jsonpatch==1.24 -jsonpickle==1.2 -jsonpointer==2.0 -jsonschema==3.1.1 -MarkupSafe==1.1.1 -mock==3.0.5 -more-itertools==7.2.0 -moto==1.3.13 -packaging==19.2 -pluggy==0.13.0 -py==1.10.0 -pyasn1==0.4.7 -pycparser==2.19 -pyparsing==2.4.2 -pyrsistent==0.15.4 -pytest==5.2.1 -pytest-cov==2.8.1 -pytest-forked==1.0.2 -pytest-mock==1.11.1 -pytest-xdist==1.30.0 -python-dateutil==2.8.0 -python-jose==3.0.1 -pytz==2019.3 -PyYAML==5.4 -requests==2.22.0 -responses==0.10.6 -rsa==4.5 -s3transfer==0.2.1 -six==1.12.0 -sshpubkeys==3.1.0 -urllib3==1.25.8 -wcwidth==0.1.7 -websocket-client==0.56.0 -Werkzeug==0.16.0 -wrapt==1.11.2 -xmltodict==0.12.0 -zipp==0.6.0 -zope.interface==4.6.0 diff --git a/test/upstream.md b/test/upstream.md new file mode 100644 index 00000000..aeb3ed63 --- /dev/null +++ b/test/upstream.md @@ -0,0 +1,7 @@ +AWS Crypto Tools maintains `test/upstream-requirements-py.txt` in our Python products such that +our Cryptographic Primitive Provider for Python ([pyca/cryptography](https://github.com/pyca/cryptography)) +may execute downstream tests against AWS Crypto Tools Python products. +These files allow pyca to install and test the Crypto Tools products. +Additionally, Crypto Tools should maintain a test configuration that can be completed without using any AWS resources. +If Crypto Tools needs to contact pyca about this expectation, +they should cut a issue to the pyca/cryptography repo. diff --git a/tox.ini b/tox.ini index 9f752505..9024f22b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] envlist = - py{27,35,36,37,38,39}-{local,integ,ddb,examples}-fast, + py{38,39,310,311,312}-{local,integ,ddb,examples}-fast, nocmk, sourcebuildcheck, docs, bandit, doc8, readme, flake8{,-tests,-examples}, pylint{,-tests,-examples}, vulture, - test-upstream-requirements-py{2,3}7 + test-upstream-requirements-py3{11} # Additional environments: # @@ -40,27 +40,29 @@ envlist = # coverage :: Runs code coverage, failing the build if coverage is below the configured threshold [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l {posargs} +commands = pytest -n auto --basetemp={envtmpdir} -l {posargs} [testenv] passenv = # Identifies AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID # Identifies AWS KMS MRK key ids to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID \ - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 # DynamoDB Table to use in integration tests - DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME \ + DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME # Pass through AWS credentials - AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN # AWS Role access in CodeBuild is via the contaner URI - AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI # Pass through AWS profile name (useful for local testing) - AWS_PROFILE \ + AWS_PROFILE # Pass through the default AWS region (used for integration tests) AWS_DEFAULT_REGION sitepackages = False -deps = -rtest/requirements.txt +deps = -rdev_requirements/test-requirements.txt # 'download' forces tox to always upgrade pip to the latest download = true commands = @@ -91,7 +93,7 @@ commands = # Do not select any specific markers manual: {[testenv:base-command]commands} # Only run examples tests - examples: {[testenv:base-command]commands} examples/test/ -m "examples" + examples: {[testenv:base-command]commands} examples/test -m "examples" # Run code coverage on the unit tests [testenv:coverage] @@ -106,7 +108,7 @@ sitepackages = False passenv = setenv = ######################################################### -deps = -rtest/requirements.txt +deps = -rdev_requirements/test-requirements.txt commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" --ignore=examples # Collect requirements for use in upstream tests @@ -117,23 +119,14 @@ recreate = True deps = commands = {toxinidir}/test/freeze-upstream-requirements.sh -# Freeze for Python 2.7 -[testenv:freeze-upstream-requirements-py27] -basepython = python2.7 +# Freeze for Python 3.11 +[testenv:freeze-upstream-requirements-py311] +basepython = python3.11 sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} recreate = {[testenv:freeze-upstream-requirements-base]recreate} deps = {[testenv:freeze-upstream-requirements-base]deps} -commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py27.txt - -# Freeze for Python 3.7 -[testenv:freeze-upstream-requirements-py37] -basepython = python3.7 -sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} -skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} -recreate = {[testenv:freeze-upstream-requirements-base]recreate} -deps = {[testenv:freeze-upstream-requirements-base]deps} -commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py37.txt +commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py311.txt # Test frozen upstream requirements [testenv:test-upstream-requirements-base] @@ -142,20 +135,11 @@ recreate = True passenv = commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" --ignore=examples -# Test frozen upstream requirements for Python 2.7 -[testenv:test-upstream-requirements-py27] -basepython = python2.7 -passenv = -deps = -rtest/upstream-requirements-py27.txt -sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} -recreate = {[testenv:test-upstream-requirements-base]recreate} -commands = {[testenv:test-upstream-requirements-base]commands} - -# Test frozen upstream requirements for Python 3.7 -[testenv:test-upstream-requirements-py37] -basepython = python3.7 +# Test frozen upstream requirements for Python 3.11 +[testenv:test-upstream-requirements-py311] +basepython = python3.11 passenv = -deps = -rtest/upstream-requirements-py37.txt +deps = -rtest/upstream-requirements-py311.txt sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} @@ -167,6 +151,7 @@ sitepackages = False recreate = True deps = {[testenv:build]deps} + -rdev_requirements/test-requirements.txt commands = {[testenv:build]commands} {toxinidir}/test/source-build-check.sh {envtmpdir} {toxinidir}/dist @@ -201,28 +186,10 @@ commands = {posargs} {[testenv:mypy-coverage]commands} -[testenv:mypy-py2] -basepython = python2.7 -deps = {[testenv:mypy-common]deps} -commands = - python -m mypy \ - --py2 \ - --linecoverage-report build \ - src/dynamodb_encryption_sdk/ \ - {posargs} - {[testenv:mypy-coverage]commands} - # Linters [testenv:flake8] basepython = python3 -deps = - flake8 - flake8-docstrings - flake8-isort - # https://github.com/PyCQA/pydocstyle/issues/375 - pydocstyle<4.0.0 - # https://github.com/JBKahn/flake8-print/pull/30 - flake8-print>=3.1.0 +deps = -rdev_requirements/linter-requirements.txt commands = flake8 \ src/dynamodb_encryption_sdk/ \ @@ -231,7 +198,7 @@ commands = [testenv:flake8-tests] basepython = {[testenv:flake8]basepython} -deps = {[testenv:flake8]deps} +deps = -rdev_requirements/linter-requirements.txt commands = flake8 \ # Ignore F811 redefinition errors in tests (breaks with pytest-mock use) @@ -261,8 +228,7 @@ commands = basepython = python3 deps = {[testenv]deps} - pyflakes - pylint + -rdev_requirements/linter-requirements.txt commands = pylint \ --rcfile=src/pylintrc \ @@ -290,8 +256,7 @@ commands = [testenv:blacken-src] basepython = python3 -deps = - black +deps = -rdev_requirements/linter-requirements.txt commands = black --line-length 120 \ src/dynamodb_encryption_sdk/ \ @@ -305,28 +270,24 @@ commands = [testenv:blacken] basepython = python3 -deps = - {[testenv:blacken-src]deps} +deps = {[testenv:blacken-src]deps} commands = {[testenv:blacken-src]commands} [testenv:black-check] basepython = python3 -deps = - {[testenv:blacken]deps} +deps = {[testenv:blacken]deps} commands = {[testenv:blacken-src]commands} --diff [testenv:isort-seed] basepython = python3 -deps = seed-isort-config +deps = -rdev_requirements/linter-requirements.txt commands = seed-isort-config [testenv:isort] basepython = python3 -# We need >=5.0.0 because -# several configuration settings changed with 5.0.0 -deps = isort>=5.0.0 +deps = -rdev_requirements/linter-requirements.txt commands = isort \ src \ test \ @@ -345,8 +306,8 @@ commands = {[testenv:isort]commands} -c [testenv:autoformat] basepython = python3 deps = - {[testenv:isort]deps} {[testenv:blacken]deps} + {[testenv:isort]deps} commands = {[testenv:isort]commands} {[testenv:blacken]commands} @@ -366,27 +327,27 @@ commands = basepython = python3 whitelist_externals = {[testenv:resetdocs]whitelist_externals} deps = - sphinx - doc8 -commands = + -rdev_requirements/doc-requirements.txt + -rdev_requirements/linter-requirements.txt +commands = {[testenv:resetdocs]commands} doc8 doc/index.rst doc/lib/ README.rst CHANGELOG.rst - + [testenv:readme] basepython = python3 -deps = readme_renderer +deps = -rdev_requirements/linter-requirements.txt commands = python setup.py check -r -s [testenv:bandit] basepython = python3 -deps = bandit +deps = -rdev_requirements/linter-requirements.txt commands = bandit -r src/dynamodb_encryption_sdk/ # Prone to false positives: only run independently [testenv:vulture] basepython = python3 -deps = vulture +deps = -rdev_requirements/linter-requirements.txt commands = vulture src/dynamodb_encryption_sdk/ [testenv:linters] @@ -416,7 +377,7 @@ commands = # Documentation [testenv:docs] basepython = python3 -deps = -rdoc/requirements.txt +deps = -rdev_requirements/doc-requirements.txt commands = sphinx-build -E -c doc/ -b html doc/ doc/build/html sphinx-build -E -c doc/ -b linkcheck doc/ doc/build/html @@ -433,27 +394,21 @@ commands = [testenv:park] basepython = python3 skip_install = true -deps = - pypi-parker - setuptools +deps = -rdev_requirements/release-requirements.txt commands = python setup.py park # Release tooling [testenv:build] basepython = python3 skip_install = true -deps = - wheel - setuptools +deps = -rdev_requirements/release-requirements.txt commands = python setup.py sdist bdist_wheel [testenv:release-base] basepython = python3 skip_install = true -deps = - {[testenv:build]deps} - twine +deps = -rdev_requirements/release-requirements.txt passenv = # Intentionally omit TWINE_REPOSITORY_URL from the passenv list, # as this overrides other ways of setting the repository and could