From c005cc76112d6a03ff1598e05b496ca0f1ca3673 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sun, 22 Nov 2020 11:10:34 +0200 Subject: [PATCH 01/20] feat: Add Ses lambda event support to Parser utility #213 --- .../utilities/parser/models/__init__.py | 3 + .../utilities/parser/models/ses.py | 71 +++++ poetry.lock | 274 +++++++++++------- pyproject.toml | 3 +- tests/functional/parser/test_ses.py | 49 ++++ 5 files changed, 289 insertions(+), 111 deletions(-) create mode 100644 aws_lambda_powertools/utilities/parser/models/ses.py create mode 100644 tests/functional/parser/test_ses.py diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index e9daded27ca..36ba05240b0 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,5 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel +from .ses import SesModel, SesRecordModel from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel @@ -8,6 +9,8 @@ "EventBridgeModel", "DynamoDBStreamChangedRecordModel", "DynamoDBStreamRecordModel", + "SesModel", + "SesRecordModel", "SnsModel", "SnsNotificationModel", "SnsRecordModel", diff --git a/aws_lambda_powertools/utilities/parser/models/ses.py b/aws_lambda_powertools/utilities/parser/models/ses.py new file mode 100644 index 00000000000..c82ae03a6c6 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/ses.py @@ -0,0 +1,71 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, Field +from pydantic.networks import EmailStr +from pydantic.types import PositiveInt +from typing_extensions import Literal + + +class SesReceiptVerdict(BaseModel): + status: Literal["PASS", "FAIL", "GRAY", "PROCESSING_FAILED"] + + +class SesReceiptAction(BaseModel): + type: Literal["Lambda"] # noqa A003,VNE003 + invocationType: Literal["Event"] + functionArn: str + + +class SesReceipt(BaseModel): + timestamp: datetime + processingTimeMillis: PositiveInt + recipients: List[EmailStr] + spamVerdict: SesReceiptVerdict + virusVerdict: SesReceiptVerdict + spfVerdict: SesReceiptVerdict + dmarcVerdict: SesReceiptVerdict + action: SesReceiptAction + + +class SesMailHeaders(BaseModel): + name: str + value: str + + +class SesMailCommonHeaders(BaseModel): + header_from: List[str] = Field(None, alias="from") + to: List[str] + cc: Optional[List[str]] + bcc: Optional[List[str]] + sender: Optional[List[str]] + reply_to: Optional[List[str]] = Field(None, alias="reply-to") + returnPath: EmailStr + messageId: str + date: str + subject: str + + +class SesMail(BaseModel): + timestamp: datetime + source: EmailStr + messageId: str + destination: List[EmailStr] + headersTruncated: bool + headers: List[SesMailHeaders] + commonHeaders: SesMailCommonHeaders + + +class SesMessage(BaseModel): + mail: SesMail + receipt: SesReceipt + + +class SesRecordModel(BaseModel): + eventSource: Literal["aws:ses"] + eventVersion: str + ses: SesMessage + + +class SesModel(BaseModel): + Records: List[SesRecordModel] diff --git a/poetry.lock b/poetry.lock index b255194ddc5..791c9dbf7d9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -16,15 +16,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "20.2.0" +version = "20.3.0" description = "Classes Without Boilerplate" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] @@ -79,20 +79,20 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "boto3" -version = "1.15.16" +version = "1.16.23" description = "The AWS SDK for Python" category = "main" optional = false python-versions = "*" [package.dependencies] -botocore = ">=1.18.16,<1.19.0" +botocore = ">=1.19.23,<1.20.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "botocore" -version = "1.18.16" +version = "1.19.23" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -101,11 +101,11 @@ python-versions = "*" [package.dependencies] jmespath = ">=0.7.1,<1.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = {version = ">=1.20,<1.26", markers = "python_version != \"3.4\""} +urllib3 = {version = ">=1.25.4,<1.27", markers = "python_version != \"3.4\""} [[package]] name = "certifi" -version = "2020.6.20" +version = "2020.11.8" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false @@ -129,7 +129,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" -version = "0.4.3" +version = "0.4.4" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -151,12 +151,39 @@ toml = ["toml"] [[package]] name = "dataclasses" -version = "0.7" +version = "0.8" description = "A backport of the dataclasses module for Python 3.6" category = "main" optional = true python-versions = ">=3.6, <3.7" +[[package]] +name = "dnspython" +version = "2.0.0" +description = "DNS toolkit" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +dnssec = ["cryptography (>=2.6)"] +doh = ["requests", "requests-toolbelt"] +idna = ["idna (>=2.1)"] +curio = ["curio (>=1.2)", "sniffio (>=1.1)"] +trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] + +[[package]] +name = "email-validator" +version = "1.1.2" +description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +dnspython = ">=1.15.0" +idna = ">=2.0.0" + [[package]] name = "eradicate" version = "1.0" @@ -230,7 +257,7 @@ test = ["coverage", "coveralls", "mock", "pytest", "pytest-cov"] [[package]] name = "flake8-comprehensions" -version = "3.2.3" +version = "3.3.0" description = "A flake8 plugin to help you write better list/set/dict comprehensions." category = "dev" optional = false @@ -329,7 +356,7 @@ smmap = ">=3.0.1,<4" [[package]] name = "gitpython" -version = "3.1.9" +version = "3.1.11" description = "Python Git Library" category = "dev" optional = false @@ -342,7 +369,7 @@ gitdb = ">=4.0.1,<5" name = "idna" version = "2.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -399,7 +426,7 @@ importlib-metadata = "*" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["coverage (<5)", "pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-black-multipy", "pytest-cov", "ecdsa", "feedparser", "numpy", "pandas", "pymongo", "sqlalchemy", "enum34", "jsonlib"] +testing = ["coverage (<5)", "pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-black-multipy", "pytest-cov", "ecdsa", "feedparser", "numpy", "pandas", "pymongo", "sqlalchemy", "enum34", "jsonlib"] "testing.libs" = ["demjson", "simplejson", "ujson", "yajl"] [[package]] @@ -433,7 +460,7 @@ restructuredText = ["rst2ansi"] [[package]] name = "markdown" -version = "3.3" +version = "3.3.3" description = "Python implementation of Markdown." category = "dev" optional = false @@ -463,7 +490,7 @@ python-versions = "*" [[package]] name = "more-itertools" -version = "8.5.0" +version = "8.6.0" description = "More routines for operating on iterables, beyond itertools" category = "dev" optional = false @@ -483,7 +510,7 @@ six = "*" [[package]] name = "pathspec" -version = "0.8.0" +version = "0.8.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false @@ -491,7 +518,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pbr" -version = "5.5.0" +version = "5.5.1" description = "Python Build Reasonableness" category = "dev" optional = false @@ -541,7 +568,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.6.1" +version = "1.7.2" description = "Data validation and settings management using python 3.6 type hinting" category = "main" optional = true @@ -591,7 +618,7 @@ py = ">=1.5.0" wcwidth = "*" [package.extras] -checkqa-mypy = ["mypy (v0.761)"] +checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -621,7 +648,7 @@ coverage = ">=4.4" pytest = ">=4.6" [package.extras] -testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] name = "pytest-mock" @@ -675,7 +702,7 @@ flake8 = ["flake8-polyfill"] [[package]] name = "regex" -version = "2020.10.11" +version = "2020.11.13" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -683,7 +710,7 @@ python-versions = "*" [[package]] name = "requests" -version = "2.24.0" +version = "2.25.0" description = "Python HTTP for Humans." category = "dev" optional = false @@ -693,11 +720,11 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" certifi = ">=2017.4.17" chardet = ">=3.0.2,<4" idna = ">=2.5,<3" -urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" +urllib3 = ">=1.21.1,<1.27" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "s3transfer" @@ -753,11 +780,11 @@ test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybi [[package]] name = "toml" -version = "0.10.1" +version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false -python-versions = "*" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typed-ast" @@ -777,7 +804,7 @@ python-versions = "*" [[package]] name = "urllib3" -version = "1.25.10" +version = "1.26.2" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -785,8 +812,8 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" @@ -819,7 +846,7 @@ requests = ">=2.0,<3.0" [[package]] name = "zipp" -version = "3.3.0" +version = "3.4.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false @@ -827,15 +854,15 @@ python-versions = ">=3.6" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [extras] -pydantic = ["pydantic", "typing_extensions"] +pydantic = ["pydantic", "typing_extensions", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "e18b9f99b7876adb78623fd8b2acb9a6f76a5e427c30d0c9ec7ebb5786bc4a52" +content-hash = "a135429f1982397757348571e17ba6435fb18c110e986038bb44a909548db005" [metadata.files] appdirs = [ @@ -847,8 +874,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, - {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] aws-xray-sdk = [ {file = "aws-xray-sdk-2.6.0.tar.gz", hash = "sha256:abf5b90f740e1f402e23414c9670e59cb9772e235e271fef2bce62b9100cbc77"}, @@ -863,16 +890,16 @@ black = [ {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] boto3 = [ - {file = "boto3-1.15.16-py2.py3-none-any.whl", hash = "sha256:557320fe8b65cfc85953e6a63d2328e8efec95bf4ec383b92fa2d01119209716"}, - {file = "boto3-1.15.16.tar.gz", hash = "sha256:454a8dfb7b367a058c7967ef6b4e2a192c318f10761769fd1003cf7f2f5a7db9"}, + {file = "boto3-1.16.23-py2.py3-none-any.whl", hash = "sha256:22a6f11383965d7ece9e391722b2989780960c62997b1aa464ffa1f886e1cfa8"}, + {file = "boto3-1.16.23.tar.gz", hash = "sha256:6e6bd178f930309c2ec79643436aae5cf6f26d51e35aa5e58162675a04785e62"}, ] botocore = [ - {file = "botocore-1.18.16-py2.py3-none-any.whl", hash = "sha256:e586e4d6eddbca31e6447a25df9972329ea3de64b1fb0eb17e7ab0c9b91f7720"}, - {file = "botocore-1.18.16.tar.gz", hash = "sha256:f0616d2c719691b94470307cee8adf89ceb1657b7b6f9aa1bf61f9de5543dbbb"}, + {file = "botocore-1.19.23-py2.py3-none-any.whl", hash = "sha256:d73a223bf88d067c3ae0a9a3199abe56e99c94267da77d7fed4c39f572f522c0"}, + {file = "botocore-1.19.23.tar.gz", hash = "sha256:9f9efca44b2ab2d9c133ceeafa377e4b3d260310109284123ebfffc15e28481e"}, ] certifi = [ - {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, - {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, + {file = "certifi-2020.11.8-py2.py3-none-any.whl", hash = "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd"}, + {file = "certifi-2020.11.8.tar.gz", hash = "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, @@ -883,8 +910,8 @@ click = [ {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, - {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ {file = "coverage-5.3-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270"}, @@ -923,8 +950,16 @@ coverage = [ {file = "coverage-5.3.tar.gz", hash = "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0"}, ] dataclasses = [ - {file = "dataclasses-0.7-py3-none-any.whl", hash = "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836"}, - {file = "dataclasses-0.7.tar.gz", hash = "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6"}, + {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, + {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, +] +dnspython = [ + {file = "dnspython-2.0.0-py3-none-any.whl", hash = "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"}, + {file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"}, +] +email-validator = [ + {file = "email-validator-1.1.2.tar.gz", hash = "sha256:1a13bd6050d1db4475f13e444e169b6fe872434922d38968c67cea9568cce2f0"}, + {file = "email_validator-1.1.2-py2.py3-none-any.whl", hash = "sha256:094b1d1c60d790649989d38d34f69e1ef07792366277a2cf88684d03495d018f"}, ] eradicate = [ {file = "eradicate-1.0.tar.gz", hash = "sha256:4ffda82aae6fd49dfffa777a857cb758d77502a1f2e0f54c9ac5155a39d2d01a"}, @@ -949,8 +984,8 @@ flake8-builtins = [ {file = "flake8_builtins-1.5.3-py2.py3-none-any.whl", hash = "sha256:7706babee43879320376861897e5d1468e396a40b8918ed7bccf70e5f90b8687"}, ] flake8-comprehensions = [ - {file = "flake8-comprehensions-3.2.3.tar.gz", hash = "sha256:d5751acc0f7364794c71d06f113f4686d6e2e26146a50fa93130b9f200fe160d"}, - {file = "flake8_comprehensions-3.2.3-py3-none-any.whl", hash = "sha256:44eaae9894aa15f86e0c86df1e218e7917494fab6f96d28f96a029c460f17d92"}, + {file = "flake8-comprehensions-3.3.0.tar.gz", hash = "sha256:355ef47288523cad7977cb9c1bc81b71c82b7091e425cd9fbcd7e5c19a613677"}, + {file = "flake8_comprehensions-3.3.0-py3-none-any.whl", hash = "sha256:c1dd6d8a00e9722619a5c5e0e6c5747f5cf23c089032c86eaf614c14a2e40adb"}, ] flake8-debugger = [ {file = "flake8-debugger-3.2.1.tar.gz", hash = "sha256:712d7c1ff69ddf3f0130e94cc88c2519e720760bce45e8c330bfdcb61ab4090d"}, @@ -982,8 +1017,8 @@ gitdb = [ {file = "gitdb-4.0.5.tar.gz", hash = "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9"}, ] gitpython = [ - {file = "GitPython-3.1.9-py3-none-any.whl", hash = "sha256:138016d519bf4dd55b22c682c904ed2fd0235c3612b2f8f65ce218ff358deed8"}, - {file = "GitPython-3.1.9.tar.gz", hash = "sha256:a03f728b49ce9597a6655793207c6ab0da55519368ff5961e4a74ae475b9fa8e"}, + {file = "GitPython-3.1.11-py3-none-any.whl", hash = "sha256:6eea89b655917b500437e9668e4a12eabdcf00229a0df1762aabd692ef9b746b"}, + {file = "GitPython-3.1.11.tar.gz", hash = "sha256:befa4d101f91bad1b632df4308ec64555db684c360bd7d2130b4807d49ce86b8"}, ] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, @@ -1014,8 +1049,8 @@ mando = [ {file = "mando-0.6.4.tar.gz", hash = "sha256:79feb19dc0f097daa64a1243db578e7674909b75f88ac2220f1c065c10a0d960"}, ] markdown = [ - {file = "Markdown-3.3-py3-none-any.whl", hash = "sha256:fbb1ba54ca41e8991dc5a561d9c6f752f5e4546f8750e56413ea50f2385761d3"}, - {file = "Markdown-3.3.tar.gz", hash = "sha256:4f4172a4e989b97f96860fa434b89895069c576e2b537c4b4eed265266a7affc"}, + {file = "Markdown-3.3.3-py3-none-any.whl", hash = "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328"}, + {file = "Markdown-3.3.3.tar.gz", hash = "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1057,20 +1092,20 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] more-itertools = [ - {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"}, - {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"}, + {file = "more-itertools-8.6.0.tar.gz", hash = "sha256:b3a9005928e5bed54076e6e549c792b306fddfe72b2d1d22dd63d42d5d3899cf"}, + {file = "more_itertools-8.6.0-py3-none-any.whl", hash = "sha256:8e1a2a43b2f2727425f2b5839587ae37093f19153dc26c0927d1048ff6557330"}, ] packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, ] pathspec = [ - {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, - {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, ] pbr = [ - {file = "pbr-5.5.0-py2.py3-none-any.whl", hash = "sha256:5adc0f9fc64319d8df5ca1e4e06eea674c26b80e6f00c530b18ce6a6592ead15"}, - {file = "pbr-5.5.0.tar.gz", hash = "sha256:14bfd98f51c78a3dd22a1ef45cf194ad79eee4a19e8e1a0d5c7f8e81ffe182ea"}, + {file = "pbr-5.5.1-py2.py3-none-any.whl", hash = "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00"}, + {file = "pbr-5.5.1.tar.gz", hash = "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9"}, ] pdoc3 = [ {file = "pdoc3-0.7.5.tar.gz", hash = "sha256:ebca75b7fcf23f3b4320abe23339834d3f08c28517718e9d29e555fc38eeb33c"}, @@ -1088,23 +1123,28 @@ pycodestyle = [ {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, ] pydantic = [ - {file = "pydantic-1.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614"}, - {file = "pydantic-1.6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99"}, - {file = "pydantic-1.6.1-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b"}, - {file = "pydantic-1.6.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e"}, - {file = "pydantic-1.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1"}, - {file = "pydantic-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e"}, - {file = "pydantic-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1"}, - {file = "pydantic-1.6.1-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c"}, - {file = "pydantic-1.6.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df"}, - {file = "pydantic-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b"}, - {file = "pydantic-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9"}, - {file = "pydantic-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d"}, - {file = "pydantic-1.6.1-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7"}, - {file = "pydantic-1.6.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20"}, - {file = "pydantic-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633"}, - {file = "pydantic-1.6.1-py36.py37.py38-none-any.whl", hash = "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d"}, - {file = "pydantic-1.6.1.tar.gz", hash = "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73"}, + {file = "pydantic-1.7.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dfaa6ed1d509b5aef4142084206584280bb6e9014f01df931ec6febdad5b200a"}, + {file = "pydantic-1.7.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:2182ba2a9290964b278bcc07a8d24207de709125d520efec9ad6fa6f92ee058d"}, + {file = "pydantic-1.7.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:0fe8b45d31ae53d74a6aa0bf801587bd49970070eac6a6326f9fa2a302703b8a"}, + {file = "pydantic-1.7.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:01f0291f4951580f320f7ae3f2ecaf0044cdebcc9b45c5f882a7e84453362420"}, + {file = "pydantic-1.7.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4ba6b903e1b7bd3eb5df0e78d7364b7e831ed8b4cd781ebc3c4f1077fbcb72a4"}, + {file = "pydantic-1.7.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b11fc9530bf0698c8014b2bdb3bbc50243e82a7fa2577c8cfba660bcc819e768"}, + {file = "pydantic-1.7.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a3c274c49930dc047a75ecc865e435f3df89715c775db75ddb0186804d9b04d0"}, + {file = "pydantic-1.7.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:c68b5edf4da53c98bb1ccb556ae8f655575cb2e676aef066c12b08c724a3f1a1"}, + {file = "pydantic-1.7.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:95d4410c4e429480c736bba0db6cce5aaa311304aea685ebcf9ee47571bfd7c8"}, + {file = "pydantic-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a2fc7bf77ed4a7a961d7684afe177ff59971828141e608f142e4af858e07dddc"}, + {file = "pydantic-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9572c0db13c8658b4a4cb705dcaae6983aeb9842248b36761b3fbc9010b740f"}, + {file = "pydantic-1.7.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f83f679e727742b0c465e7ef992d6da4a7e5268b8edd8fdaf5303276374bef52"}, + {file = "pydantic-1.7.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:e5fece30e80087d9b7986104e2ac150647ec1658c4789c89893b03b100ca3164"}, + {file = "pydantic-1.7.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ce2d452961352ba229fe1e0b925b41c0c37128f08dddb788d0fd73fd87ea0f66"}, + {file = "pydantic-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:fc21a37ff3f545de80b166e1735c4172b41b017948a3fb2d5e2f03c219eac50a"}, + {file = "pydantic-1.7.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c9760d1556ec59ff745f88269a8f357e2b7afc75c556b3a87b8dda5bc62da8ba"}, + {file = "pydantic-1.7.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c1673633ad1eea78b1c5c420a47cd48717d2ef214c8230d96ca2591e9e00958"}, + {file = "pydantic-1.7.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:388c0c26c574ff49bad7d0fd6ed82fbccd86a0473fa3900397d3354c533d6ebb"}, + {file = "pydantic-1.7.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:ab1d5e4d8de00575957e1c982b951bffaedd3204ddd24694e3baca3332e53a23"}, + {file = "pydantic-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:f045cf7afb3352a03bc6cb993578a34560ac24c5d004fa33c76efec6ada1361a"}, + {file = "pydantic-1.7.2-py3-none-any.whl", hash = "sha256:6665f7ab7fbbf4d3c1040925ff4d42d7549a8c15fe041164adfe4fc2134d4cce"}, + {file = "pydantic-1.7.2.tar.gz", hash = "sha256:c8200aecbd1fb914e1bd061d71a4d1d79ecb553165296af0c14989b89e90d09b"}, ] pyflakes = [ {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, @@ -1151,37 +1191,51 @@ radon = [ {file = "radon-4.3.2.tar.gz", hash = "sha256:758b3ab345aa86e95f642713612a57da7c7da6d552c4dbfbe397a67601ace7dd"}, ] regex = [ - {file = "regex-2020.10.11-cp27-cp27m-win32.whl", hash = "sha256:4f5c0fe46fb79a7adf766b365cae56cafbf352c27358fda811e4a1dc8216d0db"}, - {file = "regex-2020.10.11-cp27-cp27m-win_amd64.whl", hash = "sha256:39a5ef30bca911f5a8a3d4476f5713ed4d66e313d9fb6755b32bec8a2e519635"}, - {file = "regex-2020.10.11-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:7c4fc5a8ec91a2254bb459db27dbd9e16bba1dabff638f425d736888d34aaefa"}, - {file = "regex-2020.10.11-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d537e270b3e6bfaea4f49eaf267984bfb3628c86670e9ad2a257358d3b8f0955"}, - {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a8240df4957a5b0e641998a5d78b3c4ea762c845d8cb8997bf820626826fde9a"}, - {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:4302153abb96859beb2c778cc4662607a34175065fc2f33a21f49eb3fbd1ccd3"}, - {file = "regex-2020.10.11-cp36-cp36m-win32.whl", hash = "sha256:c077c9d04a040dba001cf62b3aff08fd85be86bccf2c51a770c77377662a2d55"}, - {file = "regex-2020.10.11-cp36-cp36m-win_amd64.whl", hash = "sha256:46ab6070b0d2cb85700b8863b3f5504c7f75d8af44289e9562195fe02a8dd72d"}, - {file = "regex-2020.10.11-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:d629d750ebe75a88184db98f759633b0a7772c2e6f4da529f0027b4a402c0e2f"}, - {file = "regex-2020.10.11-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e7ef296b84d44425760fe813cabd7afbb48c8dd62023018b338bbd9d7d6f2f0"}, - {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:e490f08897cb44e54bddf5c6e27deca9b58c4076849f32aaa7a0b9f1730f2c20"}, - {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:850339226aa4fec04916386577674bb9d69abe0048f5d1a99f91b0004bfdcc01"}, - {file = "regex-2020.10.11-cp37-cp37m-win32.whl", hash = "sha256:60c4f64d9a326fe48e8738c3dbc068e1edc41ff7895a9e3723840deec4bc1c28"}, - {file = "regex-2020.10.11-cp37-cp37m-win_amd64.whl", hash = "sha256:8ba3efdd60bfee1aa784dbcea175eb442d059b576934c9d099e381e5a9f48930"}, - {file = "regex-2020.10.11-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2308491b3e6c530a3bb38a8a4bb1dc5fd32cbf1e11ca623f2172ba17a81acef1"}, - {file = "regex-2020.10.11-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b8806649983a1c78874ec7e04393ef076805740f6319e87a56f91f1767960212"}, - {file = "regex-2020.10.11-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a2a31ee8a354fa3036d12804730e1e20d58bc4e250365ead34b9c30bbe9908c3"}, - {file = "regex-2020.10.11-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d9d53518eeed12190744d366ec4a3f39b99d7daa705abca95f87dd8b442df4ad"}, - {file = "regex-2020.10.11-cp38-cp38-win32.whl", hash = "sha256:3d5a8d007116021cf65355ada47bf405656c4b3b9a988493d26688275fde1f1c"}, - {file = "regex-2020.10.11-cp38-cp38-win_amd64.whl", hash = "sha256:f579caecbbca291b0fcc7d473664c8c08635da2f9b1567c22ea32311c86ef68c"}, - {file = "regex-2020.10.11-cp39-cp39-manylinux1_i686.whl", hash = "sha256:8c8c42aa5d3ac9a49829c4b28a81bebfa0378996f9e0ca5b5ab8a36870c3e5ee"}, - {file = "regex-2020.10.11-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:c529ba90c1775697a65b46c83d47a2d3de70f24d96da5d41d05a761c73b063af"}, - {file = "regex-2020.10.11-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:6cf527ec2f3565248408b61dd36e380d799c2a1047eab04e13a2b0c15dd9c767"}, - {file = "regex-2020.10.11-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:671c51d352cfb146e48baee82b1ee8d6ffe357c292f5e13300cdc5c00867ebfc"}, - {file = "regex-2020.10.11-cp39-cp39-win32.whl", hash = "sha256:a63907332531a499b8cdfd18953febb5a4c525e9e7ca4ac147423b917244b260"}, - {file = "regex-2020.10.11-cp39-cp39-win_amd64.whl", hash = "sha256:1a16afbfadaadc1397353f9b32e19a65dc1d1804c80ad73a14f435348ca017ad"}, - {file = "regex-2020.10.11.tar.gz", hash = "sha256:463e770c48da76a8da82b8d4a48a541f314e0df91cbb6d873a341dbe578efafd"}, + {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"}, + {file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"}, + {file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"}, + {file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"}, + {file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"}, + {file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"}, + {file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"}, + {file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"}, + {file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"}, + {file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"}, + {file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"}, + {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"}, + {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"}, ] requests = [ - {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, - {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, + {file = "requests-2.25.0-py2.py3-none-any.whl", hash = "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"}, + {file = "requests-2.25.0.tar.gz", hash = "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8"}, ] s3transfer = [ {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, @@ -1204,8 +1258,8 @@ testfixtures = [ {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, ] toml = [ - {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, - {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typed-ast = [ {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, @@ -1236,8 +1290,8 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, ] urllib3 = [ - {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"}, - {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"}, + {file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"}, + {file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, @@ -1251,6 +1305,6 @@ xenon = [ {file = "xenon-0.7.1.tar.gz", hash = "sha256:38bf283135f0636355ecf6054b6f37226af12faab152161bda1a4f9e4dc5b701"}, ] zipp = [ - {file = "zipp-3.3.0-py3-none-any.whl", hash = "sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"}, - {file = "zipp-3.3.0.tar.gz", hash = "sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b"}, + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, ] diff --git a/pyproject.toml b/pyproject.toml index f37c5784645..6c53461286f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ fastjsonschema = "^2.14.5" boto3 = "^1.12" jmespath = "^0.10.0" pydantic = {version = "^1.6.0", optional = true } +email-validator = {version = "*", optional = true } typing_extensions = {version = "^3.7.4.2", optional = true } [tool.poetry.dev-dependencies] @@ -51,7 +52,7 @@ flake8-bugbear = "^20.1.4" [tool.poetry.extras] -pydantic = ["pydantic", "typing_extensions"] +pydantic = ["pydantic", "typing_extensions", "email-validator"] [tool.coverage.run] source = ["aws_lambda_powertools"] diff --git a/tests/functional/parser/test_ses.py b/tests/functional/parser/test_ses.py new file mode 100644 index 00000000000..f96da7bad66 --- /dev/null +++ b/tests/functional/parser/test_ses.py @@ -0,0 +1,49 @@ +from aws_lambda_powertools.utilities.parser import event_parser +from aws_lambda_powertools.utilities.parser.models import SesModel, SesRecordModel +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.utils import load_event + + +@event_parser(model=SesModel) +def handle_ses(event: SesModel, _: LambdaContext): + expected_address = "johndoe@example.com" + records = event.Records + record: SesRecordModel = records[0] + assert record.eventSource == "aws:ses" + assert record.eventVersion == "1.0" + mail = record.ses.mail + convert_time = int(round(mail.timestamp.timestamp() * 1000)) + assert convert_time == 0 + assert mail.source == "janedoe@example.com" + assert mail.messageId == "o3vrnil0e2ic28tr" + assert mail.destination == [expected_address] + assert mail.headersTruncated is False + headers = list(mail.headers) + assert len(headers) == 10 + assert headers[0].name == "Return-Path" + assert headers[0].value == "" + common_headers = mail.commonHeaders + assert common_headers.returnPath == "janedoe@example.com" + assert common_headers.header_from == ["Jane Doe "] + assert common_headers.date == "Wed, 7 Oct 2015 12:34:56 -0700" + assert common_headers.to == [expected_address] + assert common_headers.messageId == "<0123456789example.com>" + assert common_headers.subject == "Test Subject" + receipt = record.ses.receipt + convert_time = int(round(receipt.timestamp.timestamp() * 1000)) + assert convert_time == 0 + assert receipt.processingTimeMillis == 574 + assert receipt.recipients == [expected_address] + assert receipt.spamVerdict.status == "PASS" + assert receipt.virusVerdict.status == "PASS" + assert receipt.spfVerdict.status == "PASS" + assert receipt.dmarcVerdict.status == "PASS" + action = receipt.action + assert action.type == "Lambda" + assert action.functionArn == "arn:aws:lambda:us-west-2:012345678912:function:Example" + assert action.invocationType == "Event" + + +def test_ses_trigger_event(): + event_dict = load_event("sesEvent.json") + handle_ses(event_dict, LambdaContext()) From d94d32ab307987075e75217c31a04159c6ba7e76 Mon Sep 17 00:00:00 2001 From: Igor Gentil Date: Wed, 25 Nov 2020 14:33:58 +1100 Subject: [PATCH 02/20] Add clarification on the documents around Log keys that cannot be supressed --- docs/content/core/logger.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/core/logger.mdx b/docs/content/core/logger.mdx index 43ef93f8a85..baabd9c2faf 100644 --- a/docs/content/core/logger.mdx +++ b/docs/content/core/logger.mdx @@ -353,6 +353,8 @@ logger = Logger(stream=stdout, log_record_order=["message"]) # highlight-line logger = Logger(stream=stdout, log_record_order=["level","location","message","timestamp"]) # highlight-line ``` +Some keys cannot be supressed in the Log records: `sampling_rate` is part of the specification and cannot be supressed; `xray_trace_id` is supressed automatically if X-Ray is not enabled in the Lambda function, and added automatically if it is. + ### Logging exceptions When logging exceptions, Logger will add a new key named `exception`, and will serialize the full traceback as a string. From 8bccc9e6eab5c965209f88ae1ce8e46c36986b2d Mon Sep 17 00:00:00 2001 From: Pankaj Agrawal Date: Thu, 26 Nov 2020 16:30:14 +0100 Subject: [PATCH 03/20] docs: fix broken link for github --- docs/src/gatsby-theme-apollo-docs/components/page-content.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/gatsby-theme-apollo-docs/components/page-content.js b/docs/src/gatsby-theme-apollo-docs/components/page-content.js index 3c752c4782f..4cf8fce7111 100644 --- a/docs/src/gatsby-theme-apollo-docs/components/page-content.js +++ b/docs/src/gatsby-theme-apollo-docs/components/page-content.js @@ -179,7 +179,9 @@ export default function PageContent(props) { ); }); - const githubUrl = props.githubUrl.replace("master", "master/docs") + const githubUrl = props.githubUrl.replace("tree/", "blob/") + .replace("/content/", "/docs/content/") + const editLink = githubUrl && ( Edit on GitHub From e9722574276180d8e3afdf82396d4bc11a18df98 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 27 Nov 2020 09:54:49 +0100 Subject: [PATCH 04/20] docs: add source code link in nav bar --- .../gatsby-theme-apollo-docs/components/page-content.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/src/gatsby-theme-apollo-docs/components/page-content.js b/docs/src/gatsby-theme-apollo-docs/components/page-content.js index 4cf8fce7111..30c26c00752 100644 --- a/docs/src/gatsby-theme-apollo-docs/components/page-content.js +++ b/docs/src/gatsby-theme-apollo-docs/components/page-content.js @@ -181,6 +181,7 @@ export default function PageContent(props) { const githubUrl = props.githubUrl.replace("tree/", "blob/") .replace("/content/", "/docs/content/") + const sourceUrl = /.+?(?=tree)/.exec(props.githubUrl) const editLink = githubUrl && ( @@ -210,6 +211,9 @@ export default function PageContent(props) { /> )} {editLink} + + Source code + ); @@ -218,11 +222,11 @@ export default function PageContent(props) { PageContent.propTypes = { children: PropTypes.node.isRequired, pathname: PropTypes.string.isRequired, - githubUrl: PropTypes.string, + githubUrl: PropTypes.string.isRequired, pages: PropTypes.array.isRequired, hash: PropTypes.string.isRequired, title: PropTypes.string.isRequired, - graphManagerUrl: PropTypes.string.isRequired, + graphManagerUrl: PropTypes.string, headings: PropTypes.array.isRequired, spectrumUrl: PropTypes.string }; From 240dd609d3bc7db3d086d2beae8b8bfd1968c600 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sun, 29 Nov 2020 12:06:49 +0200 Subject: [PATCH 05/20] feat: Add S3 lambda event support to Parser utility #224 --- .../utilities/parser/models/__init__.py | 3 + .../utilities/parser/models/s3.py | 72 +++++++++++++++ tests/events/s3EventGlacier.json | 44 +++++++++ tests/functional/parser/test_s3.py | 89 +++++++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 aws_lambda_powertools/utilities/parser/models/s3.py create mode 100644 tests/events/s3EventGlacier.json create mode 100644 tests/functional/parser/test_s3.py diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index 36ba05240b0..996175d13df 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,5 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel +from .s3 import S3Model, S3RecordModel from .ses import SesModel, SesRecordModel from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel @@ -9,6 +10,8 @@ "EventBridgeModel", "DynamoDBStreamChangedRecordModel", "DynamoDBStreamRecordModel", + "S3Model", + "S3RecordModel", "SesModel", "SesRecordModel", "SnsModel", diff --git a/aws_lambda_powertools/utilities/parser/models/s3.py b/aws_lambda_powertools/utilities/parser/models/s3.py new file mode 100644 index 00000000000..14ea250b35b --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/s3.py @@ -0,0 +1,72 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel +from pydantic.fields import Field +from pydantic.networks import IPvAnyNetwork +from pydantic.types import PositiveInt +from typing_extensions import Literal + + +class S3EventRecordGlacierRestoreEventData(BaseModel): + lifecycleRestorationExpiryTime: datetime + lifecycleRestoreStorageClass: str + + +class S3EventRecordGlacierEventData(BaseModel): + restoreEventData: S3EventRecordGlacierRestoreEventData + + +class S3Identity(BaseModel): + principalId: str + + +class S3RequestParameters(BaseModel): + sourceIPAddress: IPvAnyNetwork + + +class S3ResponseElements(BaseModel): + x_amz_request_id: str = Field(None, alias="x-amz-request-id") + x_amz_id_2: str = Field(None, alias="x-amz-id-2") + + +class S3OwnerIdentify(BaseModel): + principalId: str + + +class S3Bucket(BaseModel): + name: str + ownerIdentity: S3OwnerIdentify + arn: str + + +class S3Object(BaseModel): + key: str + size: PositiveInt + eTag: str + sequencer: str + versionId: Optional[str] + + +class S3Message(BaseModel): + s3SchemaVersion: str + configurationId: str + bucket: S3Bucket + object: S3Object # noqa: A003,VNE003 + + +class S3RecordModel(BaseModel): + eventVersion: str + eventSource: Literal["aws:s3"] + awsRegion: str + eventTime: datetime + eventName: str + userIdentity: S3Identity + requestParameters: S3RequestParameters + responseElements: S3ResponseElements + s3: S3Message + glacierEventData: Optional[S3EventRecordGlacierEventData] + + +class S3Model(BaseModel): + Records: List[S3RecordModel] diff --git a/tests/events/s3EventGlacier.json b/tests/events/s3EventGlacier.json new file mode 100644 index 00000000000..2fbc447b308 --- /dev/null +++ b/tests/events/s3EventGlacier.json @@ -0,0 +1,44 @@ +{ + "Records": [ + { + "eventVersion": "2.1", + "eventSource": "aws:s3", + "awsRegion": "us-east-2", + "eventTime": "2019-09-03T19:37:27.192Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "AWS:AIDAINPONIXQXHT3IKHL2" + }, + "requestParameters": { + "sourceIPAddress": "205.255.255.255" + }, + "responseElements": { + "x-amz-request-id": "D82B88E5F771F645", + "x-amz-id-2": "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo=" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "828aa6fc-f7b5-4305-8584-487c791949c1", + "bucket": { + "name": "lambda-artifacts-deafc19498e3f2df", + "ownerIdentity": { + "principalId": "A3I5XTEXAMAI3E" + }, + "arn": "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df" + }, + "object": { + "key": "b21b84d653bb07b05b1e6b33684dc11b", + "size": 1305107, + "eTag": "b21b84d653bb07b05b1e6b33684dc11b", + "sequencer": "0C0F6F405D6ED209E1" + } + }, + "glacierEventData": { + "restoreEventData": { + "lifecycleRestorationExpiryTime": "1970-01-01T00:01:00.000Z", + "lifecycleRestoreStorageClass": "standard" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/functional/parser/test_s3.py b/tests/functional/parser/test_s3.py new file mode 100644 index 00000000000..5d8a19a933e --- /dev/null +++ b/tests/functional/parser/test_s3.py @@ -0,0 +1,89 @@ +from aws_lambda_powertools.utilities.parser import event_parser +from aws_lambda_powertools.utilities.parser.models import S3Model, S3RecordModel +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.utils import load_event + + +@event_parser(model=S3Model) +def handle_s3(event: S3Model, _: LambdaContext): + records = list(event.Records) + assert len(records) == 1 + record: S3RecordModel = records[0] + assert record.eventVersion == "2.1" + assert record.eventSource == "aws:s3" + assert record.awsRegion == "us-east-2" + convert_time = int(round(record.eventTime.timestamp() * 1000)) + assert convert_time == 1567539447192 + assert record.eventName == "ObjectCreated:Put" + user_identity = record.userIdentity + assert user_identity.principalId == "AWS:AIDAINPONIXQXHT3IKHL2" + request_parameters = record.requestParameters + assert str(request_parameters.sourceIPAddress) == "205.255.255.255/32" + assert record.responseElements.x_amz_request_id == "D82B88E5F771F645" + assert ( + record.responseElements.x_amz_id_2 + == "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo=" + ) + s3 = record.s3 + assert s3.s3SchemaVersion == "1.0" + assert s3.configurationId == "828aa6fc-f7b5-4305-8584-487c791949c1" + bucket = s3.bucket + assert bucket.name == "lambda-artifacts-deafc19498e3f2df" + assert bucket.ownerIdentity.principalId == "A3I5XTEXAMAI3E" + assert bucket.arn == "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df" + assert s3.object.key == "b21b84d653bb07b05b1e6b33684dc11b" + assert s3.object.size == 1305107 + assert s3.object.eTag == "b21b84d653bb07b05b1e6b33684dc11b" + assert s3.object.versionId is None + assert s3.object.sequencer == "0C0F6F405D6ED209E1" + assert record.glacierEventData is None + + +@event_parser(model=S3Model) +def handle_s3_glacier(event: S3Model, _: LambdaContext): + records = list(event.Records) + assert len(records) == 1 + record: S3RecordModel = records[0] + assert record.eventVersion == "2.1" + assert record.eventSource == "aws:s3" + assert record.awsRegion == "us-east-2" + convert_time = int(round(record.eventTime.timestamp() * 1000)) + assert convert_time == 1567539447192 + assert record.eventName == "ObjectCreated:Put" + user_identity = record.userIdentity + assert user_identity.principalId == "AWS:AIDAINPONIXQXHT3IKHL2" + request_parameters = record.requestParameters + assert str(request_parameters.sourceIPAddress) == "205.255.255.255/32" + assert record.responseElements.x_amz_request_id == "D82B88E5F771F645" + assert ( + record.responseElements.x_amz_id_2 + == "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo=" + ) + s3 = record.s3 + assert s3.s3SchemaVersion == "1.0" + assert s3.configurationId == "828aa6fc-f7b5-4305-8584-487c791949c1" + bucket = s3.bucket + assert bucket.name == "lambda-artifacts-deafc19498e3f2df" + assert bucket.ownerIdentity.principalId == "A3I5XTEXAMAI3E" + assert bucket.arn == "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df" + assert s3.object.key == "b21b84d653bb07b05b1e6b33684dc11b" + assert s3.object.size == 1305107 + assert s3.object.eTag == "b21b84d653bb07b05b1e6b33684dc11b" + assert s3.object.versionId is None + assert s3.object.sequencer == "0C0F6F405D6ED209E1" + assert record.glacierEventData is not None + convert_time = int( + round(record.glacierEventData.restoreEventData.lifecycleRestorationExpiryTime.timestamp() * 1000) + ) + assert convert_time == 60000 + assert record.glacierEventData.restoreEventData.lifecycleRestoreStorageClass == "standard" + + +def test_s3_trigger_event(): + event_dict = load_event("s3Event.json") + handle_s3(event_dict, LambdaContext()) + + +def test_s3_glacier_trigger_event(): + event_dict = load_event("s3EventGlacier.json") + handle_s3_glacier(event_dict, LambdaContext()) From 33f07b98b9bc66cb2d968bbb880b99b181fbd4a4 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Mon, 30 Nov 2020 11:30:46 +0200 Subject: [PATCH 06/20] feat: Add Kinesis lambda event support to Parser utility --- .../utilities/parser/envelopes/__init__.py | 10 +- .../utilities/parser/envelopes/kinesis.py | 43 +++++++ .../utilities/parser/models/__init__.py | 4 + .../utilities/parser/models/kinesis.py | 37 ++++++ tests/functional/parser/schemas.py | 5 + tests/functional/parser/test_kinesis.py | 106 ++++++++++++++++++ 6 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 aws_lambda_powertools/utilities/parser/envelopes/kinesis.py create mode 100644 aws_lambda_powertools/utilities/parser/models/kinesis.py create mode 100644 tests/functional/parser/test_kinesis.py diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index 4be73363b0f..4bd157a7070 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,7 +1,15 @@ from .base import BaseEnvelope from .dynamodb import DynamoDBStreamEnvelope from .event_bridge import EventBridgeEnvelope +from .kinesis import KinesisEnvelope from .sns import SnsEnvelope from .sqs import SqsEnvelope -__all__ = ["DynamoDBStreamEnvelope", "EventBridgeEnvelope", "SnsEnvelope", "SqsEnvelope", "BaseEnvelope"] +__all__ = [ + "DynamoDBStreamEnvelope", + "EventBridgeEnvelope", + "KinesisEnvelope", + "SnsEnvelope", + "SqsEnvelope", + "BaseEnvelope", +] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py new file mode 100644 index 00000000000..cd520a437be --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py @@ -0,0 +1,43 @@ +import logging +from typing import Any, Dict, List, Optional, Union + +from ..models import KinesisStreamModel +from ..types import Model +from .base import BaseEnvelope + +logger = logging.getLogger(__name__) + + +class KinesisEnvelope(BaseEnvelope): + """Kinesis Envelope to extract array of Records + + The record's data parameter is a base64 encoded string which is parsed into a bytes array, + though it can also be a JSON encoded string. + Regardless of its type it'll be parsed into a BaseModel object. + + Note: Records will be parsed the same way so if model is str, + all items in the list will be parsed as str and npt as JSON (and vice versa) + """ + + def parse(self, data: Optional[Union[Dict[str, Any], Any]], model: Model) -> List[Optional[Model]]: + """Parses records found with model provided + + Parameters + ---------- + data : Dict + Lambda event to be parsed + model : Model + Data model provided to parse after extracting data using envelope + + Returns + ------- + List + List of records parsed with model provided + """ + logger.debug(f"Parsing incoming data with Kinesis model {KinesisStreamModel}") + parsed_envelope: KinesisStreamModel = KinesisStreamModel.parse_obj(data) + output = [] + logger.debug(f"Parsing Kinesis records in `body` with {model}") + for record in parsed_envelope.Records: + output.append(self._parse(data=record.kinesis.data.decode("utf-8"), model=model)) + return output diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index 36ba05240b0..6c09aff364d 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,5 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel +from .kinesis import KinesisStreamModel, KinesisStreamRecord, KinesisStreamRecordPayload from .ses import SesModel, SesRecordModel from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel @@ -9,6 +10,9 @@ "EventBridgeModel", "DynamoDBStreamChangedRecordModel", "DynamoDBStreamRecordModel", + "KinesisStreamModel", + "KinesisStreamRecord", + "KinesisStreamRecordPayload", "SesModel", "SesRecordModel", "SnsModel", diff --git a/aws_lambda_powertools/utilities/parser/models/kinesis.py b/aws_lambda_powertools/utilities/parser/models/kinesis.py new file mode 100644 index 00000000000..11a2c86bf16 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/kinesis.py @@ -0,0 +1,37 @@ +import base64 +from binascii import Error as BinAsciiError +from typing import List + +from pydantic import BaseModel, validator +from pydantic.types import PositiveInt +from typing_extensions import Literal + + +class KinesisStreamRecordPayload(BaseModel): + kinesisSchemaVersion: str + partitionKey: str + sequenceNumber: PositiveInt + data: bytes # base64 encoded str is parsed into bytes + approximateArrivalTimestamp: float + + @validator("data", pre=True) + def data_base64_decode(cls, value): + try: + return base64.b64decode(value) + except (BinAsciiError, TypeError): + raise ValueError("base64 decode failed") + + +class KinesisStreamRecord(BaseModel): + eventSource: Literal["aws:kinesis"] + eventVersion: str + eventID: str + eventName: Literal["aws:kinesis:record"] + invokeIdentityArn: str + awsRegion: str + eventSourceARN: str + kinesis: KinesisStreamRecordPayload + + +class KinesisStreamModel(BaseModel): + Records: List[KinesisStreamRecord] diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index bfc601e3537..9f7bfa38ff3 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -71,3 +71,8 @@ class MyAdvancedSnsRecordModel(SnsRecordModel): class MyAdvancedSnsBusiness(SnsModel): Records: List[MyAdvancedSnsRecordModel] + + +class MyKinesisBusiness(BaseModel): + message: str + username: str diff --git a/tests/functional/parser/test_kinesis.py b/tests/functional/parser/test_kinesis.py new file mode 100644 index 00000000000..a218f1aecf0 --- /dev/null +++ b/tests/functional/parser/test_kinesis.py @@ -0,0 +1,106 @@ +from typing import Any, List + +import pytest + +from aws_lambda_powertools.utilities.parser import ValidationError, envelopes, event_parser +from aws_lambda_powertools.utilities.parser.models import KinesisStreamModel, KinesisStreamRecordPayload +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.schemas import MyKinesisBusiness +from tests.functional.parser.utils import load_event + + +@event_parser(model=MyKinesisBusiness, envelope=envelopes.KinesisEnvelope) +def handle_kinesis(event: List[MyKinesisBusiness], _: LambdaContext): + assert len(event) == 1 + record: KinesisStreamModel = event[0] + assert record.message == "test message" + assert record.username == "test" + + +@event_parser(model=KinesisStreamModel) +def handle_kinesis_no_envelope(event: KinesisStreamModel, _: LambdaContext): + records = event.Records + assert len(records) == 2 + record: KinesisStreamModel = records[0] + + assert record.awsRegion == "us-east-2" + assert record.eventID == "shardId-000000000006:49590338271490256608559692538361571095921575989136588898" + assert record.eventName == "aws:kinesis:record" + assert record.eventSource == "aws:kinesis" + assert record.eventSourceARN == "arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream" + assert record.eventVersion == "1.0" + assert record.invokeIdentityArn == "arn:aws:iam::123456789012:role/lambda-role" + + kinesis: KinesisStreamRecordPayload = record.kinesis + assert kinesis.approximateArrivalTimestamp == 1545084650.987 + assert kinesis.kinesisSchemaVersion == "1.0" + assert kinesis.partitionKey == "1" + assert kinesis.sequenceNumber == 49590338271490256608559692538361571095921575989136588898 + assert kinesis.data == b"Hello, this is a test." + + +def test_kinesis_trigger_event(): + event_dict = { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "1", + "sequenceNumber": "49590338271490256608559692538361571095921575989136588898", + "data": "eyJtZXNzYWdlIjogInRlc3QgbWVzc2FnZSIsICJ1c2VybmFtZSI6ICJ0ZXN0In0=", + "approximateArrivalTimestamp": 1545084650.987, + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000006:49590338271490256608559692538361571095921575989136588898", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-role", + "awsRegion": "us-east-2", + "eventSourceARN": "arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream", + } + ] + } + + handle_kinesis(event_dict, LambdaContext()) + + +def test_kinesis_trigger_bad_base64_event(): + event_dict = { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "1", + "sequenceNumber": "49590338271490256608559692538361571095921575989136588898", + "data": "bad", + "approximateArrivalTimestamp": 1545084650.987, + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000006:49590338271490256608559692538361571095921575989136588898", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-role", + "awsRegion": "us-east-2", + "eventSourceARN": "arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream", + } + ] + } + with pytest.raises(ValidationError): + handle_kinesis_no_envelope(event_dict, LambdaContext()) + + +def test_kinesis_trigger_event_no_envelope(): + event_dict = load_event("kinesisStreamEvent.json") + handle_kinesis_no_envelope(event_dict, LambdaContext()) + + +def test_validate_event_does_not_conform_with_model_no_envelope(): + event_dict: Any = {"hello": "s"} + with pytest.raises(ValidationError): + handle_kinesis_no_envelope(event_dict, LambdaContext()) + + +def test_validate_event_does_not_conform_with_model(): + event_dict: Any = {"hello": "s"} + with pytest.raises(ValidationError): + handle_kinesis(event_dict, LambdaContext()) From 346850de95fb56a32af393007f263e859ead51f1 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Mon, 30 Nov 2020 14:25:15 +0200 Subject: [PATCH 07/20] feat: Add alb lambda event support to Parser utility #228 --- .../utilities/parser/models/__init__.py | 4 ++ .../utilities/parser/models/alb.py | 21 +++++++++ tests/functional/parser/test_alb.py | 44 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 aws_lambda_powertools/utilities/parser/models/alb.py create mode 100644 tests/functional/parser/test_alb.py diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index 36ba05240b0..5c5c0b817c9 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,3 +1,4 @@ +from .alb import AlbModel, AlbRequestContext, AlbRequestContextData from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel from .ses import SesModel, SesRecordModel @@ -5,6 +6,9 @@ from .sqs import SqsModel, SqsRecordModel __all__ = [ + "AlbModel", + "AlbRequestContext", + "AlbRequestContextData", "DynamoDBStreamModel", "EventBridgeModel", "DynamoDBStreamChangedRecordModel", diff --git a/aws_lambda_powertools/utilities/parser/models/alb.py b/aws_lambda_powertools/utilities/parser/models/alb.py new file mode 100644 index 00000000000..d4ea5fde2a1 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/alb.py @@ -0,0 +1,21 @@ +from typing import Dict + +from pydantic import BaseModel + + +class AlbRequestContextData(BaseModel): + targetGroupArn: str + + +class AlbRequestContext(BaseModel): + elb: AlbRequestContextData + + +class AlbModel(BaseModel): + httpMethod: str + path: str + body: str + isBase64Encoded: bool + headers: Dict[str, str] + queryStringParameters: Dict[str, str] + requestContext: AlbRequestContext diff --git a/tests/functional/parser/test_alb.py b/tests/functional/parser/test_alb.py new file mode 100644 index 00000000000..88631c7194c --- /dev/null +++ b/tests/functional/parser/test_alb.py @@ -0,0 +1,44 @@ +import pytest + +from aws_lambda_powertools.utilities.parser import ValidationError, event_parser +from aws_lambda_powertools.utilities.parser.models import AlbModel +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.utils import load_event + + +@event_parser(model=AlbModel) +def handle_alb(event: AlbModel, _: LambdaContext): + assert ( + event.requestContext.elb.targetGroupArn + == "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/lambda-279XGJDqGZ5rsrHC2Fjr/49e9d65c45c6791a" # noqa E501 + ) + assert event.httpMethod == "GET" + assert event.path == "/lambda" + assert event.queryStringParameters == {"query": "1234ABCD"} + assert event.headers == { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "accept-encoding": "gzip", + "accept-language": "en-US,en;q=0.9", + "connection": "keep-alive", + "host": "lambda-alb-123578498.us-east-2.elb.amazonaws.com", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", # noqa E501 + "x-amzn-trace-id": "Root=1-5c536348-3d683b8b04734faae651f476", + "x-forwarded-for": "72.12.164.125", + "x-forwarded-port": "80", + "x-forwarded-proto": "http", + "x-imforwards": "20", + } + assert event.body == "Test" + assert not event.isBase64Encoded + + +def test_alb_trigger_event(): + event_dict = load_event("albEvent.json") + handle_alb(event_dict, LambdaContext()) + + +def test_validate_event_does_not_conform_with_model(): + event = {"invalid": "event"} + with pytest.raises(ValidationError): + handle_alb(event, LambdaContext()) From 836c066add5722bc08ee0c5dff743561a05bff63 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Mon, 30 Nov 2020 15:55:52 +0200 Subject: [PATCH 08/20] feat: Add cloudwatch lambda event support to Parser utility --- .../utilities/parser/envelopes/__init__.py | 10 ++- .../utilities/parser/envelopes/cloudwatch.py | 42 +++++++++++ .../utilities/parser/models/__init__.py | 5 ++ .../utilities/parser/models/cloudwatch.py | 39 ++++++++++ tests/functional/parser/schemas.py | 5 ++ tests/functional/parser/test_cloudwatch.py | 74 +++++++++++++++++++ 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py create mode 100644 aws_lambda_powertools/utilities/parser/models/cloudwatch.py create mode 100644 tests/functional/parser/test_cloudwatch.py diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index 4be73363b0f..dc1c51d71c2 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,7 +1,15 @@ from .base import BaseEnvelope +from .cloudwatch import CloudatchEnvelope from .dynamodb import DynamoDBStreamEnvelope from .event_bridge import EventBridgeEnvelope from .sns import SnsEnvelope from .sqs import SqsEnvelope -__all__ = ["DynamoDBStreamEnvelope", "EventBridgeEnvelope", "SnsEnvelope", "SqsEnvelope", "BaseEnvelope"] +__all__ = [ + "CloudatchEnvelope", + "DynamoDBStreamEnvelope", + "EventBridgeEnvelope", + "SnsEnvelope", + "SqsEnvelope", + "BaseEnvelope", +] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py b/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py new file mode 100644 index 00000000000..74f9d6acc4e --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py @@ -0,0 +1,42 @@ +import logging +from typing import Any, Dict, List, Optional, Union + +from ..models import CloudWatchLogsModel +from ..types import Model +from .base import BaseEnvelope + +logger = logging.getLogger(__name__) + + +class CloudatchEnvelope(BaseEnvelope): + """Cloudatch Envelope to extract a List of log records. + + The record's body parameter is a string (after being base64 decoded and gzipped), + though it can also be a JSON encoded string. + Regardless of its type it'll be parsed into a BaseModel object. + + Note: The record will be parsed the same way so if model is str + """ + + def parse(self, data: Optional[Union[Dict[str, Any], Any]], model: Model) -> List[Optional[Model]]: + """Parses records found with model provided + + Parameters + ---------- + data : Dict + Lambda event to be parsed + model : Model + Data model provided to parse after extracting data using envelope + + Returns + ------- + List + List of records parsed with model provided + """ + logger.debug(f"Parsing incoming data with SNS model {CloudWatchLogsModel}") + parsed_envelope = CloudWatchLogsModel.parse_obj(data) + logger.debug(f"Parsing CloudWatch records in `body` with {model}") + output = [] + for record in parsed_envelope.awslogs.decoded_data.logEvents: + output.append(self._parse(data=record.message, model=model)) + return output diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index 36ba05240b0..aa4a9e83e35 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,3 +1,4 @@ +from .cloudwatch import CloudWatchLogsData, CloudWatchLogsDecode, CloudWatchLogsLogEvent, CloudWatchLogsModel from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel from .ses import SesModel, SesRecordModel @@ -5,6 +6,10 @@ from .sqs import SqsModel, SqsRecordModel __all__ = [ + "CloudWatchLogsData", + "CloudWatchLogsDecode", + "CloudWatchLogsLogEvent", + "CloudWatchLogsModel", "DynamoDBStreamModel", "EventBridgeModel", "DynamoDBStreamChangedRecordModel", diff --git a/aws_lambda_powertools/utilities/parser/models/cloudwatch.py b/aws_lambda_powertools/utilities/parser/models/cloudwatch.py new file mode 100644 index 00000000000..d43cde4e793 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/cloudwatch.py @@ -0,0 +1,39 @@ +import base64 +import json +import zlib +from datetime import datetime +from typing import List + +from pydantic import BaseModel, Field, validator + + +class CloudWatchLogsLogEvent(BaseModel): + id: str # noqa AA03 VNE003 + timestamp: datetime + message: str + + +class CloudWatchLogsDecode(BaseModel): + messageType: str + owner: str + logGroup: str + logStream: str + subscriptionFilters: List[str] + logEvents: List[CloudWatchLogsLogEvent] + + +class CloudWatchLogsData(BaseModel): + decoded_data: CloudWatchLogsDecode = Field(None, alias="data") + + @validator("decoded_data", pre=True) + def prepare_data(cls, value): + try: + payload = base64.b64decode(value) + uncompressed = zlib.decompress(payload, zlib.MAX_WBITS | 32) + return json.loads(uncompressed.decode("UTF-8")) + except Exception: + raise ValueError("unable to decompress data") + + +class CloudWatchLogsModel(BaseModel): + awslogs: CloudWatchLogsData diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index bfc601e3537..23614a138b4 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -71,3 +71,8 @@ class MyAdvancedSnsRecordModel(SnsRecordModel): class MyAdvancedSnsBusiness(SnsModel): Records: List[MyAdvancedSnsRecordModel] + + +class MyCloudWatchBusiness(BaseModel): + my_message: str + user: str diff --git a/tests/functional/parser/test_cloudwatch.py b/tests/functional/parser/test_cloudwatch.py new file mode 100644 index 00000000000..786ebebb89b --- /dev/null +++ b/tests/functional/parser/test_cloudwatch.py @@ -0,0 +1,74 @@ +import base64 +import json +import zlib +from typing import List + +import pytest + +from aws_lambda_powertools.utilities.parser import ValidationError, envelopes, event_parser +from aws_lambda_powertools.utilities.parser.models import CloudWatchLogsLogEvent, CloudWatchLogsModel +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.schemas import MyCloudWatchBusiness +from tests.functional.parser.utils import load_event + + +@event_parser(model=MyCloudWatchBusiness, envelope=envelopes.CloudatchEnvelope) +def handle_cloudwatch_logs(event: List[MyCloudWatchBusiness], _: LambdaContext): + assert len(event) == 1 + log: MyCloudWatchBusiness = event[0] + assert log.my_message == "hello" + assert log.user == "test" + + +@event_parser(model=CloudWatchLogsModel) +def handle_cloudwatch_logs_no_envelope(event: CloudWatchLogsModel, _: LambdaContext): + assert event.awslogs.decoded_data.owner == "123456789123" + assert event.awslogs.decoded_data.logGroup == "testLogGroup" + assert event.awslogs.decoded_data.logStream == "testLogStream" + assert event.awslogs.decoded_data.subscriptionFilters == ["testFilter"] + assert event.awslogs.decoded_data.messageType == "DATA_MESSAGE" + + assert len(event.awslogs.decoded_data.logEvents) == 2 + log_record: CloudWatchLogsLogEvent = event.awslogs.decoded_data.logEvents[0] + assert log_record.id == "eventId1" + convert_time = int(round(log_record.timestamp.timestamp() * 1000)) + assert convert_time == 1440442987000 + assert log_record.message == "[ERROR] First test message" + log_record: CloudWatchLogsLogEvent = event.awslogs.decoded_data.logEvents[1] + assert log_record.id == "eventId2" + convert_time = int(round(log_record.timestamp.timestamp() * 1000)) + assert convert_time == 1440442987001 + assert log_record.message == "[ERROR] Second test message" + + +def test_validate_event_user_model_with_envelope(): + my_log_message = {"my_message": "hello", "user": "test"} + inner_event_dict = { + "messageType": "DATA_MESSAGE", + "owner": "123456789123", + "logGroup": "testLogGroup", + "logStream": "testLogStream", + "subscriptionFilters": ["testFilter"], + "logEvents": [{"id": "eventId1", "timestamp": 1440442987000, "message": json.dumps(my_log_message)}], + } + dict_str = json.dumps(inner_event_dict) + compressesd_str = zlib.compress(str.encode(dict_str), -1) + event_dict = {"awslogs": {"data": base64.b64encode(compressesd_str)}} + + handle_cloudwatch_logs(event_dict, LambdaContext()) + + +def test_validate_event_does_not_conform_with_user_dict_model(): + event_dict = load_event("cloudWatchLogEvent.json") + with pytest.raises(ValidationError): + handle_cloudwatch_logs(event_dict, LambdaContext()) + + +def test_handle_cloudwatch_trigger_event_no_envelope(): + event_dict = load_event("cloudWatchLogEvent.json") + handle_cloudwatch_logs_no_envelope(event_dict, LambdaContext()) + + +def test_handle_invalid_event_with_envelope(): + with pytest.raises(ValidationError): + handle_cloudwatch_logs(event={}, context=LambdaContext()) From 4716aa3062cf969e7fdd5c75e6ddcd6a7f3248f1 Mon Sep 17 00:00:00 2001 From: Ran Isenberg <60175085+risenberg-cyberark@users.noreply.github.com> Date: Tue, 1 Dec 2020 10:48:11 +0200 Subject: [PATCH 09/20] feat: Add Kinesis lambda event support to Parser utility --- .../utilities/parser/envelopes/__init__.py | 4 ++-- .../utilities/parser/envelopes/kinesis.py | 4 ++-- aws_lambda_powertools/utilities/parser/models/__init__.py | 4 ++-- aws_lambda_powertools/utilities/parser/models/kinesis.py | 8 ++++++-- tests/functional/parser/test_kinesis.py | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index 4bd157a7070..c963342ffd5 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,14 +1,14 @@ from .base import BaseEnvelope from .dynamodb import DynamoDBStreamEnvelope from .event_bridge import EventBridgeEnvelope -from .kinesis import KinesisEnvelope +from .kinesis import KinesisDataStreamEnvelope from .sns import SnsEnvelope from .sqs import SqsEnvelope __all__ = [ "DynamoDBStreamEnvelope", "EventBridgeEnvelope", - "KinesisEnvelope", + "KinesisDataStreamEnvelope", "SnsEnvelope", "SqsEnvelope", "BaseEnvelope", diff --git a/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py index cd520a437be..360126cf03c 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py @@ -8,8 +8,8 @@ logger = logging.getLogger(__name__) -class KinesisEnvelope(BaseEnvelope): - """Kinesis Envelope to extract array of Records +class KinesisDataStreamEnvelope(BaseEnvelope): + """Kinesis Data Stream Envelope to extract array of Records The record's data parameter is a base64 encoded string which is parsed into a bytes array, though it can also be a JSON encoded string. diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index 6c09aff364d..ee71f194bf0 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,6 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel -from .kinesis import KinesisStreamModel, KinesisStreamRecord, KinesisStreamRecordPayload +from .kinesis import KinesisDataStreamRecordPayload, KinesisStreamModel, KinesisStreamRecord from .ses import SesModel, SesRecordModel from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel @@ -12,7 +12,7 @@ "DynamoDBStreamRecordModel", "KinesisStreamModel", "KinesisStreamRecord", - "KinesisStreamRecordPayload", + "KinesisDataStreamRecordPayload", "SesModel", "SesRecordModel", "SnsModel", diff --git a/aws_lambda_powertools/utilities/parser/models/kinesis.py b/aws_lambda_powertools/utilities/parser/models/kinesis.py index 11a2c86bf16..fa11129431f 100644 --- a/aws_lambda_powertools/utilities/parser/models/kinesis.py +++ b/aws_lambda_powertools/utilities/parser/models/kinesis.py @@ -1,4 +1,5 @@ import base64 +import logging from binascii import Error as BinAsciiError from typing import List @@ -6,8 +7,10 @@ from pydantic.types import PositiveInt from typing_extensions import Literal +logger = logging.getLogger(__name__) -class KinesisStreamRecordPayload(BaseModel): + +class KinesisDataStreamRecordPayload(BaseModel): kinesisSchemaVersion: str partitionKey: str sequenceNumber: PositiveInt @@ -17,6 +20,7 @@ class KinesisStreamRecordPayload(BaseModel): @validator("data", pre=True) def data_base64_decode(cls, value): try: + logger.debug("Decoding base64 Kinesis data record before parsing") return base64.b64decode(value) except (BinAsciiError, TypeError): raise ValueError("base64 decode failed") @@ -30,7 +34,7 @@ class KinesisStreamRecord(BaseModel): invokeIdentityArn: str awsRegion: str eventSourceARN: str - kinesis: KinesisStreamRecordPayload + kinesis: KinesisDataStreamRecordPayload class KinesisStreamModel(BaseModel): diff --git a/tests/functional/parser/test_kinesis.py b/tests/functional/parser/test_kinesis.py index a218f1aecf0..83a8d4ea290 100644 --- a/tests/functional/parser/test_kinesis.py +++ b/tests/functional/parser/test_kinesis.py @@ -3,13 +3,13 @@ import pytest from aws_lambda_powertools.utilities.parser import ValidationError, envelopes, event_parser -from aws_lambda_powertools.utilities.parser.models import KinesisStreamModel, KinesisStreamRecordPayload +from aws_lambda_powertools.utilities.parser.models import KinesisDataStreamRecordPayload, KinesisStreamModel from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyKinesisBusiness from tests.functional.parser.utils import load_event -@event_parser(model=MyKinesisBusiness, envelope=envelopes.KinesisEnvelope) +@event_parser(model=MyKinesisBusiness, envelope=envelopes.KinesisDataStreamEnvelope) def handle_kinesis(event: List[MyKinesisBusiness], _: LambdaContext): assert len(event) == 1 record: KinesisStreamModel = event[0] @@ -31,7 +31,7 @@ def handle_kinesis_no_envelope(event: KinesisStreamModel, _: LambdaContext): assert record.eventVersion == "1.0" assert record.invokeIdentityArn == "arn:aws:iam::123456789012:role/lambda-role" - kinesis: KinesisStreamRecordPayload = record.kinesis + kinesis: KinesisDataStreamRecordPayload = record.kinesis assert kinesis.approximateArrivalTimestamp == 1545084650.987 assert kinesis.kinesisSchemaVersion == "1.0" assert kinesis.partitionKey == "1" From b95a833d05e100160fdd93f649d8c17d77aeed17 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Tue, 1 Dec 2020 16:06:55 +0200 Subject: [PATCH 10/20] cr fixes --- .../utilities/parser/envelopes/__init__.py | 4 ++-- .../utilities/parser/envelopes/cloudwatch.py | 2 +- .../utilities/parser/models/cloudwatch.py | 7 ++++++- tests/functional/parser/test_cloudwatch.py | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index dc1c51d71c2..3338dd661d0 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,12 +1,12 @@ from .base import BaseEnvelope -from .cloudwatch import CloudatchEnvelope +from .cloudwatch import CloudatchLogsEnvelope from .dynamodb import DynamoDBStreamEnvelope from .event_bridge import EventBridgeEnvelope from .sns import SnsEnvelope from .sqs import SqsEnvelope __all__ = [ - "CloudatchEnvelope", + "CloudatchLogsEnvelope", "DynamoDBStreamEnvelope", "EventBridgeEnvelope", "SnsEnvelope", diff --git a/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py b/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py index 74f9d6acc4e..163e32b837a 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/cloudwatch.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -class CloudatchEnvelope(BaseEnvelope): +class CloudatchLogsEnvelope(BaseEnvelope): """Cloudatch Envelope to extract a List of log records. The record's body parameter is a string (after being base64 decoded and gzipped), diff --git a/aws_lambda_powertools/utilities/parser/models/cloudwatch.py b/aws_lambda_powertools/utilities/parser/models/cloudwatch.py index d43cde4e793..26eeef5b56f 100644 --- a/aws_lambda_powertools/utilities/parser/models/cloudwatch.py +++ b/aws_lambda_powertools/utilities/parser/models/cloudwatch.py @@ -1,11 +1,14 @@ import base64 import json +import logging import zlib from datetime import datetime from typing import List from pydantic import BaseModel, Field, validator +logger = logging.getLogger(__name__) + class CloudWatchLogsLogEvent(BaseModel): id: str # noqa AA03 VNE003 @@ -28,9 +31,11 @@ class CloudWatchLogsData(BaseModel): @validator("decoded_data", pre=True) def prepare_data(cls, value): try: + logger.debug("Decoding base64 cloudwatch log data before parsing") payload = base64.b64decode(value) + logger.debug("Decompressing cloudwatch log data before parsing") uncompressed = zlib.decompress(payload, zlib.MAX_WBITS | 32) - return json.loads(uncompressed.decode("UTF-8")) + return json.loads(uncompressed.decode("utf-8")) except Exception: raise ValueError("unable to decompress data") diff --git a/tests/functional/parser/test_cloudwatch.py b/tests/functional/parser/test_cloudwatch.py index 786ebebb89b..c4083e42631 100644 --- a/tests/functional/parser/test_cloudwatch.py +++ b/tests/functional/parser/test_cloudwatch.py @@ -12,7 +12,7 @@ from tests.functional.parser.utils import load_event -@event_parser(model=MyCloudWatchBusiness, envelope=envelopes.CloudatchEnvelope) +@event_parser(model=MyCloudWatchBusiness, envelope=envelopes.CloudatchLogsEnvelope) def handle_cloudwatch_logs(event: List[MyCloudWatchBusiness], _: LambdaContext): assert len(event) == 1 log: MyCloudWatchBusiness = event[0] From 9b5581703c7a17f355766df4751b9b534cf4785e Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Tue, 1 Dec 2020 16:11:06 +0200 Subject: [PATCH 11/20] cr fixes --- .../utilities/parser/envelopes/kinesis.py | 6 +++--- .../utilities/parser/models/__init__.py | 6 +++--- .../utilities/parser/models/kinesis.py | 6 +++--- tests/functional/parser/test_kinesis.py | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py index 360126cf03c..97ad7bffec7 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/kinesis.py @@ -1,7 +1,7 @@ import logging from typing import Any, Dict, List, Optional, Union -from ..models import KinesisStreamModel +from ..models import KinesisDataStreamModel from ..types import Model from .base import BaseEnvelope @@ -34,8 +34,8 @@ def parse(self, data: Optional[Union[Dict[str, Any], Any]], model: Model) -> Lis List List of records parsed with model provided """ - logger.debug(f"Parsing incoming data with Kinesis model {KinesisStreamModel}") - parsed_envelope: KinesisStreamModel = KinesisStreamModel.parse_obj(data) + logger.debug(f"Parsing incoming data with Kinesis model {KinesisDataStreamModel}") + parsed_envelope: KinesisDataStreamModel = KinesisDataStreamModel.parse_obj(data) output = [] logger.debug(f"Parsing Kinesis records in `body` with {model}") for record in parsed_envelope.Records: diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index ee71f194bf0..00a2d375e26 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,6 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel -from .kinesis import KinesisDataStreamRecordPayload, KinesisStreamModel, KinesisStreamRecord +from .kinesis import KinesisDataStreamModel, KinesisDataStreamRecord, KinesisDataStreamRecordPayload from .ses import SesModel, SesRecordModel from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel @@ -10,8 +10,8 @@ "EventBridgeModel", "DynamoDBStreamChangedRecordModel", "DynamoDBStreamRecordModel", - "KinesisStreamModel", - "KinesisStreamRecord", + "KinesisDataStreamModel", + "KinesisDataStreamRecord", "KinesisDataStreamRecordPayload", "SesModel", "SesRecordModel", diff --git a/aws_lambda_powertools/utilities/parser/models/kinesis.py b/aws_lambda_powertools/utilities/parser/models/kinesis.py index fa11129431f..d2852e9f4a8 100644 --- a/aws_lambda_powertools/utilities/parser/models/kinesis.py +++ b/aws_lambda_powertools/utilities/parser/models/kinesis.py @@ -26,7 +26,7 @@ def data_base64_decode(cls, value): raise ValueError("base64 decode failed") -class KinesisStreamRecord(BaseModel): +class KinesisDataStreamRecord(BaseModel): eventSource: Literal["aws:kinesis"] eventVersion: str eventID: str @@ -37,5 +37,5 @@ class KinesisStreamRecord(BaseModel): kinesis: KinesisDataStreamRecordPayload -class KinesisStreamModel(BaseModel): - Records: List[KinesisStreamRecord] +class KinesisDataStreamModel(BaseModel): + Records: List[KinesisDataStreamRecord] diff --git a/tests/functional/parser/test_kinesis.py b/tests/functional/parser/test_kinesis.py index 83a8d4ea290..5a7a94e0dac 100644 --- a/tests/functional/parser/test_kinesis.py +++ b/tests/functional/parser/test_kinesis.py @@ -3,7 +3,7 @@ import pytest from aws_lambda_powertools.utilities.parser import ValidationError, envelopes, event_parser -from aws_lambda_powertools.utilities.parser.models import KinesisDataStreamRecordPayload, KinesisStreamModel +from aws_lambda_powertools.utilities.parser.models import KinesisDataStreamModel, KinesisDataStreamRecordPayload from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyKinesisBusiness from tests.functional.parser.utils import load_event @@ -12,16 +12,16 @@ @event_parser(model=MyKinesisBusiness, envelope=envelopes.KinesisDataStreamEnvelope) def handle_kinesis(event: List[MyKinesisBusiness], _: LambdaContext): assert len(event) == 1 - record: KinesisStreamModel = event[0] + record: KinesisDataStreamModel = event[0] assert record.message == "test message" assert record.username == "test" -@event_parser(model=KinesisStreamModel) -def handle_kinesis_no_envelope(event: KinesisStreamModel, _: LambdaContext): +@event_parser(model=KinesisDataStreamModel) +def handle_kinesis_no_envelope(event: KinesisDataStreamModel, _: LambdaContext): records = event.Records assert len(records) == 2 - record: KinesisStreamModel = records[0] + record: KinesisDataStreamModel = records[0] assert record.awsRegion == "us-east-2" assert record.eventID == "shardId-000000000006:49590338271490256608559692538361571095921575989136588898" From 16e5882be2cea880b8930961ae639298ea99ccb0 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 4 Dec 2020 13:37:58 +0100 Subject: [PATCH 12/20] docs: shadow sidebar to remain expanded --- .../components/multi-code-block.js | 127 +++++++ .../components/page-layout.js | 312 ++++++++++++++++++ .../components/select.js | 133 ++++++++ docs/src/gatsby-theme-apollo-docs/prism.less | 191 +++++++++++ 4 files changed, 763 insertions(+) create mode 100644 docs/src/gatsby-theme-apollo-docs/components/multi-code-block.js create mode 100644 docs/src/gatsby-theme-apollo-docs/components/page-layout.js create mode 100644 docs/src/gatsby-theme-apollo-docs/components/select.js create mode 100644 docs/src/gatsby-theme-apollo-docs/prism.less diff --git a/docs/src/gatsby-theme-apollo-docs/components/multi-code-block.js b/docs/src/gatsby-theme-apollo-docs/components/multi-code-block.js new file mode 100644 index 00000000000..c9346ebb52e --- /dev/null +++ b/docs/src/gatsby-theme-apollo-docs/components/multi-code-block.js @@ -0,0 +1,127 @@ +import PropTypes from 'prop-types'; +import React, {createContext, useContext, useMemo} from 'react'; +import styled from '@emotion/styled'; +import {trackCustomEvent} from 'gatsby-plugin-google-analytics'; + +export const GA_EVENT_CATEGORY_CODE_BLOCK = 'Code Block'; +export const MultiCodeBlockContext = createContext({}); +export const SelectedLanguageContext = createContext(); + +const Container = styled.div({ + position: 'relative' +}); + +const langLabels = { + js: 'JavaScript', + ts: 'TypeScript', + 'hooks-js': 'Hooks (JS)', + 'hooks-ts': 'Hooks (TS)' +}; + +function getUnifiedLang(language) { + switch (language) { + case 'js': + case 'jsx': + case 'javascript': + return 'js'; + case 'ts': + case 'tsx': + case 'typescript': + return 'ts'; + default: + return language; + } +} + +function getLang(child) { + return getUnifiedLang(child.props['data-language']); +} + +export function MultiCodeBlock(props) { + const {codeBlocks, titles} = useMemo(() => { + const defaultState = { + codeBlocks: {}, + titles: {} + }; + + if (!Array.isArray(props.children)) { + return defaultState; + } + + return props.children.reduce((acc, child, index, array) => { + const lang = getLang(child); + if (lang) { + return { + ...acc, + codeBlocks: { + ...acc.codeBlocks, + [lang]: child + } + }; + } + + if (child.props.className === 'gatsby-code-title') { + const nextNode = array[index + 1]; + const title = child.props.children; + const lang = getLang(nextNode); + if (nextNode && title && lang) { + return { + ...acc, + titles: { + ...acc.titles, + [lang]: title + } + }; + } + } + + return acc; + }, defaultState); + }, [props.children]); + + const languages = useMemo(() => Object.keys(codeBlocks), [codeBlocks]); + const [selectedLanguage, setSelectedLanguage] = useContext( + SelectedLanguageContext + ); + + if (!languages.length) { + return props.children; + } + + function handleLanguageChange(language) { + setSelectedLanguage(language); + trackCustomEvent({ + category: GA_EVENT_CATEGORY_CODE_BLOCK, + action: 'Change language', + label: language + }); + } + + const defaultLanguage = languages[0]; + const renderedLanguage = + selectedLanguage in codeBlocks ? selectedLanguage : defaultLanguage; + + return ( + + ({ + lang, + label: + // try to find a label or capitalize the provided lang + langLabels[lang] || lang.charAt(0).toUpperCase() + lang.slice(1) + })), + onLanguageChange: handleLanguageChange + }} + > +
{titles[renderedLanguage]}
+ {codeBlocks[renderedLanguage]} + + + ); +} + +MultiCodeBlock.propTypes = { + children: PropTypes.node.isRequired +}; diff --git a/docs/src/gatsby-theme-apollo-docs/components/page-layout.js b/docs/src/gatsby-theme-apollo-docs/components/page-layout.js new file mode 100644 index 00000000000..7c97481ab7f --- /dev/null +++ b/docs/src/gatsby-theme-apollo-docs/components/page-layout.js @@ -0,0 +1,312 @@ +import '../prism.less'; +import 'prismjs/plugins/line-numbers/prism-line-numbers.css'; +import DocsetSwitcher from './docset-switcher'; +import Header from './header'; +import HeaderButton from './header-button'; +import PropTypes from 'prop-types'; +import React, {createContext, useMemo, useRef, useState} from 'react'; +import Search from './search'; +import styled from '@emotion/styled'; +import useLocalStorage from 'react-use/lib/useLocalStorage'; +import {Button} from '@apollo/space-kit/Button'; +import { + FlexWrapper, + Layout, + MenuButton, + Sidebar, + SidebarNav, + breakpoints, + colors, + useResponsiveSidebar +} from 'gatsby-theme-apollo-core'; +import {Helmet} from 'react-helmet'; +import {IconLayoutModule} from '@apollo/space-kit/icons/IconLayoutModule'; +import {Link, graphql, navigate, useStaticQuery} from 'gatsby'; +import {MobileLogo} from './mobile-logo'; +import {Select} from './select'; +import {SelectedLanguageContext} from './multi-code-block'; +import {getSpectrumUrl, getVersionBasePath} from '../utils'; +import {groupBy} from 'lodash'; +import {size} from 'polished'; +import {trackCustomEvent} from 'gatsby-plugin-google-analytics'; + +const Main = styled.main({ + flexGrow: 1 +}); + +const ButtonWrapper = styled.div({ + flexGrow: 1 +}); + +const StyledButton = styled(Button)({ + width: '100%', + ':not(:hover)': { + backgroundColor: colors.background + } +}); + +const StyledIcon = styled(IconLayoutModule)(size(16), { + marginLeft: 'auto' +}); + +const MobileNav = styled.div({ + display: 'none', + [breakpoints.md]: { + display: 'flex', + alignItems: 'center', + marginRight: 32, + color: colors.text1 + } +}); + +const HeaderInner = styled.span({ + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: 32 +}); + +const Eyebrow = styled.div({ + flexShrink: 0, + padding: '8px 56px', + backgroundColor: colors.background, + color: colors.primary, + fontSize: 14, + position: 'sticky', + top: 0, + a: { + color: 'inherit', + fontWeight: 600 + }, + [breakpoints.md]: { + padding: '8px 24px' + } +}); + +function getVersionLabel(version) { + return `v${version}`; +} + +const GA_EVENT_CATEGORY_SIDEBAR = 'Sidebar'; + +function handleToggleAll(expanded) { + trackCustomEvent({ + category: GA_EVENT_CATEGORY_SIDEBAR, + action: 'Toggle all', + label: expanded ? 'expand' : 'collapse' + }); +} + +function handleToggleCategory(label, expanded) { + trackCustomEvent({ + category: GA_EVENT_CATEGORY_SIDEBAR, + action: 'Toggle category', + label, + value: Number(expanded) + }); +} + +export const NavItemsContext = createContext(); + +export default function PageLayout(props) { + const data = useStaticQuery( + graphql` + { + site { + siteMetadata { + title + siteName + } + } + } + ` + ); + + const { + sidebarRef, + openSidebar, + sidebarOpen, + handleWrapperClick, + handleSidebarNavLinkClick + } = useResponsiveSidebar(); + + const buttonRef = useRef(null); + const [menuOpen, setMenuOpen] = useState(false); + const selectedLanguageState = useLocalStorage('docs-lang'); + + function openMenu() { + setMenuOpen(true); + } + + function closeMenu() { + setMenuOpen(false); + } + + const {pathname} = props.location; + const {siteName, title} = data.site.siteMetadata; + const { + subtitle, + sidebarContents, + versions, + versionDifference, + versionBasePath, + defaultVersion + } = props.pageContext; + const { + spectrumHandle, + twitterHandle, + youtubeUrl, + navConfig = {}, + footerNavConfig, + logoLink, + algoliaApiKey, + algoliaIndexName, + menuTitle + } = props.pluginOptions; + + const {navItems, navCategories} = useMemo(() => { + const navItems = Object.entries(navConfig).map(([title, navItem]) => ({ + ...navItem, + title + })); + return { + navItems, + navCategories: Object.entries(groupBy(navItems, 'category')) + }; + }, [navConfig]); + + const hasNavItems = navItems.length > 0; + const sidebarTitle = ( + {subtitle || siteName} + ); + + return ( + + + + + + + + + {hasNavItems ? ( + + + {sidebarTitle} + + + + ) : ( + sidebarTitle + )} + {versions && versions.length > 0 && ( +