From 4b6a3816bb7147e7cbe68febd771540c7049e952 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 9 May 2023 11:31:48 +0200 Subject: [PATCH 01/13] Add `db.operation` to Redis and MongoDB spans. (#2089) * Set db.operation in Redis and MongoDB spans --- sentry_sdk/consts.py | 9 ++++++++- sentry_sdk/integrations/pymongo.py | 4 ++-- sentry_sdk/integrations/redis.py | 1 + tests/integrations/redis/test_redis.py | 3 +++ tests/integrations/rediscluster/test_rediscluster.py | 1 + 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 35c02cda1e..7a76a507eb 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -57,10 +57,17 @@ class SPANDATA: See: https://develop.sentry.dev/sdk/performance/span-data-conventions/ """ + DB_OPERATION = "db.operation" + """ + The name of the operation being executed, e.g. the MongoDB command name such as findAndModify, or the SQL keyword. + See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md + Example: findAndModify, HMSET, SELECT + """ + DB_SYSTEM = "db.system" """ An identifier for the database management system (DBMS) product being used. - See: https://github.com/open-telemetry/opentelemetry-specification/blob/24de67b3827a4e3ab2515cd8ab62d5bcf837c586/specification/trace/semantic_conventions/database.md + See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md Example: postgresql """ diff --git a/sentry_sdk/integrations/pymongo.py b/sentry_sdk/integrations/pymongo.py index 0b057fe548..391219c75e 100644 --- a/sentry_sdk/integrations/pymongo.py +++ b/sentry_sdk/integrations/pymongo.py @@ -110,8 +110,8 @@ def started(self, event): tags = { "db.name": event.database_name, - "db.system": "mongodb", - "db.operation": event.command_name, + SPANDATA.DB_SYSTEM: "mongodb", + SPANDATA.DB_OPERATION: event.command_name, } try: diff --git a/sentry_sdk/integrations/redis.py b/sentry_sdk/integrations/redis.py index 8d196d00b2..b05bc741f1 100644 --- a/sentry_sdk/integrations/redis.py +++ b/sentry_sdk/integrations/redis.py @@ -196,6 +196,7 @@ def sentry_patched_execute_command(self, name, *args, **kwargs): if name: span.set_tag("redis.command", name) + span.set_tag(SPANDATA.DB_OPERATION, name) if name and args: name_low = name.lower() diff --git a/tests/integrations/redis/test_redis.py b/tests/integrations/redis/test_redis.py index beb7901122..a596319c8b 100644 --- a/tests/integrations/redis/test_redis.py +++ b/tests/integrations/redis/test_redis.py @@ -27,6 +27,7 @@ def test_basic(sentry_init, capture_events): "redis.key": "foobar", "redis.command": "GET", "redis.is_cluster": False, + "db.operation": "GET", }, "timestamp": crumb["timestamp"], "type": "redis", @@ -207,6 +208,7 @@ def test_breadcrumbs(sentry_init, capture_events): "type": "redis", "category": "redis", "data": { + "db.operation": "SET", "redis.is_cluster": False, "redis.command": "SET", "redis.key": "somekey1", @@ -218,6 +220,7 @@ def test_breadcrumbs(sentry_init, capture_events): "type": "redis", "category": "redis", "data": { + "db.operation": "SET", "redis.is_cluster": False, "redis.command": "SET", "redis.key": "somekey2", diff --git a/tests/integrations/rediscluster/test_rediscluster.py b/tests/integrations/rediscluster/test_rediscluster.py index 6425ca15e6..d00aeca350 100644 --- a/tests/integrations/rediscluster/test_rediscluster.py +++ b/tests/integrations/rediscluster/test_rediscluster.py @@ -43,6 +43,7 @@ def test_rediscluster_basic(rediscluster_cls, sentry_init, capture_events): "category": "redis", "message": "GET 'foobar'", "data": { + "db.operation": "GET", "redis.key": "foobar", "redis.command": "GET", "redis.is_cluster": True, From 8a2b74f58e97205233717c379b0d78f85d697365 Mon Sep 17 00:00:00 2001 From: Perchun Pak Date: Tue, 9 May 2023 13:18:53 +0200 Subject: [PATCH 02/13] Add `loguru` integration (#1994) * Add `loguru` integration Actually, this is the solution in comments under #653 adapted to codebase and tested as well. https://github.com/getsentry/sentry-python/issues/653#issuecomment-788854865 I also changed `logging` integration to use methods instead of functions in handlers, as in that way we can easily overwrite parts that are different in `loguru` integration. It shouldn't be a problem, as those methods are private and used only in that file. --------- Co-authored-by: Anton Pirker --- .github/workflows/test-integration-loguru.yml | 78 ++++++++++ linter-requirements.txt | 1 + sentry_sdk/integrations/logging.py | 137 +++++++++--------- sentry_sdk/integrations/loguru.py | 89 ++++++++++++ setup.py | 3 +- tests/integrations/loguru/__init__.py | 3 + tests/integrations/loguru/test_loguru.py | 77 ++++++++++ tox.ini | 9 ++ 8 files changed, 326 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/test-integration-loguru.yml create mode 100644 sentry_sdk/integrations/loguru.py create mode 100644 tests/integrations/loguru/__init__.py create mode 100644 tests/integrations/loguru/test_loguru.py diff --git a/.github/workflows/test-integration-loguru.yml b/.github/workflows/test-integration-loguru.yml new file mode 100644 index 0000000000..3fe09a8213 --- /dev/null +++ b/.github/workflows/test-integration-loguru.yml @@ -0,0 +1,78 @@ +name: Test loguru + +on: + push: + branches: + - master + - release/** + + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless + +jobs: + test: + name: loguru, python ${{ matrix.python-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + python-version: ["3.5","3.6","3.7","3.8","3.9","3.10","3.11"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-20.04] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Test Env + run: | + pip install coverage "tox>=3,<4" + + - name: Test loguru + timeout-minutes: 45 + shell: bash + run: | + set -x # print commands that are executed + coverage erase + + # Run tests + ./scripts/runtox.sh "py${{ matrix.python-version }}-loguru" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + coverage combine .coverage* + coverage xml -i + + - uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + + check_required_tests: + name: All loguru tests passed or skipped + needs: test + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Check for failures + if: contains(needs.test.result, 'failure') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/linter-requirements.txt b/linter-requirements.txt index 32f7fe8bc8..5e7ec1c52e 100644 --- a/linter-requirements.txt +++ b/linter-requirements.txt @@ -5,6 +5,7 @@ types-certifi types-redis types-setuptools pymongo # There is no separate types module. +loguru # There is no separate types module. flake8-bugbear==22.12.6 pep8-naming==0.13.2 pre-commit # local linting diff --git a/sentry_sdk/integrations/logging.py b/sentry_sdk/integrations/logging.py index 782180eea7..d4f34d085c 100644 --- a/sentry_sdk/integrations/logging.py +++ b/sentry_sdk/integrations/logging.py @@ -107,75 +107,61 @@ def sentry_patched_callhandlers(self, record): logging.Logger.callHandlers = sentry_patched_callhandlers # type: ignore -def _can_record(record): - # type: (LogRecord) -> bool - """Prevents ignored loggers from recording""" - for logger in _IGNORED_LOGGERS: - if fnmatch(record.name, logger): - return False - return True - - -def _breadcrumb_from_record(record): - # type: (LogRecord) -> Dict[str, Any] - return { - "type": "log", - "level": _logging_to_event_level(record), - "category": record.name, - "message": record.message, - "timestamp": datetime.datetime.utcfromtimestamp(record.created), - "data": _extra_from_record(record), - } - - -def _logging_to_event_level(record): - # type: (LogRecord) -> str - return LOGGING_TO_EVENT_LEVEL.get( - record.levelno, record.levelname.lower() if record.levelname else "" +class _BaseHandler(logging.Handler, object): + COMMON_RECORD_ATTRS = frozenset( + ( + "args", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "linenno", + "lineno", + "message", + "module", + "msecs", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack", + "tags", + "thread", + "threadName", + "stack_info", + ) ) + def _can_record(self, record): + # type: (LogRecord) -> bool + """Prevents ignored loggers from recording""" + for logger in _IGNORED_LOGGERS: + if fnmatch(record.name, logger): + return False + return True + + def _logging_to_event_level(self, record): + # type: (LogRecord) -> str + return LOGGING_TO_EVENT_LEVEL.get( + record.levelno, record.levelname.lower() if record.levelname else "" + ) -COMMON_RECORD_ATTRS = frozenset( - ( - "args", - "created", - "exc_info", - "exc_text", - "filename", - "funcName", - "levelname", - "levelno", - "linenno", - "lineno", - "message", - "module", - "msecs", - "msg", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "stack", - "tags", - "thread", - "threadName", - "stack_info", - ) -) - - -def _extra_from_record(record): - # type: (LogRecord) -> Dict[str, None] - return { - k: v - for k, v in iteritems(vars(record)) - if k not in COMMON_RECORD_ATTRS - and (not isinstance(k, str) or not k.startswith("_")) - } + def _extra_from_record(self, record): + # type: (LogRecord) -> Dict[str, None] + return { + k: v + for k, v in iteritems(vars(record)) + if k not in self.COMMON_RECORD_ATTRS + and (not isinstance(k, str) or not k.startswith("_")) + } -class EventHandler(logging.Handler, object): +class EventHandler(_BaseHandler): """ A logging handler that emits Sentry events for each log record @@ -190,7 +176,7 @@ def emit(self, record): def _emit(self, record): # type: (LogRecord) -> None - if not _can_record(record): + if not self._can_record(record): return hub = Hub.current @@ -232,7 +218,7 @@ def _emit(self, record): hint["log_record"] = record - event["level"] = _logging_to_event_level(record) + event["level"] = self._logging_to_event_level(record) event["logger"] = record.name # Log records from `warnings` module as separate issues @@ -255,7 +241,7 @@ def _emit(self, record): "params": record.args, } - event["extra"] = _extra_from_record(record) + event["extra"] = self._extra_from_record(record) hub.capture_event(event, hint=hint) @@ -264,7 +250,7 @@ def _emit(self, record): SentryHandler = EventHandler -class BreadcrumbHandler(logging.Handler, object): +class BreadcrumbHandler(_BaseHandler): """ A logging handler that records breadcrumbs for each log record. @@ -279,9 +265,20 @@ def emit(self, record): def _emit(self, record): # type: (LogRecord) -> None - if not _can_record(record): + if not self._can_record(record): return Hub.current.add_breadcrumb( - _breadcrumb_from_record(record), hint={"log_record": record} + self._breadcrumb_from_record(record), hint={"log_record": record} ) + + def _breadcrumb_from_record(self, record): + # type: (LogRecord) -> Dict[str, Any] + return { + "type": "log", + "level": self._logging_to_event_level(record), + "category": record.name, + "message": record.message, + "timestamp": datetime.datetime.utcfromtimestamp(record.created), + "data": self._extra_from_record(record), + } diff --git a/sentry_sdk/integrations/loguru.py b/sentry_sdk/integrations/loguru.py new file mode 100644 index 0000000000..47ad9a36c4 --- /dev/null +++ b/sentry_sdk/integrations/loguru.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import + +import enum + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.integrations import Integration, DidNotEnable +from sentry_sdk.integrations.logging import ( + BreadcrumbHandler, + EventHandler, + _BaseHandler, +) + +if TYPE_CHECKING: + from logging import LogRecord + from typing import Optional, Tuple + +try: + from loguru import logger +except ImportError: + raise DidNotEnable("LOGURU is not installed") + + +class LoggingLevels(enum.IntEnum): + TRACE = 5 + DEBUG = 10 + INFO = 20 + SUCCESS = 25 + WARNING = 30 + ERROR = 40 + CRITICAL = 50 + + +DEFAULT_LEVEL = LoggingLevels.INFO.value +DEFAULT_EVENT_LEVEL = LoggingLevels.ERROR.value +# We need to save the handlers to be able to remove them later +# in tests (they call `LoguruIntegration.__init__` multiple times, +# and we can't use `setup_once` because it's called before +# than we get configuration). +_ADDED_HANDLERS = (None, None) # type: Tuple[Optional[int], Optional[int]] + + +class LoguruIntegration(Integration): + identifier = "loguru" + + def __init__(self, level=DEFAULT_LEVEL, event_level=DEFAULT_EVENT_LEVEL): + # type: (Optional[int], Optional[int]) -> None + global _ADDED_HANDLERS + breadcrumb_handler, event_handler = _ADDED_HANDLERS + + if breadcrumb_handler is not None: + logger.remove(breadcrumb_handler) + breadcrumb_handler = None + if event_handler is not None: + logger.remove(event_handler) + event_handler = None + + if level is not None: + breadcrumb_handler = logger.add( + LoguruBreadcrumbHandler(level=level), level=level + ) + + if event_level is not None: + event_handler = logger.add( + LoguruEventHandler(level=event_level), level=event_level + ) + + _ADDED_HANDLERS = (breadcrumb_handler, event_handler) + + @staticmethod + def setup_once(): + # type: () -> None + pass # we do everything in __init__ + + +class _LoguruBaseHandler(_BaseHandler): + def _logging_to_event_level(self, record): + # type: (LogRecord) -> str + try: + return LoggingLevels(record.levelno).name.lower() + except ValueError: + return record.levelname.lower() if record.levelname else "" + + +class LoguruEventHandler(_LoguruBaseHandler, EventHandler): + """Modified version of :class:`sentry_sdk.integrations.logging.EventHandler` to use loguru's level names.""" + + +class LoguruBreadcrumbHandler(_LoguruBaseHandler, BreadcrumbHandler): + """Modified version of :class:`sentry_sdk.integrations.logging.BreadcrumbHandler` to use loguru's level names.""" diff --git a/setup.py b/setup.py index 81474ed54f..2e116c783e 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,8 @@ def get_file_text(file_name): "fastapi": ["fastapi>=0.79.0"], "pymongo": ["pymongo>=3.1"], "opentelemetry": ["opentelemetry-distro>=0.35b0"], - "grpcio": ["grpcio>=1.21.1"] + "grpcio": ["grpcio>=1.21.1"], + "loguru": ["loguru>=0.5"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/tests/integrations/loguru/__init__.py b/tests/integrations/loguru/__init__.py new file mode 100644 index 0000000000..9d67fb3799 --- /dev/null +++ b/tests/integrations/loguru/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("loguru") diff --git a/tests/integrations/loguru/test_loguru.py b/tests/integrations/loguru/test_loguru.py new file mode 100644 index 0000000000..3185f021c3 --- /dev/null +++ b/tests/integrations/loguru/test_loguru.py @@ -0,0 +1,77 @@ +import pytest +from loguru import logger + +import sentry_sdk +from sentry_sdk.integrations.loguru import LoguruIntegration, LoggingLevels + +logger.remove(0) # don't print to console + + +@pytest.mark.parametrize( + "level,created_event", + [ + # None - no breadcrumb + # False - no event + # True - event created + (LoggingLevels.TRACE, None), + (LoggingLevels.DEBUG, None), + (LoggingLevels.INFO, False), + (LoggingLevels.SUCCESS, False), + (LoggingLevels.WARNING, False), + (LoggingLevels.ERROR, True), + (LoggingLevels.CRITICAL, True), + ], +) +@pytest.mark.parametrize("disable_breadcrumbs", [True, False]) +@pytest.mark.parametrize("disable_events", [True, False]) +def test_just_log( + sentry_init, + capture_events, + level, + created_event, + disable_breadcrumbs, + disable_events, +): + sentry_init( + integrations=[ + LoguruIntegration( + level=None if disable_breadcrumbs else LoggingLevels.INFO.value, + event_level=None if disable_events else LoggingLevels.ERROR.value, + ) + ], + default_integrations=False, + ) + events = capture_events() + + getattr(logger, level.name.lower())("test") + + formatted_message = ( + " | " + + "{:9}".format(level.name.upper()) + + "| tests.integrations.loguru.test_loguru:test_just_log:46 - test" + ) + + if not created_event: + assert not events + + breadcrumbs = sentry_sdk.Hub.current.scope._breadcrumbs + if ( + not disable_breadcrumbs and created_event is not None + ): # not None == not TRACE or DEBUG level + (breadcrumb,) = breadcrumbs + assert breadcrumb["level"] == level.name.lower() + assert breadcrumb["category"] == "tests.integrations.loguru.test_loguru" + assert breadcrumb["message"][23:] == formatted_message + else: + assert not breadcrumbs + + return + + if disable_events: + assert not events + return + + (event,) = events + assert event["level"] == (level.name.lower()) + assert event["logger"] == "tests.integrations.loguru.test_loguru" + assert event["logentry"]["message"][23:] == formatted_message diff --git a/tox.ini b/tox.ini index 7632af225f..27c706796c 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,9 @@ envlist = # Huey {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-huey-2 + # Loguru + {py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-loguru-v{0.5,0.6,0.7} + # OpenTelemetry (OTel) {py3.7,py3.8,py3.9,py3.10,py3.11}-opentelemetry @@ -318,6 +321,11 @@ deps = # Huey huey-2: huey>=2.0 + # Loguru + loguru-v0.5: loguru>=0.5.0,<0.6.0 + loguru-v0.6: loguru>=0.6.0,<0.7.0 + loguru-v0.7: loguru>=0.7.0,<0.8.0 + # OpenTelemetry (OTel) opentelemetry: opentelemetry-distro @@ -452,6 +460,7 @@ setenv = gcp: TESTPATH=tests/integrations/gcp httpx: TESTPATH=tests/integrations/httpx huey: TESTPATH=tests/integrations/huey + loguru: TESTPATH=tests/integrations/loguru opentelemetry: TESTPATH=tests/integrations/opentelemetry pure_eval: TESTPATH=tests/integrations/pure_eval pymongo: TESTPATH=tests/integrations/pymongo From e0209db8076aaf4d2f90d83fe5379f8591c5d8ee Mon Sep 17 00:00:00 2001 From: Neel Shah Date: Wed, 10 May 2023 13:47:36 +0200 Subject: [PATCH 03/13] Remove relay extension from AWS Layer (#2068) we're reverting back to the older setup since the whole 'relay as AWS extension' experiment didn't really work out. * revert port override in DSN * remove gh action that bundles relay * zip in place as part of `make build_aws_lambda_layer` part of https://github.com/getsentry/team-webplatform-meta/issues/58 --- .github/workflows/ci.yml | 12 ------ Makefile | 1 + scripts/aws-delete-lamba-layer-versions.sh | 2 +- scripts/aws-deploy-local-layer.sh | 47 +++------------------- scripts/build_aws_lambda_layer.py | 28 +++++++++++-- scripts/init_serverless_sdk.py | 10 +---- 6 files changed, 33 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cbf7f36b6..8c397adabb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,18 +68,6 @@ jobs: pip install virtualenv # This will also trigger "make dist" that creates the Python packages make aws-lambda-layer - - echo "Saving SDK_VERSION for later" - export SDK_VERSION=$(grep "VERSION = " sentry_sdk/consts.py | cut -f3 -d' ' | tr -d '"') - echo "SDK_VERSION=$SDK_VERSION" - echo "SDK_VERSION=$SDK_VERSION" >> $GITHUB_ENV - - name: Upload Python AWS Lambda Layer - uses: getsentry/action-build-aws-lambda-extension@v1 - with: - artifact_name: ${{ github.sha }} - zip_file_name: sentry-python-serverless-${{ env.SDK_VERSION }}.zip - build_cache_paths: ${{ env.CACHED_BUILD_PATHS }} - build_cache_key: ${{ env.BUILD_CACHE_KEY }} - name: Upload Python Packages uses: actions/upload-artifact@v3 with: diff --git a/Makefile b/Makefile index 339a68c069..a4d07279da 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ help: dist: .venv rm -rf dist dist-serverless build + $(VENV_PATH)/bin/pip install wheel $(VENV_PATH)/bin/python setup.py sdist bdist_wheel .PHONY: dist diff --git a/scripts/aws-delete-lamba-layer-versions.sh b/scripts/aws-delete-lamba-layer-versions.sh index 5e1ea38a85..f467f9398b 100755 --- a/scripts/aws-delete-lamba-layer-versions.sh +++ b/scripts/aws-delete-lamba-layer-versions.sh @@ -8,7 +8,7 @@ set -euo pipefail # override default AWS region export AWS_REGION=eu-central-1 -LAYER_NAME=SentryPythonServerlessSDKLocalDev +LAYER_NAME=SentryPythonServerlessSDK-local-dev VERSION="0" while [[ $VERSION != "1" ]] diff --git a/scripts/aws-deploy-local-layer.sh b/scripts/aws-deploy-local-layer.sh index 9e2d7c795e..3f213849f3 100755 --- a/scripts/aws-deploy-local-layer.sh +++ b/scripts/aws-deploy-local-layer.sh @@ -9,55 +9,20 @@ set -euo pipefail # Creating Lambda layer -echo "Creating Lambda layer in ./dist-serverless ..." +echo "Creating Lambda layer in ./dist ..." make aws-lambda-layer -echo "Done creating Lambda layer in ./dist-serverless." - -# IMPORTANT: -# Please make sure that this part does the same as the GitHub action that -# is building the Lambda layer in production! -# see: https://github.com/getsentry/action-build-aws-lambda-extension/blob/main/action.yml#L23-L40 - -echo "Downloading relay..." -mkdir -p dist-serverless/relay -curl -0 --silent \ - --output dist-serverless/relay/relay \ - "$(curl -s https://release-registry.services.sentry.io/apps/relay/latest | jq -r .files.\"relay-Linux-x86_64\".url)" -chmod +x dist-serverless/relay/relay -echo "Done downloading relay." - -echo "Creating start script..." -mkdir -p dist-serverless/extensions -cat > dist-serverless/extensions/sentry-lambda-extension << EOT -#!/bin/bash -set -euo pipefail -exec /opt/relay/relay run \ - --mode=proxy \ - --shutdown-timeout=2 \ - --upstream-dsn="\$SENTRY_DSN" \ - --aws-runtime-api="\$AWS_LAMBDA_RUNTIME_API" -EOT -chmod +x dist-serverless/extensions/sentry-lambda-extension -echo "Done creating start script." - -# Zip Lambda layer and included Lambda extension -echo "Zipping Lambda layer and included Lambda extension..." -cd dist-serverless/ -zip -r ../sentry-python-serverless-x.x.x-dev.zip \ - . \ - --exclude \*__pycache__\* --exclude \*.yml -cd .. -echo "Done Zipping Lambda layer and included Lambda extension to ./sentry-python-serverless-x.x.x-dev.zip." - +echo "Done creating Lambda layer in ./dist" # Deploying zipped Lambda layer to AWS -echo "Deploying zipped Lambda layer to AWS..." +ZIP=$(ls dist | grep serverless | head -n 1) +echo "Deploying zipped Lambda layer $ZIP to AWS..." aws lambda publish-layer-version \ --layer-name "SentryPythonServerlessSDK-local-dev" \ --region "eu-central-1" \ - --zip-file "fileb://sentry-python-serverless-x.x.x-dev.zip" \ + --zip-file "fileb://dist/$ZIP" \ --description "Local test build of SentryPythonServerlessSDK (can be deleted)" \ + --compatible-runtimes python3.6 python3.7 python3.8 python3.9 --no-cli-pager echo "Done deploying zipped Lambda layer to AWS as 'SentryPythonServerlessSDK-local-dev'." diff --git a/scripts/build_aws_lambda_layer.py b/scripts/build_aws_lambda_layer.py index d694d15ba7..829b7e31d9 100644 --- a/scripts/build_aws_lambda_layer.py +++ b/scripts/build_aws_lambda_layer.py @@ -17,6 +17,7 @@ def __init__( # type: (...) -> None self.base_dir = base_dir self.python_site_packages = os.path.join(self.base_dir, PYTHON_SITE_PACKAGES) + self.out_zip_filename = f"sentry-python-serverless-{SDK_VERSION}.zip" def make_directories(self): # type: (...) -> None @@ -57,16 +58,35 @@ def create_init_serverless_sdk_package(self): "scripts/init_serverless_sdk.py", f"{serverless_sdk_path}/__init__.py" ) + def zip(self): + # type: (...) -> None + subprocess.run( + [ + "zip", + "-q", # Quiet + "-x", # Exclude files + "**/__pycache__/*", # Files to be excluded + "-r", # Recurse paths + self.out_zip_filename, # Output filename + PYTHON_SITE_PACKAGES, # Files to be zipped + ], + cwd=self.base_dir, + check=True, # Raises CalledProcessError if exit status is non-zero + ) -def build_layer_dir(): + shutil.copy( + os.path.join(self.base_dir, self.out_zip_filename), + os.path.abspath(DIST_PATH) + ) + +def build_packaged_zip(): with tempfile.TemporaryDirectory() as base_dir: layer_builder = LayerBuilder(base_dir) layer_builder.make_directories() layer_builder.install_python_packages() layer_builder.create_init_serverless_sdk_package() - - shutil.copytree(base_dir, "dist-serverless") + layer_builder.zip() if __name__ == "__main__": - build_layer_dir() + build_packaged_zip() diff --git a/scripts/init_serverless_sdk.py b/scripts/init_serverless_sdk.py index 05dd8c767a..e2c9f536f8 100644 --- a/scripts/init_serverless_sdk.py +++ b/scripts/init_serverless_sdk.py @@ -18,17 +18,9 @@ from typing import Any -def extension_relay_dsn(original_dsn): - dsn = Dsn(original_dsn) - dsn.host = "localhost" - dsn.port = 5333 - dsn.scheme = "http" - return str(dsn) - - # Configure Sentry SDK sentry_sdk.init( - dsn=extension_relay_dsn(os.environ["SENTRY_DSN"]), + dsn=os.environ["SENTRY_DSN"], integrations=[AwsLambdaIntegration(timeout_warning=True)], traces_sample_rate=float(os.environ["SENTRY_TRACES_SAMPLE_RATE"]), ) From eb5ee4acf1556a9973ef1fe7d0ae63bab150059d Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova <131587164+sentrivana@users.noreply.github.com> Date: Thu, 11 May 2023 09:54:26 +0200 Subject: [PATCH 04/13] Do not truncate request body if `request_bodies` is `"always"` (#2092) --- sentry_sdk/client.py | 2 +- sentry_sdk/serializer.py | 54 +++++++++++++++++----- tests/integrations/bottle/test_bottle.py | 32 +++++++++++++ tests/integrations/flask/test_flask.py | 27 +++++++++++ tests/integrations/pyramid/test_pyramid.py | 26 +++++++++++ tests/test_serializer.py | 42 +++++++++++++++-- 6 files changed, 168 insertions(+), 15 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 1182922dd4..204b99ce0c 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -320,7 +320,7 @@ def _prepare_event( # Postprocess the event here so that annotated types do # generally not surface in before_send if event is not None: - event = serialize(event) + event = serialize(event, request_bodies=self.options.get("request_bodies")) before_send = self.options["before_send"] if ( diff --git a/sentry_sdk/serializer.py b/sentry_sdk/serializer.py index 22eec490ae..b3f8012c28 100644 --- a/sentry_sdk/serializer.py +++ b/sentry_sdk/serializer.py @@ -67,6 +67,8 @@ # this value due to attached metadata, so keep the number conservative. MAX_EVENT_BYTES = 10**6 +# Maximum depth and breadth of databags. Excess data will be trimmed. If +# request_bodies is "always", request bodies won't be trimmed. MAX_DATABAG_DEPTH = 5 MAX_DATABAG_BREADTH = 10 CYCLE_MARKER = "" @@ -118,6 +120,8 @@ def serialize(event, **kwargs): path = [] # type: List[Segment] meta_stack = [] # type: List[Dict[str, Any]] + keep_request_bodies = kwargs.pop("request_bodies", None) == "always" # type: bool + def _annotate(**meta): # type: (**Any) -> None while len(meta_stack) <= len(path): @@ -182,10 +186,11 @@ def _is_databag(): if rv in (True, None): return rv - p0 = path[0] - if p0 == "request" and path[1] == "data": - return True + is_request_body = _is_request_body() + if is_request_body in (True, None): + return is_request_body + p0 = path[0] if p0 == "breadcrumbs" and path[1] == "values": path[2] return True @@ -198,13 +203,24 @@ def _is_databag(): return False + def _is_request_body(): + # type: () -> Optional[bool] + try: + if path[0] == "request" and path[1] == "data": + return True + except IndexError: + return None + + return False + def _serialize_node( obj, # type: Any is_databag=None, # type: Optional[bool] + is_request_body=None, # type: Optional[bool] should_repr_strings=None, # type: Optional[bool] segment=None, # type: Optional[Segment] - remaining_breadth=None, # type: Optional[int] - remaining_depth=None, # type: Optional[int] + remaining_breadth=None, # type: Optional[Union[int, float]] + remaining_depth=None, # type: Optional[Union[int, float]] ): # type: (...) -> Any if segment is not None: @@ -218,6 +234,7 @@ def _serialize_node( return _serialize_node_impl( obj, is_databag=is_databag, + is_request_body=is_request_body, should_repr_strings=should_repr_strings, remaining_depth=remaining_depth, remaining_breadth=remaining_breadth, @@ -242,9 +259,14 @@ def _flatten_annotated(obj): return obj def _serialize_node_impl( - obj, is_databag, should_repr_strings, remaining_depth, remaining_breadth + obj, + is_databag, + is_request_body, + should_repr_strings, + remaining_depth, + remaining_breadth, ): - # type: (Any, Optional[bool], Optional[bool], Optional[int], Optional[int]) -> Any + # type: (Any, Optional[bool], Optional[bool], Optional[bool], Optional[Union[float, int]], Optional[Union[float, int]]) -> Any if isinstance(obj, AnnotatedValue): should_repr_strings = False if should_repr_strings is None: @@ -253,10 +275,18 @@ def _serialize_node_impl( if is_databag is None: is_databag = _is_databag() - if is_databag and remaining_depth is None: - remaining_depth = MAX_DATABAG_DEPTH - if is_databag and remaining_breadth is None: - remaining_breadth = MAX_DATABAG_BREADTH + if is_request_body is None: + is_request_body = _is_request_body() + + if is_databag: + if is_request_body and keep_request_bodies: + remaining_depth = float("inf") + remaining_breadth = float("inf") + else: + if remaining_depth is None: + remaining_depth = MAX_DATABAG_DEPTH + if remaining_breadth is None: + remaining_breadth = MAX_DATABAG_BREADTH obj = _flatten_annotated(obj) @@ -312,6 +342,7 @@ def _serialize_node_impl( segment=str_k, should_repr_strings=should_repr_strings, is_databag=is_databag, + is_request_body=is_request_body, remaining_depth=remaining_depth - 1 if remaining_depth is not None else None, @@ -338,6 +369,7 @@ def _serialize_node_impl( segment=i, should_repr_strings=should_repr_strings, is_databag=is_databag, + is_request_body=is_request_body, remaining_depth=remaining_depth - 1 if remaining_depth is not None else None, diff --git a/tests/integrations/bottle/test_bottle.py b/tests/integrations/bottle/test_bottle.py index dfd6e52f80..206ba1cefd 100644 --- a/tests/integrations/bottle/test_bottle.py +++ b/tests/integrations/bottle/test_bottle.py @@ -8,6 +8,7 @@ from io import BytesIO from bottle import Bottle, debug as set_debug, abort, redirect from sentry_sdk import capture_message +from sentry_sdk.serializer import MAX_DATABAG_BREADTH from sentry_sdk.integrations.logging import LoggingIntegration from werkzeug.test import Client @@ -275,6 +276,37 @@ def index(): assert not event["request"]["data"]["file"] +def test_json_not_truncated_if_request_bodies_is_always( + sentry_init, capture_events, app, get_client +): + sentry_init( + integrations=[bottle_sentry.BottleIntegration()], request_bodies="always" + ) + + data = { + "key{}".format(i): "value{}".format(i) for i in range(MAX_DATABAG_BREADTH + 10) + } + + @app.route("/", method="POST") + def index(): + import bottle + + assert bottle.request.json == data + assert bottle.request.body.read() == json.dumps(data).encode("ascii") + capture_message("hi") + return "ok" + + events = capture_events() + + client = get_client() + + response = client.post("/", content_type="application/json", data=json.dumps(data)) + assert response[1] == "200 OK" + + (event,) = events + assert event["request"]["data"] == data + + @pytest.mark.parametrize( "integrations", [ diff --git a/tests/integrations/flask/test_flask.py b/tests/integrations/flask/test_flask.py index 8983c4e5ff..b5ac498dd6 100644 --- a/tests/integrations/flask/test_flask.py +++ b/tests/integrations/flask/test_flask.py @@ -28,6 +28,7 @@ ) from sentry_sdk.integrations.logging import LoggingIntegration import sentry_sdk.integrations.flask as flask_sentry +from sentry_sdk.serializer import MAX_DATABAG_BREADTH login_manager = LoginManager() @@ -447,6 +448,32 @@ def index(): assert not event["request"]["data"]["file"] +def test_json_not_truncated_if_request_bodies_is_always( + sentry_init, capture_events, app +): + sentry_init(integrations=[flask_sentry.FlaskIntegration()], request_bodies="always") + + data = { + "key{}".format(i): "value{}".format(i) for i in range(MAX_DATABAG_BREADTH + 10) + } + + @app.route("/", methods=["POST"]) + def index(): + assert request.get_json() == data + assert request.get_data() == json.dumps(data).encode("ascii") + capture_message("hi") + return "ok" + + events = capture_events() + + client = app.test_client() + response = client.post("/", content_type="application/json", data=json.dumps(data)) + assert response.status_code == 200 + + (event,) = events + assert event["request"]["data"] == data + + @pytest.mark.parametrize( "integrations", [ diff --git a/tests/integrations/pyramid/test_pyramid.py b/tests/integrations/pyramid/test_pyramid.py index 0f8755ac6b..01dd1c6a04 100644 --- a/tests/integrations/pyramid/test_pyramid.py +++ b/tests/integrations/pyramid/test_pyramid.py @@ -12,6 +12,7 @@ from sentry_sdk import capture_message, add_breadcrumb from sentry_sdk.integrations.pyramid import PyramidIntegration +from sentry_sdk.serializer import MAX_DATABAG_BREADTH from werkzeug.test import Client @@ -192,6 +193,31 @@ def index(request): assert event["request"]["data"] == data +def test_json_not_truncated_if_request_bodies_is_always( + sentry_init, capture_events, route, get_client +): + sentry_init(integrations=[PyramidIntegration()], request_bodies="always") + + data = { + "key{}".format(i): "value{}".format(i) for i in range(MAX_DATABAG_BREADTH + 10) + } + + @route("/") + def index(request): + assert request.json == data + assert request.text == json.dumps(data) + capture_message("hi") + return Response("ok") + + events = capture_events() + + client = get_client() + client.post("/", content_type="application/json", data=json.dumps(data)) + + (event,) = events + assert event["request"]["data"] == data + + def test_files_and_form(sentry_init, capture_events, route, get_client): sentry_init(integrations=[PyramidIntegration()], request_bodies="always") diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 1e28daa2f1..5bb0579d5a 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -2,7 +2,7 @@ import sys import pytest -from sentry_sdk.serializer import serialize +from sentry_sdk.serializer import MAX_DATABAG_BREADTH, MAX_DATABAG_DEPTH, serialize try: from hypothesis import given @@ -40,14 +40,24 @@ def inner(message, **kwargs): @pytest.fixture def extra_normalizer(validate_event_schema): - def inner(message, **kwargs): - event = serialize({"extra": {"foo": message}}, **kwargs) + def inner(extra, **kwargs): + event = serialize({"extra": {"foo": extra}}, **kwargs) validate_event_schema(event) return event["extra"]["foo"] return inner +@pytest.fixture +def body_normalizer(validate_event_schema): + def inner(body, **kwargs): + event = serialize({"request": {"data": body}}, **kwargs) + validate_event_schema(event) + return event["request"]["data"] + + return inner + + def test_bytes_serialization_decode(message_normalizer): binary = b"abc123\x80\xf0\x9f\x8d\x95" result = message_normalizer(binary, should_repr_strings=False) @@ -106,3 +116,29 @@ def test_custom_mapping_doesnt_mess_with_mock(extra_normalizer): m = mock.Mock() extra_normalizer(m) assert len(m.mock_calls) == 0 + + +def test_trim_databag_breadth(body_normalizer): + data = { + "key{}".format(i): "value{}".format(i) for i in range(MAX_DATABAG_BREADTH + 10) + } + + result = body_normalizer(data) + + assert len(result) == MAX_DATABAG_BREADTH + for key, value in result.items(): + assert data.get(key) == value + + +def test_no_trimming_if_request_bodies_is_always(body_normalizer): + data = { + "key{}".format(i): "value{}".format(i) for i in range(MAX_DATABAG_BREADTH + 10) + } + curr = data + for _ in range(MAX_DATABAG_DEPTH + 5): + curr["nested"] = {} + curr = curr["nested"] + + result = body_normalizer(data, request_bodies="always") + + assert result == data From fbd7d1a849666cd5e200e63a215394ffc2941eb2 Mon Sep 17 00:00:00 2001 From: Farhat Nawaz <68388692+farhat-nawaz@users.noreply.github.com> Date: Thu, 11 May 2023 13:10:25 +0500 Subject: [PATCH 05/13] Ref: Add `include_source_context` option in utils (#2020) Some users do not like the source context to be there, and so add `include_source_context` option to opt-out. --------- Co-authored-by: Farhat Nawaz Co-authored-by: Anton Pirker Co-authored-by: Ivana Kellyerova <131587164+sentrivana@users.noreply.github.com> --- sentry_sdk/utils.py | 18 ++++++++++-------- tests/test_utils.py | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index e1a0273ef1..fc9ec19480 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -594,8 +594,10 @@ def filename_for_module(module, abs_path): return abs_path -def serialize_frame(frame, tb_lineno=None, include_local_variables=True): - # type: (FrameType, Optional[int], bool) -> Dict[str, Any] +def serialize_frame( + frame, tb_lineno=None, include_local_variables=True, include_source_context=True +): + # type: (FrameType, Optional[int], bool, bool) -> Dict[str, Any] f_code = getattr(frame, "f_code", None) if not f_code: abs_path = None @@ -611,18 +613,19 @@ def serialize_frame(frame, tb_lineno=None, include_local_variables=True): if tb_lineno is None: tb_lineno = frame.f_lineno - pre_context, context_line, post_context = get_source_context(frame, tb_lineno) - rv = { "filename": filename_for_module(module, abs_path) or None, "abs_path": os.path.abspath(abs_path) if abs_path else None, "function": function or "", "module": module, "lineno": tb_lineno, - "pre_context": pre_context, - "context_line": context_line, - "post_context": post_context, } # type: Dict[str, Any] + + if include_source_context: + rv["pre_context"], rv["context_line"], rv["post_context"] = get_source_context( + frame, tb_lineno + ) + if include_local_variables: rv["vars"] = frame.f_locals @@ -1240,7 +1243,6 @@ def sanitize_url(url, remove_authority=True, remove_query_values=True): def parse_url(url, sanitize=True): - # type: (str, bool) -> ParsedUrl """ Splits a URL into a url (including path), query and fragment. If sanitize is True, the query diff --git a/tests/test_utils.py b/tests/test_utils.py index 7578e6255b..aa88d26c44 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,14 @@ import pytest import re +import sys -from sentry_sdk.utils import is_valid_sample_rate, logger, parse_url, sanitize_url +from sentry_sdk.utils import ( + is_valid_sample_rate, + logger, + parse_url, + sanitize_url, + serialize_frame, +) try: from unittest import mock # python 3.3 and above @@ -221,3 +228,16 @@ def test_warns_on_invalid_sample_rate(rate, StringContaining): # noqa: N803 result = is_valid_sample_rate(rate, source="Testing") logger.warning.assert_any_call(StringContaining("Given sample rate is invalid")) assert result is False + + +@pytest.mark.parametrize( + "include_source_context", + [True, False], +) +def test_include_source_context_when_serializing_frame(include_source_context): + frame = sys._getframe() + result = serialize_frame(frame, include_source_context=include_source_context) + + assert include_source_context ^ ("pre_context" in result) ^ True + assert include_source_context ^ ("context_line" in result) ^ True + assert include_source_context ^ ("post_context" in result) ^ True From ad3bde9804db61c17271ae3e9bd4148f14492158 Mon Sep 17 00:00:00 2001 From: Neel Shah Date: Thu, 11 May 2023 19:03:26 +0200 Subject: [PATCH 06/13] Fix __qualname__ missing attribute in asyncio integration (#2105) --- sentry_sdk/integrations/asyncio.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/asyncio.py b/sentry_sdk/integrations/asyncio.py index 3fde7ed257..03e320adc7 100644 --- a/sentry_sdk/integrations/asyncio.py +++ b/sentry_sdk/integrations/asyncio.py @@ -21,6 +21,15 @@ from sentry_sdk._types import ExcInfo +def get_name(coro): + # type: (Any) -> str + return ( + getattr(coro, "__qualname__", None) + or getattr(coro, "__name__", None) + or "coroutine without __name__" + ) + + def patch_asyncio(): # type: () -> None orig_task_factory = None @@ -37,7 +46,7 @@ async def _coro_creating_hub_and_span(): result = None with hub: - with hub.start_span(op=OP.FUNCTION, description=coro.__qualname__): + with hub.start_span(op=OP.FUNCTION, description=get_name(coro)): try: result = await coro except Exception: From e8f47929041a048af88ac25ef092bcbf15915935 Mon Sep 17 00:00:00 2001 From: rco-ableton <11273197+rco-ableton@users.noreply.github.com> Date: Fri, 12 May 2023 11:07:40 +0200 Subject: [PATCH 07/13] Import Markup from markupsafe (#2047) Flask v2.3.0 deprecates importing Markup from flask, indicating that it should be imported from markupsafe instead. --------- Co-authored-by: Anton Pirker Co-authored-by: Ivana Kellyerova --- sentry_sdk/integrations/flask.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/flask.py b/sentry_sdk/integrations/flask.py index c60f6437fd..ea5a3c081a 100644 --- a/sentry_sdk/integrations/flask.py +++ b/sentry_sdk/integrations/flask.py @@ -26,7 +26,7 @@ flask_login = None try: - from flask import Flask, Markup, Request # type: ignore + from flask import Flask, Request # type: ignore from flask import __version__ as FLASK_VERSION from flask import request as flask_request from flask.signals import ( @@ -34,6 +34,7 @@ got_request_exception, request_started, ) + from markupsafe import Markup except ImportError: raise DidNotEnable("Flask is not installed") diff --git a/setup.py b/setup.py index 2e116c783e..abd49b0854 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_file_text(file_name): "certifi", ], extras_require={ - "flask": ["flask>=0.11", "blinker>=1.1"], + "flask": ["flask>=0.11", "blinker>=1.1", "markupsafe"], "quart": ["quart>=0.16.1", "blinker>=1.1"], "bottle": ["bottle>=0.12.13"], "falcon": ["falcon>=1.4"], From f80523939576cba84cbdf9e54044acf159559eb3 Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Fri, 12 May 2023 12:36:18 +0200 Subject: [PATCH 08/13] Surface `include_source_context` as an option (#2100) --- sentry_sdk/consts.py | 1 + sentry_sdk/utils.py | 13 ++++++++++--- tests/test_client.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 7a76a507eb..33f72651e3 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -185,6 +185,7 @@ def __init__( project_root=None, # type: Optional[str] enable_tracing=None, # type: Optional[bool] include_local_variables=True, # type: Optional[bool] + include_source_context=True, # type: Optional[bool] trace_propagation_targets=[ # noqa: B006 MATCH_ALL ], # type: Optional[Sequence[str]] diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index fc9ec19480..ddbc329932 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -632,8 +632,8 @@ def serialize_frame( return rv -def current_stacktrace(include_local_variables=True): - # type: (bool) -> Any +def current_stacktrace(include_local_variables=True, include_source_context=True): + # type: (bool, bool) -> Any __tracebackhide__ = True frames = [] @@ -641,7 +641,11 @@ def current_stacktrace(include_local_variables=True): while f is not None: if not should_hide_frame(f): frames.append( - serialize_frame(f, include_local_variables=include_local_variables) + serialize_frame( + f, + include_local_variables=include_local_variables, + include_source_context=include_source_context, + ) ) f = f.f_back @@ -677,14 +681,17 @@ def single_exception_from_error_tuple( if client_options is None: include_local_variables = True + include_source_context = True else: include_local_variables = client_options["include_local_variables"] + include_source_context = client_options["include_source_context"] frames = [ serialize_frame( tb.tb_frame, tb_lineno=tb.tb_lineno, include_local_variables=include_local_variables, + include_source_context=include_source_context, ) for tb in iter_stacks(tb) ] diff --git a/tests/test_client.py b/tests/test_client.py index 167cb7347c..1a932c65f2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -365,6 +365,38 @@ def test_include_local_variables_disabled(sentry_init, capture_events): ) +def test_include_source_context_enabled(sentry_init, capture_events): + sentry_init(include_source_context=True) + events = capture_events() + try: + 1 / 0 + except Exception: + capture_exception() + + (event,) = events + + frame = event["exception"]["values"][0]["stacktrace"]["frames"][0] + assert "post_context" in frame + assert "pre_context" in frame + assert "context_line" in frame + + +def test_include_source_context_disabled(sentry_init, capture_events): + sentry_init(include_source_context=False) + events = capture_events() + try: + 1 / 0 + except Exception: + capture_exception() + + (event,) = events + + frame = event["exception"]["values"][0]["stacktrace"]["frames"][0] + assert "post_context" not in frame + assert "pre_context" not in frame + assert "context_line" not in frame + + @pytest.mark.parametrize("integrations", [[], [ExecutingIntegration()]]) def test_function_names(sentry_init, capture_events, integrations): sentry_init(integrations=integrations) From ccdaed397293009c942da35a28a1a44c7d1872c8 Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Fri, 12 May 2023 12:46:11 +0200 Subject: [PATCH 09/13] Make sure we're importing redis the library (#2106) ...not the module, if there is one present. --- sentry_sdk/integrations/redis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/redis.py b/sentry_sdk/integrations/redis.py index b05bc741f1..22464d8b4c 100644 --- a/sentry_sdk/integrations/redis.py +++ b/sentry_sdk/integrations/redis.py @@ -115,14 +115,14 @@ def __init__(self, max_data_size=_DEFAULT_MAX_DATA_SIZE): def setup_once(): # type: () -> None try: - import redis + from redis import StrictRedis, client except ImportError: raise DidNotEnable("Redis client not installed") - patch_redis_client(redis.StrictRedis, is_cluster=False) - patch_redis_pipeline(redis.client.Pipeline, False, _get_redis_command_args) + patch_redis_client(StrictRedis, is_cluster=False) + patch_redis_pipeline(client.Pipeline, False, _get_redis_command_args) try: - strict_pipeline = redis.client.StrictPipeline # type: ignore + strict_pipeline = client.StrictPipeline # type: ignore except AttributeError: pass else: From 041534db42178a7d3babee1c04e89e6c6fc6be5c Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Fri, 12 May 2023 13:10:02 +0200 Subject: [PATCH 10/13] Add a note about `pip freeze` to the bug template (#2103) --- .github/ISSUE_TEMPLATE/bug.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index f6e47929eb..78f1e03d21 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -27,6 +27,8 @@ body: 1. What 2. you 3. did. + + Extra points for also including the output of `pip freeze --all`. validations: required: true - type: textarea From f8f53b873e1513cc243eb38981651184108dd378 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 15 May 2023 10:12:58 +0200 Subject: [PATCH 11/13] Fixed Celery headers for Beat auto-instrumentation (#2102) * Fixed celery headers for beat auto instrumentation --------- Co-authored-by: Ivana Kellyerova --- sentry_sdk/integrations/celery.py | 11 ++++++++- tests/integrations/celery/test_celery.py | 30 +++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/celery.py b/sentry_sdk/integrations/celery.py index 8c9484e2f0..c2dc4e1e74 100644 --- a/sentry_sdk/integrations/celery.py +++ b/sentry_sdk/integrations/celery.py @@ -157,6 +157,13 @@ def apply_async(*args, **kwargs): # tracing tools (dd-trace-py) also employ this exact # workaround and we don't want to break them. kwarg_headers.setdefault("headers", {}).update(headers) + + # Add the Sentry options potentially added in `sentry_apply_entry` + # to the headers (done when auto-instrumenting Celery Beat tasks) + for key, value in kwarg_headers.items(): + if key.startswith("sentry-"): + kwarg_headers["headers"][key] = value + kwargs["headers"] = kwarg_headers return f(*args, **kwargs) @@ -431,7 +438,9 @@ def sentry_apply_entry(*args, **kwargs): ) headers.update({"sentry-monitor-check-in-id": check_in_id}) - schedule_entry.options.update(headers) + # Set the Sentry configuration in the options of the ScheduleEntry. + # Those will be picked up in `apply_async` and added to the headers. + schedule_entry.options["headers"] = headers return original_apply_entry(*args, **kwargs) Scheduler.apply_entry = sentry_apply_entry diff --git a/tests/integrations/celery/test_celery.py b/tests/integrations/celery/test_celery.py index a2c8fa1594..fc77d9c5e1 100644 --- a/tests/integrations/celery/test_celery.py +++ b/tests/integrations/celery/test_celery.py @@ -5,11 +5,13 @@ pytest.importorskip("celery") from sentry_sdk import Hub, configure_scope, start_transaction -from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.celery import CeleryIntegration, _get_headers + from sentry_sdk._compat import text_type from celery import Celery, VERSION from celery.bin import worker +from celery.signals import task_success try: from unittest import mock # python 3.3 and above @@ -437,3 +439,29 @@ def dummy_task(x, y): celery_invocation(dummy_task, 1, 0) assert not events + + +def test_task_headers(celery): + """ + Test that the headers set in the Celery Beat auto-instrumentation are passed to the celery signal handlers + """ + sentry_crons_setup = { + "sentry-monitor-slug": "some-slug", + "sentry-monitor-config": {"some": "config"}, + "sentry-monitor-check-in-id": "123abc", + } + + @celery.task(name="dummy_task") + def dummy_task(x, y): + return x + y + + def crons_task_success(sender, **kwargs): + headers = _get_headers(sender) + assert headers == sentry_crons_setup + + task_success.connect(crons_task_success) + + # This is how the Celery Beat auto-instrumentation starts a task + # in the monkey patched version of `apply_async` + # in `sentry_sdk/integrations/celery.py::_wrap_apply_async()` + dummy_task.apply_async(args=(1, 0), headers=sentry_crons_setup) From e82e4db1e6b4a9c6af523284f62e5328f6b11850 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Mon, 15 May 2023 12:23:38 +0000 Subject: [PATCH 12/13] release: 1.23.0 --- CHANGELOG.md | 16 ++++++++++++++++ docs/conf.py | 2 +- sentry_sdk/consts.py | 2 +- setup.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc55492d86..5eec50fd9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 1.23.0 + +### Various fixes & improvements + +- Fixed Celery headers for Beat auto-instrumentation (#2102) by @antonpirker +- Add a note about `pip freeze` to the bug template (#2103) by @sentrivana +- Make sure we're importing redis the library (#2106) by @sentrivana +- Surface `include_source_context` as an option (#2100) by @sentrivana +- Import Markup from markupsafe (#2047) by @rco-ableton +- Fix __qualname__ missing attribute in asyncio integration (#2105) by @sl0thentr0py +- Ref: Add `include_source_context` option in utils (#2020) by @farhat-nawaz +- Do not truncate request body if `request_bodies` is `"always"` (#2092) by @sentrivana +- Remove relay extension from AWS Layer (#2068) by @sl0thentr0py +- Add `loguru` integration (#1994) by @PerchunPak +- Add `db.operation` to Redis and MongoDB spans. (#2089) by @antonpirker + ## 1.22.2 ### Various fixes & improvements diff --git a/docs/conf.py b/docs/conf.py index 21a9c5e0be..1af3a24b02 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ copyright = "2019, Sentry Team and Contributors" author = "Sentry Team and Contributors" -release = "1.22.2" +release = "1.23.0" version = ".".join(release.split(".")[:2]) # The short X.Y version. diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 33f72651e3..258cb527fa 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -214,4 +214,4 @@ def _get_default_options(): del _get_default_options -VERSION = "1.22.2" +VERSION = "1.23.0" diff --git a/setup.py b/setup.py index abd49b0854..05504bf198 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def get_file_text(file_name): setup( name="sentry-sdk", - version="1.22.2", + version="1.23.0", author="Sentry Team and Contributors", author_email="hello@sentry.io", url="https://github.com/getsentry/sentry-python", From 8480e474e608d8e2b0323ee83a8f667c144b816d Mon Sep 17 00:00:00 2001 From: Ivana Kellyerova Date: Mon, 15 May 2023 14:34:35 +0200 Subject: [PATCH 13/13] Update CHANGELOG.md --- CHANGELOG.md | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eec50fd9d..ea0bff7c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,45 @@ ### Various fixes & improvements -- Fixed Celery headers for Beat auto-instrumentation (#2102) by @antonpirker -- Add a note about `pip freeze` to the bug template (#2103) by @sentrivana -- Make sure we're importing redis the library (#2106) by @sentrivana -- Surface `include_source_context` as an option (#2100) by @sentrivana -- Import Markup from markupsafe (#2047) by @rco-ableton -- Fix __qualname__ missing attribute in asyncio integration (#2105) by @sl0thentr0py -- Ref: Add `include_source_context` option in utils (#2020) by @farhat-nawaz +- **New:** Add `loguru` integration (#1994) by @PerchunPak + + Check [the documentation](https://docs.sentry.io/platforms/python/configuration/integrations/loguru/) for more information. + + Usage: + + ```python + from loguru import logger + import sentry_sdk + from sentry_sdk.integrations.loguru import LoguruIntegration + + sentry_sdk.init( + dsn="___PUBLIC_DSN___", + integrations=[ + LoguruIntegration(), + ], + ) + + logger.debug("I am ignored") + logger.info("I am a breadcrumb") + logger.error("I am an event", extra=dict(bar=43)) + logger.exception("An exception happened") + ``` + + - An error event with the message `"I am an event"` will be created. + - `"I am a breadcrumb"` will be attached as a breadcrumb to that event. + - `bar` will end up in the `extra` attributes of that event. + - `"An exception happened"` will send the current exception from `sys.exc_info()` with the stack trace to Sentry. If there's no exception, the current stack will be attached. + - The debug message `"I am ignored"` will not be captured by Sentry. To capture it, set `level` to `DEBUG` or lower in `LoguruIntegration`. + - Do not truncate request body if `request_bodies` is `"always"` (#2092) by @sentrivana +- Fixed Celery headers for Beat auto-instrumentation (#2102) by @antonpirker +- Add `db.operation` to Redis and MongoDB spans (#2089) by @antonpirker +- Make sure we're importing `redis` the library (#2106) by @sentrivana +- Add `include_source_context` option (#2020) by @farhat-nawaz and @sentrivana +- Import `Markup` from `markupsafe` (#2047) by @rco-ableton +- Fix `__qualname__` missing attribute in asyncio integration (#2105) by @sl0thentr0py - Remove relay extension from AWS Layer (#2068) by @sl0thentr0py -- Add `loguru` integration (#1994) by @PerchunPak -- Add `db.operation` to Redis and MongoDB spans. (#2089) by @antonpirker +- Add a note about `pip freeze` to the bug template (#2103) by @sentrivana ## 1.22.2