From 8a2b74f58e97205233717c379b0d78f85d697365 Mon Sep 17 00:00:00 2001 From: Perchun Pak Date: Tue, 9 May 2023 13:18:53 +0200 Subject: [PATCH 1/3] Add `loguru` integration (#1994) * Add `loguru` integration Actually, this is the solution in comments under #653 adapted to codebase and tested as well. https://github.com/getsentry/sentry-python/issues/653#issuecomment-788854865 I also changed `logging` integration to use methods instead of functions in handlers, as in that way we can easily overwrite parts that are different in `loguru` integration. It shouldn't be a problem, as those methods are private and used only in that file. --------- Co-authored-by: Anton Pirker --- .github/workflows/test-integration-loguru.yml | 78 ++++++++++ linter-requirements.txt | 1 + sentry_sdk/integrations/logging.py | 137 +++++++++--------- sentry_sdk/integrations/loguru.py | 89 ++++++++++++ setup.py | 3 +- tests/integrations/loguru/__init__.py | 3 + tests/integrations/loguru/test_loguru.py | 77 ++++++++++ tox.ini | 9 ++ 8 files changed, 326 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/test-integration-loguru.yml create mode 100644 sentry_sdk/integrations/loguru.py create mode 100644 tests/integrations/loguru/__init__.py create mode 100644 tests/integrations/loguru/test_loguru.py diff --git a/.github/workflows/test-integration-loguru.yml b/.github/workflows/test-integration-loguru.yml new file mode 100644 index 0000000000..3fe09a8213 --- /dev/null +++ b/.github/workflows/test-integration-loguru.yml @@ -0,0 +1,78 @@ +name: Test loguru + +on: + push: + branches: + - master + - release/** + + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless + +jobs: + test: + name: loguru, python ${{ matrix.python-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + python-version: ["3.5","3.6","3.7","3.8","3.9","3.10","3.11"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-20.04] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Test Env + run: | + pip install coverage "tox>=3,<4" + + - name: Test loguru + timeout-minutes: 45 + shell: bash + run: | + set -x # print commands that are executed + coverage erase + + # Run tests + ./scripts/runtox.sh "py${{ matrix.python-version }}-loguru" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + coverage combine .coverage* + coverage xml -i + + - uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + + check_required_tests: + name: All loguru tests passed or skipped + needs: test + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Check for failures + if: contains(needs.test.result, 'failure') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/linter-requirements.txt b/linter-requirements.txt index 32f7fe8bc8..5e7ec1c52e 100644 --- a/linter-requirements.txt +++ b/linter-requirements.txt @@ -5,6 +5,7 @@ types-certifi types-redis types-setuptools pymongo # There is no separate types module. +loguru # There is no separate types module. flake8-bugbear==22.12.6 pep8-naming==0.13.2 pre-commit # local linting diff --git a/sentry_sdk/integrations/logging.py b/sentry_sdk/integrations/logging.py index 782180eea7..d4f34d085c 100644 --- a/sentry_sdk/integrations/logging.py +++ b/sentry_sdk/integrations/logging.py @@ -107,75 +107,61 @@ def sentry_patched_callhandlers(self, record): logging.Logger.callHandlers = sentry_patched_callhandlers # type: ignore -def _can_record(record): - # type: (LogRecord) -> bool - """Prevents ignored loggers from recording""" - for logger in _IGNORED_LOGGERS: - if fnmatch(record.name, logger): - return False - return True - - -def _breadcrumb_from_record(record): - # type: (LogRecord) -> Dict[str, Any] - return { - "type": "log", - "level": _logging_to_event_level(record), - "category": record.name, - "message": record.message, - "timestamp": datetime.datetime.utcfromtimestamp(record.created), - "data": _extra_from_record(record), - } - - -def _logging_to_event_level(record): - # type: (LogRecord) -> str - return LOGGING_TO_EVENT_LEVEL.get( - record.levelno, record.levelname.lower() if record.levelname else "" +class _BaseHandler(logging.Handler, object): + COMMON_RECORD_ATTRS = frozenset( + ( + "args", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "linenno", + "lineno", + "message", + "module", + "msecs", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack", + "tags", + "thread", + "threadName", + "stack_info", + ) ) + def _can_record(self, record): + # type: (LogRecord) -> bool + """Prevents ignored loggers from recording""" + for logger in _IGNORED_LOGGERS: + if fnmatch(record.name, logger): + return False + return True + + def _logging_to_event_level(self, record): + # type: (LogRecord) -> str + return LOGGING_TO_EVENT_LEVEL.get( + record.levelno, record.levelname.lower() if record.levelname else "" + ) -COMMON_RECORD_ATTRS = frozenset( - ( - "args", - "created", - "exc_info", - "exc_text", - "filename", - "funcName", - "levelname", - "levelno", - "linenno", - "lineno", - "message", - "module", - "msecs", - "msg", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "stack", - "tags", - "thread", - "threadName", - "stack_info", - ) -) - - -def _extra_from_record(record): - # type: (LogRecord) -> Dict[str, None] - return { - k: v - for k, v in iteritems(vars(record)) - if k not in COMMON_RECORD_ATTRS - and (not isinstance(k, str) or not k.startswith("_")) - } + def _extra_from_record(self, record): + # type: (LogRecord) -> Dict[str, None] + return { + k: v + for k, v in iteritems(vars(record)) + if k not in self.COMMON_RECORD_ATTRS + and (not isinstance(k, str) or not k.startswith("_")) + } -class EventHandler(logging.Handler, object): +class EventHandler(_BaseHandler): """ A logging handler that emits Sentry events for each log record @@ -190,7 +176,7 @@ def emit(self, record): def _emit(self, record): # type: (LogRecord) -> None - if not _can_record(record): + if not self._can_record(record): return hub = Hub.current @@ -232,7 +218,7 @@ def _emit(self, record): hint["log_record"] = record - event["level"] = _logging_to_event_level(record) + event["level"] = self._logging_to_event_level(record) event["logger"] = record.name # Log records from `warnings` module as separate issues @@ -255,7 +241,7 @@ def _emit(self, record): "params": record.args, } - event["extra"] = _extra_from_record(record) + event["extra"] = self._extra_from_record(record) hub.capture_event(event, hint=hint) @@ -264,7 +250,7 @@ def _emit(self, record): SentryHandler = EventHandler -class BreadcrumbHandler(logging.Handler, object): +class BreadcrumbHandler(_BaseHandler): """ A logging handler that records breadcrumbs for each log record. @@ -279,9 +265,20 @@ def emit(self, record): def _emit(self, record): # type: (LogRecord) -> None - if not _can_record(record): + if not self._can_record(record): return Hub.current.add_breadcrumb( - _breadcrumb_from_record(record), hint={"log_record": record} + self._breadcrumb_from_record(record), hint={"log_record": record} ) + + def _breadcrumb_from_record(self, record): + # type: (LogRecord) -> Dict[str, Any] + return { + "type": "log", + "level": self._logging_to_event_level(record), + "category": record.name, + "message": record.message, + "timestamp": datetime.datetime.utcfromtimestamp(record.created), + "data": self._extra_from_record(record), + } diff --git a/sentry_sdk/integrations/loguru.py b/sentry_sdk/integrations/loguru.py new file mode 100644 index 0000000000..47ad9a36c4 --- /dev/null +++ b/sentry_sdk/integrations/loguru.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import + +import enum + +from sentry_sdk._types import TYPE_CHECKING +from sentry_sdk.integrations import Integration, DidNotEnable +from sentry_sdk.integrations.logging import ( + BreadcrumbHandler, + EventHandler, + _BaseHandler, +) + +if TYPE_CHECKING: + from logging import LogRecord + from typing import Optional, Tuple + +try: + from loguru import logger +except ImportError: + raise DidNotEnable("LOGURU is not installed") + + +class LoggingLevels(enum.IntEnum): + TRACE = 5 + DEBUG = 10 + INFO = 20 + SUCCESS = 25 + WARNING = 30 + ERROR = 40 + CRITICAL = 50 + + +DEFAULT_LEVEL = LoggingLevels.INFO.value +DEFAULT_EVENT_LEVEL = LoggingLevels.ERROR.value +# We need to save the handlers to be able to remove them later +# in tests (they call `LoguruIntegration.__init__` multiple times, +# and we can't use `setup_once` because it's called before +# than we get configuration). +_ADDED_HANDLERS = (None, None) # type: Tuple[Optional[int], Optional[int]] + + +class LoguruIntegration(Integration): + identifier = "loguru" + + def __init__(self, level=DEFAULT_LEVEL, event_level=DEFAULT_EVENT_LEVEL): + # type: (Optional[int], Optional[int]) -> None + global _ADDED_HANDLERS + breadcrumb_handler, event_handler = _ADDED_HANDLERS + + if breadcrumb_handler is not None: + logger.remove(breadcrumb_handler) + breadcrumb_handler = None + if event_handler is not None: + logger.remove(event_handler) + event_handler = None + + if level is not None: + breadcrumb_handler = logger.add( + LoguruBreadcrumbHandler(level=level), level=level + ) + + if event_level is not None: + event_handler = logger.add( + LoguruEventHandler(level=event_level), level=event_level + ) + + _ADDED_HANDLERS = (breadcrumb_handler, event_handler) + + @staticmethod + def setup_once(): + # type: () -> None + pass # we do everything in __init__ + + +class _LoguruBaseHandler(_BaseHandler): + def _logging_to_event_level(self, record): + # type: (LogRecord) -> str + try: + return LoggingLevels(record.levelno).name.lower() + except ValueError: + return record.levelname.lower() if record.levelname else "" + + +class LoguruEventHandler(_LoguruBaseHandler, EventHandler): + """Modified version of :class:`sentry_sdk.integrations.logging.EventHandler` to use loguru's level names.""" + + +class LoguruBreadcrumbHandler(_LoguruBaseHandler, BreadcrumbHandler): + """Modified version of :class:`sentry_sdk.integrations.logging.BreadcrumbHandler` to use loguru's level names.""" diff --git a/setup.py b/setup.py index 81474ed54f..2e116c783e 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,8 @@ def get_file_text(file_name): "fastapi": ["fastapi>=0.79.0"], "pymongo": ["pymongo>=3.1"], "opentelemetry": ["opentelemetry-distro>=0.35b0"], - "grpcio": ["grpcio>=1.21.1"] + "grpcio": ["grpcio>=1.21.1"], + "loguru": ["loguru>=0.5"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/tests/integrations/loguru/__init__.py b/tests/integrations/loguru/__init__.py new file mode 100644 index 0000000000..9d67fb3799 --- /dev/null +++ b/tests/integrations/loguru/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("loguru") diff --git a/tests/integrations/loguru/test_loguru.py b/tests/integrations/loguru/test_loguru.py new file mode 100644 index 0000000000..3185f021c3 --- /dev/null +++ b/tests/integrations/loguru/test_loguru.py @@ -0,0 +1,77 @@ +import pytest +from loguru import logger + +import sentry_sdk +from sentry_sdk.integrations.loguru import LoguruIntegration, LoggingLevels + +logger.remove(0) # don't print to console + + +@pytest.mark.parametrize( + "level,created_event", + [ + # None - no breadcrumb + # False - no event + # True - event created + (LoggingLevels.TRACE, None), + (LoggingLevels.DEBUG, None), + (LoggingLevels.INFO, False), + (LoggingLevels.SUCCESS, False), + (LoggingLevels.WARNING, False), + (LoggingLevels.ERROR, True), + (LoggingLevels.CRITICAL, True), + ], +) +@pytest.mark.parametrize("disable_breadcrumbs", [True, False]) +@pytest.mark.parametrize("disable_events", [True, False]) +def test_just_log( + sentry_init, + capture_events, + level, + created_event, + disable_breadcrumbs, + disable_events, +): + sentry_init( + integrations=[ + LoguruIntegration( + level=None if disable_breadcrumbs else LoggingLevels.INFO.value, + event_level=None if disable_events else LoggingLevels.ERROR.value, + ) + ], + default_integrations=False, + ) + events = capture_events() + + getattr(logger, level.name.lower())("test") + + formatted_message = ( + " | " + + "{:9}".format(level.name.upper()) + + "| tests.integrations.loguru.test_loguru:test_just_log:46 - test" + ) + + if not created_event: + assert not events + + breadcrumbs = sentry_sdk.Hub.current.scope._breadcrumbs + if ( + not disable_breadcrumbs and created_event is not None + ): # not None == not TRACE or DEBUG level + (breadcrumb,) = breadcrumbs + assert breadcrumb["level"] == level.name.lower() + assert breadcrumb["category"] == "tests.integrations.loguru.test_loguru" + assert breadcrumb["message"][23:] == formatted_message + else: + assert not breadcrumbs + + return + + if disable_events: + assert not events + return + + (event,) = events + assert event["level"] == (level.name.lower()) + assert event["logger"] == "tests.integrations.loguru.test_loguru" + assert event["logentry"]["message"][23:] == formatted_message diff --git a/tox.ini b/tox.ini index 7632af225f..27c706796c 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,9 @@ envlist = # Huey {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-huey-2 + # Loguru + {py3.5,py3.6,py3.7,py3.8,py3.9,py3.10,py3.11}-loguru-v{0.5,0.6,0.7} + # OpenTelemetry (OTel) {py3.7,py3.8,py3.9,py3.10,py3.11}-opentelemetry @@ -318,6 +321,11 @@ deps = # Huey huey-2: huey>=2.0 + # Loguru + loguru-v0.5: loguru>=0.5.0,<0.6.0 + loguru-v0.6: loguru>=0.6.0,<0.7.0 + loguru-v0.7: loguru>=0.7.0,<0.8.0 + # OpenTelemetry (OTel) opentelemetry: opentelemetry-distro @@ -452,6 +460,7 @@ setenv = gcp: TESTPATH=tests/integrations/gcp httpx: TESTPATH=tests/integrations/httpx huey: TESTPATH=tests/integrations/huey + loguru: TESTPATH=tests/integrations/loguru opentelemetry: TESTPATH=tests/integrations/opentelemetry pure_eval: TESTPATH=tests/integrations/pure_eval pymongo: TESTPATH=tests/integrations/pymongo From cc187039f91d6778d7c7f1eb0ced0547b37a7ce4 Mon Sep 17 00:00:00 2001 From: Neel Shah Date: Tue, 18 Apr 2023 12:36:10 +0200 Subject: [PATCH 2/3] Remove relay extension from AWS Layer we're reverting back to the older setup since the whole 'relay as AWS extension' experiment didn't really work out. * revert port override in DSN * remove gh action that bundles relay * zip in place as part of `make build_aws_lambda_layer` part of https://github.com/getsentry/team-webplatform-meta/issues/58 --- .github/workflows/ci.yml | 12 ------ Makefile | 1 + scripts/aws-delete-lamba-layer-versions.sh | 2 +- scripts/aws-deploy-local-layer.sh | 47 +++------------------- scripts/build_aws_lambda_layer.py | 28 +++++++++++-- scripts/init_serverless_sdk.py | 10 +---- 6 files changed, 33 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cbf7f36b6..8c397adabb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,18 +68,6 @@ jobs: pip install virtualenv # This will also trigger "make dist" that creates the Python packages make aws-lambda-layer - - echo "Saving SDK_VERSION for later" - export SDK_VERSION=$(grep "VERSION = " sentry_sdk/consts.py | cut -f3 -d' ' | tr -d '"') - echo "SDK_VERSION=$SDK_VERSION" - echo "SDK_VERSION=$SDK_VERSION" >> $GITHUB_ENV - - name: Upload Python AWS Lambda Layer - uses: getsentry/action-build-aws-lambda-extension@v1 - with: - artifact_name: ${{ github.sha }} - zip_file_name: sentry-python-serverless-${{ env.SDK_VERSION }}.zip - build_cache_paths: ${{ env.CACHED_BUILD_PATHS }} - build_cache_key: ${{ env.BUILD_CACHE_KEY }} - name: Upload Python Packages uses: actions/upload-artifact@v3 with: diff --git a/Makefile b/Makefile index 339a68c069..a4d07279da 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ help: dist: .venv rm -rf dist dist-serverless build + $(VENV_PATH)/bin/pip install wheel $(VENV_PATH)/bin/python setup.py sdist bdist_wheel .PHONY: dist diff --git a/scripts/aws-delete-lamba-layer-versions.sh b/scripts/aws-delete-lamba-layer-versions.sh index 5e1ea38a85..f467f9398b 100755 --- a/scripts/aws-delete-lamba-layer-versions.sh +++ b/scripts/aws-delete-lamba-layer-versions.sh @@ -8,7 +8,7 @@ set -euo pipefail # override default AWS region export AWS_REGION=eu-central-1 -LAYER_NAME=SentryPythonServerlessSDKLocalDev +LAYER_NAME=SentryPythonServerlessSDK-local-dev VERSION="0" while [[ $VERSION != "1" ]] diff --git a/scripts/aws-deploy-local-layer.sh b/scripts/aws-deploy-local-layer.sh index 9e2d7c795e..3f213849f3 100755 --- a/scripts/aws-deploy-local-layer.sh +++ b/scripts/aws-deploy-local-layer.sh @@ -9,55 +9,20 @@ set -euo pipefail # Creating Lambda layer -echo "Creating Lambda layer in ./dist-serverless ..." +echo "Creating Lambda layer in ./dist ..." make aws-lambda-layer -echo "Done creating Lambda layer in ./dist-serverless." - -# IMPORTANT: -# Please make sure that this part does the same as the GitHub action that -# is building the Lambda layer in production! -# see: https://github.com/getsentry/action-build-aws-lambda-extension/blob/main/action.yml#L23-L40 - -echo "Downloading relay..." -mkdir -p dist-serverless/relay -curl -0 --silent \ - --output dist-serverless/relay/relay \ - "$(curl -s https://release-registry.services.sentry.io/apps/relay/latest | jq -r .files.\"relay-Linux-x86_64\".url)" -chmod +x dist-serverless/relay/relay -echo "Done downloading relay." - -echo "Creating start script..." -mkdir -p dist-serverless/extensions -cat > dist-serverless/extensions/sentry-lambda-extension << EOT -#!/bin/bash -set -euo pipefail -exec /opt/relay/relay run \ - --mode=proxy \ - --shutdown-timeout=2 \ - --upstream-dsn="\$SENTRY_DSN" \ - --aws-runtime-api="\$AWS_LAMBDA_RUNTIME_API" -EOT -chmod +x dist-serverless/extensions/sentry-lambda-extension -echo "Done creating start script." - -# Zip Lambda layer and included Lambda extension -echo "Zipping Lambda layer and included Lambda extension..." -cd dist-serverless/ -zip -r ../sentry-python-serverless-x.x.x-dev.zip \ - . \ - --exclude \*__pycache__\* --exclude \*.yml -cd .. -echo "Done Zipping Lambda layer and included Lambda extension to ./sentry-python-serverless-x.x.x-dev.zip." - +echo "Done creating Lambda layer in ./dist" # Deploying zipped Lambda layer to AWS -echo "Deploying zipped Lambda layer to AWS..." +ZIP=$(ls dist | grep serverless | head -n 1) +echo "Deploying zipped Lambda layer $ZIP to AWS..." aws lambda publish-layer-version \ --layer-name "SentryPythonServerlessSDK-local-dev" \ --region "eu-central-1" \ - --zip-file "fileb://sentry-python-serverless-x.x.x-dev.zip" \ + --zip-file "fileb://dist/$ZIP" \ --description "Local test build of SentryPythonServerlessSDK (can be deleted)" \ + --compatible-runtimes python3.6 python3.7 python3.8 python3.9 --no-cli-pager echo "Done deploying zipped Lambda layer to AWS as 'SentryPythonServerlessSDK-local-dev'." diff --git a/scripts/build_aws_lambda_layer.py b/scripts/build_aws_lambda_layer.py index d694d15ba7..829b7e31d9 100644 --- a/scripts/build_aws_lambda_layer.py +++ b/scripts/build_aws_lambda_layer.py @@ -17,6 +17,7 @@ def __init__( # type: (...) -> None self.base_dir = base_dir self.python_site_packages = os.path.join(self.base_dir, PYTHON_SITE_PACKAGES) + self.out_zip_filename = f"sentry-python-serverless-{SDK_VERSION}.zip" def make_directories(self): # type: (...) -> None @@ -57,16 +58,35 @@ def create_init_serverless_sdk_package(self): "scripts/init_serverless_sdk.py", f"{serverless_sdk_path}/__init__.py" ) + def zip(self): + # type: (...) -> None + subprocess.run( + [ + "zip", + "-q", # Quiet + "-x", # Exclude files + "**/__pycache__/*", # Files to be excluded + "-r", # Recurse paths + self.out_zip_filename, # Output filename + PYTHON_SITE_PACKAGES, # Files to be zipped + ], + cwd=self.base_dir, + check=True, # Raises CalledProcessError if exit status is non-zero + ) -def build_layer_dir(): + shutil.copy( + os.path.join(self.base_dir, self.out_zip_filename), + os.path.abspath(DIST_PATH) + ) + +def build_packaged_zip(): with tempfile.TemporaryDirectory() as base_dir: layer_builder = LayerBuilder(base_dir) layer_builder.make_directories() layer_builder.install_python_packages() layer_builder.create_init_serverless_sdk_package() - - shutil.copytree(base_dir, "dist-serverless") + layer_builder.zip() if __name__ == "__main__": - build_layer_dir() + build_packaged_zip() diff --git a/scripts/init_serverless_sdk.py b/scripts/init_serverless_sdk.py index 05dd8c767a..e2c9f536f8 100644 --- a/scripts/init_serverless_sdk.py +++ b/scripts/init_serverless_sdk.py @@ -18,17 +18,9 @@ from typing import Any -def extension_relay_dsn(original_dsn): - dsn = Dsn(original_dsn) - dsn.host = "localhost" - dsn.port = 5333 - dsn.scheme = "http" - return str(dsn) - - # Configure Sentry SDK sentry_sdk.init( - dsn=extension_relay_dsn(os.environ["SENTRY_DSN"]), + dsn=os.environ["SENTRY_DSN"], integrations=[AwsLambdaIntegration(timeout_warning=True)], traces_sample_rate=float(os.environ["SENTRY_TRACES_SAMPLE_RATE"]), ) From 157c59ff0ba03d1674b6cfd8f00a74823e98f54c Mon Sep 17 00:00:00 2001 From: Neel Shah Date: Wed, 10 May 2023 13:25:57 +0200 Subject: [PATCH 3/3] Trigger gh