Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(serverless): Python Serverless nocode instrumentation #1004

Merged
merged 29 commits into from
Feb 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
32a97c8
Moved logic from aws_lambda.py to aws_lambda.__init__
ahmedetefy Feb 10, 2021
913be92
Added init function that revokes original handler
ahmedetefy Feb 10, 2021
77d57e8
Added documentation
ahmedetefy Feb 10, 2021
1c03288
fix: Formatting
Feb 10, 2021
d0e21f8
Added test definition for serverless no code instrumentation
ahmedetefy Feb 11, 2021
e31ef7c
TODO comments
ahmedetefy Feb 11, 2021
6a6c669
Merge branch 'master' of github.com:getsentry/sentry-python into feat…
ahmedetefy Feb 11, 2021
80ce8eb
Merge branch 'master' of github.com:getsentry/sentry-python into feat…
ahmedetefy Feb 11, 2021
092a182
Refactored AWSLambda Layer script and fixed missing dir bug
ahmedetefy Feb 12, 2021
4a74db6
Removed redunant line
ahmedetefy Feb 12, 2021
90b5a2d
Organized import
ahmedetefy Feb 12, 2021
ae39bd2
Moved build-aws-layer script to integrations/aws_lambda
ahmedetefy Feb 12, 2021
6c69ddb
Added check if path fails
ahmedetefy Feb 12, 2021
66d9495
Renamed script to have underscore rather than dashes
ahmedetefy Feb 12, 2021
37dd471
Fixed naming change for calling script
ahmedetefy Feb 12, 2021
20181ff
Tests to ensure lambda check does not fail existing tests
ahmedetefy Feb 12, 2021
a38e022
Merging changes in master
ahmedetefy Feb 15, 2021
d0cfa8e
Added dest abs path as an arg
ahmedetefy Feb 15, 2021
0a5702e
Testing init script
ahmedetefy Feb 15, 2021
2a0128a
Modifying tests to accomodate addtion of layer
ahmedetefy Feb 15, 2021
f9cecdc
Added test that ensures serverless auto instrumentation works as expe…
ahmedetefy Feb 15, 2021
9db56e0
Removed redundant test arg from sentry_sdk init in serverless init
ahmedetefy Feb 15, 2021
9b8ebc2
Removed redundant todo statement
ahmedetefy Feb 15, 2021
6ce581e
Refactored layer and function creation into its own function
ahmedetefy Feb 15, 2021
18d2124
Linting fixes
ahmedetefy Feb 15, 2021
9de62fd
Linting fixes
ahmedetefy Feb 15, 2021
cf0738e
Moved scripts from within sdk to scripts dir
ahmedetefy Feb 17, 2021
77a9f08
Updated documentation
ahmedetefy Feb 17, 2021
383951c
Pinned dependency to fix CI issue
ahmedetefy Feb 17, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -63,5 +63,5 @@ apidocs-hotfix: apidocs
aws-lambda-layer-build: dist
$(VENV_PATH)/bin/pip install urllib3
$(VENV_PATH)/bin/pip install certifi
$(VENV_PATH)/bin/python -m scripts.build-awslambda-layer
$(VENV_PATH)/bin/python -m scripts.build_awslambda_layer
.PHONY: aws-lambda-layer-build
77 changes: 0 additions & 77 deletions scripts/build-awslambda-layer.py

This file was deleted.

115 changes: 115 additions & 0 deletions scripts/build_awslambda_layer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import os
import subprocess
import tempfile
import shutil

from sentry_sdk.consts import VERSION as SDK_VERSION
from sentry_sdk._types import MYPY

if MYPY:
from typing import Union


class PackageBuilder:
def __init__(
self,
base_dir, # type: str
pkg_parent_dir, # type: str
dist_rel_path, # type: str
):
# type: (...) -> None
self.base_dir = base_dir
self.pkg_parent_dir = pkg_parent_dir
self.dist_rel_path = dist_rel_path
self.packages_dir = self.get_relative_path_of(pkg_parent_dir)

def make_directories(self):
# type: (...) -> None
os.makedirs(self.packages_dir)

def install_python_binaries(self):
# type: (...) -> None
wheels_filepath = os.path.join(
self.dist_rel_path, f"sentry_sdk-{SDK_VERSION}-py2.py3-none-any.whl"
)
subprocess.run(
[
"pip",
"install",
"--no-cache-dir", # Disables the cache -> always accesses PyPI
"-q", # Quiet
wheels_filepath, # Copied to the target directory before installation
"-t", # Target directory flag
self.packages_dir,
],
check=True,
)

def create_init_serverless_sdk_package(self):
# type: (...) -> None
"""
Method that creates the init_serverless_sdk pkg in the
sentry-python-serverless zip
"""
serverless_sdk_path = f'{self.packages_dir}/sentry_sdk/' \
f'integrations/init_serverless_sdk'
if not os.path.exists(serverless_sdk_path):
os.makedirs(serverless_sdk_path)
shutil.copy('scripts/init_serverless_sdk.py',
f'{serverless_sdk_path}/__init__.py')

def zip(
self, filename # type: str
):
# type: (...) -> None
subprocess.run(
[
"zip",
"-q", # Quiet
"-x", # Exclude files
"**/__pycache__/*", # Files to be excluded
"-r", # Recurse paths
filename, # Output filename
self.pkg_parent_dir, # Files to be zipped
],
cwd=self.base_dir,
check=True, # Raises CalledProcessError if exit status is non-zero
)

def get_relative_path_of(
self, subfile # type: str
):
# type: (...) -> str
return os.path.join(self.base_dir, subfile)


# Ref to `pkg_parent_dir` Top directory in the ZIP file.
# Placing the Sentry package in `/python` avoids
# creating a directory for a specific version. For more information, see
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
def build_packaged_zip(
dist_rel_path="dist", # type: str
dest_zip_filename=f"sentry-python-serverless-{SDK_VERSION}.zip", # type: str
pkg_parent_dir="python", # type: str
dest_abs_path=None, # type: Union[str, None]
):
# type: (...) -> None
if dest_abs_path is None:
dest_abs_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", dist_rel_path)
)
with tempfile.TemporaryDirectory() as tmp_dir:
package_builder = PackageBuilder(tmp_dir, pkg_parent_dir, dist_rel_path)
package_builder.make_directories()
package_builder.install_python_binaries()
package_builder.create_init_serverless_sdk_package()
package_builder.zip(dest_zip_filename)
if not os.path.exists(dist_rel_path):
os.makedirs(dist_rel_path)
shutil.copy(
package_builder.get_relative_path_of(dest_zip_filename), dest_abs_path
)


if __name__ == "__main__":
build_packaged_zip()
37 changes: 37 additions & 0 deletions scripts/init_serverless_sdk.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""
For manual instrumentation,
The Handler function string of an aws lambda function should be added as an
environment variable with a key of 'INITIAL_HANDLER' along with the 'DSN'
Then the Handler function sstring should be replaced with
'sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler'
"""
import os

import sentry_sdk
from sentry_sdk._types import MYPY
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration

if MYPY:
from typing import Any


# Configure Sentry SDK
sentry_sdk.init(
dsn=os.environ["DSN"],
integrations=[AwsLambdaIntegration(timeout_warning=True)],
)


def sentry_lambda_handler(event, context):
# type: (Any, Any) -> None
"""
Handler function that invokes a lambda handler which path is defined in
environment vairables as "INITIAL_HANDLER"
"""
try:
module_name, handler_name = os.environ["INITIAL_HANDLER"].rsplit(".", 1)
except ValueError:
raise ValueError("Incorrect AWS Handler path (Not a path)")
lambda_function = __import__(module_name)
lambda_handler = getattr(lambda_function, handler_name)
lambda_handler(event, context)
111 changes: 83 additions & 28 deletions tests/integrations/aws_lambda/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,46 @@ def get_boto_client():
)


def build_no_code_serverless_function_and_layer(
client, tmpdir, fn_name, runtime, timeout
):
"""
Util function that auto instruments the no code implementation of the python
sdk by creating a layer containing the Python-sdk, and then creating a func
that uses that layer
"""
from scripts.build_awslambda_layer import (
build_packaged_zip,
)

build_packaged_zip(dest_abs_path=tmpdir, dest_zip_filename="serverless-ball.zip")

with open(os.path.join(tmpdir, "serverless-ball.zip"), "rb") as serverless_zip:
response = client.publish_layer_version(
LayerName="python-serverless-sdk-test",
Description="Created as part of testsuite for getsentry/sentry-python",
Content={"ZipFile": serverless_zip.read()},
)

with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
client.create_function(
FunctionName=fn_name,
Runtime=runtime,
Timeout=timeout,
Environment={
"Variables": {
"INITIAL_HANDLER": "test_lambda.test_handler",
"DSN": "https://123abc@example.com/123",
}
},
Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
Handler="sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler",
Layers=[response["LayerVersionArn"]],
Code={"ZipFile": zip.read()},
Description="Created as part of testsuite for getsentry/sentry-python",
)


def run_lambda_function(
client,
runtime,
Expand All @@ -25,6 +65,7 @@ def run_lambda_function(
add_finalizer,
syntax_check=True,
timeout=30,
layer=None,
subprocess_kwargs=(),
):
subprocess_kwargs = dict(subprocess_kwargs)
Expand All @@ -40,39 +81,53 @@ def run_lambda_function(
# such as chalice's)
subprocess.check_call([sys.executable, test_lambda_py])

setup_cfg = os.path.join(tmpdir, "setup.cfg")
with open(setup_cfg, "w") as f:
f.write("[install]\nprefix=")
fn_name = "test_function_{}".format(uuid.uuid4())

subprocess.check_call(
[sys.executable, "setup.py", "sdist", "-d", os.path.join(tmpdir, "..")],
**subprocess_kwargs
)
if layer is None:
setup_cfg = os.path.join(tmpdir, "setup.cfg")
with open(setup_cfg, "w") as f:
f.write("[install]\nprefix=")

subprocess.check_call(
"pip install mock==3.0.0 funcsigs -t .",
cwd=tmpdir,
shell=True,
**subprocess_kwargs
)
subprocess.check_call(
[sys.executable, "setup.py", "sdist", "-d", os.path.join(tmpdir, "..")],
**subprocess_kwargs
)

# https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
subprocess.check_call(
"pip install ../*.tar.gz -t .", cwd=tmpdir, shell=True, **subprocess_kwargs
)
shutil.make_archive(os.path.join(tmpdir, "ball"), "zip", tmpdir)
subprocess.check_call(
"pip install mock==3.0.0 funcsigs -t .",
cwd=tmpdir,
shell=True,
**subprocess_kwargs
)

fn_name = "test_function_{}".format(uuid.uuid4())
# https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
subprocess.check_call(
"pip install ../*.tar.gz -t .",
cwd=tmpdir,
shell=True,
**subprocess_kwargs
)

with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
client.create_function(
FunctionName=fn_name,
Runtime=runtime,
Timeout=timeout,
Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
Handler="test_lambda.test_handler",
Code={"ZipFile": zip.read()},
Description="Created as part of testsuite for getsentry/sentry-python",
shutil.make_archive(os.path.join(tmpdir, "ball"), "zip", tmpdir)

with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
client.create_function(
FunctionName=fn_name,
Runtime=runtime,
Timeout=timeout,
Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
Handler="test_lambda.test_handler",
Code={"ZipFile": zip.read()},
Description="Created as part of testsuite for getsentry/sentry-python",
)
else:
subprocess.run(
["zip", "-q", "-x", "**/__pycache__/*", "-r", "ball.zip", "./"],
cwd=tmpdir,
check=True,
)
build_no_code_serverless_function_and_layer(
client, tmpdir, fn_name, runtime, timeout
)

@add_finalizer
Expand Down
Loading