Merge main into mlops

.devcontainer/Dockerfile

@@ -1,4 +1,14 @@
-ARG IMAGE=ghcr.io/newrelic-experimental/pyenv-devcontainer:latest
-
 # To target other architectures, change the --platform directive in the Dockerfile.
-FROM --platform=linux/amd64 ${IMAGE}
+ARG IMAGE_TAG=latest
+FROM ghcr.io/newrelic/newrelic-python-agent-ci:${IMAGE_TAG}
+
+# Setup non-root user
+USER root
+ARG UID=1000
+ARG GID=$UID
+ENV HOME /home/vscode
+RUN mkdir -p ${HOME} && \
+    groupadd --gid ${GID} vscode && \
+    useradd --uid ${UID} --gid ${GID} --home ${HOME} vscode && \
+    chown -R ${UID}:${GID} /home/vscode
+USER ${UID}:${GID}

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
 		// To target other architectures, change the --platform directive in the Dockerfile.
 		"dockerfile": "Dockerfile",
 		"args": {
-			"IMAGE": "ghcr.io/newrelic-experimental/pyenv-devcontainer:latest"
+			"IMAGE_TAG": "latest"
 		}
 	},
 	"remoteUser": "vscode",

.github/containers/Dockerfile

@@ -23,6 +23,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         build-essential \
         curl \
         expat \
+        fish \
+        fontconfig \
         freetds-common \
         freetds-dev \
         gcc \
@@ -46,13 +48,16 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         python2-dev \
         python3-dev \
         python3-pip \
+        sudo \
         tzdata \
         unixodbc-dev \
         unzip \
+        vim \
         wget \
         zip \
         zlib1g \
-        zlib1g-dev && \
+        zlib1g-dev \
+        zsh && \
     rm -rf /var/lib/apt/lists/*
 
 # Build librdkafka from source
@@ -93,6 +98,10 @@ RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \
 # Install Python
 ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.12 pypy3.8-7.3.11"
 COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh
-COPY ./requirements.txt /requirements.txt
 RUN /tmp/install-python.sh && \
     rm /tmp/install-python.sh
+
+# Install dependencies for main python installation
+COPY ./requirements.txt /tmp/requirements.txt
+RUN pyenv exec pip install --upgrade -r /tmp/requirements.txt && \
+    rm /tmp/requirements.txt

.github/containers/Makefile

@@ -19,16 +19,16 @@ REPO_ROOT:=$(realpath $(MAKEFILE_DIR)../../)
 .PHONY: default
 default: test
 
+# Perform a shortened build for testing
 .PHONY: build
 build:
-	@# Perform a shortened build for testing
 	@docker build $(MAKEFILE_DIR) \
 		-t ghcr.io/newrelic/newrelic-python-agent-ci:local \
 		--build-arg='PYTHON_VERSIONS=3.10 2.7'
 
+# Ensure python versions are usable
 .PHONY: test
 test: build
-	@# Ensure python versions are usable
 	@docker run --rm ghcr.io/newrelic/python-agent-ci:local /bin/bash -c '\
 		python3.10 --version && \
 		python2.7 --version && \

.github/containers/requirements.txt

@@ -1,5 +1,9 @@
+bandit
+black
+flake8
+isort
 pip
 setuptools
-wheel
+tox
 virtualenv<20.22.0
-tox
+wheel

.github/stale.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
+daysUntilStale: 365
 # Number of days of inactivity before a stale issue is closed
 # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
 daysUntilClose: false

.github/workflows/build-ci-image.yml

@@ -63,6 +63,6 @@ jobs:
         with:
           push: ${{ github.event_name != 'pull_request' }}
           context: .github/containers
-          platforms: linux/amd64
+          platforms: ${{ (github.ref == 'refs/head/main') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yml

@@ -46,6 +46,7 @@ jobs:
       - postgres
       - rabbitmq
       - redis
+      - rediscluster
       - solr
 
     steps:
@@ -384,6 +385,105 @@ jobs:
           path: ./**/.coverage.*
           retention-days: 1
 
+  rediscluster:
+    env:
+      TOTAL_GROUPS: 1
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group-number: [1]
+
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
+      options: >-
+        --add-host=host.docker.internal:host-gateway
+    timeout-minutes: 30
+
+    services:
+      redis1:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6379:6379
+          - 16379:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      redis2:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6380:6379
+          - 16380:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      redis3:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6381:6379
+          - 16381:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      redis4:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6382:6379
+          - 16382:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      redis5:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6383:6379
+          - 16383:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      redis6:
+        image: hmstepanek/redis-cluster-node:1.0.0
+        ports:
+          - 6384:6379
+          - 16384:16379
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+      cluster-setup:
+        image: hmstepanek/redis-cluster:1.0.0    
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Fetch git tags
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch --tags origin
+
+      - name: Get Environments
+        id: get-envs
+        run: |
+          echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
+        env:
+          GROUP_NUMBER: ${{ matrix.group-number }}
+
+      - name: Test
+        run: |
+          tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
+        env:
+          TOX_PARALLEL_NO_SPINNER: 1
+          PY_COLORS: 0
+
+      - name: Upload Coverage Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ github.job }}-${{ strategy.job-index }}
+          path: ./**/.coverage.*
+          retention-days: 1
+
   redis:
     env:
       TOTAL_GROUPS: 2
@@ -924,3 +1024,68 @@ jobs:
           name: coverage-${{ github.job }}-${{ strategy.job-index }}
           path: ./**/.coverage.*
           retention-days: 1
+
+  firestore:
+    env:
+      TOTAL_GROUPS: 1
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group-number: [1]
+
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
+      options: >-
+        --add-host=host.docker.internal:host-gateway
+    timeout-minutes: 30
+
+    services:
+      firestore:
+        # Image set here MUST be repeated down below in options. See comment below.
+        image: gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators
+        ports:
+          - 8080:8080
+        # Set health checks to wait 5 seconds in lieu of an actual healthcheck
+        options: >-
+          --health-cmd "echo success"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 5s
+          gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators /bin/bash -c "gcloud emulators firestore start --host-port=0.0.0.0:8080" ||
+        # This is a very hacky solution. GitHub Actions doesn't provide APIs for setting commands on services, but allows adding arbitrary options.
+        # --entrypoint won't work as it only accepts an executable and not the [] syntax.
+        # Instead, we specify the image again the command afterwards like a call to docker create. The result is a few environment variables
+        # and the original command being appended to our hijacked docker create command. We can avoid any issues by adding || to prevent that
+        # from every being executed as bash commands.
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Fetch git tags
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch --tags origin
+
+      - name: Get Environments
+        id: get-envs
+        run: |
+          echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
+        env:
+          GROUP_NUMBER: ${{ matrix.group-number }}
+
+      - name: Test
+        run: |
+          tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
+        env:
+          TOX_PARALLEL_NO_SPINNER: 1
+          PY_COLORS: 0
+
+      - name: Upload Coverage Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ github.job }}-${{ strategy.job-index }}
+          path: ./**/.coverage.*
+          retention-days: 1

newrelic/api/database_trace.py

@@ -16,7 +16,7 @@
 import logging
 
 from newrelic.api.time_trace import TimeTrace, current_trace
-from newrelic.common.async_wrapper import async_wrapper
+from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper
 from newrelic.common.object_wrapper import FunctionWrapper, wrap_object
 from newrelic.core.database_node import DatabaseNode
 from newrelic.core.stack_trace import current_stack
@@ -44,11 +44,6 @@ def register_database_client(
     dbapi2_module._nr_explain_query = explain_query
     dbapi2_module._nr_explain_stmts = explain_stmts
     dbapi2_module._nr_instance_info = instance_info
-    dbapi2_module._nr_datastore_instance_feature_flag = False
-
-
-def enable_datastore_instance_feature(dbapi2_module):
-    dbapi2_module._nr_datastore_instance_feature_flag = True
 
 
 class DatabaseTrace(TimeTrace):
@@ -153,12 +148,7 @@ def finalize_data(self, transaction, exc=None, value=None, tb=None):
 
         if instance_enabled or db_name_enabled:
 
-            if (
-                self.dbapi2_module
-                and self.connect_params
-                and self.dbapi2_module._nr_datastore_instance_feature_flag
-                and self.dbapi2_module._nr_instance_info is not None
-            ):
+            if self.dbapi2_module and self.connect_params and self.dbapi2_module._nr_instance_info is not None:
 
                 instance_info = self.dbapi2_module._nr_instance_info(*self.connect_params)
 
@@ -244,9 +234,9 @@ def create_node(self):
         )
 
 
-def DatabaseTraceWrapper(wrapped, sql, dbapi2_module=None):
+def DatabaseTraceWrapper(wrapped, sql, dbapi2_module=None, async_wrapper=None):
     def _nr_database_trace_wrapper_(wrapped, instance, args, kwargs):
-        wrapper = async_wrapper(wrapped)
+        wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped)
         if not wrapper:
             parent = current_trace()
             if not parent:
@@ -273,9 +263,9 @@ def _nr_database_trace_wrapper_(wrapped, instance, args, kwargs):
     return FunctionWrapper(wrapped, _nr_database_trace_wrapper_)
 
 
-def database_trace(sql, dbapi2_module=None):
-    return functools.partial(DatabaseTraceWrapper, sql=sql, dbapi2_module=dbapi2_module)
+def database_trace(sql, dbapi2_module=None, async_wrapper=None):
+    return functools.partial(DatabaseTraceWrapper, sql=sql, dbapi2_module=dbapi2_module, async_wrapper=async_wrapper)
 
 
-def wrap_database_trace(module, object_path, sql, dbapi2_module=None):
-    wrap_object(module, object_path, DatabaseTraceWrapper, (sql, dbapi2_module))
+def wrap_database_trace(module, object_path, sql, dbapi2_module=None, async_wrapper=None):
+    wrap_object(module, object_path, DatabaseTraceWrapper, (sql, dbapi2_module, async_wrapper))