From ca6d7224c699ebf56d3d53990d64fb48b6fd49dc Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:10:49 +0200 Subject: [PATCH] :sparkles: Source Sentry: migrate to low code (#35755) --- .../connectors/source-sentry/.coveragerc | 2 +- .../connectors/source-sentry/README.md | 2 +- .../source-sentry/acceptance-test-config.yml | 6 +- .../configured_catalog_full_refresh.json | 31 ++ .../connectors/source-sentry/metadata.yaml | 2 +- .../connectors/source-sentry/poetry.lock | 41 ++- .../connectors/source-sentry/pyproject.toml | 2 +- .../source-sentry/source_sentry/manifest.yaml | 203 +++++++++----- .../source_sentry/schemas/project_detail.json | 38 +++ .../source_sentry/schemas/releases.json | 18 ++ .../source-sentry/source_sentry/source.py | 45 +-- .../source-sentry/source_sentry/streams.py | 265 ------------------ .../unit_tests/integration/config_builder.py | 17 ++ .../integration/test_events_stream.py | 53 ++++ .../integration/test_issues_stream.py | 54 ++++ .../http/response/events_full_refresh.json | 47 ++++ .../http/response/events_incremental.json | 50 ++++ .../http/response/issues_full_refresh.json | 42 +++ .../http/response/issues_incremental.json | 82 ++++++ .../source-sentry/unit_tests/test_source.py | 10 +- .../source-sentry/unit_tests/test_streams.py | 191 +++++-------- docs/integrations/sources/sentry.md | 1 + 22 files changed, 676 insertions(+), 526 deletions(-) create mode 100644 airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog_full_refresh.json delete mode 100644 airbyte-integrations/connectors/source-sentry/source_sentry/streams.py create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_full_refresh.json create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_incremental.json create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_full_refresh.json create mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_incremental.json diff --git a/airbyte-integrations/connectors/source-sentry/.coveragerc b/airbyte-integrations/connectors/source-sentry/.coveragerc index 748997278499..58f2b7bbbf83 100644 --- a/airbyte-integrations/connectors/source-sentry/.coveragerc +++ b/airbyte-integrations/connectors/source-sentry/.coveragerc @@ -1,3 +1,3 @@ [run] -omit = +omit = source_sentry/run.py diff --git a/airbyte-integrations/connectors/source-sentry/README.md b/airbyte-integrations/connectors/source-sentry/README.md index 5646c9f4be76..a5651e8ee8e6 100644 --- a/airbyte-integrations/connectors/source-sentry/README.md +++ b/airbyte-integrations/connectors/source-sentry/README.md @@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file. poetry run source-sentry spec poetry run source-sentry check --config secrets/config.json poetry run source-sentry discover --config secrets/config.json -poetry run source-sentry read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-sentry read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Running unit tests diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index e694c493e7c5..aa5b998b91cb 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -4,7 +4,7 @@ acceptance_tests: - config_path: secrets/config.json empty_streams: - name: issues - bypass_reason: "Project sssues are not being returned by the Sentry API." + bypass_reason: "Project issues are not being returned by the Sentry API." - name: events bypass_reason: "No event records exist for the test project." timeout_seconds: 1200 @@ -22,10 +22,10 @@ acceptance_tests: full_refresh: tests: - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json + configured_catalog_path: integration_tests/configured_catalog_full_refresh.json # test 403 exception is not breaking the sync - config_path: secrets/config_limited_scopes.json - configured_catalog_path: integration_tests/configured_catalog.json + configured_catalog_path: integration_tests/configured_catalog_full_refresh.json incremental: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..e319b768ed97 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog_full_refresh.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "stream": { + "name": "project_detail", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "stream": { + "name": "releases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index d6220aa98435..b198bc351431 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 - dockerImageTag: 0.4.2 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-sentry documentationUrl: https://docs.airbyte.com/integrations/sources/sentry githubIssueLabel: source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/poetry.lock b/airbyte-integrations/connectors/source-sentry/poetry.lock index a58864674c96..24d9697ea250 100644 --- a/airbyte-integrations/connectors/source-sentry/poetry.lock +++ b/airbyte-integrations/connectors/source-sentry/poetry.lock @@ -2,39 +2,38 @@ [[package]] name = "airbyte-cdk" -version = "0.74.0" +version = "0.77.1" description = "A framework for writing Airbyte Connectors." optional = false -python-versions = ">=3.9" +python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte-cdk-0.74.0.tar.gz", hash = "sha256:74241a055c205403a951383f43801067b7f451370e14d553d13d0cc476cbfff7"}, - {file = "airbyte_cdk-0.74.0-py3-none-any.whl", hash = "sha256:7e5b201d69ec0e7daab7e627dbc6add4dbba4a2f779132e86aaf6713650ff4d5"}, + {file = "airbyte_cdk-0.77.1-py3-none-any.whl", hash = "sha256:1530f4a5e44fc8a3e8f81132658222d9b89930385f7ecd9ef0a17a06cc16ea0b"}, + {file = "airbyte_cdk-0.77.1.tar.gz", hash = "sha256:5a4526c3e83cae8144170ec823093b51962c21db8038058e467574ad7574e6c5"}, ] [package.dependencies] airbyte-protocol-models = "0.5.1" backoff = "*" cachetools = "*" -Deprecated = ">=1.2,<2.0" +Deprecated = ">=1.2,<1.3" dpath = ">=2.0.1,<2.1.0" genson = "1.2.2" isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" -jsonref = ">=0.2,<1.0" +jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" -PyYAML = ">=6.0.1" +PyYAML = ">=6.0.1,<7.0.0" requests = "*" -requests-cache = "*" +requests_cache = "*" wcmatch = "8.4" [package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] @@ -366,13 +365,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jsonref" -version = "0.3.0" -description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +version = "0.2" +description = "An implementation of JSON Reference for Python" optional = false -python-versions = ">=3.3,<4.0" +python-versions = "*" files = [ - {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, - {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, ] [[package]] @@ -838,22 +837,20 @@ yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" -version = "1.11.0" +version = "1.12.0" description = "Mock out responses from the requests package" optional = false python-versions = "*" files = [ - {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, - {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, + {file = "requests-mock-1.12.0.tar.gz", hash = "sha256:4e34f2a2752f0b78397fb414526605d95fcdeab021ac1f26d18960e7eb41f6a8"}, + {file = "requests_mock-1.12.0-py2.py3-none-any.whl", hash = "sha256:4f6fdf956de568e0bac99eee4ad96b391c602e614cc0ad33e7f5c72edd699e70"}, ] [package.dependencies] -requests = ">=2.3,<3" -six = "*" +requests = ">=2.22,<3" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] [[package]] name = "setuptools" diff --git a/airbyte-integrations/connectors/source-sentry/pyproject.toml b/airbyte-integrations/connectors/source-sentry/pyproject.toml index b1adb0c46ac3..f54e9970c9be 100644 --- a/airbyte-integrations/connectors/source-sentry/pyproject.toml +++ b/airbyte-integrations/connectors/source-sentry/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.4.2" +version = "0.5.0" name = "source-sentry" description = "Source implementation for Sentry." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml index 4cd5508a78eb..2e49246fe98d 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml @@ -1,105 +1,174 @@ -version: "0.29.0" +version: 0.57.0 +type: DeclarativeSource + definitions: - page_size: 50 schema_loader: type: JsonFileSchemaLoader - file_path: "./source_sentry/schemas/{{ parameters.name }}.json" - selector: + file_path: "./source_sentry/schemas/{{ parameters['name'] }}.json" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['auth_token'] }}" + requester: + type: HttpRequester + url_base: "https://{{ config['hostname'] }}/api/0/" + http_method: GET + request_headers: {} + authenticator: + $ref: "#/definitions/authenticator" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + filter_record_selector: type: RecordSelector extractor: type: DpathExtractor field_path: [] - requester: - type: HttpRequester - url_base: "https://{{ config.hostname }}/api/0/" - http_method: "GET" - authenticator: - type: "BearerAuthenticator" - api_token: "{{ config.auth_token }}" paginator: type: DefaultPaginator - page_size: "#/definitions/page_size" - limit_option: - inject_into: "request_parameter" - field_name: "" page_token_option: type: RequestOption - inject_into: "request_parameter" - field_name: "cursor" + inject_into: request_parameter + field_name: cursor pagination_strategy: - type: "CursorPagination" - cursor_value: "{{ headers.link.next.cursor }}" - stop_condition: "{{ headers.link.next.results != 'true' }}" + type: CursorPagination + cursor_value: '{{ headers["link"].get("next", {}).get("cursor", {}) }}' + stop_condition: '{{ headers["link"]["next"]["results"] == "false" }}' retriever: type: SimpleRetriever - -streams: - - type: DeclarativeStream - $parameters: - # https://docs.sentry.io/api/events/list-a-projects-events/ - name: "events" - primary_key: "id" + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/record_selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: [] + retriever_with_filter: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/record_selector" + record_filter: + condition: "{{ record[parameters['cursor_field']] > stream_state.get(parameters['cursor_field'], '') }}" + paginator: + $ref: "#/definitions/paginator" + partition_router: [] + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "{{ parameters['cursor_field'] }}" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: "1900-01-01T00:00:00.0Z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + is_data_feed: true + base_stream_full_refresh: schema_loader: $ref: "#/definitions/schema_loader" retriever: $ref: "#/definitions/retriever" - record_selector: - $ref: "#/definitions/selector" + base_stream_incremental: + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_with_filter" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + # Stream Events https://docs.sentry.io/api/events/list-a-projects-error-events/ + events: + type: DeclarativeStream + $parameters: + name: "events" + primary_key: "id" + path: "projects/{{ config['organization'] }}/{{ config['project'] }}/events/" + cursor_field: "dateCreated" + retriever: + type: SimpleRetriever requester: $ref: "#/definitions/requester" - path: "projects/{{config.organization}}/{{config.project}}/events/" request_parameters: full: "true" + record_selector: + $ref: "#/definitions/record_selector" + record_filter: + condition: "{{ record[parameters['cursor_field']] > stream_state.get(parameters['cursor_field'], '') }}" paginator: $ref: "#/definitions/paginator" - - type: DeclarativeStream + incremental_sync: + $ref: "#/definitions/incremental_sync" + + # Stream Issues https://docs.sentry.io/api/events/list-a-projects-issues/ + issues: + type: DeclarativeStream $parameters: name: "issues" - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" + primary_key: "id" + path: "projects/{{ config['organization'] }}/{{ config['project'] }}/issues/" + cursor_field: "lastSeen" retriever: - $ref: "#/definitions/retriever" - record_selector: - $ref: "#/definitions/selector" + type: SimpleRetriever requester: $ref: "#/definitions/requester" - path: "projects/{{config.organization}}/{{config.project}}/issues/" request_parameters: - statsPeriod: "" - query: "" + query: "lastSeen:>{{ stream_state.get(parameters['cursor_field']) or '1900-01-01T00:00:00.0Z' if stream_state else '1900-01-01T00:00:00.0Z' }}" + record_selector: + $ref: "#/definitions/record_selector" paginator: $ref: "#/definitions/paginator" - - type: DeclarativeStream + incremental_sync: + $ref: "#/definitions/incremental_sync" + + # Stream Projects https://docs.sentry.io/api/projects/list-your-projects/ + projects: + type: DeclarativeStream + $ref: "#/definitions/base_stream_incremental" $parameters: name: "projects" - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" - retriever: - $ref: "#/definitions/retriever" - record_selector: - $ref: "#/definitions/selector" - requester: - $ref: "#/definitions/requester" - path: "projects/" - paginator: - $ref: "#/definitions/paginator" - - type: DeclarativeStream + primary_key: "id" + path: "projects/" + cursor_field: "dateCreated" + + # Stream Project Detail https://docs.sentry.io/api/projects/retrieve-a-project/ + project_detail: + type: DeclarativeStream + $ref: "#/definitions/base_stream_full_refresh" $parameters: name: "project_detail" - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" - retriever: - $ref: "#/definitions/retriever" - record_selector: - $ref: "#/definitions/selector" - requester: - $ref: "#/definitions/requester" - path: "projects/{{config.organization}}/{{config.project}}/" - paginator: - type: NoPagination + primary_key: "id" + path: "projects/{{ config['organization'] }}/{{ config['project'] }}/" + + # Stream Releases https://docs.sentry.io/api/projects/retrieve-a-project/ + releases: + type: DeclarativeStream + $ref: "#/definitions/base_stream_incremental" + $parameters: + name: "releases" + primary_key: "id" + path: "organizations/{{ config['organization'] }}/releases/" + cursor_field: "dateCreated" + +streams: + - $ref: "#/definitions/events" + - $ref: "#/definitions/issues" + - $ref: "#/definitions/projects" + - $ref: "#/definitions/project_detail" + - $ref: "#/definitions/releases" + check: type: CheckStream - stream_names: ["project_detail"] + stream_names: + - project_detail + +metadata: + autoImportSchema: + events: true + issues: true + projects: true + project_detail: true + releases: true diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json index 16e132f4bfff..2e413aa8dab2 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json @@ -174,6 +174,21 @@ }, "sentry:reprocessing_active": { "type": ["boolean", "null"] + }, + "filters:chunk-load-error": { + "type": ["boolean", "null"] + }, + "filters:react-hydration-errors": { + "type": ["boolean", "null"] + }, + "quotas:spike-protection-disabled": { + "type": ["boolean", "null"] + }, + "sentry:feedback_user_report_notification": { + "type": ["boolean", "null"] + }, + "sentry:replay_rage_click_issues": { + "type": ["integer", "null"] } } }, @@ -223,6 +238,29 @@ "type": ["string", "null"] } } + }, + "hasAuthProvider": { + "type": ["null", "boolean"] + }, + "features": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "links": { + "type": ["null", "object"], + "properties": { + "organizationUrl": { + "type": ["null", "string"] + }, + "regionUrl": { + "type": ["null", "string"] + } + } + }, + "requireEmailVerification": { + "type": ["null", "boolean"] } } }, diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json index 5ecc0f17f0ef..d657acba2ebb 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json @@ -67,6 +67,24 @@ }, "slug": { "type": ["null", "string"] + }, + "hasHealthData": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "newGroups": { + "type": ["null", "integer"] + }, + "platform": { + "type": ["null", "string"] + }, + "platforms": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py index 464f6188496a..5f86fd62dd08 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py @@ -2,46 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -from .streams import Events, Issues, ProjectDetail, Projects, Releases - - -# Source -class SourceSentry(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, Any]: - try: - stream = ProjectDetail( - authenticator=TokenAuthenticator(token=config["auth_token"]), - hostname=config.get("hostname"), - organization=config.get("organization"), - project=config.get("project"), - ) - next(stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - stream_args = { - "authenticator": TokenAuthenticator(token=config["auth_token"]), - "hostname": config.get("hostname"), - } - project_stream_args = { - **stream_args, - "organization": config["organization"], - "project": config["project"], - } - return [ - Events(**project_stream_args), - Issues(**project_stream_args), - ProjectDetail(**project_stream_args), - Projects(**stream_args), - Releases(**project_stream_args), - ] +class SourceSentry(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py deleted file mode 100644 index 1482228b362a..000000000000 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ /dev/null @@ -1,265 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.http import HttpStream - - -class SentryStream(HttpStream, ABC): - API_VERSION = "0" - URL_TEMPLATE = "https://{hostname}/api/{api_version}/" - primary_key = "id" - - def __init__(self, hostname: str, **kwargs): - super().__init__(**kwargs) - self._url_base = self.URL_TEMPLATE.format(hostname=hostname, api_version=self.API_VERSION) - # hardcode the start_date default value, since it's not present in spec. - self.start_date = "1900-01-01T00:00:00.0Z" - - @property - def url_base(self) -> str: - return self._url_base - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - return {} - - -class SentryStreamPagination(SentryStream): - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Expect the link header field to always contain the values ​​for `rel`, `results`, and `cursor`. - If there is actually the next page, rel="next"; results="true"; cursor="". - """ - if response.links["next"]["results"] == "true": - return {"cursor": response.links["next"]["cursor"]} - else: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - -class SentryIncremental(SentryStreamPagination, IncrementalMixin): - def __init__(self, *args, **kwargs): - super(SentryIncremental, self).__init__(*args, **kwargs) - self._cursor_value = None - - def validate_state_value(self, state_value: str = None) -> str: - none_or_empty = state_value == "None" if state_value else True - return self.start_date if none_or_empty else state_value - - def get_state_value(self, stream_state: Mapping[str, Any] = None) -> str: - state_value = self.validate_state_value(stream_state.get(self.cursor_field, self.start_date) if stream_state else self.start_date) - return pendulum.parse(state_value) - - def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: - """ - Endpoint does not provide query filtering params, but they provide us - cursor field in most cases, so we used that as incremental filtering - during the parsing. - """ - if pendulum.parse(record[self.cursor_field]) > self.get_state_value(stream_state): - # Persist state. - # There is a bug in state setter: because of self._cursor_value is not defined it raises Attribute error - # which is ignored in airbyte_cdk/sources/abstract_source.py:320 and we have an empty state in return - # See: https://github.com/airbytehq/oncall/issues/1317 - self.state = record - yield record - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[MutableMapping]: - json_response = response.json() or [] - - for record in json_response: - yield from self.filter_by_state(stream_state=stream_state, record=record) - - @property - def state(self) -> Mapping[str, Any]: - return {self.cursor_field: self._cursor_value} - - @state.setter - def state(self, value: Mapping[str, Any]): - """ - Define state as a max between given value and current state - """ - if not self._cursor_value: - self._cursor_value = value.get(self.cursor_field) - else: - current_value = value.get(self.cursor_field) or self.start_date - current_state = str(self.get_state_value(self.state)) - self._cursor_value = max(current_value, current_state) - - -class Events(SentryIncremental): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-error-events/ - """ - - primary_key = "id" - cursor_field = "dateCreated" - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/events/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"full": "true"}) - - return params - - -class Issues(SentryIncremental): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-issues/ - """ - - primary_key = "id" - cursor_field = "lastSeen" - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/issues/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - filter_date = self._get_filter_date(stream_state) - params.update(self._build_query_params(filter_date)) - return params - - def _get_filter_date(self, stream_state: Optional[Mapping[str, Any]]) -> str: - """Retrieve the filter date from the stream state or use the start_date.""" - return stream_state.get(self.cursor_field) or self.start_date if stream_state else self.start_date - - def _build_query_params(self, filter_date: str) -> Dict[str, str]: - """Generate query parameters for the request.""" - filter_date_iso = pendulum.parse(filter_date).to_iso8601_string() - return {"statsPeriod": "", "query": f"lastSeen:>{filter_date_iso}"} - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[MutableMapping]: - json_response = response.json() or [] - - for record in json_response: - cursor_value = self._get_cursor_value(record, stream_state) - self.state = {self.cursor_field: cursor_value} - yield record - - def _get_cursor_value(self, record: Dict[str, Any], stream_state: Mapping[str, Any]) -> pendulum.datetime: - """Compute the maximum cursor value based on the record and stream state.""" - record_time = record[self.cursor_field] - state_time = str(self.get_state_value(stream_state)) - return max(record_time, state_time) - - -class Projects(SentryIncremental): - """ - Docs: https://docs.sentry.io/api/projects/list-your-projects/ - """ - - primary_key = "id" - cursor_field = "dateCreated" - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return "projects/" - - -class ProjectDetail(SentryStream): - """ - Docs: https://docs.sentry.io/api/projects/retrieve-a-project/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() - - -class Releases(SentryIncremental): - """ - Docs: https://docs.sentry.io/api/releases/list-an-organizations-releases/ - """ - - primary_key = "id" - cursor_field = "dateCreated" - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"organizations/{self._organization}/releases/" diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py new file mode 100644 index 000000000000..0c5af692c5af --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "auth_token": "test token", + "organization": "test organization", + "project": "test project", + "hostname": "sentry.io" + } + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py new file mode 100644 index 000000000000..7e13aabbab22 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import SyncMode +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + + +class TestEvents(TestCase): + fr_read_file = "events_full_refresh" + inc_read_file = "events_incremental" + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="events", sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().build() + + def state(self): + return StateBuilder().with_stream_state(stream_name="events", state={"dateCreated": "2023-01-01T00:00:00.0Z"}).build() + + @HttpMocker() + def test_read(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", + query_params={"full": "true"} + ), + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) + + ) + output = read(SourceSentry(), self.config(), self.catalog()) + assert len(output.records) == 1 + + @HttpMocker() + def test_read_incremental(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", + query_params={"full": "true"} + ), + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) + + ) + output = read(SourceSentry(), self.config(), self.catalog(SyncMode.incremental), self.state()) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py new file mode 100644 index 000000000000..e9665a7854bb --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py @@ -0,0 +1,54 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import SyncMode +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + + +class TestEvents(TestCase): + fr_read_file = "issues_full_refresh" + inc_read_file = "issues_incremental" + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="issues", sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().build() + + def state(self): + return StateBuilder().with_stream_state(stream_name="issues", state={"lastSeen": "2023-01-01T00:00:00.0Z"}).build() + + @HttpMocker() + def test_read(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", + query_params={"query": "lastSeen:>1900-01-01T00:00:00.0Z"} + ), + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) + + ) + # https://sentry.io/api/1/projects/airbyte-09/airbyte-09/issues/?query=lastSeen%3A%3E2022-01-01T00%3A00%3A00.0Z + output = read(SourceSentry(), self.config(), self.catalog()) + assert len(output.records) == 1 + + @HttpMocker() + def test_read_incremental(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", + query_params={"query": "lastSeen:>2023-01-01T00:00:00.0Z"} + ), + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) + + ) + output = read(SourceSentry(), self.config(), self.catalog(SyncMode.incremental), self.state()) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_full_refresh.json b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_full_refresh.json new file mode 100644 index 000000000000..fe17f7bb7bb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_full_refresh.json @@ -0,0 +1,47 @@ +[ + { + "eventID": "9fac2ceed9344f2bbfdd1fdacb0ed9b1", + "tags": [ + { + "key": "browser", + "value": "Chrome 60.0" + }, + { + "key": "device", + "value": "Other" + }, + { + "key": "environment", + "value": "production" + }, + { + "value": "fatal", + "key": "level" + }, + { + "key": "os", + "value": "Mac OS X 10.12.6" + }, + { + "value": "CPython 2.7.16", + "key": "runtime" + }, + { + "key": "release", + "value": "17642328ead24b51867165985996d04b29310337" + }, + { + "key": "server_name", + "value": "web1.example.com" + } + ], + "dateCreated": "2022-09-02T15:01:28.946777Z", + "user": null, + "message": "", + "title": "This is an example Python exception", + "id": "dfb1a2d057194e76a4186cc8a5271553", + "platform": "python", + "event.type": "error", + "groupID": "1889724436" + } +] diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_incremental.json b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_incremental.json new file mode 100644 index 000000000000..8fed688dfb4d --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/events_incremental.json @@ -0,0 +1,50 @@ +[ + { + "eventID": "9fac2ceed9344f2bbfdd1fdacb0ed9b1", + "tags": [ + { "key": "browser", "value": "Chrome 60.0" }, + { + "key": "device", + "value": "Other" + }, + { "key": "environment", "value": "production" }, + { "value": "fatal", "key": "level" }, + { "key": "os", "value": "Mac OS X 10.12.6" }, + { "value": "CPython 2.7.16", "key": "runtime" }, + { "key": "release", "value": "17642328ead24b51867165985996d04b29310337" }, + { "key": "server_name", "value": "web1.example.com" } + ], + "dateCreated": "2023-02-01T00:00:00.0Z", + "user": null, + "message": "", + "title": "This is an example Python exception", + "id": "dfb1a2d057194e76a4186cc8a5271553", + "platform": "python", + "event.type": "error", + "groupID": "1889724436" + }, + { + "eventID": "9fac2ceed9344f2bbfdd1fdacb0ed9b1", + "tags": [ + { "key": "browser", "value": "Chrome 60.0" }, + { + "key": "device", + "value": "Other" + }, + { "key": "environment", "value": "production" }, + { "value": "fatal", "key": "level" }, + { "key": "os", "value": "Mac OS X 10.12.6" }, + { "value": "CPython 2.7.16", "key": "runtime" }, + { "key": "release", "value": "17642328ead24b51867165985996d04b29310337" }, + { "key": "server_name", "value": "web1.example.com" } + ], + "dateCreated": "2024-01-02T15:01:28.946777Z", + "user": null, + "message": "", + "title": "This is an example Python exception", + "id": "dfb1a2d057194e76a4186cc8a5271553", + "platform": "python", + "event.type": "error", + "groupID": "1889724436" + } +] diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_full_refresh.json b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_full_refresh.json new file mode 100644 index 000000000000..248792c45c5d --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_full_refresh.json @@ -0,0 +1,42 @@ +[ + { + "annotations": [], + "assignedTo": null, + "count": "1", + "culprit": "raven.scripts.runner in main", + "firstSeen": "2018-11-06T21:19:55Z", + "hasSeen": false, + "id": "1", + "isBookmarked": false, + "isPublic": false, + "isSubscribed": true, + "lastSeen": "2018-11-06T21:19:55Z", + "level": "error", + "logger": null, + "metadata": { + "title": "This is an example Python exception" + }, + "numComments": 0, + "permalink": "https://sentry.io/the-interstellar-jurisdiction/pump-station/issues/1/", + "project": { + "id": "2", + "name": "Pump Station", + "slug": "pump-station" + }, + "shareId": null, + "shortId": "PUMP-STATION-1", + "stats": { + "24h": [ + [1541455200, 473], + [1541458800, 914], + [1541462400, 991] + ] + }, + "status": "unresolved", + "statusDetails": {}, + "subscriptionDetails": null, + "title": "This is an example Python exception", + "type": "default", + "userCount": 0 + } +] diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_incremental.json b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_incremental.json new file mode 100644 index 000000000000..8a61b03d3572 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/resource/http/response/issues_incremental.json @@ -0,0 +1,82 @@ +[ + { + "annotations": [], + "assignedTo": null, + "count": "1", + "culprit": "raven.scripts.runner in main", + "firstSeen": "2018-11-06T21:19:55Z", + "hasSeen": false, + "id": "1", + "isBookmarked": false, + "isPublic": false, + "isSubscribed": true, + "lastSeen": "2023-02-02T00:00:00.0Z", + "level": "error", + "logger": null, + "metadata": { + "title": "This is an example Python exception" + }, + "numComments": 0, + "permalink": "https://sentry.io/the-interstellar-jurisdiction/pump-station/issues/1/", + "project": { + "id": "2", + "name": "Pump Station", + "slug": "pump-station" + }, + "shareId": null, + "shortId": "PUMP-STATION-1", + "stats": { + "24h": [ + [1541455200, 473], + [1541458800, 914], + [1541462400, 991] + ] + }, + "status": "unresolved", + "statusDetails": {}, + "subscriptionDetails": null, + "title": "This is an example Python exception", + "type": "default", + "userCount": 0 + }, + { + "annotations": [], + "assignedTo": null, + "count": "1", + "culprit": "raven.scripts.runner in main", + "firstSeen": "2018-11-06T21:19:55Z", + "hasSeen": false, + "id": "1", + "isBookmarked": false, + "isPublic": false, + "isSubscribed": true, + "lastSeen": "2023-01-02T00:00:00.0Z", + "level": "error", + "logger": null, + "metadata": { + "title": "This is an example Python exception" + }, + "numComments": 0, + "permalink": "https://sentry.io/the-interstellar-jurisdiction/pump-station/issues/1/", + "project": { + "id": "2", + "name": "Pump Station", + "slug": "pump-station" + }, + "shareId": null, + "shortId": "PUMP-STATION-1", + "stats": { + "24h": [ + [1541455200, 473], + [1541458800, 914], + [1541462400, 991] + ] + }, + "status": "unresolved", + "statusDetails": {}, + "subscriptionDetails": null, + "title": "This is an example Python exception", + "type": "default", + "userCount": 0 + } +] diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py index 385f2625e45e..c28cd169d0fa 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py @@ -6,7 +6,6 @@ from airbyte_cdk.logger import AirbyteLogger from source_sentry.source import SourceSentry -from source_sentry.streams import ProjectDetail def test_source_wrong_credentials(requests_mock): @@ -15,11 +14,12 @@ def test_source_wrong_credentials(requests_mock): assert not status -def test_check_connection(mocker): +def test_check_connection(requests_mock): source = SourceSentry() - logger_mock, config_mock = MagicMock(), MagicMock() - mocker.patch.object(ProjectDetail, "read_records", return_value=iter([{"id": "1", "name": "test"}])) - assert source.check_connection(logger_mock, config_mock) == (True, None) + logger_mock = MagicMock() + requests_mock.get(url="https://sentry.io/api/0/projects/test-org/test-project/", json={"id": "id", "name": "test-project"}) + config = {"auth_token": "token", "organization": "test-org", "project": "test-project", "hostname": "sentry.io"} + assert source.check_connection(logger_mock, config) == (True, None) def test_streams(mocker): diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 87376d158902..4f41688c2990 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -2,186 +2,139 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock -import pendulum as pdm import pytest -import requests -from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryIncremental, SentryStreamPagination +from airbyte_protocol.models import SyncMode +from source_sentry import SourceSentry INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(SentryStreamPagination, "path", "test_endpoint") - mocker.patch.object(SentryStreamPagination, "__abstractmethods__", set()) +def get_stream_by_name(stream_name): + streams = SourceSentry().streams(config=INIT_ARGS) + for stream in streams: + if stream.name == stream_name: + return stream + raise ValueError(f"Stream {stream_name} not found") -def test_next_page_token(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() - cursor = "next_page_num" - resp.links = {"next": {"results": "true", "cursor": cursor}} - inputs = {"response": resp} - expected_token = {"cursor": cursor} - assert stream.next_page_token(**inputs) == expected_token +def test_next_page_token(): + stream = get_stream_by_name("events") + response_mock = MagicMock() + response_mock.headers = {} + response_mock.links = {"next": {"cursor": "next-page"}} + assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_records=[]) == "next-page" -def test_next_page_token_is_none(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() - resp.links = {"next": {"results": "false", "cursor": "no_next"}} - inputs = {"response": resp} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def next_page_token_inputs(): - links_headers = [ - {}, - {"next": {}}, - ] - responses = [MagicMock() for _ in links_headers] - for mock, header in zip(responses, links_headers): - mock.links = header - - return responses - - -@pytest.mark.parametrize("response", next_page_token_inputs()) -def test_next_page_token_raises(patch_base_class, response): - stream = SentryStreamPagination(hostname="sentry.io") - inputs = {"response": response} - with pytest.raises(KeyError): - stream.next_page_token(**inputs) +def test_next_page_token_is_none(): + stream = get_stream_by_name("events") + response_mock = MagicMock() + response_mock.headers = {} + # stop condition: "results": "false" + response_mock.links = {"next": {"cursor": "", "results": "false"}} + assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_records=[]) is None def test_events_path(): - stream = Events(**INIT_ARGS) + stream = get_stream_by_name("events") expected = "projects/test-org/test-project/events/" - assert stream.path() == expected + assert stream.retriever.requester.get_path(stream_state=None, stream_slice=None, next_page_token=None) == expected def test_issues_path(): - stream = Issues(**INIT_ARGS) + stream = get_stream_by_name("issues") expected = "projects/test-org/test-project/issues/" - assert stream.path() == expected + assert stream.retriever.requester.get_path(stream_state=None, stream_slice=None, next_page_token=None) == expected def test_projects_path(): - stream = Projects(hostname="sentry.io") + stream = get_stream_by_name("projects") expected = "projects/" - assert stream.path() == expected + assert stream.retriever.requester.get_path(stream_state=None, stream_slice=None, next_page_token=None) == expected def test_project_detail_path(): - stream = ProjectDetail(**INIT_ARGS) + stream = get_stream_by_name("project_detail") expected = "projects/test-org/test-project/" - assert stream.path() == expected - - -def test_sentry_stream_pagination_request_params(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected + assert stream.retriever.requester.get_path(stream_state=None, stream_slice=None, next_page_token=None) == expected def test_events_request_params(): - stream = Events(**INIT_ARGS) - expected = {"cursor": "next-page", "full": "true"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected + stream = get_stream_by_name("events") + assert stream.retriever.requester.get_request_params(stream_state=None, stream_slice=None, next_page_token=None) == {"full": "true"} def test_issues_request_params(): - stream = Issues(**INIT_ARGS) - expected = {"cursor": "next-page", "statsPeriod": "", "query": "lastSeen:>1900-01-01T00:00:00Z"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected + stream = get_stream_by_name("issues") + expected = {"query": "lastSeen:>1900-01-01T00:00:00.0Z"} + assert stream.retriever.requester.get_request_params(stream_state=None, stream_slice=None, next_page_token=None) == expected def test_projects_request_params(): - stream = Projects(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected + stream = get_stream_by_name("projects") + expected = "next-page" + response_mock = MagicMock() + response_mock.headers = {} + response_mock.links = {"next": {"cursor": expected}} + assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_records=[]) == expected def test_project_detail_request_params(): - stream = ProjectDetail(**INIT_ARGS) + stream = get_stream_by_name("project_detail") expected = {} - assert stream.request_params(stream_state=None, next_page_token=None) == expected - -def test_issues_parse_response(mocker): - with patch('source_sentry.streams.Issues._get_cursor_value') as mock_get_cursor_value: - stream = Issues(**INIT_ARGS) - mock_get_cursor_value.return_value = "time" - state = {} - response = requests.Response() - mocker.patch.object(response, "json", return_value=[{"id": "1"}]) - result = list(stream.parse_response(response, state)) - assert result[0] == {"id": "1"} - -def test_project_detail_parse_response(mocker): - stream = ProjectDetail(organization="test_org", project="test_proj", hostname="sentry.io") - response = requests.Response() - response.json = Mock(return_value={"id": "1"}) - result = list(stream.parse_response(response)) - assert result[0] == {"id": "1"} - -class MockSentryIncremental(SentryIncremental): - def path(): - return '/test/path' - -def test_sentry_incremental_parse_response(mocker): - with patch('source_sentry.streams.SentryIncremental.filter_by_state') as mock_filter_by_state: - stream = MockSentryIncremental(hostname="sentry.io") - mock_filter_by_state.return_value = True - state = None - response = requests.Response() - mocker.patch.object(response, "json", return_value=[{"id": "1"}]) - mock_filter_by_state.return_value = iter(response.json()) - result = list(stream.parse_response(response, state)) - print(result) - assert result[0] == {"id": "1"} + assert stream.retriever.requester.get_request_params(stream_state=None, stream_slice=None, next_page_token=None) == expected + + +def test_project_detail_parse_response(requests_mock): + expected = {"id": "1", "name": "test project"} + stream = get_stream_by_name("project_detail") + requests_mock.get( + "https://sentry.io/api/0/projects/test-org/test-project/", + json=expected + ) + result = list(stream.read_records(sync_mode=SyncMode.full_refresh))[0] + assert expected == result.data @pytest.mark.parametrize( "state, expected", [ - ({}, "1900-01-01T00:00:00.0Z"), - ({"dateCreated": ""}, "1900-01-01T00:00:00.0Z"), - ({"dateCreated": "None"}, "1900-01-01T00:00:00.0Z"), + ({}, None), + ({"dateCreated": ""}, None), ({"dateCreated": "2023-01-01T00:00:00.0Z"}, "2023-01-01T00:00:00.0Z"), ], ids=[ "No State", "State is Empty String", - "State is 'None'", "State is present", ], ) -def test_validate_state_value(state, expected): - stream = Events(**INIT_ARGS) - state_value = state.get(stream.cursor_field) - assert stream.validate_state_value(state_value) == expected +def test_events_validate_state_value(state, expected): + # low code cdk sets state to none if it does not exist, py version used 1900-01-01 as state in this case. + # Instead, record condition will pass all records that were fetched and state will be updated after. + stream = get_stream_by_name("events") + stream.retriever.state = state + assert stream.state.get(stream.cursor_field) == expected @pytest.mark.parametrize( "state, expected", [ - ({}, "1900-01-01T00:00:00.0Z"), - ({"dateCreated": ""}, "1900-01-01T00:00:00.0Z"), - ({"dateCreated": "None"}, "1900-01-01T00:00:00.0Z"), - ({"dateCreated": "2023-01-01T00:00:00.0Z"}, "2023-01-01T00:00:00.0Z"), + ({}, None), + ({"lastSeen": ""}, None), + ({"lastSeen": "2023-01-01T00:00:00.0Z"}, "2023-01-01T00:00:00.0Z"), ], ids=[ "No State", "State is Empty String", - "State is 'None'", "State is present", ], ) -def test_get_state_value(state, expected): - stream = Events(**INIT_ARGS) - # we expect the datetime object out of get_state_value method. - assert stream.get_state_value(state) == pdm.parse(expected) +def test_issues_validate_state_value(state, expected): + # low code cdk sets state to none if it does not exist, py version used 1900-01-01 as state in this case. + # Instead, record condition will pass all records that were fetched and state will be updated after. + stream = get_stream_by_name("issues") + stream.retriever.state = state + assert stream.state.get(stream.cursor_field) == expected + diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index 993bd893daf6..5bc2728a1920 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -47,6 +47,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.5.0 | 2024-03-27 | [35755](https://github.com/airbytehq/airbyte/pull/35755) | Migrate to low-code. | | 0.4.2 | 2024-03-25 | [36448](https://github.com/airbytehq/airbyte/pull/36448) | Unpin CDK version | | 0.4.1 | 2024-02-12 | [35145](https://github.com/airbytehq/airbyte/pull/35145) | Manage dependencies with Poetry | | 0.4.0 | 2024-01-05 | [32957](https://github.com/airbytehq/airbyte/pull/32957) | Added undeclared fields to schema and migrated to base image |