diff --git a/airbyte-integrations/connectors/source-shortio/Dockerfile b/airbyte-integrations/connectors/source-shortio/Dockerfile index 33aa353896c3..9650d6ff1014 100644 --- a/airbyte-integrations/connectors/source-shortio/Dockerfile +++ b/airbyte-integrations/connectors/source-shortio/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_shortio ./source_shortio + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_shortio ./source_shortio ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-shortio diff --git a/airbyte-integrations/connectors/source-shortio/README.md b/airbyte-integrations/connectors/source-shortio/README.md index 51a9f9902398..3e8a6bdb3870 100644 --- a/airbyte-integrations/connectors/source-shortio/README.md +++ b/airbyte-integrations/connectors/source-shortio/README.md @@ -1,34 +1,10 @@ # Shortio Source -This is the repository for the Shortio source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/shortio). +This is the repository for the Shortio configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/shortio). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shortio) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shortio/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/shortio) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shortio/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source shortio test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shortio:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shortio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. @@ -129,7 +80,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - -## Notes specific to the connector - -1. The links stream output doesn't match exactly what the documentation in the official website say (e.g. an owner object is returned as part of the response but that isn't listed there.) diff --git a/airbyte-integrations/connectors/source-shortio/__init__.py b/airbyte-integrations/connectors/source-shortio/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml index 0e03f5d44cbb..3314e7f6cb11 100644 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml @@ -1,25 +1,39 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-shortio:dev -tests: +acceptance_tests: spec: - - spec_path: "source_shortio/spec.json" + tests: + - spec_path: "source_shortio/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["clicks"] - # TODO: uncomment when any of incremental streams has records - # incremental: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: clicks + bypass_reason: "Sandbox account cannot seed the stream" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + # bypass_reason: "This connector does not implement incremental sync" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py b/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json index b5425313f16e..0771b31e498b 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json @@ -1,5 +1,16 @@ -{ - "clicks": { - "dt": "2052-07-17 14:03:43.449925" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2099-07-31T03:43:59.244Z" }, + "stream_descriptor": { "name": "links" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "dt": "2099-09-10T12:44:55.000Z" }, + "stream_descriptor": { "name": "clicks" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py index 82823254d266..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py @@ -10,5 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json index 025b5475ee2f..483dfc373454 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json @@ -1,25 +1,24 @@ { "streams": [ { - "destination_sync_mode": "append", - "sync_mode": "incremental", "stream": { "name": "clicks", "source_defined_cursor": true, "default_cursor_field": ["dt"], "supported_sync_modes": ["incremental"], "json_schema": {} - } + }, + "destination_sync_mode": "append", + "sync_mode": "incremental" }, { - "destination_sync_mode": "append", - "sync_mode": "full_refresh", "stream": { "name": "links", - "source_defined_cursor": true, - "supported_sync_modes": ["full_refresh"], - "json_schema": {} - } + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..1b7a7d79ca97 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl @@ -0,0 +1,4 @@ +{"stream": "links", "data": {"lcpath": "gem9bt", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://jk-genesis.com.ua/promotions/kvartiry-rjadom-s-metro-shuljavskaja-ot-980-00-grn?utm_source=lun.ua&utm_medium=referral&utm_content=239563406.1628516963&utm_term=5105", "cloaking": false, "path": "geM9Bt", "idString": "lnk_ZhP_Tw4Ye", "shortURL": "https://1hsf.short.gy/geM9Bt", "secureShortURL": "https://1hsf.short.gy/geM9Bt", "id": "lnk_ZhP_Tw4Ye", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031178} +{"stream": "links", "data": {"lcpath": "4sfi0i", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://airbyte.io/connector-development-kit", "cloaking": false, "path": "4SfI0I", "idString": "lnk_ZhP_Tw4Y3", "shortURL": "https://1hsf.short.gy/4SfI0I", "secureShortURL": "https://1hsf.short.gy/4SfI0I", "id": "lnk_ZhP_Tw4Y3", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031183} +{"stream": "links", "data": {"lcpath": "saeipy", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://great.com.ua/ua/news/11/znizhki-do-10-u-zhitlovomu-kompleksi-great/?utm_source=lun.ua&utm_medium=referral&utm_content=239563406.1628516963&utm_term=6405", "cloaking": false, "path": "Saeipy", "idString": "lnk_ZhP_Tw4Y9", "shortURL": "https://1hsf.short.gy/Saeipy", "secureShortURL": "https://1hsf.short.gy/Saeipy", "id": "lnk_ZhP_Tw4Y9", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031187} +{"stream": "links", "data": {"lcpath": "48ne6k", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "http://www.redstar.ru/2005/03/10_03/1_02.html", "cloaking": false, "path": "48ne6k", "idString": "lnk_ZhP_Tw4Y4", "shortURL": "https://1hsf.short.gy/48ne6k", "secureShortURL": "https://1hsf.short.gy/48ne6k", "id": "lnk_ZhP_Tw4Y4", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031191} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json index 2017a4d76fa9..68bb7cb0e995 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { "secret_key": "RANDOMKEY", - "domain_id": "123456", - "start_date": "2021-07-01" + "domain_id": "99999999", + "start_date": "2099-07-01" } diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json new file mode 100644 index 000000000000..83e947a41857 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "secret_key": "KEY", + "domain_id": "123456", + "start_date": "2023-07-30T03:43:59.244Z" +} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_state.json similarity index 100% rename from airbyte-integrations/connectors/source-shortio/integration_tests/state.json rename to airbyte-integrations/connectors/source-shortio/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-shortio/metadata.yaml b/airbyte-integrations/connectors/source-shortio/metadata.yaml index db5299f4e0ef..6e831983879d 100644 --- a/airbyte-integrations/connectors/source-shortio/metadata.yaml +++ b/airbyte-integrations/connectors/source-shortio/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - https://api.short.io + - https://api-v2.short.cm + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 2fed2292-5586-480c-af92-9944e39fe12d - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-shortio githubIssueLabel: source-shortio - icon: short.svg + icon: shortio.svg license: MIT - name: Short.io - registries: - cloud: - enabled: true - oss: - enabled: true + name: Shortio + releaseDate: 2023-08-02 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/shortio tags: - - language:python - _ab_internal: - _sl: 100 - _ql: 200 - supportLevel: community + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-shortio/setup.py b/airbyte-integrations/connectors/source-shortio/setup.py index 4c3679af61e2..608b9feb7862 100644 --- a/airbyte-integrations/connectors/source-shortio/setup.py +++ b/airbyte-integrations/connectors/source-shortio/setup.py @@ -5,10 +5,13 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "pytest~=6.2.5", + "pytest~=6.2", + "pytest-mock~=3.6.1", "connector-acceptance-test", ] @@ -19,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py b/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py index 73f38b078831..8560123a8d78 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml b/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml new file mode 100644 index 000000000000..b0f7e60366c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml @@ -0,0 +1,106 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractor_path }}"] + + v1_api_requester: + type: HttpRequester + url_base: "https://api.short.io/api/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "Authorization" + api_token: "{{ config['secret_key'] }}" + request_parameters: + domain_id: "{{ config['domain_id'] }}" + + v2_api_requester: + type: HttpRequester + url_base: "https://api-v2.short.cm/statistics/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "Authorization" + api_token: "{{ config['secret_key'] }}" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['nextPageToken'] }}" + page_token_option: + type: "RequestPath" + field_name: "pageToken" + inject_into: "request_parameter" + + v1_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/v1_api_requester" + + v2_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/v2_api_requester" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "updatedAt" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + end_time_option: + field_name: "beforeDate" + inject_into: "request_parameter" + start_time_option: + field_name: "afterDate" + inject_into: "request_parameter" + + links_stream: + $ref: "#/definitions/v1_base_stream" + name: "links" + incremental_sync: + $ref: "#/definitions/incremental_base" + primary_key: "id" + $parameters: + extractor_path: "links" + path: "links" + + clicks_stream: + $ref: "#/definitions/v2_base_stream" + name: "clicks" + $parameters: + path: "domain/{{ config['domain_id'] }}/link_clicks" + +streams: + - "#/definitions/links_stream" + - "#/definitions/clicks_stream" + +check: + type: CheckStream + stream_names: + - "links" + - "clicks" diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json index 6f3eed302b60..f1d34afa1540 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "default_cursor_field": ["dt"], + "additionalProperties": true, "properties": { "host": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json index ae204f7aa53f..3c35d4ab4ace 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json @@ -1,7 +1,23 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "additionalProperties": true, "properties": { + "lcpath":{ + "type": ["null", "string"] + }, + "passwordContact": { + "type": ["null", "string"] + }, + "hasPassword": { + "type": ["null", "boolean"] + }, + "OwnerId": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, "path": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py index 384baccfa7e5..6fe21b6789db 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py @@ -2,228 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import contextlib -import datetime -import json -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -class BasicAuthenticator(TokenAuthenticator): - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self._token}"} - - -class Links(HttpStream, ABC): - - url_base = "https://api.short.io/api/" - limit = 150 - primary_key = "idString" - before_id = None - domain_id = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - - links = json.loads(response.text)["links"] - try: - earliest_id_string = sorted(links, key=lambda k: k["createdAt"], reverse=False)[0]["idString"] - if self.before_id != earliest_id_string: - self.before_id = earliest_id_string - return earliest_id_string - else: - return None - except IndexError: - return None - - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - return { - "limit": self.limit, - "domain_id": self.domain_id, - "before": next_page_token or None, - } - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - # Get all the links - return "links" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - The short.io API can be inconsistent in its inclusion of UTM parameters. - Here, we check if they've been provided and if they haven't, attempt to extract it from the original url. - """ - utm_response_fields_to_utm_params = { - # Passing secondary UTM Campaign in order to capture either or of the 2 args. - "utmSource": "utm_source", - "utmMedium": "utm_medium", - "utmCampaign": "utm_campaign", - "utmCampaignId": "utm_id", - "utmTerm": "utm_term", - "utmContent": "utm_content", - } - links = json.loads(response.text)["links"] - for item in links: - for resp_field, param in utm_response_fields_to_utm_params.items(): - if resp_field not in item.keys(): - param = f"{param}=" - original_url = item["originalURL"] - param_value = None - with contextlib.suppress(IndexError): - # Extracting parameter value from original URL - # i.e "talent" from http://airbyte.io/?utm_source=talent - param_value = original_url.split(param, 2)[1].split("&", 1)[0] - item[resp_field] = param_value - yield item - - -# Clicks stream -class Clicks(HttpStream, ABC): - """ - This stream attempts to return the list of raw clicks from shortio. - """ - - url_base = "https://api-v2.short.cm/statistics/domain/" - before_dt = datetime.datetime.now().__str__() - domain_id = None - start_date = None - - @property - def http_method(self) -> str: - return "POST" - - @property - def cursor_field(self) -> str: - """ - :return str: The name of the cursor field. - """ - return "dt" - - @property - def primary_key(self) -> Optional[Any]: - return None - - @property - def limit(self) -> int: - return 1000 - - state_checkpoint_interval = limit - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - # Get all the links - return f"{self.domain_id}/last_clicks" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - This function goes through the API responses and ensures that no more requests are left to take place - :return str: min(dt) object from the previous API response. - """ - clicks = json.loads(response.text) - try: - before_dt = sorted(clicks, key=lambda k: k["dt"], reverse=False)[0]["dt"] - return None if self.limit > len(clicks) else before_dt - except IndexError: - return None - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, any]: - """ - Here we keep track of the state between different syncs to ensure that the data fetched is correct. - When the object is created, the datetime is taken and records are fetched until that point. - Due to varying duration possibilities this allows to a reproducable set of results" - """ - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - if current_stream_state is not None and "dt" in current_stream_state: - return {"dt": self.before_dt} - else: - return {"dt": self.start_date} - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - """ - This method passes the arguments necessary to get the clicks from the shortio API. - No parameters have been implemented here at all with the exception of hardcoding human clicks only to come through. - Human clicks are hardcoded to reduce unnecessary clicks from coming through. Some resources from short.io: - https://help.short.io/en/articles/4065954-how-short-io-tracks-clicks - https://help.short.io/en/articles/4890644-what-are-the-redirects - - :return dict: json body for the request - """ - return { - "limit": self.limit, - "include": {"human": True}, - "beforeDate": next_page_token or self.before_dt, - "afterDate": stream_state["dt"] if stream_state and "dt" in stream_state.keys() else self.start_date, - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from json.loads(response.text) - - -# Source -class SourceShortio(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - CHeck whether configuration is correct. - - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - - url = "https://api.short.io/api/domains" - api_secret = config["secret_key"] - domain_id = int(config["domain_id"]) - headers = {"Accept": "application/json", "Authorization": api_secret} - - response = requests.request("GET", url, headers=headers) - response.raise_for_status() - for domain in response.json(): - if domain_id == domain["id"]: - return True, None - except Exception as e: - return False, e - - return False, "Domain not found" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - key = config["secret_key"] - auth = BasicAuthenticator(token=key, auth_method=None) - links = Links(authenticator=auth) - links.domain_id = config["domain_id"] - clicks = Clicks(authenticator=auth) - clicks.domain_id = config["domain_id"] - clicks.start_date = config["start_date"] - return [clicks, links] +# Declarative Source +class SourceShortio(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json deleted file mode 100644 index 27e39c4a96ef..000000000000 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "documentationUrl": "https://developers.short.io/reference", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shortio Spec", - "type": "object", - "required": ["domain_id", "secret_key", "start_date"], - "properties": { - "domain_id": { - "type": "string", - "desciprtion": "Short.io Domain ID", - "title": "Domain ID", - "airbyte_secret": false - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Short.io Secret Key", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "airbyte_secret": false - } - } - } -} diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml new file mode 100644 index 000000000000..6ec62eff50cb --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml @@ -0,0 +1,29 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/shortio/ +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Shortio Spec + type: object + additionalProperties: true + required: + - domain_id + - secret_key + - start_date + properties: + domain_id: + type: string + desciprtion: Short.io Domain ID + title: Domain ID + airbyte_secret: false + secret_key: + type: string + title: Secret Key + description: Short.io Secret Key + airbyte_secret: true + start_date: + type: string + title: Start Date + description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data + before this date will not be replicated. + examples: + - '2023-07-30T03:43:59.244Z' + airbyte_secret: false diff --git a/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py deleted file mode 100644 index 9961dce6365d..000000000000 --- a/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import Status -from source_shortio.source import SourceShortio - - -@pytest.fixture -def config(): - return {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} - - -def test_source_shortio_client_wrong_credentials(): - source = SourceShortio() - result = source.check(logger=AirbyteLogger, config={"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"}) - assert result.status == Status.FAILED - - -def test_streams(): - source = SourceShortio() - config_mock = {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/docs/integrations/sources/shortio.md b/docs/integrations/sources/shortio.md index 4bf26cee51fe..ad2f692dfd12 100644 --- a/docs/integrations/sources/shortio.md +++ b/docs/integrations/sources/shortio.md @@ -39,10 +39,11 @@ This Source is capable of syncing the following Streams: ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2022-08-01 | [15066](https://github.com/airbytehq/airbyte/pull/15066) | Update primary key to `idString` | -| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-08-16 | [3787](https://github.com/airbytehq/airbyte/pull/5418) | Add Native Shortio Source Connector | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------- | +| 0.2.0 | 2023-08-02 | [28950](https://github.com/airbytehq/airbyte/pull/28950) | Migrate to Low-Code CDK | +| 0.1.3 | 2022-08-01 | [15066](https://github.com/airbytehq/airbyte/pull/15066) | Update primary key to `idString` | +| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-08-16 | [3787](https://github.com/airbytehq/airbyte/pull/5418) | Add Native Shortio Source Connector |