Merge branch 'main' into edgarrmondragon/refactor/limit-usage-of-pend…

…ulum

.pre-commit-config.yaml

@@ -8,6 +8,10 @@ repos:
   rev: v4.5.0
   hooks:
   - id: check-json
+    exclude: |
+        (?x)^(
+            .*/launch.json
+        )$
   - id: check-toml
     exclude: |
         (?x)^(

cookiecutter/.gitignore

@@ -0,0 +1,2 @@
+# Include template VSCode directory
+!*/*/.vscode/

cookiecutter/tap-template/hooks/post_gen_project.py

@@ -28,4 +28,4 @@
         shutil.rmtree(".github")
 
     if "{{ cookiecutter.ide }}" != "VSCode":
-        shutil.rmtree(".vscode")
+        shutil.rmtree(".vscode", ignore_errors=True)

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml

@@ -8,6 +8,10 @@ repos:
   rev: v4.5.0
   hooks:
   - id: check-json
+    exclude: |
+      (?x)^(
+        \.vscode/.*\.json
+      )$
   - id: check-toml
   - id: check-yaml
   - id: end-of-file-fixer
@@ -20,7 +24,7 @@ repos:
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.11
+  rev: v0.1.14
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "{{ cookiecutter.tap_id }}",
+            "type": "python",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "program": "{{ cookiecutter.library_name }}",
+            "justMyCode": false,
+            "args": [
+                "--config",
+                ".secrets/config.json",
+            ],
+        },
+    ]
+}

docs/conf.py

@@ -117,7 +117,12 @@
     "css/custom.css",
 ]
 
+# -- Options for MyST --------------------------------------------------------
+# https://myst-parser.readthedocs.io/en/latest/configuration.html
 myst_heading_anchors = 3
+myst_enable_extensions = {
+    "colon_fence",
+}
 
 redirects = {
     "porting.html": "guides/porting.html",

docs/dev_guide.md

@@ -55,6 +55,10 @@ pipx install poetry
 pipx install tox
 ```
 
+:::{tip}
+The minimum recommended version of cookiecutter is `2.2.0` (released 2023-07-06).
+:::
+
 Now you can initialize your new project with the Cookiecutter template for taps:
 
 ```bash

pyproject.toml

@@ -187,6 +187,7 @@ exclude_also = [
     '''if (t\.)?TYPE_CHECKING:''',
 ]
 fail_under = 82
+show_missing = true
 
 [tool.mypy]
 exclude = "tests"

singer_sdk/_singerlib/messages.py

@@ -226,7 +226,12 @@ def format_message(message: Message) -> str:
     Returns:
         The formatted message.
     """
-    return json.dumps(message.to_dict(), use_decimal=True, default=_default_encoding)
+    return json.dumps(
+        message.to_dict(),
+        use_decimal=True,
+        default=_default_encoding,
+        separators=(",", ":"),
+    )
 
 
 def write_message(message: Message) -> None:

singer_sdk/about.py

@@ -9,6 +9,9 @@
 from collections import OrderedDict
 from textwrap import dedent
 
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
 if t.TYPE_CHECKING:
     from singer_sdk.helpers.capabilities import CapabilitiesEnum
 
@@ -19,6 +22,38 @@
     "MarkdownFormatter",
 ]
 
+# Keep these in sync with the supported Python versions in pyproject.toml
+_PY_MIN_VERSION = 8
+_PY_MAX_VERSION = 12
+
+
+def _get_min_version(specifiers: SpecifierSet) -> int:
+    min_version: list[int] = []
+    for specifier in specifiers:
+        if specifier.operator == ">=":
+            min_version.append(Version(specifier.version).minor)
+        if specifier.operator == ">":
+            min_version.append(Version(specifier.version).minor + 1)
+    return min(min_version, default=_PY_MIN_VERSION)
+
+
+def _get_max_version(specifiers: SpecifierSet) -> int:
+    max_version: list[int] = []
+    for specifier in specifiers:
+        if specifier.operator == "<=":
+            max_version.append(Version(specifier.version).minor)
+        if specifier.operator == "<":
+            max_version.append(Version(specifier.version).minor - 1)
+    return max(max_version, default=_PY_MAX_VERSION)
+
+
+def get_supported_pythons(requires_python: str) -> t.Generator[str, None, None]:
+    specifiers = SpecifierSet(requires_python)
+    min_version = _get_min_version(specifiers)
+    max_version = _get_max_version(specifiers)
+
+    yield from specifiers.filter(f"3.{v}" for v in range(min_version, max_version + 1))
+
 
 @dataclasses.dataclass
 class AboutInfo:

singer_sdk/plugin_base.py

@@ -14,7 +14,6 @@
 
 import click
 from jsonschema import Draft7Validator
-from packaging.specifiers import SpecifierSet
 
 from singer_sdk import about, metrics
 from singer_sdk.cli import plugin_cli
@@ -36,30 +35,6 @@
 from singer_sdk.typing import extend_validator_with_defaults
 
 SDK_PACKAGE_NAME = "singer_sdk"
-CHECK_SUPPORTED_PYTHON_VERSIONS = (
-    # unsupported versions
-    "2.7",
-    "3.0",
-    "3.1",
-    "3.2",
-    "3.3",
-    "3.4",
-    "3.5",
-    "3.6",
-    "3.7",
-    # current supported versions
-    "3.8",
-    "3.9",
-    "3.10",
-    "3.11",
-    "3.12",
-    # future supported versions
-    "3.13",
-    "3.14",
-    "3.15",
-    "3.16",
-)
-
 
 JSONSchemaValidator = extend_validator_with_defaults(Draft7Validator)
 
@@ -300,12 +275,7 @@ def _get_supported_python_versions(package: str) -> list[str] | None:
         except metadata.PackageNotFoundError:
             return None
 
-        reported_python_versions = SpecifierSet(package_metadata["Requires-Python"])
-        return [
-            version
-            for version in CHECK_SUPPORTED_PYTHON_VERSIONS
-            if version in reported_python_versions
-        ]
+        return list(about.get_supported_pythons(package_metadata["Requires-Python"]))
 
     @classmethod
     def get_plugin_version(cls) -> str:

tests/_singerlib/test_messages.py

@@ -25,7 +25,7 @@ def test_format_message():
         record={"id": 1, "name": "test"},
     )
     assert singerwriter.format_message(message) == (
-        '{"type": "RECORD", "stream": "test", "record": {"id": 1, "name": "test"}}'
+        '{"type":"RECORD","stream":"test","record":{"id":1,"name":"test"}}'
     )
 
 
@@ -39,7 +39,7 @@ def test_write_message():
         singerwriter.write_message(message)
 
     assert out.getvalue() == (
-        '{"type": "RECORD", "stream": "test", "record": {"id": 1, "name": "test"}}\n'
+        '{"type":"RECORD","stream":"test","record":{"id":1,"name":"test"}}\n'
     )
 
 

tests/core/test_about.py

@@ -3,11 +3,19 @@
 from __future__ import annotations
 
 import typing as t
+from importlib import metadata
 
 import pytest
 
-from singer_sdk.about import AboutFormatter, AboutInfo
+from singer_sdk.about import (
+    _PY_MAX_VERSION,
+    _PY_MIN_VERSION,
+    AboutFormatter,
+    AboutInfo,
+    get_supported_pythons,
+)
 from singer_sdk.helpers.capabilities import TapCapabilities
+from singer_sdk.plugin_base import SDK_PACKAGE_NAME
 
 if t.TYPE_CHECKING:
     from pathlib import Path
@@ -72,3 +80,26 @@ def test_about_format(
     output = formatter.format_about(about_info)
     snapshot_name = f"{about_format}.snap.{_format_to_extension[about_format]}"
     snapshot.assert_match(output, snapshot_name)
+
+
+def test_get_supported_pythons_sdk():
+    package_metadata = metadata.metadata(SDK_PACKAGE_NAME)
+    requires_python = package_metadata["Requires-Python"]
+
+    supported_pythons = list(get_supported_pythons(requires_python))
+    assert supported_pythons[0] == f"3.{_PY_MIN_VERSION}"
+    assert supported_pythons[-1] == f"3.{_PY_MAX_VERSION}"
+
+
+@pytest.mark.parametrize(
+    "specifiers,expected",
+    [
+        (">=3.7,<3.12", ["3.7", "3.8", "3.9", "3.10", "3.11"]),
+        (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]),
+        (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12"]),
+        (">3.7,<=3.11", ["3.8", "3.9", "3.10", "3.11"]),
+    ],
+)
+def test_get_supported_pythons(specifiers: str, expected: list[str]):
+    supported_pythons = list(get_supported_pythons(specifiers))
+    assert supported_pythons == expected

tests/core/test_plugin_base.py

@@ -5,7 +5,7 @@
 
 import pytest
 
-from singer_sdk.plugin_base import MapperNotInitialized, PluginBase
+from singer_sdk.plugin_base import SDK_PACKAGE_NAME, MapperNotInitialized, PluginBase
 from singer_sdk.typing import IntegerType, PropertiesList, Property, StringType
 
 
@@ -53,3 +53,8 @@ def test_mapper_not_initialized():
     )
     with pytest.raises(MapperNotInitialized):
         _ = plugin.mapper
+
+
+def test_supported_python_versions():
+    """Test that supported python versions are correctly parsed."""
+    assert PluginBase._get_supported_python_versions(SDK_PACKAGE_NAME)

tests/snapshots/countries_write_schemas/countries_write_schemas

@@ -1,2 +1,2 @@
-{"type": "SCHEMA", "stream": "continents", "schema": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": ["code"]}
-{"type": "SCHEMA", "stream": "countries", "schema": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}, "native": {"type": ["string", "null"]}, "phone": {"type": ["string", "null"]}, "capital": {"type": ["string", "null"]}, "currency": {"type": ["string", "null"]}, "emoji": {"type": ["string", "null"]}, "continent": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": ["object", "null"]}, "languages": {"items": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": "object"}, "type": ["array", "null"]}}, "type": "object"}, "key_properties": ["code"]}
+{"type":"SCHEMA","stream":"continents","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"key_properties":["code"]}
+{"type":"SCHEMA","stream":"countries","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]},"native":{"type":["string","null"]},"phone":{"type":["string","null"]},"capital":{"type":["string","null"]},"currency":{"type":["string","null"]},"emoji":{"type":["string","null"]},"continent":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":["object","null"]},"languages":{"items":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"type":["array","null"]}},"type":"object"},"key_properties":["code"]}

tests/snapshots/mapped_stream/aliased_stream.jsonl

@@ -1,6 +1,6 @@
-{"type": "STATE", "value": {}}
-{"type": "SCHEMA", "stream": "aliased_stream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []}
-{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}}
+{"type":"STATE","value":{}}
+{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

tests/snapshots/mapped_stream/changed_key_properties.jsonl

@@ -1,6 +1,6 @@
-{"type": "STATE", "value": {}}
-{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}}}, "key_properties": ["email_hash"]}
-{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "426b189df1e2f359efe6ee90f2d2030f"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}}
+{"type":"STATE","value":{}}
+{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}}},"key_properties":["email_hash"]}
+{"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

tests/snapshots/mapped_stream/drop_property.jsonl

@@ -1,6 +1,6 @@
-{"type": "STATE", "value": {}}
-{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}}
+{"type":"STATE","value":{}}
+{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]}
+{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

tests/snapshots/mapped_stream/drop_property_null_string.jsonl

@@ -1,6 +1,6 @@
-{"type": "STATE", "value": {}}
-{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}}
+{"type":"STATE","value":{}}
+{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]}
+{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

tests/snapshots/mapped_stream/flatten_all.jsonl

@@ -1,6 +1,6 @@
-{"type": "STATE", "value": {}}
-{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub__num": {"type": ["integer", "null"]}, "user__some_numbers": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": []}
-{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub__num": 1, "user__some_numbers": "[3.14, 2.718]"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub__num": 2, "user__some_numbers": "[10.32, 1.618]"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user__id": 3, "user__sub__num": 3, "user__some_numbers": "[1.414, 1.732]"}, "time_extracted": "2022-01-01T00:00:00+00:00"}
-{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}}
+{"type":"STATE","value":{}}
+{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]}
+{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}