diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e466499a8..0501629fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,10 @@ repos: rev: v4.5.0 hooks: - id: check-json + exclude: | + (?x)^( + .*/launch.json + )$ - id: check-toml exclude: | (?x)^( diff --git a/cookiecutter/.gitignore b/cookiecutter/.gitignore new file mode 100644 index 000000000..d6ea64289 --- /dev/null +++ b/cookiecutter/.gitignore @@ -0,0 +1,2 @@ +# Include template VSCode directory +!*/*/.vscode/ diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py index ca51924f2..dc7134238 100644 --- a/cookiecutter/tap-template/hooks/post_gen_project.py +++ b/cookiecutter/tap-template/hooks/post_gen_project.py @@ -28,4 +28,4 @@ shutil.rmtree(".github") if "{{ cookiecutter.ide }}" != "VSCode": - shutil.rmtree(".vscode") + shutil.rmtree(".vscode", ignore_errors=True) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml index 5e3f6cb83..640e62aa2 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml @@ -8,6 +8,10 @@ repos: rev: v4.5.0 hooks: - id: check-json + exclude: | + (?x)^( + \.vscode/.*\.json + )$ - id: check-toml - id: check-yaml - id: end-of-file-fixer @@ -20,7 +24,7 @@ repos: - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.11 + rev: v0.1.14 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json new file mode 100644 index 000000000..2605119c4 --- /dev/null +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "{{ cookiecutter.tap_id }}", + "type": "python", + "request": "launch", + "cwd": "${workspaceFolder}", + "program": "{{ cookiecutter.library_name }}", + "justMyCode": false, + "args": [ + "--config", + ".secrets/config.json", + ], + }, + ] +} diff --git a/docs/conf.py b/docs/conf.py index a784a46f5..519b2af0c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,7 +117,12 @@ "css/custom.css", ] +# -- Options for MyST -------------------------------------------------------- +# https://myst-parser.readthedocs.io/en/latest/configuration.html myst_heading_anchors = 3 +myst_enable_extensions = { + "colon_fence", +} redirects = { "porting.html": "guides/porting.html", diff --git a/docs/dev_guide.md b/docs/dev_guide.md index 316cca8ad..75dcf60dc 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -55,6 +55,10 @@ pipx install poetry pipx install tox ``` +:::{tip} +The minimum recommended version of cookiecutter is `2.2.0` (released 2023-07-06). +::: + Now you can initialize your new project with the Cookiecutter template for taps: ```bash diff --git a/pyproject.toml b/pyproject.toml index 856d534f9..99e9b0375 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,6 +187,7 @@ exclude_also = [ '''if (t\.)?TYPE_CHECKING:''', ] fail_under = 82 +show_missing = true [tool.mypy] exclude = "tests" diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py index 6a9fc24ef..f9a93b76b 100644 --- a/singer_sdk/_singerlib/messages.py +++ b/singer_sdk/_singerlib/messages.py @@ -226,7 +226,12 @@ def format_message(message: Message) -> str: Returns: The formatted message. """ - return json.dumps(message.to_dict(), use_decimal=True, default=_default_encoding) + return json.dumps( + message.to_dict(), + use_decimal=True, + default=_default_encoding, + separators=(",", ":"), + ) def write_message(message: Message) -> None: diff --git a/singer_sdk/about.py b/singer_sdk/about.py index e5bbf60de..5585f55dd 100644 --- a/singer_sdk/about.py +++ b/singer_sdk/about.py @@ -9,6 +9,9 @@ from collections import OrderedDict from textwrap import dedent +from packaging.specifiers import SpecifierSet +from packaging.version import Version + if t.TYPE_CHECKING: from singer_sdk.helpers.capabilities import CapabilitiesEnum @@ -19,6 +22,38 @@ "MarkdownFormatter", ] +# Keep these in sync with the supported Python versions in pyproject.toml +_PY_MIN_VERSION = 8 +_PY_MAX_VERSION = 12 + + +def _get_min_version(specifiers: SpecifierSet) -> int: + min_version: list[int] = [] + for specifier in specifiers: + if specifier.operator == ">=": + min_version.append(Version(specifier.version).minor) + if specifier.operator == ">": + min_version.append(Version(specifier.version).minor + 1) + return min(min_version, default=_PY_MIN_VERSION) + + +def _get_max_version(specifiers: SpecifierSet) -> int: + max_version: list[int] = [] + for specifier in specifiers: + if specifier.operator == "<=": + max_version.append(Version(specifier.version).minor) + if specifier.operator == "<": + max_version.append(Version(specifier.version).minor - 1) + return max(max_version, default=_PY_MAX_VERSION) + + +def get_supported_pythons(requires_python: str) -> t.Generator[str, None, None]: + specifiers = SpecifierSet(requires_python) + min_version = _get_min_version(specifiers) + max_version = _get_max_version(specifiers) + + yield from specifiers.filter(f"3.{v}" for v in range(min_version, max_version + 1)) + @dataclasses.dataclass class AboutInfo: diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 271e248c7..fc1ce9415 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -14,7 +14,6 @@ import click from jsonschema import Draft7Validator -from packaging.specifiers import SpecifierSet from singer_sdk import about, metrics from singer_sdk.cli import plugin_cli @@ -36,30 +35,6 @@ from singer_sdk.typing import extend_validator_with_defaults SDK_PACKAGE_NAME = "singer_sdk" -CHECK_SUPPORTED_PYTHON_VERSIONS = ( - # unsupported versions - "2.7", - "3.0", - "3.1", - "3.2", - "3.3", - "3.4", - "3.5", - "3.6", - "3.7", - # current supported versions - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - # future supported versions - "3.13", - "3.14", - "3.15", - "3.16", -) - JSONSchemaValidator = extend_validator_with_defaults(Draft7Validator) @@ -300,12 +275,7 @@ def _get_supported_python_versions(package: str) -> list[str] | None: except metadata.PackageNotFoundError: return None - reported_python_versions = SpecifierSet(package_metadata["Requires-Python"]) - return [ - version - for version in CHECK_SUPPORTED_PYTHON_VERSIONS - if version in reported_python_versions - ] + return list(about.get_supported_pythons(package_metadata["Requires-Python"])) @classmethod def get_plugin_version(cls) -> str: diff --git a/tests/_singerlib/test_messages.py b/tests/_singerlib/test_messages.py index e10259497..491573545 100644 --- a/tests/_singerlib/test_messages.py +++ b/tests/_singerlib/test_messages.py @@ -25,7 +25,7 @@ def test_format_message(): record={"id": 1, "name": "test"}, ) assert singerwriter.format_message(message) == ( - '{"type": "RECORD", "stream": "test", "record": {"id": 1, "name": "test"}}' + '{"type":"RECORD","stream":"test","record":{"id":1,"name":"test"}}' ) @@ -39,7 +39,7 @@ def test_write_message(): singerwriter.write_message(message) assert out.getvalue() == ( - '{"type": "RECORD", "stream": "test", "record": {"id": 1, "name": "test"}}\n' + '{"type":"RECORD","stream":"test","record":{"id":1,"name":"test"}}\n' ) diff --git a/tests/core/test_about.py b/tests/core/test_about.py index cd458ceab..415077227 100644 --- a/tests/core/test_about.py +++ b/tests/core/test_about.py @@ -3,11 +3,19 @@ from __future__ import annotations import typing as t +from importlib import metadata import pytest -from singer_sdk.about import AboutFormatter, AboutInfo +from singer_sdk.about import ( + _PY_MAX_VERSION, + _PY_MIN_VERSION, + AboutFormatter, + AboutInfo, + get_supported_pythons, +) from singer_sdk.helpers.capabilities import TapCapabilities +from singer_sdk.plugin_base import SDK_PACKAGE_NAME if t.TYPE_CHECKING: from pathlib import Path @@ -72,3 +80,26 @@ def test_about_format( output = formatter.format_about(about_info) snapshot_name = f"{about_format}.snap.{_format_to_extension[about_format]}" snapshot.assert_match(output, snapshot_name) + + +def test_get_supported_pythons_sdk(): + package_metadata = metadata.metadata(SDK_PACKAGE_NAME) + requires_python = package_metadata["Requires-Python"] + + supported_pythons = list(get_supported_pythons(requires_python)) + assert supported_pythons[0] == f"3.{_PY_MIN_VERSION}" + assert supported_pythons[-1] == f"3.{_PY_MAX_VERSION}" + + +@pytest.mark.parametrize( + "specifiers,expected", + [ + (">=3.7,<3.12", ["3.7", "3.8", "3.9", "3.10", "3.11"]), + (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]), + (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12"]), + (">3.7,<=3.11", ["3.8", "3.9", "3.10", "3.11"]), + ], +) +def test_get_supported_pythons(specifiers: str, expected: list[str]): + supported_pythons = list(get_supported_pythons(specifiers)) + assert supported_pythons == expected diff --git a/tests/core/test_plugin_base.py b/tests/core/test_plugin_base.py index 7d1021c11..0ea87f036 100644 --- a/tests/core/test_plugin_base.py +++ b/tests/core/test_plugin_base.py @@ -5,7 +5,7 @@ import pytest -from singer_sdk.plugin_base import MapperNotInitialized, PluginBase +from singer_sdk.plugin_base import SDK_PACKAGE_NAME, MapperNotInitialized, PluginBase from singer_sdk.typing import IntegerType, PropertiesList, Property, StringType @@ -53,3 +53,8 @@ def test_mapper_not_initialized(): ) with pytest.raises(MapperNotInitialized): _ = plugin.mapper + + +def test_supported_python_versions(): + """Test that supported python versions are correctly parsed.""" + assert PluginBase._get_supported_python_versions(SDK_PACKAGE_NAME) diff --git a/tests/snapshots/countries_write_schemas/countries_write_schemas b/tests/snapshots/countries_write_schemas/countries_write_schemas index 02b567fc9..71a900f1b 100644 --- a/tests/snapshots/countries_write_schemas/countries_write_schemas +++ b/tests/snapshots/countries_write_schemas/countries_write_schemas @@ -1,2 +1,2 @@ -{"type": "SCHEMA", "stream": "continents", "schema": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": ["code"]} -{"type": "SCHEMA", "stream": "countries", "schema": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}, "native": {"type": ["string", "null"]}, "phone": {"type": ["string", "null"]}, "capital": {"type": ["string", "null"]}, "currency": {"type": ["string", "null"]}, "emoji": {"type": ["string", "null"]}, "continent": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": ["object", "null"]}, "languages": {"items": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": "object"}, "type": ["array", "null"]}}, "type": "object"}, "key_properties": ["code"]} +{"type":"SCHEMA","stream":"continents","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"key_properties":["code"]} +{"type":"SCHEMA","stream":"countries","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]},"native":{"type":["string","null"]},"phone":{"type":["string","null"]},"capital":{"type":["string","null"]},"currency":{"type":["string","null"]},"emoji":{"type":["string","null"]},"continent":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":["object","null"]},"languages":{"items":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"type":["array","null"]}},"type":"object"},"key_properties":["code"]} diff --git a/tests/snapshots/mapped_stream/aliased_stream.jsonl b/tests/snapshots/mapped_stream/aliased_stream.jsonl index 8df28ddf4..e353c0717 100644 --- a/tests/snapshots/mapped_stream/aliased_stream.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "aliased_stream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "aliased_stream", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/changed_key_properties.jsonl b/tests/snapshots/mapped_stream/changed_key_properties.jsonl index c5168a45b..d90219138 100644 --- a/tests/snapshots/mapped_stream/changed_key_properties.jsonl +++ b/tests/snapshots/mapped_stream/changed_key_properties.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}}}, "key_properties": ["email_hash"]} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "426b189df1e2f359efe6ee90f2d2030f"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}}},"key_properties":["email_hash"]} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/drop_property.jsonl b/tests/snapshots/mapped_stream/drop_property.jsonl index aece20a30..03c663b05 100644 --- a/tests/snapshots/mapped_stream/drop_property.jsonl +++ b/tests/snapshots/mapped_stream/drop_property.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl index aece20a30..03c663b05 100644 --- a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl +++ b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_all.jsonl b/tests/snapshots/mapped_stream/flatten_all.jsonl index e588c9dc1..ddb313b01 100644 --- a/tests/snapshots/mapped_stream/flatten_all.jsonl +++ b/tests/snapshots/mapped_stream/flatten_all.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub__num": {"type": ["integer", "null"]}, "user__some_numbers": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub__num": 1, "user__some_numbers": "[3.14, 2.718]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub__num": 2, "user__some_numbers": "[10.32, 1.618]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user__id": 3, "user__sub__num": 3, "user__some_numbers": "[1.414, 1.732]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl index 7a8b54581..4107f9d19 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl index e3f6cd05f..68022e1f0 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub": {"type": ["string", "null"]}, "user__some_numbers": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub": "{\"num\": 1}", "user__some_numbers": "[3.14, 2.718]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub": "{\"num\": 2}", "user__some_numbers": "[10.32, 1.618]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user__id": 3, "user__sub": "{\"num\": 3}", "user__some_numbers": "[1.414, 1.732]"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub":"{\"num\": 1}","user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub":"{\"num\": 2}","user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub":"{\"num\": 3}","user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/keep_all_fields.jsonl b/tests/snapshots/mapped_stream/keep_all_fields.jsonl index da476f6a5..4c337d0fa 100644 --- a/tests/snapshots/mapped_stream/keep_all_fields.jsonl +++ b/tests/snapshots/mapped_stream/keep_all_fields.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}, "email_hash": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}, "email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}, "email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}, "email_hash": "426b189df1e2f359efe6ee90f2d2030f"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]},"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]},"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]},"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/map_and_flatten.jsonl b/tests/snapshots/mapped_stream/map_and_flatten.jsonl index 921094d05..4d1ecd5d4 100644 --- a/tests/snapshots/mapped_stream/map_and_flatten.jsonl +++ b/tests/snapshots/mapped_stream/map_and_flatten.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub__num": {"type": ["integer", "null"]}, "user__some_numbers": {"type": ["string", "null"]}, "email_hash": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": ["email_hash"]} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub__num": 1, "user__some_numbers": "[3.14, 2.718]", "email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub__num": 2, "user__some_numbers": "[10.32, 1.618]", "email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user__id": 3, "user__sub__num": 3, "user__some_numbers": "[1.414, 1.732]", "email_hash": "426b189df1e2f359efe6ee90f2d2030f"}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__some_numbers":{"type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":["email_hash"]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__some_numbers":"[3.14, 2.718]","email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__some_numbers":"[10.32, 1.618]","email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__some_numbers":"[1.414, 1.732]","email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/no_map.jsonl b/tests/snapshots/mapped_stream/no_map.jsonl index 7a8b54581..4107f9d19 100644 --- a/tests/snapshots/mapped_stream/no_map.jsonl +++ b/tests/snapshots/mapped_stream/no_map.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl index 0cbbf451a..78d332dce 100644 --- a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl +++ b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"count": {"type": ["integer", "null"]}}}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"count": 21}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 13}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"count": 19}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"count":21},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":13},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":19},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl index e53042958..df053aa52 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}, "fixed_count": {"type": ["integer", "null"]}}}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060", "fixed_count": 20}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d", "fixed_count": 12}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "426b189df1e2f359efe6ee90f2d2030f", "fixed_count": 18}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl index e53042958..df053aa52 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}, "fixed_count": {"type": ["integer", "null"]}}}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060", "fixed_count": 20}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d", "fixed_count": 12}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"email_hash": "426b189df1e2f359efe6ee90f2d2030f", "fixed_count": 18}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/record_to_column.jsonl b/tests/snapshots/mapped_stream/record_to_column.jsonl index 8fc3efb21..74e0326b7 100644 --- a/tests/snapshots/mapped_stream/record_to_column.jsonl +++ b/tests/snapshots/mapped_stream/record_to_column.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"_data": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}}, "key_properties": []} -{"type": "RECORD", "stream": "mystream", "record": {"_data": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"_data": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "mystream", "record": {"_data": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"_data":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":["object","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl index a68add19c..8868c39cd 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "sourced_stream_1", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl index a68add19c..8868c39cd 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "sourced_stream_1", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl index 234551006..3bdc0db77 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl @@ -1,6 +1,6 @@ -{"type": "STATE", "value": {}} -{"type": "SCHEMA", "stream": "sourced_stream_2", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}, "some_numbers": {"items": {"type": ["number"]}, "type": ["array", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} -{"type": "RECORD", "stream": "sourced_stream_2", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}, "some_numbers": [3.14, 2.718]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_2", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}, "some_numbers": [10.32, 1.618]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "RECORD", "stream": "sourced_stream_2", "record": {"email": "charlie@example.com", "count": 19, "user": {"id": 3, "sub": {"num": 3}, "some_numbers": [1.414, 1.732]}}, "time_extracted": "2022-01-01T00:00:00+00:00"} -{"type": "STATE", "value": {"bookmarks": {"mystream": {}}}} +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}