Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#2110 adds several not found error message patterns to sqlalchemy destination #2213

Open
wants to merge 2 commits into
base: devel
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 30 additions & 11 deletions dlt/destinations/impl/sqlalchemy/db_api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,17 +435,36 @@ def _make_database_exception(e: Exception) -> Exception:
return DatabaseUndefinedRelation(e)
msg = str(e).lower()
if isinstance(e, (sa.exc.ProgrammingError, sa.exc.OperationalError)):
if "exist" in msg: # TODO: Hack
return DatabaseUndefinedRelation(e)
elif "unknown table" in msg:
return DatabaseUndefinedRelation(e)
elif "unknown database" in msg:
return DatabaseUndefinedRelation(e)
elif "no such table" in msg: # sqlite # TODO: Hack
return DatabaseUndefinedRelation(e)
elif "no such database" in msg: # sqlite # TODO: Hack
return DatabaseUndefinedRelation(e)
elif "syntax" in msg:
patterns = [
# MySQL / MariaDB
r"unknown database", # Missing schema
r"doesn't exist", # Missing table
r"unknown table", # Missing table
# SQLite
r"no such table", # Missing table
r"no such database", # Missing table
# PostgreSQL / Trino / Vertica
r"does not exist", # Missing schema, relation
# r"does not exist", # Missing table
# MSSQL
r"invalid object name", # Missing schema or table
# Oracle
r"ora-00942: table or view does not exist", # Missing schema or table
# SAP HANA
r"invalid schema name", # Missing schema
r"invalid table name", # Missing table
# DB2
r"is an undefined name", # SQL0204N... Missing schema or table
# Apache Hive
r"table not found", # Missing table
r"database does not exist",
]
# entity not found
for pat_ in patterns:
if pat_ in msg:
return DatabaseUndefinedRelation(e)

if "syntax" in msg:
return DatabaseTransientException(e)
elif isinstance(e, (sa.exc.OperationalError, sa.exc.IntegrityError)):
return DatabaseTerminalException(e)
Expand Down
14 changes: 14 additions & 0 deletions tests/load/filesystem/test_filesystem_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,20 @@ def test_glob_files(with_gdrive_buckets_env: str, load_content: bool, glob_filte
assert_sample_files(all_file_items, filesystem, config, load_content, glob_filter)


@pytest.mark.parametrize("load_content", (True, False))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this looks completely unrelated but also good :)

def test_glob_single_file(with_gdrive_buckets_env: str, load_content: bool) -> None:
bucket_url = os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"]
bucket_url, config, filesystem = glob_test_setup(
bucket_url, "standard_source/samples/met_csv/A803"
)
# use glob to get data
all_file_items = list(glob_files(filesystem, bucket_url, "A803_20230919.csv"))
assert len(all_file_items) == 1
assert (
all_file_items[0]["relative_path"] == all_file_items[0]["file_name"] == "A803_20230919.csv"
)


def test_glob_overlapping_path_files(with_gdrive_buckets_env: str) -> None:
bucket_url = os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"]
# "standard_source/sample" overlaps with a real existing "standard_source/samples". walk operation on azure
Expand Down
Loading