[Redshift] Improve Sweep query (#428)

artie-labs · Apr 11, 2024 · 64792a0 · 64792a0
1 parent 61f6333
commit 64792a0
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 6 deletions.
diff --git a/clients/redshift/redshift.go b/clients/redshift/redshift.go
@@ -73,6 +73,7 @@ func (s *Store) Sweep() error {
 		return err
 	}
 
+	// `relkind` will filter for only ordinary tables and exclude sequences, views, etc.
 	queryFunc := func(dbAndSchemaPair kafkalib.DatabaseSchemaPair) (string, []any) {
 		return `
 SELECT 
@@ -82,7 +83,7 @@ FROM
 JOIN 
     PG_CATALOG.PG_NAMESPACE n ON n.oid = c.relnamespace
 WHERE 
-    n.nspname = $1 AND c.relname ILIKE $2;`, []any{dbAndSchemaPair.Schema, "%" + constants.ArtiePrefix + "%"}
+    n.nspname = $1 AND c.relname ILIKE $2 AND c.relkind = 'r';`, []any{dbAndSchemaPair.Schema, "%" + constants.ArtiePrefix + "%"}
 	}
 
 	return shared.Sweep(s, tcs, queryFunc)

diff --git a/lib/destination/ddl/expiry.go b/lib/destination/ddl/expiry.go
@@ -13,13 +13,10 @@ func ShouldDeleteFromName(name string) bool {
 		return false
 	}
 
-	return shouldDeleteUnix(nameParts[len(nameParts)-1])
-}
-
-func shouldDeleteUnix(unixString string) bool {
+	unixString := nameParts[len(nameParts)-1]
 	unix, err := strconv.Atoi(unixString)
 	if err != nil {
-		slog.Warn("Failed to parse unix string", slog.Any("err", err), slog.String("unixString", unixString))
+		slog.Error("Failed to parse unix string", slog.Any("err", err), slog.String("tableName", name), slog.String("unixString", unixString))
 		return false
 	}