Skip to content

Commit

Permalink
remove error and some wrong validation
Browse files Browse the repository at this point in the history
  • Loading branch information
eveleighoj committed Jan 23, 2025
1 parent 0bf17f2 commit 0183d9b
Showing 1 changed file with 0 additions and 9 deletions.
9 changes: 0 additions & 9 deletions digital_land/package/dataset_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,6 @@ def load_entities_range(
) = 1
)
"""
logging.error(query)

pivot_query = f"""
PIVOT (
Expand Down Expand Up @@ -392,14 +391,6 @@ def combine_parquet_files(self, input_path, output_path):
This method combines multiple parquet files into a single parquet file
"""
logger.info(f"combining parquet files from {input_path} into {output_path}")
# check input path is a directory using Path
if not Path(input_path).is_dir():
raise ValueError("Input path must be a directory")

# check output_path is a file that doesn't exist
if not Path(output_path).is_file():
raise ValueError("Output path must be a file")

# use self.conn to use duckdb to combine files
sql = f"""
COPY (select * from parquet_scan('{input_path}/*.parquet')) TO '{output_path}' (FORMAT PARQUET);
Expand Down

0 comments on commit 0183d9b

Please sign in to comment.