From 35f620b10b534be11171d9cc5966bfe5ba4ad8ae Mon Sep 17 00:00:00 2001 From: Caleb Santangelo Date: Thu, 18 Nov 2021 10:01:01 -0600 Subject: [PATCH 1/3] map_java_column will use defined target_column_name --- CONTRIBUTING.md | 2 +- pipewrench/merge.py | 12 ++++++++---- setup.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 449c3c3..a38f896 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ ## Submitting issues Use the github [issue tracker](https://github.com/Cargill/pipewrench/issues) to submit bugs or ask questions. -## Submitting features and fixex +## Submitting features and fixes To contribute features or bug fixes, create a [fork](https://help.github.com/articles/fork-a-repo/) then a [pull request](https://help.github.com/articles/creating-a-pull-request/) back to this repository. diff --git a/pipewrench/merge.py b/pipewrench/merge.py index 2be788f..3879b5d 100644 --- a/pipewrench/merge.py +++ b/pipewrench/merge.py @@ -26,6 +26,7 @@ from json import dumps from jinja2 import Template import yaml +from yaml.loader import SafeLoader @@ -186,7 +187,7 @@ def load_mapping_file(mapping, templates_dir, conf): try: with codecs.open(os.path.join(templates_dir, conf[mapping]), 'r', 'UTF-8') as mapping_file: logging.debug('mapping file: %s', mapping) - type_mappings[mapping] = yaml.load(mapping_file.read()) + type_mappings[mapping] = yaml.load(mapping_file.read(), Loader=SafeLoader) logging.debug("type mappings: %s", type_mappings) except KeyError: logging.warning( @@ -224,7 +225,7 @@ def get_conf(path, env): env_applied = render(str_conf, **env) # Load the conf yaml into Python data structures - conf = yaml.load(env_applied) + conf = yaml.load(env_applied, Loader=SafeLoader) conf.update(env) logging.debug('conf: %s', conf) @@ -240,7 +241,7 @@ def get_env(path): with codecs.open(path, 'r', 'UTF-8') as env_file: conf_string = env_file.read() - env = yaml.load(conf_string) + env = yaml.load(conf_string, Loader=SafeLoader) logging.debug('env: %s', env) return env @@ -351,7 +352,10 @@ def sqoop_map_java_column(columns, clean_column=False): map_java_column = "--map-column-java " for column in columns: - column_name = column['name'] + if 'target_column_name' in column.keys(): + column_name = column['target_column_name'] + else: + column_name = column['name'] if clean_column: column_name = cleanse_column(column_name) diff --git a/setup.py b/setup.py index e7e9f91..8d9a9e3 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ import os import sys -version = '1.1.0' +version = '1.1.1' def pipewrench_test_suite(): test_loader = unittest.TestLoader() From 4e632a3ddce3b33c2a1601fce813f8102c1c122d Mon Sep 17 00:00:00 2001 From: Caleb Santangelo Date: Tue, 21 Dec 2021 13:07:11 -0600 Subject: [PATCH 2/3] readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 572a956..665eae2 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,6 @@ Generate scripts using the `generate-scripts` helper: ```bash $ cd project-scripts $ ./generate-scripts - ``` Running `pipewrench-merge` from the `examples` directory manually would look like: From ff2c8c3f59ac8d2d38071e8f7ffbb60f5fc0f56d Mon Sep 17 00:00:00 2001 From: aaverbec <43386278+aaverbec@users.noreply.github.com> Date: Fri, 21 Jan 2022 13:41:57 -0600 Subject: [PATCH 3/3] Update merge.py --- pipewrench/merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipewrench/merge.py b/pipewrench/merge.py index 3879b5d..55249b1 100644 --- a/pipewrench/merge.py +++ b/pipewrench/merge.py @@ -29,7 +29,6 @@ from yaml.loader import SafeLoader - OUT_DIR = "output" LOGGING_FORMAT = "[%(levelname) s:%(filename)s:%(lineno)s in %(funcName)20s() ] %(message)s"