From f8b52feb7b55d809acff7fc1382d8ba512509ada Mon Sep 17 00:00:00 2001 From: caltangelo Date: Fri, 21 Jan 2022 14:09:18 -0600 Subject: [PATCH] map_java_column will use target_column_name if defined (#87) * map_java_column will use defined target_column_name * readme * Update merge.py Co-authored-by: Caleb Santangelo Co-authored-by: aaverbec <43386278+aaverbec@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- README.md | 1 - pipewrench/merge.py | 13 ++++++++----- setup.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 449c3c3..a38f896 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ ## Submitting issues Use the github [issue tracker](https://github.com/Cargill/pipewrench/issues) to submit bugs or ask questions. -## Submitting features and fixex +## Submitting features and fixes To contribute features or bug fixes, create a [fork](https://help.github.com/articles/fork-a-repo/) then a [pull request](https://help.github.com/articles/creating-a-pull-request/) back to this repository. diff --git a/README.md b/README.md index 572a956..665eae2 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,6 @@ Generate scripts using the `generate-scripts` helper: ```bash $ cd project-scripts $ ./generate-scripts - ``` Running `pipewrench-merge` from the `examples` directory manually would look like: diff --git a/pipewrench/merge.py b/pipewrench/merge.py index 2be788f..55249b1 100644 --- a/pipewrench/merge.py +++ b/pipewrench/merge.py @@ -26,7 +26,7 @@ from json import dumps from jinja2 import Template import yaml - +from yaml.loader import SafeLoader OUT_DIR = "output" @@ -186,7 +186,7 @@ def load_mapping_file(mapping, templates_dir, conf): try: with codecs.open(os.path.join(templates_dir, conf[mapping]), 'r', 'UTF-8') as mapping_file: logging.debug('mapping file: %s', mapping) - type_mappings[mapping] = yaml.load(mapping_file.read()) + type_mappings[mapping] = yaml.load(mapping_file.read(), Loader=SafeLoader) logging.debug("type mappings: %s", type_mappings) except KeyError: logging.warning( @@ -224,7 +224,7 @@ def get_conf(path, env): env_applied = render(str_conf, **env) # Load the conf yaml into Python data structures - conf = yaml.load(env_applied) + conf = yaml.load(env_applied, Loader=SafeLoader) conf.update(env) logging.debug('conf: %s', conf) @@ -240,7 +240,7 @@ def get_env(path): with codecs.open(path, 'r', 'UTF-8') as env_file: conf_string = env_file.read() - env = yaml.load(conf_string) + env = yaml.load(conf_string, Loader=SafeLoader) logging.debug('env: %s', env) return env @@ -351,7 +351,10 @@ def sqoop_map_java_column(columns, clean_column=False): map_java_column = "--map-column-java " for column in columns: - column_name = column['name'] + if 'target_column_name' in column.keys(): + column_name = column['target_column_name'] + else: + column_name = column['name'] if clean_column: column_name = cleanse_column(column_name) diff --git a/setup.py b/setup.py index e7e9f91..8d9a9e3 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ import os import sys -version = '1.1.0' +version = '1.1.1' def pipewrench_test_suite(): test_loader = unittest.TestLoader()