Skip to content

Commit

Permalink
map_java_column will use target_column_name if defined (#87)
Browse files Browse the repository at this point in the history
* map_java_column will use defined target_column_name

* readme

* Update merge.py

Co-authored-by: Caleb Santangelo <caleb_santangelo@cargill.com>
Co-authored-by: aaverbec <43386278+aaverbec@users.noreply.github.com>
  • Loading branch information
3 people authored Jan 21, 2022
1 parent d47bf47 commit f8b52fe
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 8 deletions.
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
## Submitting issues
Use the github [issue tracker](https://github.com/Cargill/pipewrench/issues) to submit bugs or ask questions.

## Submitting features and fixex
## Submitting features and fixes
To contribute features or bug fixes, create a [fork](https://help.github.com/articles/fork-a-repo/) then a
[pull request](https://help.github.com/articles/creating-a-pull-request/) back to this repository.

Expand Down
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,6 @@ Generate scripts using the `generate-scripts` helper:
```bash
$ cd project-scripts
$ ./generate-scripts

```

Running `pipewrench-merge` from the `examples` directory manually would look like:
Expand Down
13 changes: 8 additions & 5 deletions pipewrench/merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from json import dumps
from jinja2 import Template
import yaml

from yaml.loader import SafeLoader


OUT_DIR = "output"
Expand Down Expand Up @@ -186,7 +186,7 @@ def load_mapping_file(mapping, templates_dir, conf):
try:
with codecs.open(os.path.join(templates_dir, conf[mapping]), 'r', 'UTF-8') as mapping_file:
logging.debug('mapping file: %s', mapping)
type_mappings[mapping] = yaml.load(mapping_file.read())
type_mappings[mapping] = yaml.load(mapping_file.read(), Loader=SafeLoader)
logging.debug("type mappings: %s", type_mappings)
except KeyError:
logging.warning(
Expand Down Expand Up @@ -224,7 +224,7 @@ def get_conf(path, env):
env_applied = render(str_conf, **env)

# Load the conf yaml into Python data structures
conf = yaml.load(env_applied)
conf = yaml.load(env_applied, Loader=SafeLoader)
conf.update(env)
logging.debug('conf: %s', conf)

Expand All @@ -240,7 +240,7 @@ def get_env(path):
with codecs.open(path, 'r', 'UTF-8') as env_file:
conf_string = env_file.read()

env = yaml.load(conf_string)
env = yaml.load(conf_string, Loader=SafeLoader)
logging.debug('env: %s', env)
return env

Expand Down Expand Up @@ -351,7 +351,10 @@ def sqoop_map_java_column(columns, clean_column=False):
map_java_column = "--map-column-java "

for column in columns:
column_name = column['name']
if 'target_column_name' in column.keys():
column_name = column['target_column_name']
else:
column_name = column['name']
if clean_column:
column_name = cleanse_column(column_name)

Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import os
import sys

version = '1.1.0'
version = '1.1.1'

def pipewrench_test_suite():
test_loader = unittest.TestLoader()
Expand Down

0 comments on commit f8b52fe

Please sign in to comment.