From 150756e53413220b4e9521c755045e712e43cc95 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 10:00:23 -0700 Subject: [PATCH 01/14] add entrypoint for readxml --- pyhf/commandline.py | 13 +++++++++++++ setup.py | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 pyhf/commandline.py diff --git a/pyhf/commandline.py b/pyhf/commandline.py new file mode 100644 index 0000000000..2fa2b302e8 --- /dev/null +++ b/pyhf/commandline.py @@ -0,0 +1,13 @@ +import logging +logging.basicConfig() + +import click +import json +from . import readxml + +@click.command() +@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.') +@click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.') +def xml2json(entrypoint_xml, workspace): + spec = readxml(entrypoint_xml, workspace) + import pdb; pdb.set_trace() diff --git a/setup.py b/setup.py index 2dc7467ea0..616bb07bff 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,8 @@ include_package_data = True, install_requires = [ 'numpy<=1.14.5,>=1.14.3', # required by tensorflow, mxnet, and us - 'scipy' + 'scipy', + 'click>=6.0', # for console scripts ], extras_require = { 'xmlimport': [ @@ -52,6 +53,7 @@ ] }, entry_points = { + 'console_scripts': ['pyhf_xml2json=pyhf.commandline:xml2json'] }, dependency_links = [ ] From a54b77fd7f1fd0fc8e24b01ea9be0cb01431be7b Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 10:42:42 -0700 Subject: [PATCH 02/14] use click for commandline, tqdm for readxml (disabled by default) --- pyhf/commandline.py | 13 ++++++++----- pyhf/readxml.py | 22 +++++++++++++--------- setup.py | 3 ++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 2fa2b302e8..612128bcde 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -1,13 +1,16 @@ import logging logging.basicConfig() +log = logging.getLogger(__name__) import click import json from . import readxml @click.command() -@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.') -@click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.') -def xml2json(entrypoint_xml, workspace): - spec = readxml(entrypoint_xml, workspace) - import pdb; pdb.set_trace() +@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.', type=click.Path(exists=True)) +@click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.', type=click.Path(exists=True)) +@click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.', type=click.Path(exists=False)) +def xml2json(entrypoint_xml, workspace, output_file=None): + spec = readxml.parse(entrypoint_xml, workspace, enable_tqdm=True) + json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) + log.info("Written to {0:s}".format(output_file)) diff --git a/pyhf/readxml.py b/pyhf/readxml.py index beba837ac8..3089d59be1 100644 --- a/pyhf/readxml.py +++ b/pyhf/readxml.py @@ -1,9 +1,10 @@ +import logging +log = logging.getLogger(__name__) + import os import xml.etree.ElementTree as ET import numpy as np -import logging - -log = logging.getLogger(__name__) +import tqdm def import_root_histogram(rootdir, filename, path, name): import uproot @@ -51,7 +52,6 @@ def process_sample(sample,rootdir,inputfile, histopath, channelname): 'type': 'normfactor', 'data': None }) - elif modtag.tag == 'HistoSys': lo,_ = import_root_histogram(rootdir, modtag.attrib.get('HistoFileLow',inputfile), @@ -111,12 +111,16 @@ def process_channel(channelxml,rootdir): channelname = channel.attrib['Name'] return channelname, process_data(data, rootdir, inputfile, histopath), [process_sample(x, rootdir, inputfile, histopath, channelname) for x in samples] -def parse(configfile,rootdir): +def parse(configfile, rootdir, enable_tqdm=False): toplvl = ET.parse(configfile) - inputs = [ET.parse(os.path.join(rootdir,x.text)) for x in toplvl.findall('Input')] - channels = { - k:{'data': d, 'samples': v} for k,d,v in [process_channel(inp,rootdir) for inp in inputs] - } + inputs = tqdm.tqdm([x.text for x in toplvl.findall('Input')], unit='channel', disable=not(enable_tqdm)) + + channels = {} + for inp in inputs: + inputs.set_description('Processing {}'.format(inp)) + k, d, v = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir) + channels[k] = {'data': d, 'samples': v} + return { 'toplvl':{ 'resultprefix':toplvl.getroot().attrib['OutputFilePrefix'], diff --git a/setup.py b/setup.py index 616bb07bff..719d2cf742 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,8 @@ install_requires = [ 'numpy<=1.14.5,>=1.14.3', # required by tensorflow, mxnet, and us 'scipy', - 'click>=6.0', # for console scripts + 'click>=6.0', # for console scripts, + 'tqdm', # for readxml ], extras_require = { 'xmlimport': [ From 2c12f0929e7225b7598486a71e96652430dad53c Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 10:48:26 -0700 Subject: [PATCH 03/14] no need for checking if path exists for output json file --- pyhf/commandline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 612128bcde..247553011f 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -9,7 +9,7 @@ @click.command() @click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.', type=click.Path(exists=True)) @click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.', type=click.Path(exists=True)) -@click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.', type=click.Path(exists=False)) +@click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') def xml2json(entrypoint_xml, workspace, output_file=None): spec = readxml.parse(entrypoint_xml, workspace, enable_tqdm=True) json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) From 54b378c424c5b7650f0920583c49afe4d612d49d Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 11:37:19 -0700 Subject: [PATCH 04/14] add more coverage for testing the entry point script. see #198. --- pyhf/commandline.py | 5 +++-- setup.py | 1 + tests/test_scripts.py | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/test_scripts.py diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 247553011f..01f521ad15 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -10,7 +10,8 @@ @click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.', type=click.Path(exists=True)) @click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.', type=click.Path(exists=True)) @click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') -def xml2json(entrypoint_xml, workspace, output_file=None): - spec = readxml.parse(entrypoint_xml, workspace, enable_tqdm=True) +@click.option('--tqdm/--no-tqdm', default=True) +def xml2json(entrypoint_xml, workspace, output_file, tqdm): + spec = readxml.parse(entrypoint_xml, workspace, enable_tqdm=tqdm) json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) log.info("Written to {0:s}".format(output_file)) diff --git a/setup.py b/setup.py index 719d2cf742..96e8cd4301 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ 'pytest>=3.5.1', 'pytest-cov>=2.5.1', 'pytest-benchmark[histogram]', + 'pytest-console-scripts', 'python-coveralls', 'coverage==4.0.3', # coveralls 'matplotlib', diff --git a/tests/test_scripts.py b/tests/test_scripts.py new file mode 100644 index 0000000000..9e6f36757b --- /dev/null +++ b/tests/test_scripts.py @@ -0,0 +1,27 @@ +import pytest +import json +import shlex + +import pyhf + +# see test_import.py for the same (detailed) test +def test_import_prepHistFactory(tmpdir, script_runner): + temp = tmpdir.join("parsed_output.json") + command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --workspace validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) + ret = script_runner.run(*shlex.split(command)) + assert ret.success + assert ret.stdout == '' + assert ret.stderr == '' + + parsed_xml = json.loads(temp.read()) + spec = {'channels': parsed_xml['channels']} + pyhf.utils.validate(spec, pyhf.utils.get_default_schema()) + +def test_import_prepHistFactory_TQDM(tmpdir, script_runner): + temp = tmpdir.join("parsed_output.json") + command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --workspace validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) + ret = script_runner.run(*shlex.split(command)) + assert ret.success + assert ret.stdout == '' + assert ret.stderr != '' + From de9a38297ca29d62241e25b7f4a340b0c92eae90 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 11:56:30 -0700 Subject: [PATCH 05/14] make pyflakes happy --- pyhf/readxml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyhf/readxml.py b/pyhf/readxml.py index 3089d59be1..068e8ef238 100644 --- a/pyhf/readxml.py +++ b/pyhf/readxml.py @@ -118,8 +118,8 @@ def parse(configfile, rootdir, enable_tqdm=False): channels = {} for inp in inputs: inputs.set_description('Processing {}'.format(inp)) - k, d, v = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir) - channels[k] = {'data': d, 'samples': v} + channel, data, samples = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir) + channels[channel] = {'data': data, 'samples': samples} return { 'toplvl':{ From eb945797f475f45df4e3410f5fb4760433f1de75 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 16:17:56 -0700 Subject: [PATCH 06/14] change to click group --- pyhf/commandline.py | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 01f521ad15..cf6d52e115 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -6,7 +6,11 @@ import json from . import readxml -@click.command() +@click.group(context_settings=dict(help_option_names=['-h', '--help'])) +def pyhf(): + pass + +@pyhf.command() @click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.', type=click.Path(exists=True)) @click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.', type=click.Path(exists=True)) @click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') diff --git a/setup.py b/setup.py index 96e8cd4301..05da26c971 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ ] }, entry_points = { - 'console_scripts': ['pyhf_xml2json=pyhf.commandline:xml2json'] + 'console_scripts': ['pyhf=pyhf.commandline:pyhf'] }, dependency_links = [ ] From b9b294cb0d41a06d737b961127d120d2c3607b6e Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 16:19:10 -0700 Subject: [PATCH 07/14] workspace to basedir --- pyhf/commandline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index cf6d52e115..8b9f774261 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -11,11 +11,11 @@ def pyhf(): pass @pyhf.command() -@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the workspace definition.', type=click.Path(exists=True)) -@click.option('--workspace', required=True, prompt='Workspace directory', help='The location of workspace.', type=click.Path(exists=True)) +@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the PDF definition.', type=click.Path(exists=True)) +@click.option('--basedir', required=True, prompt='Base directory', help='The base directory for the XML files to point relative to.', type=click.Path(exists=True)) @click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') @click.option('--tqdm/--no-tqdm', default=True) -def xml2json(entrypoint_xml, workspace, output_file, tqdm): - spec = readxml.parse(entrypoint_xml, workspace, enable_tqdm=tqdm) +def xml2json(entrypoint_xml, basedir, output_file, tqdm): + spec = readxml.parse(entrypoint_xml, basedir, enable_tqdm=tqdm) json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) log.info("Written to {0:s}".format(output_file)) From aeffd4f8a7af51556c7752da94035c4ad91d8ebe Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 16:19:45 -0700 Subject: [PATCH 08/14] update tests --- tests/test_scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 9e6f36757b..fcb8bd38cd 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -7,7 +7,7 @@ # see test_import.py for the same (detailed) test def test_import_prepHistFactory(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --workspace validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) + command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' @@ -19,7 +19,7 @@ def test_import_prepHistFactory(tmpdir, script_runner): def test_import_prepHistFactory_TQDM(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --workspace validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) + command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' From 616f5c6e31228a413061776614d3b10832c65139 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 16:33:07 -0700 Subject: [PATCH 09/14] update tests again --- tests/test_scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index fcb8bd38cd..ab794c49e2 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -7,7 +7,7 @@ # see test_import.py for the same (detailed) test def test_import_prepHistFactory(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) + command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' @@ -19,7 +19,7 @@ def test_import_prepHistFactory(tmpdir, script_runner): def test_import_prepHistFactory_TQDM(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf_xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) + command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' From 6a2883672b642aa00bd4a3857f9e5e6280a92f2e Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 18:07:24 -0700 Subject: [PATCH 10/14] more tqdms. it's soo pretty --- pyhf/readxml.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pyhf/readxml.py b/pyhf/readxml.py index 068e8ef238..46caedeaa5 100644 --- a/pyhf/readxml.py +++ b/pyhf/readxml.py @@ -11,6 +11,7 @@ def import_root_histogram(rootdir, filename, path, name): #import pdb; pdb.set_trace() #assert path == '' # strip leading slashes as uproot doesn't use "/" for top-level + if path is None: path = '' path = path.strip('/') f = uproot.open(os.path.join(rootdir, filename)) try: @@ -27,7 +28,7 @@ def import_root_histogram(rootdir, filename, path, name): raise KeyError('Both {0:s} and {1:s} were tried and not found in {2:s}'.format(name, os.path.join(path, name), os.path.join(rootdir, filename))) -def process_sample(sample,rootdir,inputfile, histopath, channelname): +def process_sample(sample,rootdir,inputfile, histopath, channelname, enable_tqdm=False): if 'InputFile' in sample.attrib: inputfile = sample.attrib.get('InputFile') if 'HistoPath' in sample.attrib: @@ -37,7 +38,11 @@ def process_sample(sample,rootdir,inputfile, histopath, channelname): data,err = import_root_histogram(rootdir, inputfile, histopath, histoname) modifiers = [] - for modtag in sample.iter(): + + modtags = tqdm.tqdm(sample.iter(), unit='modifier', disable=not(enable_tqdm), total=len(sample)) + + for modtag in modtags: + modtags.set_description(' - modifier {0:s}({1:s})'.format(modtag.attrib.get('Name', 'n/a'), modtag.tag)) if modtag == sample: continue if modtag.tag == 'OverallSys': @@ -97,19 +102,24 @@ def process_data(sample,rootdir,inputfile, histopath): data,_ = import_root_histogram(rootdir, inputfile, histopath, histoname) return data -def process_channel(channelxml,rootdir): +def process_channel(channelxml, rootdir, enable_tqdm=False): channel = channelxml.getroot() inputfile = channel.attrib.get('InputFile') histopath = channel.attrib.get('HistoPath') - samples = channel.findall('Sample') - + samples = tqdm.tqdm(channel.findall('Sample'), unit='sample', disable=not(enable_tqdm)) data = channel.findall('Data')[0] - channelname = channel.attrib['Name'] - return channelname, process_data(data, rootdir, inputfile, histopath), [process_sample(x, rootdir, inputfile, histopath, channelname) for x in samples] + + results = [] + for sample in samples: + samples.set_description(' - sample {}'.format(sample.attrib.get('Name'))) + result = process_sample(sample, rootdir, inputfile, histopath, channelname, enable_tqdm) + results.append(result) + + return channelname, process_data(data, rootdir, inputfile, histopath), results def parse(configfile, rootdir, enable_tqdm=False): toplvl = ET.parse(configfile) @@ -118,7 +128,7 @@ def parse(configfile, rootdir, enable_tqdm=False): channels = {} for inp in inputs: inputs.set_description('Processing {}'.format(inp)) - channel, data, samples = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir) + channel, data, samples = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir, enable_tqdm) channels[channel] = {'data': data, 'samples': samples} return { From f591c196cf7592421bca49adf25ff766e2519ef8 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 22:56:06 -0700 Subject: [PATCH 11/14] tqdm -> track-progress/hide-progress --- pyhf/commandline.py | 6 +++--- pyhf/readxml.py | 16 ++++++++-------- tests/test_scripts.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 8b9f774261..607ee709be 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -14,8 +14,8 @@ def pyhf(): @click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the PDF definition.', type=click.Path(exists=True)) @click.option('--basedir', required=True, prompt='Base directory', help='The base directory for the XML files to point relative to.', type=click.Path(exists=True)) @click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') -@click.option('--tqdm/--no-tqdm', default=True) -def xml2json(entrypoint_xml, basedir, output_file, tqdm): - spec = readxml.parse(entrypoint_xml, basedir, enable_tqdm=tqdm) +@click.option('--track-progress/--hide-progress', default=True) +def xml2json(entrypoint_xml, basedir, output_file, track_progress): + spec = readxml.parse(entrypoint_xml, basedir, track_progress=track_progress) json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) log.info("Written to {0:s}".format(output_file)) diff --git a/pyhf/readxml.py b/pyhf/readxml.py index 46caedeaa5..be8b4c7380 100644 --- a/pyhf/readxml.py +++ b/pyhf/readxml.py @@ -28,7 +28,7 @@ def import_root_histogram(rootdir, filename, path, name): raise KeyError('Both {0:s} and {1:s} were tried and not found in {2:s}'.format(name, os.path.join(path, name), os.path.join(rootdir, filename))) -def process_sample(sample,rootdir,inputfile, histopath, channelname, enable_tqdm=False): +def process_sample(sample,rootdir,inputfile, histopath, channelname, track_progress=False): if 'InputFile' in sample.attrib: inputfile = sample.attrib.get('InputFile') if 'HistoPath' in sample.attrib: @@ -39,7 +39,7 @@ def process_sample(sample,rootdir,inputfile, histopath, channelname, enable_tqdm modifiers = [] - modtags = tqdm.tqdm(sample.iter(), unit='modifier', disable=not(enable_tqdm), total=len(sample)) + modtags = tqdm.tqdm(sample.iter(), unit='modifier', disable=not(track_progress), total=len(sample)) for modtag in modtags: modtags.set_description(' - modifier {0:s}({1:s})'.format(modtag.attrib.get('Name', 'n/a'), modtag.tag)) @@ -102,13 +102,13 @@ def process_data(sample,rootdir,inputfile, histopath): data,_ = import_root_histogram(rootdir, inputfile, histopath, histoname) return data -def process_channel(channelxml, rootdir, enable_tqdm=False): +def process_channel(channelxml, rootdir, track_progress=False): channel = channelxml.getroot() inputfile = channel.attrib.get('InputFile') histopath = channel.attrib.get('HistoPath') - samples = tqdm.tqdm(channel.findall('Sample'), unit='sample', disable=not(enable_tqdm)) + samples = tqdm.tqdm(channel.findall('Sample'), unit='sample', disable=not(track_progress)) data = channel.findall('Data')[0] channelname = channel.attrib['Name'] @@ -116,19 +116,19 @@ def process_channel(channelxml, rootdir, enable_tqdm=False): results = [] for sample in samples: samples.set_description(' - sample {}'.format(sample.attrib.get('Name'))) - result = process_sample(sample, rootdir, inputfile, histopath, channelname, enable_tqdm) + result = process_sample(sample, rootdir, inputfile, histopath, channelname, track_progress) results.append(result) return channelname, process_data(data, rootdir, inputfile, histopath), results -def parse(configfile, rootdir, enable_tqdm=False): +def parse(configfile, rootdir, track_progress=False): toplvl = ET.parse(configfile) - inputs = tqdm.tqdm([x.text for x in toplvl.findall('Input')], unit='channel', disable=not(enable_tqdm)) + inputs = tqdm.tqdm([x.text for x in toplvl.findall('Input')], unit='channel', disable=not(track_progress)) channels = {} for inp in inputs: inputs.set_description('Processing {}'.format(inp)) - channel, data, samples = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir, enable_tqdm) + channel, data, samples = process_channel(ET.parse(os.path.join(rootdir,inp)), rootdir, track_progress) channels[channel] = {'data': data, 'samples': samples} return { diff --git a/tests/test_scripts.py b/tests/test_scripts.py index ab794c49e2..e759c75842 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -7,7 +7,7 @@ # see test_import.py for the same (detailed) test def test_import_prepHistFactory(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --no-tqdm'.format(temp.strpath) + command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --hide-progress'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' From a9041119af5a919a1403b63575862737047f4e51 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 22:56:50 -0700 Subject: [PATCH 12/14] more pythonic --- pyhf/readxml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhf/readxml.py b/pyhf/readxml.py index be8b4c7380..fcbc370482 100644 --- a/pyhf/readxml.py +++ b/pyhf/readxml.py @@ -11,7 +11,7 @@ def import_root_histogram(rootdir, filename, path, name): #import pdb; pdb.set_trace() #assert path == '' # strip leading slashes as uproot doesn't use "/" for top-level - if path is None: path = '' + path = path or '' path = path.strip('/') f = uproot.open(os.path.join(rootdir, filename)) try: From 2dc2a02561c55be985ff419a692a915beda069fd Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 23:01:21 -0700 Subject: [PATCH 13/14] entrypoint-xml is an argument. basedir defaults to cwd. output json if not input specified --- pyhf/commandline.py | 15 ++++++++++----- tests/test_scripts.py | 7 +++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index 607ee709be..b2f0ebc08a 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -4,6 +4,8 @@ import click import json +import os + from . import readxml @click.group(context_settings=dict(help_option_names=['-h', '--help'])) @@ -11,11 +13,14 @@ def pyhf(): pass @pyhf.command() -@click.option('--entrypoint-xml', required=True, prompt='Top-level XML', help='The top-level XML file for the PDF definition.', type=click.Path(exists=True)) -@click.option('--basedir', required=True, prompt='Base directory', help='The base directory for the XML files to point relative to.', type=click.Path(exists=True)) -@click.option('--output-file', required=True, prompt='Output file', help='The location of the output json file. If not specified, prints to screen.') +@click.argument('entrypoint-xml', help='The top-level XML file for the PDF definition.', type=click.Path(exists=True)) +@click.option('--basedir', help='The base directory for the XML files to point relative to.', type=click.Path(exists=True), default=os.getcwd()) +@click.option('--output-file', help='The location of the output json file. If not specified, prints to screen.', default=None) @click.option('--track-progress/--hide-progress', default=True) def xml2json(entrypoint_xml, basedir, output_file, track_progress): spec = readxml.parse(entrypoint_xml, basedir, track_progress=track_progress) - json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) - log.info("Written to {0:s}".format(output_file)) + if output_file is None: + json.dumps(spec, indent=4, sort_keys=True) + else: + json.dump(spec, open(output_file, 'w+'), indent=4, sort_keys=True) + log.info("Written to {0:s}".format(output_file)) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index e759c75842..338bb759f8 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -7,7 +7,7 @@ # see test_import.py for the same (detailed) test def test_import_prepHistFactory(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --hide-progress'.format(temp.strpath) + command = 'pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s} --hide-progress'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' @@ -17,11 +17,10 @@ def test_import_prepHistFactory(tmpdir, script_runner): spec = {'channels': parsed_xml['channels']} pyhf.utils.validate(spec, pyhf.utils.get_default_schema()) -def test_import_prepHistFactory_TQDM(tmpdir, script_runner): +def test_import_prepHistFactory_withProgress(tmpdir, script_runner): temp = tmpdir.join("parsed_output.json") - command = 'pyhf xml2json --entrypoint-xml validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) + command = 'pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(temp.strpath) ret = script_runner.run(*shlex.split(command)) assert ret.success assert ret.stdout == '' assert ret.stderr != '' - From 29bca28b1e9bfd6c5acdde42c40a26860b530012 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 23 Aug 2018 23:17:54 -0700 Subject: [PATCH 14/14] broke click, no help in argument --- pyhf/commandline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyhf/commandline.py b/pyhf/commandline.py index b2f0ebc08a..51326ed533 100644 --- a/pyhf/commandline.py +++ b/pyhf/commandline.py @@ -13,11 +13,12 @@ def pyhf(): pass @pyhf.command() -@click.argument('entrypoint-xml', help='The top-level XML file for the PDF definition.', type=click.Path(exists=True)) +@click.argument('entrypoint-xml', type=click.Path(exists=True)) @click.option('--basedir', help='The base directory for the XML files to point relative to.', type=click.Path(exists=True), default=os.getcwd()) @click.option('--output-file', help='The location of the output json file. If not specified, prints to screen.', default=None) @click.option('--track-progress/--hide-progress', default=True) def xml2json(entrypoint_xml, basedir, output_file, track_progress): + """ Entrypoint XML: The top-level XML file for the PDF definition. """ spec = readxml.parse(entrypoint_xml, basedir, track_progress=track_progress) if output_file is None: json.dumps(spec, indent=4, sort_keys=True)