From 0a136a16613e95eaabaf121ab2539ebb228665a4 Mon Sep 17 00:00:00 2001 From: Hayden Date: Mon, 31 Jul 2017 14:36:33 +0100 Subject: [PATCH 1/4] Deal with incomplete Codelists when generating element pages This adds a check that outputs 'should' rather than 'must' where applicable. --- gen.py | 42 ++++++++++++++++++++++++++------- templates/en/schema_element.rst | 10 ++++---- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/gen.py b/gen.py index e327cce0..9c211786 100644 --- a/gen.py +++ b/gen.py @@ -48,7 +48,7 @@ def see_also(path, lang): mapping = json.load(open(os.path.join('IATI-Extra-Documentation', lang, standard, 'overview-mapping.json'))) # Loading this file is incredibly inefficient # Common 'simple' path e.g. iati-activities or budget/period-start # Using this prevents subpages of iati-activity using the activity file overview - simpler = len(path.split('/')) > 3 + simpler = len(path.split('/')) > 3 simple_path = '/'.join(path.split('/')[3:]) if simpler else path return list(lookup_see_also(standard, mapping, simple_path)) @@ -107,6 +107,30 @@ def match_codelist(path): pass # FIXME return +def is_complete_codelist(codelist_name): + """Determine whether the specified Codelist is complete. + + Args: + codelist_name (str): The name of the Codelist. This is case-sensitive and must match the mapping file. + + Returns: + bool: Whether the Codelist is complete. + + Note: + Need to manually specify which Codelists are incomplete - it is not auto-detected. This is due to the surrounding architecture making it a challenge to auto-detect this information. + + """ + # use a list of incomplete Codelists since it is shorter + incomplete_codelists = [ + 'Country', + 'HumanitarianScopeType', + 'HumanitarianScopeVocabulary', + 'IndicatorVocabulary', + 'OrganisationIdentifier', + 'OrganisationRegistrationAgency' + ] + return codelist_name not in incomplete_codelists + def path_to_ref(path): return path.replace('//','_').replace('@','.') @@ -118,7 +142,7 @@ def get_extra_docs(rst_filename): return fp.read().decode('utf8') else: return '' - + class Schema2Doc(object): @@ -170,7 +194,7 @@ def output_docs(self, element_name, path, element=None, minOccurs='', maxOccurs= element_name. path is the xpath of the context where this element was found, for the - root context, this is the empty string + root context, this is the empty string """ if element is None: @@ -299,7 +323,7 @@ def output_overview_page(self, standard, page, reference_pages): extra_docs=get_extra_docs(os.path.join(self.lang, standard, 'overview', page+'.rst')), reference_pages=reference_pages ).encode('utf8')) - + def element_loop(self, element, path): @@ -341,7 +365,7 @@ def attribute_loop(self, element): """ #if element.find("xsd:complexType[@mixed='true']", namespaces=namespaces) is not None: # print_column_info('text', indent) - + a = element.attrib type_attributes = [] type_attributeGroups = [] @@ -360,7 +384,7 @@ def attribute_loop(self, element): ) group_attributes = [] - for attributeGroup in ( + for attributeGroup in ( element.findall('xsd:complexType/xsd:attributeGroup', namespaces=namespaces) + element.findall('xsd:complexType/xsd:simpleContent/xsd:extension/xsd:attributeGroup', namespaces=namespaces) + type_attributeGroups @@ -398,9 +422,9 @@ def codelists_to_docs(lang): for fname in os.listdir(dirname): json_file = os.path.join(dirname, fname) if not fname.endswith('.json'): continue - with open(json_file) as fp: + with open(json_file) as fp: codelist_json = json.load(fp) - + fname = fname[:-5] embedded = os.path.exists(os.path.join('IATI-Codelists','xml',fname+'.xml')) if embedded: @@ -467,7 +491,7 @@ def extra_extra_docs(): filename='organisation-standard/summary-table.rst', title='Organisation Standard Summary Table') orgs.output_overview_pages('organisation-standard') - + ruleset_page(lang=language) codelists_to_docs(lang=language) extra_extra_docs() diff --git a/templates/en/schema_element.rst b/templates/en/schema_element.rst index c7e18c6d..843a18b3 100644 --- a/templates/en/schema_element.rst +++ b/templates/en/schema_element.rst @@ -60,14 +60,14 @@ Attributes {% if required %} This attribute is required. -{% endif %}{% set codelist_tuple = match_codelist(path+element_name+'/@'+attribute) %}{% if attribute_type %} +{% endif %}{% set codelist_tuple = match_codelist(path+element_name+'/@'+attribute) %}{% if attribute_type %} This value should be of type {{attribute_type}}. -{% endif %}{% if codelist_tuple[0] %} - This value should be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. +{% endif %}{% if codelist_tuple[0] %} + This value {% if is_complete_codelist(codelist_tuple[0]) %}must{% else %}should{% endif %} be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. + +{% endif %} -{% endif %} - {{ '\n\n '.join(ruleset_text(path+element_name+'/@'+attribute)) }}{% endfor %} {% endif %} From d9b055ced199267a1a5d7a2346bc1d3aa9c69b87 Mon Sep 17 00:00:00 2001 From: hayfield Date: Mon, 7 Aug 2017 13:59:56 +0100 Subject: [PATCH 2/4] should -> must --- templates/en/schema_element.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/en/schema_element.rst b/templates/en/schema_element.rst index 8a4e49af..711c8f81 100644 --- a/templates/en/schema_element.rst +++ b/templates/en/schema_element.rst @@ -62,7 +62,7 @@ Attributes {% endif %}{% set codelist_tuple = match_codelist(path+element_name+'/@'+attribute) %}{% if attribute_type %} - This value should be of type {{attribute_type}}. + This value must be of type {{attribute_type}}. {% endif %}{% if codelist_tuple[0] %} This value {% if is_complete_codelist(codelist_tuple[0]) %}must{% else %}should{% endif %} be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. From f1e092c1263451ca9203ff39dfe03ed83edb206b Mon Sep 17 00:00:00 2001 From: hayfield Date: Mon, 7 Aug 2017 15:49:36 +0100 Subject: [PATCH 3/4] Update gen.py Add a filter for detecting incomplete Codelists in templates --- gen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gen.py b/gen.py index fef1fb9d..b7be1379 100644 --- a/gen.py +++ b/gen.py @@ -161,6 +161,8 @@ def __init__(self, schema, lang): self.tree2 = ET.parse("./IATI-Schemas/iati-common.xsd") self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader('templates')) self.lang = lang + + self.jinja_env.filters['is_complete_codelist'] = is_complete_codelist def get_schema_element(self, tag_name, name_attribute): """ From 3027ebc65e4d6150de2b65f093b2762e5446c717 Mon Sep 17 00:00:00 2001 From: hayfield Date: Mon, 7 Aug 2017 15:51:49 +0100 Subject: [PATCH 4/4] Update schema_element.rst Use a filter rather than passing a parameter to a function in a template. --- templates/en/schema_element.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/en/schema_element.rst b/templates/en/schema_element.rst index 711c8f81..04b90c91 100644 --- a/templates/en/schema_element.rst +++ b/templates/en/schema_element.rst @@ -65,7 +65,7 @@ Attributes This value must be of type {{attribute_type}}. {% endif %}{% if codelist_tuple[0] %} - This value {% if is_complete_codelist(codelist_tuple[0]) %}must{% else %}should{% endif %} be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. + This value {% if codelist_tuple[0]|is_complete_codelist() %}must{% else %}should{% endif %} be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. {% endif %}