From 8447c94551f9795e48a07ae63cc50c3eac245816 Mon Sep 17 00:00:00 2001 From: Lasse Bjermeland Date: Fri, 15 Mar 2024 08:00:55 +0100 Subject: [PATCH] Handle relative paths and API cleanup + github action --- .github/workflows/publish.yml | 40 ++++++++++++ .github/workflows/testing.yml | 21 +++++++ setup.py | 2 +- src/dmtgen/__init__.py | 4 +- src/dmtgen/base_generator.py | 14 ++--- src/dmtgen/common/blueprint.py | 31 +--------- src/dmtgen/common/blueprint_attribute.py | 79 +++++++++++------------- src/dmtgen/common/enum_description.py | 28 ++------- src/dmtgen/common/package.py | 35 ++++++----- src/dmtgen/common/system_package.py | 3 +- src/dmtgen/none_generator.py | 14 +++++ src/dmtgen/template.py | 1 + src/tests/test_relative_paths.py | 4 +- 13 files changed, 155 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/testing.yml create mode 100644 src/dmtgen/none_generator.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ddadc87 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + - name: Build package + run: python setup.py clean --all sdist bdist_wheel + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..e4fa55d --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,21 @@ +name: Testing + +on: [push] + +jobs: + testing: + name: Run tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check out and setup python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies and linting + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + pylint src --errors-only + - name: Build package + run: pytest --junitxml output/report.xml diff --git a/setup.py b/setup.py index a9289e7..2ddfad7 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='dmtgen', - version='0.4.2', + version='0.5.0.dev1', author="SINTEF Ocean", description="Python generator utilities for DMT", long_description=long_description, diff --git a/src/dmtgen/__init__.py b/src/dmtgen/__init__.py index 4fc61f4..7a842fe 100644 --- a/src/dmtgen/__init__.py +++ b/src/dmtgen/__init__.py @@ -1,4 +1,6 @@ +"""This module contains the classes for the different types of generators.""" from .package_generator import PackageGenerator from .base_generator import BaseGenerator from .basic_template_generator import BasicTemplateGenerator -from .template_generator import TemplateBasedGenerator \ No newline at end of file +from .template_generator import TemplateBasedGenerator +from .none_generator import NoneGenerator \ No newline at end of file diff --git a/src/dmtgen/base_generator.py b/src/dmtgen/base_generator.py index 424410b..5c7bd8a 100644 --- a/src/dmtgen/base_generator.py +++ b/src/dmtgen/base_generator.py @@ -26,11 +26,12 @@ def __init__(self, root_dir: Path, package_name: str, self.output_dir = output_dir self.source_only = False self.root_package = root_package - self.generators = self.get_template_generators() - def get_template_generators(self) -> Dict[str,TemplateBasedGenerator]: - """ Override in subclasses """ - return {} + # pylint: disable=unused-argument, no-self-use + def get_template_generator(self, template: Path, config: Dict) -> TemplateBasedGenerator: + """ Override in subclasses to control which template generator to use""" + return BasicTemplateGenerator() + def generate_package(self, config: Dict): """ Generate package """ @@ -65,12 +66,9 @@ def post_generate(self,output_dir: Path): def __find_templates_and_generate(self, output_dir: Path, config: Dict): for path in sorted(output_dir.rglob('*.jinja')): - generator = self.generators.get(path.name,self.get_basic_generator()) + generator = self.get_template_generator(path, config) self.__generate_template(path, generator, config) - def get_basic_generator(self) -> TemplateBasedGenerator: - return BasicTemplateGenerator() - @staticmethod def __read_template(templatefile: Path): loader = jinja2.FileSystemLoader(templatefile.parents[0]) diff --git a/src/dmtgen/common/blueprint.py b/src/dmtgen/common/blueprint.py index f5649cb..d6fd738 100644 --- a/src/dmtgen/common/blueprint.py +++ b/src/dmtgen/common/blueprint.py @@ -1,11 +1,10 @@ +"""Blueprint class for SIMOS""" from __future__ import annotations from typing import Dict, Sequence from typing import TYPE_CHECKING +from .blueprint_attribute import BlueprintAttribute if TYPE_CHECKING: from .package import Package -from .blueprint_attribute import BlueprintAttribute - - class Blueprint: """ " A basic SIMOS Blueprint""" @@ -26,25 +25,6 @@ def __init__(self, bp_dict: Dict, parent: Package) -> None: # We will resolve this later self.__extensions = None - @property - def name(self) -> str: - """Entity id""" - return self.__name - - @name.setter - def name(self, value: str): - """Set name""" - self.__name = str(value) - - @property - def description(self) -> str: - """Entity id""" - return self.__description - - @description.setter - def description(self, value: str): - """Set description""" - self.__description = str(value) @property def abstract(self) -> bool: @@ -67,8 +47,6 @@ def all_attributes(self) -> Dict[str,BlueprintAttribute]: atributes.update(self.__attributes) return atributes - - @property def extensions(self) -> Sequence[Blueprint]: """Extensions""" @@ -84,11 +62,8 @@ def __resolve(self,extension: str): def get_path(self): """ Get full path to blueprint """ - parent = self.get_parent() + parent = self.parent if parent: return parent.get_path() + "/" + self.name # Then we are at root return "/" + self.name - - def get_parent(self) -> Package: - return self.parent diff --git a/src/dmtgen/common/blueprint_attribute.py b/src/dmtgen/common/blueprint_attribute.py index 7f05d51..ee28170 100644 --- a/src/dmtgen/common/blueprint_attribute.py +++ b/src/dmtgen/common/blueprint_attribute.py @@ -10,67 +10,62 @@ class BlueprintAttribute: def __init__(self, content: Dict, parent_blueprint: Blueprint) -> None: self.content = content - if "description" not in content: - content["description"] = "" name = content["name"] - if not name: + if len(name)==0: raise ValueError("Attribute has no name") self.name = name + if "description" not in content: + content["description"] = "" self.description = content["description"].replace('"',"'") - self.__is_many= "dimensions" in content - self.__contained = content.get("contained",True) - atype = content["attributeType"] + dims=content.get("dimensions") + if dims: + self.dimensions = dims.split(",") + else: + self.dimensions = [] + atype = content["attributeType"] + self.parent = parent_blueprint package = parent_blueprint.parent - self.__type = package.resolve_type(atype) - primitive_types = ['boolean', 'number', 'string', 'integer'] - self.__is_primitive = atype in primitive_types + self.type = package.resolve_type(atype) + self.is_primitive = atype in ['boolean', 'number', 'string', 'integer'] self.is_enum = self.content.get("enumType",None) is not None + self.is_blueprint = not (self.is_primitive or self.is_enum) + self.is_optional = self.content.get("optional",True) + self.is_array = len(self.dimensions)>0 + self.is_contained = content.get("contained",True) @property - def name(self) -> str: - """Entity id""" - return self.__name - - @name.setter - def name(self, value: str): - """Set name""" - self.__name = str(value) + def is_string(self) -> bool: + """Is this a string""" + return self.type == "string" @property - def type(self) -> str: - """Attribute type""" - return self.__type + def is_boolean(self) -> bool: + """Is this a boolean""" + return self.type == "boolean" @property - def description(self) -> str: - """Entity id""" - return self.__description + def is_integer(self) -> bool: + """Is this an integer""" + return self.type == "integer" @property - def contained(self) -> bool: - """Is contained""" - return self.__contained + def is_number(self) -> bool: + """Is this a number""" + return self.type == "number" @property - def is_primitive(self) -> bool: - """Is this a primitive attribute""" - return self.__is_primitive + def is_required(self) -> bool: + """Is a required relation""" + return not self.is_optional - @property - def is_many(self) -> bool: - """Is this a many relation""" - return self.__is_many - - @property - def optional(self) -> bool: - """Is this a many relation""" - return self.content.get("optional",True) + def is_fixed_array(self) -> bool: + """Is this a fixed array""" + return self.is_array and "*" not in self.dimensions - @description.setter - def description(self, value: str): - """Set description""" - self.__description = str(value) + def is_variable_array(self) -> bool: + """Is this a variable array""" + return self.is_array and "*" in self.dimensions def get(self, key, default=None): """Return the content value or an optional default""" diff --git a/src/dmtgen/common/enum_description.py b/src/dmtgen/common/enum_description.py index 89daf89..00287df 100644 --- a/src/dmtgen/common/enum_description.py +++ b/src/dmtgen/common/enum_description.py @@ -1,14 +1,13 @@ +""" A basic DMT Enum""" from __future__ import annotations from typing import TYPE_CHECKING, Dict if TYPE_CHECKING: from .package import Package - - class EnumDescription: """ " A basic DMT Enum""" - def __init__(self, enum_dict: Dict, parent) -> None: + def __init__(self, enum_dict: Dict, parent: Package) -> None: self.parent = parent self.blueprint = enum_dict self.name = self.blueprint["name"] @@ -23,33 +22,14 @@ def __init__(self, enum_dict: Dict, parent) -> None: "label": labels[i] }) - @property - def name(self) -> str: - """Entity id""" - return self.__name - - @name.setter - def name(self, value: str): - """Set name""" - self.__name = str(value) - - @property - def description(self) -> str: - """Entity id""" - return self.__description - - @description.setter - def description(self, value: str): - """Set description""" - self.__description = str(value) - def get_path(self): """ Get full path to blueprint """ - parent = self.get_parent() + parent = self.parent if parent: return parent.get_path() + "/" + self.name # Then we are at root return "/" + self.name def get_parent(self) -> Package: + """ Get parent package """ return self.parent diff --git a/src/dmtgen/common/package.py b/src/dmtgen/common/package.py index 8e32e49..2efb089 100644 --- a/src/dmtgen/common/package.py +++ b/src/dmtgen/common/package.py @@ -15,12 +15,12 @@ class Package: """ " A basic SIMOS package""" - def __init__(self, pkg_dir: Path) -> None: + def __init__(self, pkg_dir: Path, parent: Package) -> None: self.package_dir = pkg_dir self.version = 0 self.name = pkg_dir.name self.aliases = {"core":"system/SIMOS"} - self.parent = None + self.parent = parent self.__blueprints = {} self.__enums = {} self.__packages = {} @@ -36,16 +36,15 @@ def __read_package(self, pkg_dir: Path): pkg_filename = "package.json" package_file = pkg_dir / pkg_filename if package_file.exists(): - package = json.load(open(package_file, encoding="utf-8")) - self.__read_package_info(package) + with open(package_file, encoding="utf-8") as file: + package = json.load(file) + self.__read_package_info(package) for file in pkg_dir.glob("*.json"): - entity = json.load(open(file, encoding="utf-8")) - if file.name == "__versions__.json": - self.__read_version(entity) - elif file.name == pkg_filename: + if file.name == pkg_filename: continue - else: + with open(file, encoding="utf-8") as file: + entity = json.load(file) etype = self.resolve_type(entity["type"]) if etype == "system/SIMOS/Blueprint": blueprint = Blueprint(entity, self) @@ -60,7 +59,7 @@ def __read_package(self, pkg_dir: Path): for folder in pkg_dir.glob("*/"): if folder.is_dir(): - sub_package = Package(folder) + sub_package = Package(folder,self) sub_package.parent = self self.__packages[sub_package.name] = sub_package @@ -79,6 +78,7 @@ def __read_package_info(self,pkg: dict): self.aliases[alias]=dep.get("address") def resolve_type(self, etype:str) -> str: + """Resolve type to full path""" if etype.startswith("."): # This is a relative path path = self.get_path() + "/" + etype @@ -114,19 +114,23 @@ def get_paths(self) -> List[str]: @property def blueprints(self) -> Sequence[Blueprint]: + """All blueprints in package""" return self.__blueprints.values() @property def enums(self) -> Sequence[EnumDescription]: + """All enums in package""" return self.__enums.values() def blueprint(self, name:str) -> Blueprint: - bp = self.__blueprints.get(name,None) - if not bp: + """Get blueprint by name""" + blueprint = self.__blueprints.get(name,None) + if not blueprint: raise ValueError(f"Blueprint not found \"{name}\" in {self.name}") - return bp + return blueprint def enum(self, name:str) -> EnumDescription: + """Get enum by name""" enum = self.__enums.get(name,None) if not enum: raise ValueError(f"Enum not found \"{name}\" in {self.name}") @@ -146,9 +150,11 @@ def package(self, name:str) -> Package: return pkg def get_parent(self) -> Package: + """Get parent package""" return self.parent def get_root(self): + """Get root package""" parent: Package = self.parent if parent: return parent.get_root() @@ -183,9 +189,10 @@ def __get_package(self, parts: Sequence[str]) -> Package: for part in parts: if part == '.': raise ValueError("Relative path not allowed. Should have been resolved by now.") - elif part == '': + if part == '': package = self.get_root() elif part == 'system': + # pylint: disable=import-outside-toplevel from .system_package import system_package package = system_package elif package is None: diff --git a/src/dmtgen/common/system_package.py b/src/dmtgen/common/system_package.py index 3a50d8d..3352d54 100644 --- a/src/dmtgen/common/system_package.py +++ b/src/dmtgen/common/system_package.py @@ -1,6 +1,7 @@ +"""The system package contains all root level blueprints.""" from __future__ import annotations from pathlib import Path from .package import Package system_dir = Path(__file__).parent / "../data/system" -system_package = Package(system_dir) +system_package = Package(system_dir,None) diff --git a/src/dmtgen/none_generator.py b/src/dmtgen/none_generator.py new file mode 100644 index 0000000..b63ae5c --- /dev/null +++ b/src/dmtgen/none_generator.py @@ -0,0 +1,14 @@ +"""Basic generator, one template, one output file""" + +from pathlib import Path +from typing import Dict +from jinja2.environment import Template +from .package_generator import PackageGenerator +from .template_generator import TemplateBasedGenerator + + +class NoneGenerator(TemplateBasedGenerator): + """Do not generate anything, meaning the given tempate is not used for anything""" + + def generate(self,package_generator: PackageGenerator, template : Template, outputfile: Path, config: Dict): + """Do nothing""" diff --git a/src/dmtgen/template.py b/src/dmtgen/template.py index e26708a..964dfaa 100644 --- a/src/dmtgen/template.py +++ b/src/dmtgen/template.py @@ -1,3 +1,4 @@ +""" Template utilities """ def escape_string(value: str) -> str: """ Escape control characters""" if value: diff --git a/src/tests/test_relative_paths.py b/src/tests/test_relative_paths.py index 2241552..68e4d8a 100644 --- a/src/tests/test_relative_paths.py +++ b/src/tests/test_relative_paths.py @@ -7,13 +7,13 @@ def test_relative_bps(): """Test iport of Bluepritns with relative paths.""" pkg_dir = Path(__file__).parent / 'test_data' / 'apps' assert pkg_dir.exists() - pkg = Package(pkg_dir) + pkg = Package(pkg_dir, None) assert pkg.name == "apps" app = pkg.packages[0] assert app.name == "EmployeeApp" employee_app = app.blueprint("EmployeeApp") employees = employee_app.all_attributes["employees"] - assert employees.type == "EmployeeApp/Employee" + assert employees.type == "apps/EmployeeApp/Employee" employee = app.blueprint("Employee") pic = employee.all_attributes["profilePicture"] assert pic.type == "system/SIMOS/File"