Merge pull request #280 from Riminder/feature/update-subtype-paths

core: replacing all ConnectorModel.name.lower() into ConnectorModel.s…

src/hrflow_connectors/core/connector.py

@@ -778,7 +778,7 @@ def logo(self, connectors_directory: Path) -> str:
                 "PIL is not found in current environment. Mind that you need to install"
                 " the package with dev dependencies to use manifest utility"
             )
-        connector_directory = connectors_directory / self.name.lower()
+        connector_directory = connectors_directory / self.subtype
         if not connector_directory.is_dir():
             raise ValueError(
                 "No directory found for connector {} in {}".format(
@@ -923,7 +923,7 @@ def manifest(self, connectors_directory: Path) -> t.Dict:
 
             jsonmap_path = (
                 connectors_directory
-                / model.name.lower()
+                / model.subtype
                 / "mappings"
                 / "format"
                 / "{}.json".format(action.name.value)

src/hrflow_connectors/core/documentation.py

@@ -193,7 +193,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
         model = connector.model
         result = subprocess.run(
             GIT_UPDATE_DATE.format(
-                connector=model.name.lower(),
+                connector=model.subtype,
                 base_connector_path=BASE_CONNECTOR_PATH.get().rstrip("/"),
             ),
             shell=True,
@@ -204,7 +204,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
         if result.stderr:
             raise Exception(
                 "Subprocess run for Git update dates failed for connector {} with"
-                " errors {}".format(model.name.lower(), result.stderr)
+                " errors {}".format(model.subtype, result.stderr)
             )
         filtered = [
             line.split(" ")[0]
@@ -253,7 +253,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
             name=match.group("name"),
             readme_link="./{base_connector_path}/{connector}/README.md".format(
                 base_connector_path=BASE_CONNECTOR_PATH.get().strip("/"),
-                connector=model.name.lower(),
+                connector=model.subtype,
             ),
             type=model.type.value,
             release_date=match.group("release_date"),
@@ -283,7 +283,7 @@ def generate_docs(
     )
     for connector in connectors:
         model = connector.model
-        connector_directory = connectors_directory / model.name.lower()
+        connector_directory = connectors_directory / model.subtype
         if not connector_directory.is_dir():
             logging.error(
                 "Skipping documentation for {}: no directory found at {}".format(

src/hrflow_connectors/core/tests.py

@@ -41,7 +41,7 @@ class InvalidTestConfigException(Exception):
 
 
 @contextmanager
-def secrets(connector_name: str, connectors_directory: Path):
+def secrets(connector_name: str, connector_subtype: str, connectors_directory: Path):
     secrets_prefix = ENVIRON_SECRETS_PREFIX.format(
         connector_name=connector_name.upper()
     )
@@ -63,9 +63,7 @@ def secrets(connector_name: str, connectors_directory: Path):
     else:
         global_secrets = dict()
 
-    connector_secrets_file = (
-        connectors_directory / connector_name.lower() / "secrets.json"
-    )
+    connector_secrets_file = connectors_directory / connector_subtype / "secrets.json"
     if connector_secrets_file.exists():
         try:
             connector_secrets = json.loads(connector_secrets_file.read_text())
@@ -97,16 +95,16 @@ def actions(connector: Connector):
 
 
 @contextmanager
-def warehouses(connector_name: str, connectors_directory: Path):
+def warehouses(connector_subtype: str, connectors_directory: Path):
     if connectors_directory is CONNECTORS_DIRECTORY:  # pragma: no cover
         warehouse_module = import_module(
-            "hrflow_connectors.connectors.{}.warehouse".format(connector_name.lower())
+            "hrflow_connectors.connectors.{}.warehouse".format(connector_subtype)
         )
     else:
         import_from = connectors_directory.relative_to(PROJECT_DIRECTORY)
         warehouse_module = import_module(
             "{}.{}.warehouse".format(
-                str(import_from).replace("/", "."), connector_name.lower()
+                str(import_from).replace("/", "."), connector_subtype
             )
         )
     warehouse_names = []
@@ -223,9 +221,8 @@ def collect_connector_tests(
     connector: Connector, connectors_directory: Path = CONNECTORS_DIRECTORY
 ):
     connector_name = connector.model.name
-    test_config_file = (
-        connectors_directory / connector_name.lower() / "test-config.yaml"
-    )
+    connector_subtype = connector.model.subtype
+    test_config_file = connectors_directory / connector_subtype / "test-config.yaml"
     if test_config_file.exists() is False:
         raise NoTestConfigException(
             "No test configuration found for connector {} at {}".format(
@@ -243,14 +240,15 @@ def collect_connector_tests(
 
     try:
         with warehouses(
-            connector_name=connector_name,
+            connector_subtype=connector_subtype,
             connectors_directory=connectors_directory,
         ):
             with actions(
                 connector=connector,
             ):
                 with secrets(
                     connector_name=connector_name,
+                    connector_subtype=connector_subtype,
                     connectors_directory=connectors_directory,
                 ):
                     test_suite = ConnectorTestConfig(**test_config)

tests/core/test_documentation.py

@@ -111,20 +111,18 @@ def connectors_directory(root_readme: Path):
     except FileNotFoundError:
         pass
 
-    readme = path / SmartLeads.model.name.lower() / "README.md"
-    notebooks_directory = path / SmartLeads.model.name.lower() / "notebooks"
+    readme = path / SmartLeads.model.subtype / "README.md"
+    notebooks_directory = path / SmartLeads.model.subtype / "notebooks"
     keep_empty_notebooks_file = (
-        path / SmartLeads.model.name.lower() / "notebooks" / KEEP_EMPTY_FOLDER
+        path / SmartLeads.model.subtype / "notebooks" / KEEP_EMPTY_FOLDER
     )
     notebook = notebooks_directory / NOTEBOOKS_FILE
-    mappings_directory = path / SmartLeads.model.name.lower() / "mappings"
-    format_mappings_directory = (
-        path / SmartLeads.model.name.lower() / "mappings" / "format"
-    )
+    mappings_directory = path / SmartLeads.model.subtype / "mappings"
+    format_mappings_directory = path / SmartLeads.model.subtype / "mappings" / "format"
     keep_empty_format_file = format_mappings_directory / KEEP_EMPTY_FOLDER
     format_file = format_mappings_directory / FORMAT_FILE
 
-    actions_documentation_directory = path / SmartLeads.model.name.lower() / "docs"
+    actions_documentation_directory = path / SmartLeads.model.subtype / "docs"
     action_documentation = actions_documentation_directory / "{}.md".format(
         SmartLeads.model.actions[0].name.value
     )
@@ -153,29 +151,27 @@ def connectors_directory(root_readme: Path):
 
 
 def test_documentation(connectors_directory):
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
-    notebooks_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "notebooks"
-    )
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
+    notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
     keep_empty_notebooks_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "notebooks"
         / KEEP_EMPTY_FOLDER
     )
     format_mappings_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
+        connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
     )
     keep_empty_format_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "mappings"
         / "format"
         / KEEP_EMPTY_FOLDER
     )
     action_documentation = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "docs"
         / "{}.md".format(SmartLeads.model.actions[0].name.value)
     )
@@ -202,12 +198,10 @@ def test_documentation(connectors_directory):
 def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
     connectors_directory,
 ):
-    notebooks_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "notebooks"
-    )
+    notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
     keep_empty_notebooks_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "notebooks"
         / KEEP_EMPTY_FOLDER
     )
@@ -224,10 +218,10 @@ def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
     assert notebooks_directory.exists() is True
     assert keep_empty_notebooks_file.exists() is True
 
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
     action_documentation = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "docs"
         / "{}.md".format(SmartLeads.model.actions[0].name.value)
     )
@@ -238,12 +232,10 @@ def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
 def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_other_files(
     connectors_directory,
 ):
-    notebooks_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "notebooks"
-    )
+    notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
     keep_empty_notebooks_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "notebooks"
         / KEEP_EMPTY_FOLDER
     )
@@ -264,10 +256,10 @@ def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_othe
     assert other.exists() is True
     assert keep_empty_notebooks_file.exists() is False
 
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
     action_documentation = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "docs"
         / "{}.md".format(SmartLeads.model.actions[0].name.value)
     )
@@ -278,12 +270,10 @@ def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_othe
 def test_documentation_removes_keep_empty_notebooks_file_if_folder_has_other_files(
     connectors_directory,
 ):
-    notebooks_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "notebooks"
-    )
+    notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
     keep_empty_notebooks_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "notebooks"
         / KEEP_EMPTY_FOLDER
     )
@@ -305,10 +295,10 @@ def test_documentation_removes_keep_empty_notebooks_file_if_folder_has_other_fil
     assert other.exists() is True
     assert keep_empty_notebooks_file.exists() is False
 
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
     action_documentation = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "docs"
         / "{}.md".format(SmartLeads.model.actions[0].name.value)
     )
@@ -320,11 +310,11 @@ def test_documentation_adds_keep_empty_format_file_if_folder_is_empty(
     connectors_directory,
 ):
     format_mappings_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
+        connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
     )
     keep_empty_format_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "mappings"
         / "format"
         / KEEP_EMPTY_FOLDER
@@ -347,11 +337,11 @@ def test_documentation_does_not_add_keep_empty_format_file_if_folder_has_other_f
     connectors_directory,
 ):
     format_mappings_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
+        connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
     )
     keep_empty_format_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "mappings"
         / "format"
         / KEEP_EMPTY_FOLDER
@@ -377,11 +367,11 @@ def test_documentation_removes_keep_empty_format_file_if_folder_has_other_files(
     connectors_directory,
 ):
     format_mappings_directory = (
-        connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
+        connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
     )
     keep_empty_format_file = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "mappings"
         / "format"
         / KEEP_EMPTY_FOLDER
@@ -406,7 +396,7 @@ def test_documentation_removes_keep_empty_format_file_if_folder_has_other_files(
 
 
 def test_documentation_fails_if_actions_section_not_found(connectors_directory):
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
     with patched_subprocess():
         generate_docs(
             connectors=[SmartLeads], connectors_directory=connectors_directory
@@ -584,10 +574,10 @@ def test_main_readme_update_at_helper_doesnt_override_handwritten_updated_at(
 
 
 def test_documentation_with_remote_code_links(connectors_directory):
-    readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
+    readme = connectors_directory / SmartLeads.model.subtype / "README.md"
     action_documentation = (
         connectors_directory
-        / SmartLeads.model.name.lower()
+        / SmartLeads.model.subtype
         / "docs"
         / "{}.md".format(SmartLeads.model.actions[0].name.value)
     )
@@ -641,10 +631,11 @@ def test_documentation_with_remote_code_links(connectors_directory):
 
 def test_documentation_connector_directory_not_found(caplog, connectors_directory):
     mismatch_name = "NoConnectorDir"
+    subtype = mismatch_name.lower().replace(" ", "")
     NameMismatchSmartLeads = Connector(
         name=mismatch_name,
         type=ConnectorType.Other,
-        subtype=mismatch_name.lower().replace(" ", ""),
+        subtype=subtype,
         description=DESCRIPTION,
         url="https://www.smartleads.test/",
         actions=[
@@ -660,12 +651,10 @@ def test_documentation_connector_directory_not_found(caplog, connectors_director
         ],
     )
 
-    readme = (
-        connectors_directory / NameMismatchSmartLeads.model.name.lower() / "README.md"
-    )
+    readme = connectors_directory / NameMismatchSmartLeads.model.subtype / "README.md"
     action_documentation = (
         connectors_directory
-        / NameMismatchSmartLeads.model.name.lower()
+        / NameMismatchSmartLeads.model.subtype
         / "docs"
         / "{}.md".format(NameMismatchSmartLeads.model.actions[0].name.value)
     )
@@ -715,10 +704,11 @@ def test_documentation_fails_if_connector_not_already_listed_in_root_readme(
     connectors_directory,
 ):
     name = "Not Listed In Root README"
+    subtype = name.lower().replace(" ", "")
     NotListed = Connector(
         name=name,
         type=ConnectorType.Other,
-        subtype=name.lower().replace(" ", ""),
+        subtype=subtype,
         description=DESCRIPTION,
         url="https://not.listed.in.root.test/",
         actions=[