Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core: replacing all ConnectorModel.name.lower() into ConnectorModel.s… #280

Merged
merged 1 commit into from
Sep 9, 2024
Merged

core: replacing all ConnectorModel.name.lower() into ConnectorModel.s… #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/hrflow_connectors/core/connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,7 @@ def logo(self, connectors_directory: Path) -> str:
"PIL is not found in current environment. Mind that you need to install"
" the package with dev dependencies to use manifest utility"
)
connector_directory = connectors_directory / self.name.lower()
connector_directory = connectors_directory / self.subtype
if not connector_directory.is_dir():
raise ValueError(
"No directory found for connector {} in {}".format(
Expand Down Expand Up @@ -923,7 +923,7 @@ def manifest(self, connectors_directory: Path) -> t.Dict:

jsonmap_path = (
connectors_directory
/ model.name.lower()
/ model.subtype
/ "mappings"
/ "format"
/ "{}.json".format(action.name.value)
Expand Down
8 changes: 4 additions & 4 deletions src/hrflow_connectors/core/documentation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
model = connector.model
result = subprocess.run(
GIT_UPDATE_DATE.format(
connector=model.name.lower(),
connector=model.subtype,
base_connector_path=BASE_CONNECTOR_PATH.get().rstrip("/"),
),
shell=True,
Expand All @@ -204,7 +204,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
if result.stderr:
raise Exception(
"Subprocess run for Git update dates failed for connector {} with"
" errors {}".format(model.name.lower(), result.stderr)
" errors {}".format(model.subtype, result.stderr)
)
filtered = [
line.split(" ")[0]
Expand Down Expand Up @@ -253,7 +253,7 @@ def update_root_readme(connectors: t.List[Connector], root: Path) -> t.Dict:
name=match.group("name"),
readme_link="./{base_connector_path}/{connector}/README.md".format(
base_connector_path=BASE_CONNECTOR_PATH.get().strip("/"),
connector=model.name.lower(),
connector=model.subtype,
),
type=model.type.value,
release_date=match.group("release_date"),
Expand Down Expand Up @@ -283,7 +283,7 @@ def generate_docs(
)
for connector in connectors:
model = connector.model
connector_directory = connectors_directory / model.name.lower()
connector_directory = connectors_directory / model.subtype
if not connector_directory.is_dir():
logging.error(
"Skipping documentation for {}: no directory found at {}".format(
Expand Down
20 changes: 9 additions & 11 deletions src/hrflow_connectors/core/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class InvalidTestConfigException(Exception):


@contextmanager
def secrets(connector_name: str, connectors_directory: Path):
def secrets(connector_name: str, connector_subtype: str, connectors_directory: Path):
secrets_prefix = ENVIRON_SECRETS_PREFIX.format(
connector_name=connector_name.upper()
)
Expand All @@ -63,9 +63,7 @@ def secrets(connector_name: str, connectors_directory: Path):
else:
global_secrets = dict()

connector_secrets_file = (
connectors_directory / connector_name.lower() / "secrets.json"
)
connector_secrets_file = connectors_directory / connector_subtype / "secrets.json"
if connector_secrets_file.exists():
try:
connector_secrets = json.loads(connector_secrets_file.read_text())
Expand Down Expand Up @@ -97,16 +95,16 @@ def actions(connector: Connector):


@contextmanager
def warehouses(connector_name: str, connectors_directory: Path):
def warehouses(connector_subtype: str, connectors_directory: Path):
if connectors_directory is CONNECTORS_DIRECTORY: # pragma: no cover
warehouse_module = import_module(
"hrflow_connectors.connectors.{}.warehouse".format(connector_name.lower())
"hrflow_connectors.connectors.{}.warehouse".format(connector_subtype)
)
else:
import_from = connectors_directory.relative_to(PROJECT_DIRECTORY)
warehouse_module = import_module(
"{}.{}.warehouse".format(
str(import_from).replace("/", "."), connector_name.lower()
str(import_from).replace("/", "."), connector_subtype
)
)
warehouse_names = []
Expand Down Expand Up @@ -223,9 +221,8 @@ def collect_connector_tests(
connector: Connector, connectors_directory: Path = CONNECTORS_DIRECTORY
):
connector_name = connector.model.name
test_config_file = (
connectors_directory / connector_name.lower() / "test-config.yaml"
)
connector_subtype = connector.model.subtype
test_config_file = connectors_directory / connector_subtype / "test-config.yaml"
if test_config_file.exists() is False:
raise NoTestConfigException(
"No test configuration found for connector {} at {}".format(
Expand All @@ -243,14 +240,15 @@ def collect_connector_tests(

try:
with warehouses(
connector_name=connector_name,
connector_subtype=connector_subtype,
connectors_directory=connectors_directory,
):
with actions(
connector=connector,
):
with secrets(
connector_name=connector_name,
connector_subtype=connector_subtype,
connectors_directory=connectors_directory,
):
test_suite = ConnectorTestConfig(**test_config)
Expand Down
88 changes: 39 additions & 49 deletions tests/core/test_documentation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,20 +111,18 @@ def connectors_directory(root_readme: Path):
except FileNotFoundError:
pass

readme = path / SmartLeads.model.name.lower() / "README.md"
notebooks_directory = path / SmartLeads.model.name.lower() / "notebooks"
readme = path / SmartLeads.model.subtype / "README.md"
notebooks_directory = path / SmartLeads.model.subtype / "notebooks"
keep_empty_notebooks_file = (
path / SmartLeads.model.name.lower() / "notebooks" / KEEP_EMPTY_FOLDER
path / SmartLeads.model.subtype / "notebooks" / KEEP_EMPTY_FOLDER
)
notebook = notebooks_directory / NOTEBOOKS_FILE
mappings_directory = path / SmartLeads.model.name.lower() / "mappings"
format_mappings_directory = (
path / SmartLeads.model.name.lower() / "mappings" / "format"
)
mappings_directory = path / SmartLeads.model.subtype / "mappings"
format_mappings_directory = path / SmartLeads.model.subtype / "mappings" / "format"
keep_empty_format_file = format_mappings_directory / KEEP_EMPTY_FOLDER
format_file = format_mappings_directory / FORMAT_FILE

actions_documentation_directory = path / SmartLeads.model.name.lower() / "docs"
actions_documentation_directory = path / SmartLeads.model.subtype / "docs"
action_documentation = actions_documentation_directory / "{}.md".format(
SmartLeads.model.actions[0].name.value
)
Expand Down Expand Up @@ -153,29 +151,27 @@ def connectors_directory(root_readme: Path):


def test_documentation(connectors_directory):
readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
notebooks_directory = (
connectors_directory / SmartLeads.model.name.lower() / "notebooks"
)
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
keep_empty_notebooks_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "notebooks"
/ KEEP_EMPTY_FOLDER
)
format_mappings_directory = (
connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
)
keep_empty_format_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "mappings"
/ "format"
/ KEEP_EMPTY_FOLDER
)
action_documentation = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "docs"
/ "{}.md".format(SmartLeads.model.actions[0].name.value)
)
Expand All @@ -202,12 +198,10 @@ def test_documentation(connectors_directory):
def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
connectors_directory,
):
notebooks_directory = (
connectors_directory / SmartLeads.model.name.lower() / "notebooks"
)
notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
keep_empty_notebooks_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "notebooks"
/ KEEP_EMPTY_FOLDER
)
Expand All @@ -224,10 +218,10 @@ def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
assert notebooks_directory.exists() is True
assert keep_empty_notebooks_file.exists() is True

readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
action_documentation = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "docs"
/ "{}.md".format(SmartLeads.model.actions[0].name.value)
)
Expand All @@ -238,12 +232,10 @@ def test_documentation_adds_keep_empty_notebooks_file_if_folder_is_empty(
def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_other_files(
connectors_directory,
):
notebooks_directory = (
connectors_directory / SmartLeads.model.name.lower() / "notebooks"
)
notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
keep_empty_notebooks_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "notebooks"
/ KEEP_EMPTY_FOLDER
)
Expand All @@ -264,10 +256,10 @@ def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_othe
assert other.exists() is True
assert keep_empty_notebooks_file.exists() is False

readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
action_documentation = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "docs"
/ "{}.md".format(SmartLeads.model.actions[0].name.value)
)
Expand All @@ -278,12 +270,10 @@ def test_documentation_does_not_add_keep_empty_notebooks_file_if_folder_has_othe
def test_documentation_removes_keep_empty_notebooks_file_if_folder_has_other_files(
connectors_directory,
):
notebooks_directory = (
connectors_directory / SmartLeads.model.name.lower() / "notebooks"
)
notebooks_directory = connectors_directory / SmartLeads.model.subtype / "notebooks"
keep_empty_notebooks_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "notebooks"
/ KEEP_EMPTY_FOLDER
)
Expand All @@ -305,10 +295,10 @@ def test_documentation_removes_keep_empty_notebooks_file_if_folder_has_other_fil
assert other.exists() is True
assert keep_empty_notebooks_file.exists() is False

readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
action_documentation = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "docs"
/ "{}.md".format(SmartLeads.model.actions[0].name.value)
)
Expand All @@ -320,11 +310,11 @@ def test_documentation_adds_keep_empty_format_file_if_folder_is_empty(
connectors_directory,
):
format_mappings_directory = (
connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
)
keep_empty_format_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "mappings"
/ "format"
/ KEEP_EMPTY_FOLDER
Expand All @@ -347,11 +337,11 @@ def test_documentation_does_not_add_keep_empty_format_file_if_folder_has_other_f
connectors_directory,
):
format_mappings_directory = (
connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
)
keep_empty_format_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "mappings"
/ "format"
/ KEEP_EMPTY_FOLDER
Expand All @@ -377,11 +367,11 @@ def test_documentation_removes_keep_empty_format_file_if_folder_has_other_files(
connectors_directory,
):
format_mappings_directory = (
connectors_directory / SmartLeads.model.name.lower() / "mappings" / "format"
connectors_directory / SmartLeads.model.subtype / "mappings" / "format"
)
keep_empty_format_file = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "mappings"
/ "format"
/ KEEP_EMPTY_FOLDER
Expand All @@ -406,7 +396,7 @@ def test_documentation_removes_keep_empty_format_file_if_folder_has_other_files(


def test_documentation_fails_if_actions_section_not_found(connectors_directory):
readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
with patched_subprocess():
generate_docs(
connectors=[SmartLeads], connectors_directory=connectors_directory
Expand Down Expand Up @@ -584,10 +574,10 @@ def test_main_readme_update_at_helper_doesnt_override_handwritten_updated_at(


def test_documentation_with_remote_code_links(connectors_directory):
readme = connectors_directory / SmartLeads.model.name.lower() / "README.md"
readme = connectors_directory / SmartLeads.model.subtype / "README.md"
action_documentation = (
connectors_directory
/ SmartLeads.model.name.lower()
/ SmartLeads.model.subtype
/ "docs"
/ "{}.md".format(SmartLeads.model.actions[0].name.value)
)
Expand Down Expand Up @@ -641,10 +631,11 @@ def test_documentation_with_remote_code_links(connectors_directory):

def test_documentation_connector_directory_not_found(caplog, connectors_directory):
mismatch_name = "NoConnectorDir"
subtype = mismatch_name.lower().replace(" ", "")
NameMismatchSmartLeads = Connector(
name=mismatch_name,
type=ConnectorType.Other,
subtype=mismatch_name.lower().replace(" ", ""),
subtype=subtype,
description=DESCRIPTION,
url="https://www.smartleads.test/",
actions=[
Expand All @@ -660,12 +651,10 @@ def test_documentation_connector_directory_not_found(caplog, connectors_director
],
)

readme = (
connectors_directory / NameMismatchSmartLeads.model.name.lower() / "README.md"
)
readme = connectors_directory / NameMismatchSmartLeads.model.subtype / "README.md"
action_documentation = (
connectors_directory
/ NameMismatchSmartLeads.model.name.lower()
/ NameMismatchSmartLeads.model.subtype
/ "docs"
/ "{}.md".format(NameMismatchSmartLeads.model.actions[0].name.value)
)
Expand Down Expand Up @@ -715,10 +704,11 @@ def test_documentation_fails_if_connector_not_already_listed_in_root_readme(
connectors_directory,
):
name = "Not Listed In Root README"
subtype = name.lower().replace(" ", "")
NotListed = Connector(
name=name,
type=ConnectorType.Other,
subtype=name.lower().replace(" ", ""),
subtype=subtype,
description=DESCRIPTION,
url="https://not.listed.in.root.test/",
actions=[
Expand Down
Loading
Loading