Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Baris/eng 303 artifact param naming bug #300

Merged
merged 4 commits into from
Jan 12, 2022
Merged

Baris/eng 303 artifact param naming bug #300

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ef86f4f
added param names to the duplicate check
bcdurak Jan 7, 2022
1bc0ec6
Merge remote-tracking branch 'origin/main' into baris/ENG-303-artifac…
bcdurak Jan 7, 2022
d86cd88
added counters
bcdurak Jan 11, 2022
cf94ae8
formatting and mypy issues
bcdurak Jan 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions src/zenml/steps/base_step.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# or implied. See the License for the specific language governing
# permissions and limitations under the License.

import collections
import hashlib
import inspect
import json
Expand All @@ -20,6 +21,7 @@
from typing import (
Any,
ClassVar,
Counter,
Dict,
List,
Optional,
Expand Down Expand Up @@ -135,6 +137,7 @@ def __new__(
)
cls.CONFIG_PARAMETER_NAME = arg
cls.CONFIG_CLASS = arg_type

elif issubclass(arg_type, StepContext):
if cls.CONTEXT_PARAMETER_NAME is not None:
raise StepInterfaceError(
Expand Down Expand Up @@ -163,14 +166,19 @@ def __new__(
# tfx requires them to be unique
# TODO [ENG-155]: Can we prefix inputs and outputs to avoid this
# restriction?
shared_input_output_keys = set(cls.INPUT_SIGNATURE).intersection(
set(cls.OUTPUT_SIGNATURE)
)
if shared_input_output_keys:
counter: Counter[str] = collections.Counter()
counter.update(list(cls.INPUT_SIGNATURE))
counter.update(list(cls.OUTPUT_SIGNATURE))
if cls.CONFIG_CLASS:
counter.update(list(cls.CONFIG_CLASS.__fields__.keys()))

shared_keys = {k for k in counter.elements() if counter[k] > 1}
if shared_keys:
raise StepInterfaceError(
f"There is an overlap in the input and output names of "
f"step '{name}': {shared_input_output_keys}. Please make "
f"sure that your input and output names are distinct."
f"The following keys are overlapping in the input, output and "
f"config parameter names of step '{name}': {shared_keys}. "
f"Please make sure that your input, output and config "
f"parameter names are unique."
)

return cls
Expand Down