Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "respect soft_fail argument when ExternalTaskSensor runs in deferrable mode (#33196)" #33458

Merged
merged 2 commits into from
Aug 22, 2023
Merged

Revert "respect soft_fail argument when ExternalTaskSensor runs in deferrable mode (#33196)" #33458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b69f577
Revert "respect soft_fail argument when ExternalTaskSensor runs in de…
hussein-awala Aug 16, 2023
96034c9
Merge branch 'main' into revert_33196
hussein-awala Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions airflow/sensors/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,12 +339,6 @@ def reschedule(self):
def get_serialized_fields(cls):
return super().get_serialized_fields() | {"reschedule"}

def raise_failed_or_skiping_exception(self, *, failed_message: str, skipping_message: str = "") -> None:
"""Raise AirflowSkipException if self.soft_fail is set to True. Otherwise raise AirflowException."""
if self.soft_fail:
raise AirflowSkipException(skipping_message or failed_message)
raise AirflowException(failed_message)


def poke_mode_only(cls):
"""
Expand Down
46 changes: 20 additions & 26 deletions airflow/sensors/external_task.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,6 @@ def __init__(
self.deferrable = deferrable
self.poll_interval = poll_interval

self._skipping_message_postfix = " Skipping due to soft_fail."

def _get_dttm_filter(self, context):
if self.execution_delta:
dttm = context["logical_date"] - self.execution_delta
Expand Down Expand Up @@ -276,28 +274,32 @@ def poke(self, context: Context, session: Session = NEW_SESSION) -> bool:
# Fail if anything in the list has failed.
if count_failed > 0:
if self.external_task_ids:
failed_message = (
if self.soft_fail:
raise AirflowSkipException(
f"Some of the external tasks {self.external_task_ids} "
f"in DAG {self.external_dag_id} failed. Skipping due to soft_fail."
)
raise AirflowException(
f"Some of the external tasks {self.external_task_ids} "
f"in DAG {self.external_dag_id} failed."
)

self.raise_failed_or_skiping_exception(
failed_message=failed_message,
skipping_message=f"{failed_message}{self._skipping_message_postfix}",
)
elif self.external_task_group_id:
self.raise_failed_or_skiping_exception(
failed_message=(
if self.soft_fail:
raise AirflowSkipException(
f"The external task_group '{self.external_task_group_id}' "
f"in DAG '{self.external_dag_id}' failed."
f"in DAG '{self.external_dag_id}' failed. Skipping due to soft_fail."
)
raise AirflowException(
f"The external task_group '{self.external_task_group_id}' "
f"in DAG '{self.external_dag_id}' failed."
)

else:
failed_message = f"The external DAG {self.external_dag_id} failed."
self.raise_failed_or_skiping_exception(
failed_message=failed_message,
skipping_message=f"{failed_message}{self._skipping_message_postfix}",
)
if self.soft_fail:
raise AirflowSkipException(
f"The external DAG {self.external_dag_id} failed. Skipping due to soft_fail."
)
raise AirflowException(f"The external DAG {self.external_dag_id} failed.")

count_skipped = -1
if self.skipped_states:
Expand Down Expand Up @@ -349,20 +351,12 @@ def execute_complete(self, context, event=None):
self.log.info("External task %s has executed successfully.", self.external_task_id)
return None
elif event["status"] == "timeout":
failed_message = "Dag was not started within 1 minute, assuming fail."
self.raise_failed_or_skiping_exception(
failed_message=failed_message,
skipping_message=f"{failed_message}{self._skipping_message_postfix}",
)
raise AirflowException("Dag was not started within 1 minute, assuming fail.")
else:
failed_message = (
raise AirflowException(
"Error occurred while trying to retrieve task status. Please, check the "
"name of executed task and Dag."
)
self.raise_failed_or_skiping_exception(
failed_message=failed_message,
skipping_message=f"{failed_message}{self._skipping_message_postfix}",
)

def _check_for_existence(self, session) -> None:
dag_to_wait = DagModel.get_current(self.external_dag_id, session)
Expand Down