Skip to content

Commit

Permalink
Use pendulum instead of datetime for a DAG's start_date
Browse files Browse the repository at this point in the history
This aligns our DAGs with the Airflow example DAGs. See apache/airflow#21646.
  • Loading branch information
hammerhead committed Apr 1, 2022
1 parent 3634749 commit 43181b1
Show file tree
Hide file tree
Showing 6 changed files with 12 additions and 13 deletions.
4 changes: 2 additions & 2 deletions dags/data_retention_delete_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
In CrateDB, tables for storing retention policies need to be created once manually.
See the file setup/data_retention_schema.sql in this repository.
"""
import datetime
import json
import logging
from pathlib import Path
import pendulum
from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.providers.postgres.hooks.postgres import PostgresHook
Expand Down Expand Up @@ -62,7 +62,7 @@ def delete_partitions(ti):

with DAG(
dag_id="data-retention-delete-dag",
start_date=datetime.datetime(2021, 11, 19),
start_date=pendulum.datetime(2021, 11, 19, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down
4 changes: 2 additions & 2 deletions dags/data_retention_reallocate_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
In CrateDB, tables for storing retention policies need to be created once manually.
See the file setup/data_retention_schema.sql in this repository.
"""
import datetime
import json
import logging
from pathlib import Path
import pendulum
from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.providers.postgres.hooks.postgres import PostgresHook
Expand Down Expand Up @@ -85,7 +85,7 @@ def reallocate_partitions(ti):

with DAG(
dag_id="data-retention-reallocate-dag",
start_date=datetime.datetime(2021, 11, 19),
start_date=pendulum.datetime(2021, 11, 19, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down
4 changes: 2 additions & 2 deletions dags/data_retention_snapshot_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
In CrateDB, tables for storing retention policies need to be created once manually.
See the file setup/data_retention_schema.sql in this repository.
"""
import datetime
import json
import logging
from pathlib import Path
import pendulum
from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.providers.postgres.hooks.postgres import PostgresHook
Expand Down Expand Up @@ -77,7 +77,7 @@ def snapshot_partitions(ti):

with DAG(
dag_id="data-retention-snapshot-dag",
start_date=datetime.datetime(2022, 1, 31),
start_date=pendulum.datetime(2022, 1, 31, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down
3 changes: 2 additions & 1 deletion dags/financial_data_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import math
import json
import logging
import pendulum
import requests
from bs4 import BeautifulSoup
import yfinance as yf
Expand Down Expand Up @@ -109,7 +110,7 @@ def format_and_insert_data_function(ti):

with DAG(
dag_id="financial_data_dag",
start_date=datetime.datetime(2022, 1, 10),
start_date=pendulum.datetime(2022, 1, 10, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down
5 changes: 2 additions & 3 deletions dags/nyc_taxi_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@
You can retrieve the CREATE TABLE statements from the file setup/taxi-schema.sql
in this repository.
"""
import datetime
import logging

from pathlib import Path
import pendulum
from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.providers.postgres.hooks.postgres import PostgresHook
Expand Down Expand Up @@ -86,7 +85,7 @@ def process_new_files(ti):

with DAG(
dag_id="nyc-taxi",
start_date=datetime.datetime(2021, 11, 11),
start_date=pendulum.datetime(2021, 11, 11, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down
5 changes: 2 additions & 3 deletions dags/table_export_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
A detailed tutorial is available at https://community.crate.io/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901
"""
import datetime
import os

import pendulum
from airflow import DAG
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.utils.task_group import TaskGroup
Expand All @@ -16,7 +15,7 @@

with DAG(
dag_id="cratedb_table_export",
start_date=datetime.datetime(2021, 11, 11),
start_date=pendulum.datetime(2021, 11, 11, tz="UTC"),
schedule_interval="@daily",
catchup=False,
) as dag:
Expand Down

0 comments on commit 43181b1

Please sign in to comment.