Skip to content

Commit

Permalink
Add docstrings to new functions
Browse files Browse the repository at this point in the history
  • Loading branch information
jmelot committed Aug 28, 2023
1 parent 074f306 commit a14a638
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 0 deletions.
15 changes: 15 additions & 0 deletions airtable_to_bq.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@


def update_staging(dag: DAG, start_task, config: dict):
"""
Update staging tables
:param dag: Dag to add tasks to
:param start_task: Task that should happen before the first task generated by this function
:param config: Task configuration
:return: Final task defined in this function
"""
bucket = DEV_DATA_BUCKET
name = config["name"]
sql_dir = f"sql/{DATASET}/{config.get('parent_name', name)}"
Expand Down Expand Up @@ -142,6 +149,14 @@ def update_staging(dag: DAG, start_task, config: dict):


def update_production(dag: DAG, start_task, end_task, config: dict) -> None:
"""
Run production table update and backup tasks
:param dag: Dag to run tasks within
:param start_task: Task that should happen before production update
:param end_task: Task that should happen after backup
:param config: Task configuration
:return: None
"""
STAGING_DATASET = f"staging_{DATASET}"
name = config["name"]
with dag:
Expand Down
8 changes: 8 additions & 0 deletions bq_to_airtable.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@


def update_airtable(dag: DAG, start_task, end_task, config: dict):
"""
Generates tasks needed to move data from BQ to airtable
:param dag: Dag we are adding tasks to
:param start_task: Task that should run before any generated by this function
:param end_task: Task that should run after any generated by this function except the `add_to_airtable` task
:param config: Task configuration
:return: The `add_to_airtable`, to be executed in series downstream
"""
bucket = DEV_DATA_BUCKET
name = config["name"]
sql_dir = f"sql/{DATASET}/{config.get('parent_name', name)}"
Expand Down

0 comments on commit a14a638

Please sign in to comment.