Skip to content

Commit

Permalink
Fix comments from Wei
Browse files Browse the repository at this point in the history
  • Loading branch information
sunank200 committed Nov 20, 2023
1 parent 090cced commit d5b80c4
Show file tree
Hide file tree
Showing 10 changed files with 18 additions and 15 deletions.
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-airflow-docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
)
def ask_astro_load_airflow_docs():
"""
This DAG performs incremental load for any new Airflow docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
This DAG performs incremental load for any new Airflow docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
any existing documents that have been updated will be removed and re-added.
"""

Expand Down
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-blogs.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
)
def ask_astro_load_blogs():
"""
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
any existing documents that have been updated will be removed and re-added.
"""

Expand Down
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-github.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@
)
def ask_astro_load_github():
"""
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
any existing documents that have been updated will be removed and re-added.
"""

Expand Down
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
)
def ask_astro_load_registry():
"""
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
any existing documents that have been updated will be removed and re-added.
"""

Expand Down
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-slack.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
)
def ask_astro_load_slack():
"""
This DAG performs incremental load for any new slack threads. The slack archive is a point-in-time capture. This
DAG should run nightly to capture threads between archive periods. By using the upsert logic of the
This DAG performs incremental load for any new slack threads. The slack archive is a point-in-time capture. This
DAG should run nightly to capture threads between archive periods. By using the upsert logic of the
weaviate_import decorator any existing documents that have been updated will be removed and re-added.
"""

Expand Down
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-stackoverflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
)
def ask_astro_load_stackoverflow():
"""
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported
data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator
any existing documents that have been updated will be removed and re-added.
"""

Expand Down
2 changes: 1 addition & 1 deletion airflow/dags/ingestion/ask-astro-load.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

seed_baseline_url = None
stackoverflow_cutoff_date = "2021-09-01"
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "dev")

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
_GITHUB_CONN_ID = "github_ro"
Expand Down
Binary file not shown.
2 changes: 0 additions & 2 deletions airflow/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#/tmp/airflow_provider_weaviate-0.0.1-py3-none-any.whl
apache-airflow-providers-weaviate==1.0.0
apache-airflow-providers-github==2.3.1
apache-airflow-providers-slack==7.3.2
Expand All @@ -15,4 +14,3 @@ bs4==0.0.1
lxml==4.9.3
tiktoken==0.5.1
firebase-admin==6.2.0
apache-airflow-providers-weaviate
5 changes: 5 additions & 0 deletions airflow/ruff.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
line-length = 100

[format]
quote-style = "single"
indent-style = "tab"

0 comments on commit d5b80c4

Please sign in to comment.