diff --git a/airflow/dags/ingestion/ask-astro-load-airflow-docs.py b/airflow/dags/ingestion/ask-astro-load-airflow-docs.py index a7c41adb..be2ccb23 100644 --- a/airflow/dags/ingestion/ask-astro-load-airflow-docs.py +++ b/airflow/dags/ingestion/ask-astro-load-airflow-docs.py @@ -27,8 +27,8 @@ ) def ask_astro_load_airflow_docs(): """ - This DAG performs incremental load for any new Airflow docs. Initial load via ask_astro_load_bulk imported - data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator + This DAG performs incremental load for any new Airflow docs. Initial load via ask_astro_load_bulk imported + data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load-blogs.py b/airflow/dags/ingestion/ask-astro-load-blogs.py index ebee720b..53c40b94 100644 --- a/airflow/dags/ingestion/ask-astro-load-blogs.py +++ b/airflow/dags/ingestion/ask-astro-load-blogs.py @@ -27,8 +27,8 @@ ) def ask_astro_load_blogs(): """ - This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported - data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator + This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported + data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load-github.py b/airflow/dags/ingestion/ask-astro-load-github.py index 9d610e0b..6170996a 100644 --- a/airflow/dags/ingestion/ask-astro-load-github.py +++ b/airflow/dags/ingestion/ask-astro-load-github.py @@ -43,8 +43,8 @@ ) def ask_astro_load_github(): """ - This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported - data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator + This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported + data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load-registry.py b/airflow/dags/ingestion/ask-astro-load-registry.py index e41b0a6d..7ef40792 100644 --- a/airflow/dags/ingestion/ask-astro-load-registry.py +++ b/airflow/dags/ingestion/ask-astro-load-registry.py @@ -25,8 +25,8 @@ ) def ask_astro_load_registry(): """ - This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported - data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator + This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported + data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load-slack.py b/airflow/dags/ingestion/ask-astro-load-slack.py index 93e56b3f..f4a496fe 100644 --- a/airflow/dags/ingestion/ask-astro-load-slack.py +++ b/airflow/dags/ingestion/ask-astro-load-slack.py @@ -34,8 +34,8 @@ ) def ask_astro_load_slack(): """ - This DAG performs incremental load for any new slack threads. The slack archive is a point-in-time capture. This - DAG should run nightly to capture threads between archive periods. By using the upsert logic of the + This DAG performs incremental load for any new slack threads. The slack archive is a point-in-time capture. This + DAG should run nightly to capture threads between archive periods. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load-stackoverflow.py b/airflow/dags/ingestion/ask-astro-load-stackoverflow.py index 6b97bf03..105ed1f7 100644 --- a/airflow/dags/ingestion/ask-astro-load-stackoverflow.py +++ b/airflow/dags/ingestion/ask-astro-load-stackoverflow.py @@ -30,8 +30,8 @@ ) def ask_astro_load_stackoverflow(): """ - This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported - data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator + This DAG performs incremental load for any new docs. Initial load via ask_astro_load_bulk imported + data from a point-in-time data capture. By using the upsert logic of the weaviate_import decorator any existing documents that have been updated will be removed and re-added. """ diff --git a/airflow/dags/ingestion/ask-astro-load.py b/airflow/dags/ingestion/ask-astro-load.py index 32695dfa..97eca3f5 100644 --- a/airflow/dags/ingestion/ask-astro-load.py +++ b/airflow/dags/ingestion/ask-astro-load.py @@ -13,7 +13,7 @@ seed_baseline_url = None stackoverflow_cutoff_date = "2021-09-01" -ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "") +ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "dev") _WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}" _GITHUB_CONN_ID = "github_ro" diff --git a/airflow/include/airflow_provider_weaviate-0.0.1-py3-none-any.whl b/airflow/include/airflow_provider_weaviate-0.0.1-py3-none-any.whl deleted file mode 100644 index 20060ccb..00000000 Binary files a/airflow/include/airflow_provider_weaviate-0.0.1-py3-none-any.whl and /dev/null differ diff --git a/airflow/requirements.txt b/airflow/requirements.txt index a1f6f922..04700214 100644 --- a/airflow/requirements.txt +++ b/airflow/requirements.txt @@ -1,4 +1,3 @@ -#/tmp/airflow_provider_weaviate-0.0.1-py3-none-any.whl apache-airflow-providers-weaviate==1.0.0 apache-airflow-providers-github==2.3.1 apache-airflow-providers-slack==7.3.2 @@ -15,4 +14,3 @@ bs4==0.0.1 lxml==4.9.3 tiktoken==0.5.1 firebase-admin==6.2.0 -apache-airflow-providers-weaviate diff --git a/airflow/ruff.toml b/airflow/ruff.toml new file mode 100644 index 00000000..424b740e --- /dev/null +++ b/airflow/ruff.toml @@ -0,0 +1,5 @@ +line-length = 100 + +[format] +quote-style = "single" +indent-style = "tab"