Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add monitoring for APIs #98

Merged
merged 6 commits into from
Nov 9, 2023
Merged

Add monitoring for APIs #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e16fbf4
Add monitoring for APIs
sunank200 Nov 8, 2023
63cf506
add comment as per Wei's PR review
sunank200 Nov 9, 2023
b619502
Add check for base url in the DAG
sunank200 Nov 9, 2023
f24ec74
Take the weaviate class name from environment variable
sunank200 Nov 9, 2023
c623414
Change to throw error logs as to what broke in slack as well
sunank200 Nov 9, 2023
8c3a8dc
Format the error on slack
sunank200 Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# syntax=quay.io/astronomer/airflow-extensions:latest

FROM quay.io/astronomer/astro-runtime:9.3.0-base
FROM quay.io/astronomer/astro-runtime:9.5.0-base

COPY include/airflow_provider_weaviate-0.0.1-py3-none-any.whl /tmp
3 changes: 2 additions & 1 deletion airflow/dags/ingestion/ask-astro-load-blogs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")

blog_cutoff_date = datetime.strptime("2023-01-19", "%Y-%m-%d")

Expand All @@ -30,7 +31,7 @@ def ask_astro_load_blogs():
weaviate_conn_id=_WEAVIATE_CONN_ID,
retries=10,
retry_delay=30,
).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])


ask_astro_load_blogs()
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
_GITHUB_CONN_ID = "github_ro"

WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
markdown_docs_sources = [
{"doc_dir": "learn", "repo_base": "astronomer/docs"},
{"doc_dir": "astro", "repo_base": "astronomer/docs"},
Expand Down Expand Up @@ -71,7 +71,7 @@ def ask_astro_load_github():
weaviate_conn_id=_WEAVIATE_CONN_ID,
retries=10,
retry_delay=30,
).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])

issues_docs >> md_docs >> rst_docs >> code_samples

Expand Down
3 changes: 2 additions & 1 deletion airflow/dags/ingestion/ask-astro-load-registry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")


@dag(schedule_interval="0 5 * * *", start_date=datetime(2023, 9, 27), catchup=False, is_paused_upon_creation=True)
Expand All @@ -32,7 +33,7 @@ def ask_astro_load_registry():
weaviate_conn_id=_WEAVIATE_CONN_ID,
retries=10,
retry_delay=30,
).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])
).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs, split_code_docs])


ask_astro_load_registry()
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-slack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"

WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
slack_channel_sources = [
{
"channel_name": "troubleshooting",
Expand Down Expand Up @@ -38,7 +38,7 @@ def ask_astro_load_slack():
weaviate_conn_id=_WEAVIATE_CONN_ID,
retries=10,
retry_delay=30,
).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])


ask_astro_load_slack()
4 changes: 2 additions & 2 deletions airflow/dags/ingestion/ask-astro-load-stackoverflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
ask_astro_env = os.environ.get("ASK_ASTRO_ENV", "")

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"

WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")
stackoverflow_cutoff_date = "2021-09-01"

stackoverflow_tags = [
Expand Down Expand Up @@ -38,7 +38,7 @@ def ask_astro_load_stackoverflow():
weaviate_conn_id=_WEAVIATE_CONN_ID,
retries=10,
retry_delay=30,
).partial(class_name="Docs", primary_key="docLink").expand(dfs=[split_md_docs])
).partial(class_name=WEAVIATE_CLASS, primary_key="docLink").expand(dfs=[split_md_docs])


ask_astro_load_stackoverflow()
7 changes: 4 additions & 3 deletions airflow/dags/ingestion/ask-astro-load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

_WEAVIATE_CONN_ID = f"weaviate_{ask_astro_env}"
_GITHUB_CONN_ID = "github_ro"
WEAVIATE_CLASS = os.environ.get("WEAVIATE_CLASS", "DocsProd")

markdown_docs_sources = [
{"doc_dir": "learn", "repo_base": "astronomer/docs"},
Expand Down Expand Up @@ -83,7 +84,7 @@ def ask_astro_load_bulk():
task_id="create_schema",
weaviate_conn_id=_WEAVIATE_CONN_ID,
class_object_data="file://include/data/schema.json",
existing="fail",
existing="ignore",
)

@task.branch(trigger_rule="none_failed")
Expand Down Expand Up @@ -238,12 +239,12 @@ def extract_astro_blogs():
split_code_docs = task(split.split_python).expand(dfs=python_code_tasks)

task.weaviate_import(ingest.import_data, weaviate_conn_id=_WEAVIATE_CONN_ID, retries=10, retry_delay=30).partial(
class_name="Docs"
class_name=WEAVIATE_CLASS
).expand(dfs=[split_md_docs, split_code_docs])

_import_baseline = task.weaviate_import(
ingest.import_baseline, trigger_rule="none_failed", weaviate_conn_id=_WEAVIATE_CONN_ID
)(class_name="Docs", seed_baseline_url=seed_baseline_url)
)(class_name=WEAVIATE_CLASS, seed_baseline_url=seed_baseline_url)

_check_schema >> [_check_seed_baseline, _create_schema]

Expand Down
Loading
Loading