Merge pull request #12 from airflow-laminar/tkp/config

add documentation

.github/workflows/docs.yml

@@ -0,0 +1,24 @@
+name: Docs
+on:
+  push:
+    branches:
+      - main
+permissions:
+  contents: write
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - run: pip install .
+      - run: pip install yardang
+      - run: yardang build
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/html

.gitignore

@@ -126,9 +126,12 @@ dmypy.json
 
 # Documentation
 docs/_build/
+docs/src/_build/
 /site
 docs/api
 docs/index.md
+docs/html
+index.md
 _template/labextension
 
 # Jupyter

airflow_priority/plugins/aws.py

@@ -1,21 +1,28 @@
 import os
 import sys
 from datetime import datetime
+from functools import lru_cache
+from logging import getLogger
 
 from airflow.listeners import hookimpl
 from airflow.models.dagrun import DagRun
 from airflow.plugins_manager import AirflowPlugin
 from boto3 import client as Boto3Client
 from botocore.config import Config
 
-from airflow_priority import DagStatus, has_priority_tag
+from airflow_priority import AirflowPriorityConfigurationOptionNotFound, DagStatus, has_priority_tag
 
-config = Config(retries=dict(max_attempts=10, mode="adaptive"))
-client = Boto3Client("cloudwatch", config=config)
+_log = getLogger(__name__)
+
+
+@lru_cache
+def get_client():
+    config = Config(retries=dict(max_attempts=10, mode="adaptive"))
+    return Boto3Client("cloudwatch", config=config)
 
 
 def send_metric_cloudwatch(dag_id: str, priority: int, tag: DagStatus) -> None:
-    client.put_metric_data(
+    get_client().put_metric_data(
         Namespace="Airflow/Custom",
         MetricData=[
             {
@@ -60,6 +67,13 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
         send_metric_cloudwatch(dag_id, priority, "failed")
 
 
-class AWSCloudWatchPriorityPlugin(AirflowPlugin):
-    name = "AWSCloudWatchPriorityPlugin"
-    listeners = [sys.modules[__name__]]
+try:
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_client()
+
+    class AWSCloudWatchPriorityPlugin(AirflowPlugin):
+        name = "AWSCloudWatchPriorityPlugin"
+        listeners = [sys.modules[__name__]]
+except AirflowPriorityConfigurationOptionNotFound:
+    _log.exception("Plugin could not be enabled")

airflow_priority/plugins/datadog.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from datetime import datetime
 from functools import lru_cache
@@ -86,8 +87,9 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
 
 
 try:
-    # Call once to ensure plugin will work
-    get_config_option("datadog", "api_key")
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_config_option("datadog", "api_key")
 
     class DatadogPriorityPlugin(AirflowPlugin):
         name = "DatadogPriorityPlugin"

airflow_priority/plugins/discord.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from asyncio import sleep
 from functools import lru_cache
@@ -66,9 +67,10 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
 
 
 try:
-    # Call once to ensure plugin will work
-    get_config_option("discord", "channel")
-    get_config_option("discord", "token")
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_config_option("discord", "channel")
+        get_config_option("discord", "token")
 
     class DiscordPriorityPlugin(AirflowPlugin):
         name = "DiscordPriorityPlugin"

airflow_priority/plugins/newrelic.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from functools import lru_cache
 from logging import getLogger
@@ -54,8 +55,9 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
 
 
 try:
-    # Call once to ensure plugin will work
-    get_config_option("newrelic", "api_key")
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_config_option("newrelic", "api_key")
 
     class NewRelicPriorityPlugin(AirflowPlugin):
         name = "NewRelicPriorityPlugin"

airflow_priority/plugins/slack.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from functools import lru_cache
 from logging import getLogger
@@ -57,9 +58,10 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
 
 
 try:
-    # Call once to ensure plugin will work
-    get_config_option("slack", "token")
-    get_config_option("slack", "channel")
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_config_option("slack", "token")
+        get_config_option("slack", "channel")
 
     class SlackPriorityPlugin(AirflowPlugin):
         name = "SlackPriorityPlugin"

airflow_priority/plugins/symphony.py

@@ -1,3 +1,4 @@
+import os
 import ssl
 import sys
 from functools import lru_cache
@@ -97,8 +98,9 @@ def on_dag_run_failed(dag_run: DagRun, msg: str):
 
 
 try:
-    # Call once to ensure plugin will work
-    get_config_options()
+    if os.environ.get("SPHINX_BUILDING", "0") != "1":
+        # Call once to ensure plugin will work
+        get_config_options()
 
     class SymphonyPriorityPlugin(AirflowPlugin):
         name = "SymphonyPriorityPlugin"

airflow_priority/tests/test_airflow_config.py

@@ -40,8 +40,3 @@ def test_call_plugin_via_airflow_config(airflow_config, dag_run):
     client = get_client()
     assert client is not None
     assert client.token == "abc"
-    # with patch("airflow_priority.plugins.slack.send_metric_slack") as p2, \
-    #      patch("airflow_priority.plugins.slack.send_metric_slack") as p1:
-    #     on_dag_run_failed(dag_run, "test")
-
-    #     assert p1.call_args == []

docs/src/API.md

@@ -0,0 +1,14 @@
+# API Reference
+
+```{eval-rst}
+.. autosummary::
+   :toctree: _build
+
+    airflow_priority.common
+    airflow_priority.plugins.aws
+    airflow_priority.plugins.discord
+    airflow_priority.plugins.newrelic
+    airflow_priority.plugins.slack
+    airflow_priority.plugins.symphony
+
+```

pyproject.toml

@@ -107,6 +107,7 @@ ignore = [
     ".copier-answers.yml",
     "Makefile",
     "setup.py",
+    "docs/*/*",
 ]
 
 [tool.hatch.build]
@@ -125,6 +126,7 @@ exclude = [
     "/.github",
     "/.gitattributes",
     "/.gitignore",
+    "/docs",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -136,6 +138,7 @@ exclude = [
     "/.gitattributes",
     "/.gitignore",
     "/pyproject.toml",
+    "/docs",
 ]
 
 [tool.hatch.build.targets.wheel.shared-data]
@@ -160,3 +163,9 @@ section-order = [
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401", "F403"]
+
+[tool.yardang]
+title = "airflow-priority"
+root = "README.md"
+pages = ["docs/src/API.md"]
+use-autoapi = false