starting exploring the use of prefect server

.github/actions/docker__start-prefect-server/action.yml

@@ -0,0 +1,31 @@
+name: "docker__start-lambda-docker"
+description: "Start Lambda Docker"
+
+runs:
+  using: "composite"
+  steps:
+    # Set up Docker Buildx for better build performance and features
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+
+#    # Install Docker Compose
+#    - name: Install Docker Compose
+#      run: sudo apt-get update && sudo apt-get install -y docker-compose
+
+    # Navigate to the docker-compose file location and start the services
+    - name: Start Docker Compose
+      shell: bash
+      working-directory: ./deploy/docker/prefect-server-v2
+      run: | 
+        docker-compose up -d
+
+#    # Wait for services to be up and running
+#    - name: Wait for services
+#      run: |
+#        echo "Waiting for services to be up..."
+#        sleep 30
+
+#    # Test Prefect service to see if it's running
+#    - name: Test Prefect API
+#      run: curl --retry 5 --retry-delay 10 -v http://localhost:4200/api/health

.github/workflows/ci-pipeline__dev.yml

@@ -22,9 +22,28 @@ env:
   PREFECT_CLOUD__WORKSPACE_ID: ${{ secrets.PREFECT_CLOUD__WORKSPACE_ID }}
 
 jobs:
+  start-prefect-server:
+    name: "Start Prefect Server"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start Prefect Server
+        uses: ./.github/actions/docker__start-prefect-server
+
+      - name: Install Dependencies
+        uses: owasp-sbot/OSBot-GitHub-Actions/.github/actions/poetry__install@dev
+        with:
+          pip_install: '-r requirements-test.txt'
+
+      - name: Run Unit Tests
+        uses: owasp-sbot/OSBot-GitHub-Actions/.github/actions/poetry__run-unit-tests@dev
+        with:
+          test_target: 'tests/integration/observability'
+
 
   run-unit-and-integration-tests:
-    #if: False
+    if: False
     name: "Run Unit & Integration Tests"
     runs-on: ubuntu-latest
     steps:

deploy/docker/osbot-serverless-flows/docker-compose.yml

@@ -9,8 +9,5 @@ services:
     container_name: osbot-serverless-flows-web
     ports:
       - "5002:8080"
-#    env_file:
-#      - .env
-
 
 #curl "http://localhost:5002/2015-03-31/functions/function/invocations" -d '{}'

deploy/docker/prefect-server-v2/docker-compose.yml

@@ -0,0 +1,25 @@
+services:
+  prefect:
+    image: prefecthq/prefect:2-latest
+    container_name: prefect-orion
+    environment:
+      - PREFECT_API_HOST=0.0.0.0
+      - PREFECT_API_PORT=4200
+    ports:
+      - "4200:4200"
+    command: prefect server start --host 0.0.0.0 --port 4200
+
+  postgres:
+    image: postgres:13
+    container_name: prefect-postgres
+    environment:
+      POSTGRES_USER: prefect
+      POSTGRES_PASSWORD: prefect
+      POSTGRES_DB: prefect_db
+    volumes:
+      - prefect_postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+volumes:
+  prefect_postgres_data:

osbot_serverless_flows/observability/prefect/Prefect__Rest_API.py

@@ -7,6 +7,7 @@
 ENV_NAME__PREFECT_CLOUD__API_KEY      = 'PREFECT_CLOUD__API_KEY'
 ENV_NAME__PREFECT_CLOUD__ACCOUNT_ID   = 'PREFECT_CLOUD__ACCOUNT_ID'
 ENV_NAME__PREFECT_CLOUD__WORKSPACE_ID = 'PREFECT_CLOUD__WORKSPACE_ID'
+ENV_NAME__PREFECT_TARGET_SERVER       = 'PREFECT_TARGET_SERVER'
 
 class Prefect__Rest_API(Type_Safe):
 
@@ -21,7 +22,8 @@ def workspace_id(self):
         return get_env(ENV_NAME__PREFECT_CLOUD__WORKSPACE_ID)
 
     def prefect_api_url(self):
-        return f"https://api.prefect.cloud/api/accounts/{self.account_id()}/workspaces/{self.workspace_id()}/"
+        target_server = get_env(ENV_NAME__PREFECT_TARGET_SERVER,  f"https://api.prefect.cloud/api/accounts/{self.account_id()}/workspaces/{self.workspace_id()}/")
+        return target_server
 
     def get_headers(self):
         return {"Authorization": f"Bearer {self.api_key()}"}                            # Create headers dictionary including authorization token

tests/integration/observability/prefect/test_Prefect__Cloud_API.py

@@ -1,8 +1,8 @@
 from unittest                                                           import TestCase
+from osbot_utils.utils.Lists                                            import list_in_list
 from osbot_utils.utils.Misc                                             import list_set
 from osbot_serverless_flows.observability.prefect.Prefect__Cloud_API    import Prefect__Cloud_API
-from osbot_utils.utils.Env                                              import load_dotenv
-
+from osbot_utils.utils.Env                                              import load_dotenv, get_env
 
 class test_Prefect__Cloud_API(TestCase):
 
@@ -11,14 +11,17 @@ def setUpClass(cls) -> None:
         load_dotenv()
         cls.prefect_cloud_api = Prefect__Cloud_API()
 
+    def test_confirm_local_docker(self):
+        assert get_env('PREFECT_TARGET_SERVER') == 'http://localhost:4200/api'
+
     def test_flows(self):
         flows    = self.prefect_cloud_api.flows()
         for flow in flows:
-            assert list_set(flow) == ['created', 'id', 'labels', 'name', 'tags', 'updated']
+            assert list_in_list(["created", "id", "tags", "updated"], list_set(flow)) is True
 
     def test_flow(self):
         with self.prefect_cloud_api as _:
             flows_ids = _.flows_ids()
             flow_id   = flows_ids.pop()
             flow      = self.prefect_cloud_api.flow(flow_id=flow_id)
-            assert list_set(flow) == ['created', 'id', 'labels', 'name', 'tags', 'updated']
+            assert list_in_list(["created", "id", "tags", "updated"], list_set(flow)) is True