Record logs into step artifacts (#1339)

Co-authored-by: Shuchang Zheng <[email protected]> Co-authored-by: LawyZheng <[email protected]> Co-authored-by: Nick Fisher <[email protected]>
Skyvern-AI · Dec 17, 2024 · b8e2527 · b8e2527
1 parent 5e80b90
commit b8e2527
Show file tree

Hide file tree

Showing 19 changed files with 592 additions and 28 deletions.
diff --git a/.env.example b/.env.example
@@ -69,3 +69,6 @@ PORT=8000
 # Analytics configuration:
 # Distinct analytics ID (a UUID is generated if left blank).
 ANALYTICS_ID="anonymous"
+
+# Enable recording skyvern logs as artifacts
+ENABLE_LOG_ARTIFACTS=false
diff --git a/skyvern-frontend/.env.example b/skyvern-frontend/.env.example
@@ -8,4 +8,7 @@ VITE_ARTIFACT_API_BASE_URL=http://localhost:9090
 VITE_WSS_BASE_URL=ws://localhost:8000/api/v1
 
 # your api key - for x-api-key header
-VITE_SKYVERN_API_KEY=YOUR_API_KEY
+VITE_SKYVERN_API_KEY=YOUR_API_KEY
+
+# Enable recording skyvern logs as artifacts
+VITE_ENABLE_LOG_ARTIFACTS=false
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
@@ -10,6 +10,8 @@ export const ArtifactType = {
   LLMPrompt: "llm_prompt",
   LLMRequest: "llm_request",
   HTMLScrape: "html_scrape",
+  SkyvernLog: "skyvern_log",
+  SkyvernLogRaw: "skyvern_log_raw",
 } as const;
 
 export type ArtifactType = (typeof ArtifactType)[keyof typeof ArtifactType];

diff --git a/skyvern-frontend/src/routes/tasks/detail/ScrollableActionList.tsx b/skyvern-frontend/src/routes/tasks/detail/ScrollableActionList.tsx
@@ -11,7 +11,6 @@ import {
 } from "@radix-ui/react-icons";
 import { useQueryClient } from "@tanstack/react-query";
 import { ReactNode, useRef } from "react";
-import { useParams } from "react-router-dom";
 import { ActionTypePill } from "./ActionTypePill";
 
 type Props = {
@@ -33,7 +32,6 @@ function ScrollableActionList({
   showStreamOption,
   taskDetails,
 }: Props) {
-  const { taskId } = useParams();
   const queryClient = useQueryClient();
   const credentialGetter = useCredentialGetter();
   const refs = useRef<Array<HTMLDivElement | null>>(
@@ -65,11 +63,11 @@ function ScrollableActionList({
           onClick={() => onActiveIndexChange(i)}
           onMouseEnter={() => {
             queryClient.prefetchQuery({
-              queryKey: ["task", taskId, "steps", action.stepId, "artifacts"],
+              queryKey: ["step", action.stepId, "artifacts"],
               queryFn: async () => {
                 const client = await getClient(credentialGetter);
                 return client
-                  .get(`/tasks/${taskId}/steps/${action.stepId}/artifacts`)
+                  .get(`/step/${action.stepId}/artifacts`)
                   .then((response) => response.data);
               },
             });

diff --git a/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx b/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx
@@ -7,7 +7,7 @@ import {
 import { StatusBadge } from "@/components/StatusBadge";
 import { Label } from "@/components/ui/label";
 import { useQuery } from "@tanstack/react-query";
-import { useParams, useSearchParams } from "react-router-dom";
+import { useSearchParams } from "react-router-dom";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { ZoomableImage } from "@/components/ZoomableImage";
 import { Skeleton } from "@/components/ui/skeleton";
@@ -17,6 +17,9 @@ import { basicLocalTimeFormat, basicTimeFormat } from "@/util/timeFormat";
 import { useCredentialGetter } from "@/hooks/useCredentialGetter";
 import { Artifact } from "./Artifact";
 
+const enable_log_artifacts =
+  import.meta.env.VITE_ENABLE_LOG_ARTIFACTS === "true";
+
 type Props = {
   id: string;
   stepProps: StepApiResponse;
@@ -25,19 +28,18 @@ type Props = {
 function StepArtifacts({ id, stepProps }: Props) {
   const [searchParams, setSearchParams] = useSearchParams();
   const artifact = searchParams.get("artifact") ?? "info";
-  const { taskId } = useParams();
   const credentialGetter = useCredentialGetter();
   const {
     data: artifacts,
     isFetching,
     isError,
     error,
   } = useQuery<Array<ArtifactApiResponse>>({
-    queryKey: ["task", taskId, "steps", id, "artifacts"],
+    queryKey: ["step", id, "artifacts"],
     queryFn: async () => {
       const client = await getClient(credentialGetter);
       return client
-        .get(`/tasks/${taskId}/steps/${id}/artifacts`)
+        .get(`/step/${id}/artifacts`)
         .then((response) => response.data);
     },
   });
@@ -79,6 +81,10 @@ function StepArtifacts({ id, stepProps }: Props) {
     (artifact) => artifact.artifact_type === ArtifactType.HTMLScrape,
   );
 
+  const skyvernLog = artifacts?.filter(
+    (artifact) => artifact.artifact_type === ArtifactType.SkyvernLog,
+  );
+
   return (
     <Tabs
       value={artifact}
@@ -108,6 +114,9 @@ function StepArtifacts({ id, stepProps }: Props) {
         <TabsTrigger value="llm_response_parsed">Action List</TabsTrigger>
         <TabsTrigger value="html_raw">HTML (Raw)</TabsTrigger>
         <TabsTrigger value="llm_request">LLM Request (Raw)</TabsTrigger>
+        {enable_log_artifacts && (
+          <TabsTrigger value="skyvern_log">Skyvern Log</TabsTrigger>
+        )}
       </TabsList>
       <TabsContent value="info">
         <div className="flex flex-col gap-6 p-4">
@@ -209,6 +218,11 @@ function StepArtifacts({ id, stepProps }: Props) {
       <TabsContent value="llm_request">
         {llmRequest ? <Artifact type="json" artifacts={llmRequest} /> : null}
       </TabsContent>
+      {enable_log_artifacts && (
+        <TabsContent value="skyvern_log">
+          {skyvernLog ? <Artifact type="text" artifacts={skyvernLog} /> : null}
+        </TabsContent>
+      )}
     </Tabs>
   );
 }

diff --git a/skyvern/config.py b/skyvern/config.py
@@ -131,6 +131,8 @@ class Settings(BaseSettings):
 
     SVG_MAX_LENGTH: int = 100000
 
+    ENABLE_LOG_ARTIFACTS: bool = False
+
     def is_cloud_environment(self) -> bool:
         """
         :return: True if env is not local, else False

diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
@@ -50,6 +50,7 @@
 from skyvern.forge.sdk.core.security import generate_skyvern_signature
 from skyvern.forge.sdk.core.validators import prepend_scheme_and_validate_url
 from skyvern.forge.sdk.db.enums import TaskType
+from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
 from skyvern.forge.sdk.models import Step, StepStatus
 from skyvern.forge.sdk.schemas.organizations import Organization
 from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatus
@@ -1783,6 +1784,9 @@ async def update_step(
             step_id=step.step_id,
             diff=update_comparison,
         )
+
+        await save_step_logs(step.step_id)
+
         return await app.DATABASE.update_step(
             task_id=step.task_id,
             step_id=step.step_id,
@@ -1815,6 +1819,7 @@ async def update_task(
             for key, value in updates.items()
             if getattr(task, key) != value
         }
+        await save_task_logs(task.task_id)
         LOG.info("Updating task in db", task_id=task.task_id, diff=update_comparison)
         return await app.DATABASE.update_task(
             task.task_id,

diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py
@@ -1,12 +1,11 @@
 import asyncio
 import time
 from collections import defaultdict
-from typing import Literal
 
 import structlog
 
 from skyvern.forge import app
-from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
 from skyvern.forge.sdk.db.id import generate_artifact_id
 from skyvern.forge.sdk.models import Step
 from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverThought
@@ -82,6 +81,35 @@ async def create_artifact(
             path=path,
         )
 
+    async def create_log_artifact(
+        self,
+        log_entity_type: LogEntityType,
+        log_entity_id: str,
+        artifact_type: ArtifactType,
+        step_id: str | None = None,
+        task_id: str | None = None,
+        workflow_run_id: str | None = None,
+        workflow_run_block_id: str | None = None,
+        organization_id: str | None = None,
+        data: bytes | None = None,
+        path: str | None = None,
+    ) -> str:
+        artifact_id = generate_artifact_id()
+        uri = app.STORAGE.build_log_uri(log_entity_type, log_entity_id, artifact_type)
+        return await self._create_artifact(
+            aio_task_primary_key=log_entity_id,
+            artifact_id=artifact_id,
+            artifact_type=artifact_type,
+            uri=uri,
+            step_id=step_id,
+            task_id=task_id,
+            workflow_run_id=workflow_run_id,
+            workflow_run_block_id=workflow_run_block_id,
+            organization_id=organization_id,
+            data=data,
+            path=path,
+        )
+
     async def create_observer_thought_artifact(
         self,
         observer_thought: ObserverThought,
@@ -174,7 +202,7 @@ async def update_artifact_data(
         artifact_id: str | None,
         organization_id: str | None,
         data: bytes,
-        primary_key: Literal["task_id", "observer_thought_id", "observer_cruise_id"] = "task_id",
+        primary_key: str = "task_id",
     ) -> None:
         if not artifact_id or not organization_id:
             return None
@@ -183,18 +211,10 @@ async def update_artifact_data(
             return
         # Fire and forget
         aio_task = asyncio.create_task(app.STORAGE.store_artifact(artifact, data))
-        if primary_key == "task_id":
-            if not artifact.task_id:
-                raise ValueError("Task ID is required to update artifact data.")
-            self.upload_aiotasks_map[artifact.task_id].append(aio_task)
-        elif primary_key == "observer_thought_id":
-            if not artifact.observer_thought_id:
-                raise ValueError("Observer Thought ID is required to update artifact data.")
-            self.upload_aiotasks_map[artifact.observer_thought_id].append(aio_task)
-        elif primary_key == "observer_cruise_id":
-            if not artifact.observer_cruise_id:
-                raise ValueError("Observer Cruise ID is required to update artifact data.")
-            self.upload_aiotasks_map[artifact.observer_cruise_id].append(aio_task)
+
+        if not artifact[primary_key]:
+            raise ValueError(f"{primary_key} is required to update artifact data.")
+        self.upload_aiotasks_map[artifact[primary_key]].append(aio_task)
 
     async def retrieve_artifact(self, artifact: Artifact) -> bytes | None:
         return await app.STORAGE.retrieve_artifact(artifact)

diff --git a/skyvern/forge/sdk/artifact/models.py b/skyvern/forge/sdk/artifact/models.py
@@ -2,6 +2,7 @@
 
 from datetime import datetime
 from enum import StrEnum
+from typing import Any
 
 from pydantic import BaseModel, Field, field_serializer
 
@@ -10,6 +11,9 @@ class ArtifactType(StrEnum):
     RECORDING = "recording"
     BROWSER_CONSOLE_LOG = "browser_console_log"
 
+    SKYVERN_LOG = "skyvern_log"
+    SKYVERN_LOG_RAW = "skyvern_log_raw"
+
     # DEPRECATED. pls use SCREENSHOT_LLM, SCREENSHOT_ACTION or SCREENSHOT_FINAL
     SCREENSHOT = "screenshot"
 
@@ -70,3 +74,13 @@ def serialize_datetime_to_isoformat(self, value: datetime) -> str:
     observer_thought_id: str | None = None
     signed_url: str | None = None
     organization_id: str | None = None
+
+    def __getitem__(self, key: str) -> Any:
+        return getattr(self, key)
+
+
+class LogEntityType(StrEnum):
+    STEP = "step"
+    TASK = "task"
+    WORKFLOW_RUN = "workflow_run"
+    WORKFLOW_RUN_BLOCK = "workflow_run_block"
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 
-from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
 from skyvern.forge.sdk.models import Step
 from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverThought
 
@@ -11,6 +11,8 @@
     ArtifactType.SCREENSHOT_LLM: "png",
     ArtifactType.SCREENSHOT_ACTION: "png",
     ArtifactType.SCREENSHOT_FINAL: "png",
+    ArtifactType.SKYVERN_LOG: "log",
+    ArtifactType.SKYVERN_LOG_RAW: "json",
     ArtifactType.LLM_PROMPT: "txt",
     ArtifactType.LLM_REQUEST: "json",
     ArtifactType.LLM_RESPONSE: "json",
@@ -34,6 +36,10 @@ class BaseStorage(ABC):
     def build_uri(self, artifact_id: str, step: Step, artifact_type: ArtifactType) -> str:
         pass
 
+    @abstractmethod
+    def build_log_uri(self, log_entity_type: LogEntityType, log_entity_id: str, artifact_type: ArtifactType) -> str:
+        pass
+
     @abstractmethod
     def build_observer_thought_uri(
         self, artifact_id: str, observer_thought: ObserverThought, artifact_type: ArtifactType

diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.py
@@ -8,7 +8,7 @@
 
 from skyvern.config import settings
 from skyvern.forge.sdk.api.files import get_download_dir, get_skyvern_temp_dir
-from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
 from skyvern.forge.sdk.artifact.storage.base import FILE_EXTENTSION_MAP, BaseStorage
 from skyvern.forge.sdk.models import Step
 from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverThought
@@ -24,6 +24,10 @@ def build_uri(self, artifact_id: str, step: Step, artifact_type: ArtifactType) -
         file_ext = FILE_EXTENTSION_MAP[artifact_type]
         return f"file://{self.artifact_path}/{step.task_id}/{step.order:02d}_{step.retry_index}_{step.step_id}/{datetime.utcnow().isoformat()}_{artifact_id}_{artifact_type}.{file_ext}"
 
+    def build_log_uri(self, log_entity_type: LogEntityType, log_entity_id: str, artifact_type: ArtifactType) -> str:
+        file_ext = FILE_EXTENTSION_MAP[artifact_type]
+        return f"file://{self.artifact_path}/logs/{log_entity_type}/{log_entity_id}/{datetime.utcnow().isoformat()}_{artifact_type}.{file_ext}"
+
     def build_observer_thought_uri(
         self, artifact_id: str, observer_thought: ObserverThought, artifact_type: ArtifactType
     ) -> str:

diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -12,7 +12,7 @@
     make_temp_directory,
     unzip_files,
 )
-from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
 from skyvern.forge.sdk.artifact.storage.base import FILE_EXTENTSION_MAP, BaseStorage
 from skyvern.forge.sdk.models import Step
 from skyvern.forge.sdk.schemas.observers import ObserverCruise, ObserverThought
@@ -27,6 +27,10 @@ def build_uri(self, artifact_id: str, step: Step, artifact_type: ArtifactType) -
         file_ext = FILE_EXTENTSION_MAP[artifact_type]
         return f"s3://{self.bucket}/{settings.ENV}/{step.task_id}/{step.order:02d}_{step.retry_index}_{step.step_id}/{datetime.utcnow().isoformat()}_{artifact_id}_{artifact_type}.{file_ext}"
 
+    def build_log_uri(self, log_entity_type: LogEntityType, log_entity_id: str, artifact_type: ArtifactType) -> str:
+        file_ext = FILE_EXTENTSION_MAP[artifact_type]
+        return f"s3://{self.bucket}/{settings.ENV}/logs/{log_entity_type}/{log_entity_id}/{datetime.utcnow().isoformat()}_{artifact_type}.{file_ext}"
+
     def build_observer_thought_uri(
         self, artifact_id: str, observer_thought: ObserverThought, artifact_type: ArtifactType
     ) -> str:

diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
@@ -13,6 +13,7 @@ class SkyvernContext:
     max_steps_override: int | None = None
     tz_info: ZoneInfo | None = None
     totp_codes: dict[str, str | None] = field(default_factory=dict)
+    log: list[dict] = field(default_factory=list)
 
     def __repr__(self) -> str:
         return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, max_steps_override={self.max_steps_override})"