Observer code open source (#1417)

skyvern/exceptions.py

@@ -552,3 +552,8 @@ def __init__(self, element_id: str):
         super().__init__(
             f"Select on the dropdown container instead of the option, try again with another element. element_id={element_id}"
         )
+
+
+class UrlGenerationFailure(SkyvernHTTPException):
+    def __init__(self) -> None:
+        super().__init__("Failed to generate the url for the prompt")

skyvern/forge/prompts/skyvern/observer.j2

@@ -0,0 +1,43 @@
+You're to assist the user to achieve the user goal in the web, given the DOM elements in the list, the screenshots of the website and the task history list. Plan the next task the use needs to do towards the goal.
+
+You have access to the following task types to take actions:
+- navigate: this task can be used to set up a mini goal to achieve in the web which most likely results in navigating the web, like filling a form in the page, clicking a buton in the page to open or navigate to another page, interacting with some elements in the page like clicking, typing and selecting options, and so on.
+- extract: extract information users would like to output from the page. When planning such a task, you should not expect anything related to navigation mentioned in the "navigate" task. The website will remain still and the only action here is to extract information from it.
+- loop: this task can be used to generate a list of planning sessions like this. When to use a loop task? Use loop when there are multiple parallel tasks you can do with the same goal. Each task in the loop has the same goal but with different objects/values/targets/variables. Use loop task when it's in a "breadth first search" situation where you can go through a list of values and execute the same task for each value. Examples:
+  - When the goal is "Open up to 10 links from an ecomm search result page, and extract information like the price of each product.", loop task should be used to iterate through a list of links or URLs. In each iteration of the loop, the task will go to the linked page and trigger another planning session with the goal of extrating price information of the product
+  - When the goal is "download 5 documents found on a page", loop task should be used to iterate through a list of document names. Each document will trigger another planning session to download the relative document
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+  "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.
+  "thoughts": str, // Think step by step. What has been done so far and what is the next reasonable mini goal a human can do foreseeably move towards the overall goal.
+  "user_goal_achieved": bool, // True if the user goal has been completed, false otherwise. If the user is searching for something, looking for information or specifically trying to extract information along side the goal, make sure at least an extract task has happened in the task history before claiming that the goal is achieved.
+  "plan": str, // The mini goal to achieve to move towards the goal. Include all the necessary and required user information provided by the user goal to achieve the plan. DO NOT come up or hallucinate any data that's not provided in the user goal. Be accurate and precise. Return null if the user goal has been achieved.
+  "task_type": str, // One of the available task types: navigate, extract, loop
+  "loop_values": list[str], // a list of string values to iterate through for loop task. null if it's not a loop task
+  "is_loop_value_link": bool, // true if the loop_values is a list of urls to go to before for each planning session inside the loop
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+Clickable elements from the page:
+```
+{{ elements }}
+```
+
+User goal:
+```
+{{ user_goal }}
+```
+
+Task history (the earliest task is the first in the list and the latest is the last in the list):
+```
+{{ task_history }}
+```
+
+Current datetime, ISO format:
+```
+{{ local_datetime }}
+```

skyvern/forge/prompts/skyvern/observer_check_completion.j2

@@ -0,0 +1,22 @@
+You're to assist the user to achieve the user goal in the web, given the user goal and the mini tasks that have been completed by the user along the way.
+
+Reply in JSON format with the following keys:
+{
+  "thoughts": str, // Think step by step. Would completing the tasks in the task history be good enough to achieve the user goal? If more tasks need to be completed to achieve the goal, what would be the next task?
+  "user_goal_achieved": bool, // True if the user goal has been completed, false otherwise. If the user wants to extract information along side the goal, make sure the extract task has happened before claiming that the goal is achieved.
+}
+
+User goal:
+```
+{{ user_goal }}
+```
+
+Task history (the earliest task is the first in the list and the latest is the last in the list):
+```
+{{ task_history }}
+```
+
+Current datetime, ISO format:
+```
+{{ local_datetime }}
+```

skyvern/forge/prompts/skyvern/observer_generate_extraction_task.j2

@@ -0,0 +1,25 @@
+You're to assist the user to extract data from the web. Given the data extraction goal, DOM elements in the list and the screenshots of the website, design the schema for the data output.
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+  "schema": JSON, // the schema of the output data to extract. Use JSON Schema specification style.
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+Clickable elements from the page:
+```
+{{ elements }}
+```
+
+Data extraction goal:
+```
+{{ data_extraction_goal }}
+```
+
+Current datetime, ISO format:
+```
+{{ local_datetime }}
+```

skyvern/forge/prompts/skyvern/observer_generate_loop_task.j2

@@ -0,0 +1,26 @@
+You're to assist the user to achieve goals on the web, given the DOM elements in the list, the screenshots of the website. Now the user needs to iterate throught a list of values. For each value, the user has the same goal and plan to complete a task.
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+  "thoughts": str, // Think step by step. What needs to been done for each value in the list.
+  "task": str, // Describe the task complete for each value in one sentence. Be concise about what you need to do and include all the user data required to complete the task.
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+Clickable elements from the page:
+```
+{{ elements }}
+```
+
+User's overall plan right now:
+```
+{{ plan }}
+```
+
+Current datetime, ISO format:
+```
+{{ local_datetime }}
+```

skyvern/forge/prompts/skyvern/observer_generate_metadata.j2

@@ -0,0 +1,20 @@
+You're to assist the user to achieve the user goal in the browser. Given the user input, what is the url to type into the browser? Also come up with a proper title for this goal to achieve.
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+  "thoughts": str, // Think step by step. If the user specifically mentioned which website it is to use, let's use that url. If the user doesn't specifify a specific website, to achieve what the user wants to do, what is most likely website url for achieving the goal? 
+  "url": str, // The url to type into the browser. DO NOT change the url if user specified one.
+  "title": str, // A descriptive and informative title for the goal. Use no more than 5 words
+}
+
+User goal:
+```
+{{ user_goal }}
+```{% if user_url %}
+
+Starting url provided by user:
+```
+{{ user_url }}
+```{% endif %}

skyvern/forge/prompts/skyvern/observer_generate_task_block.j2

@@ -0,0 +1,26 @@
+You're to assist the user to achieve the user goal in the web, given the DOM elements in the list, the screenshots of the website and the user plan. Help user generate the navigation goal and data extraction goal if any.
+
+User is planning to {% if is_link %}go to a list of links{% else %}go through a list of values{% endif %} and and take the same task with each {% if is_link %}link{% else %}value{% endif %} in the list. Here's the specific plan:
+```
+{{ plan }}
+```
+
+{% if is_link %}Links{% else %}Values{% endif %} the user will go through:
+```
+{{ loop_values }}
+```
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+  "thoughts": str, // Think step by step. What would the use do to achieve the goal.
+  "navigation_goal": str, // What kind things the user needs to do in the web achieve the plan and finally get the data to extract. Include all the data needed to complete the goal here.{% if is_link %} The user already has the link to go to the target page first, in order to start executing the plan. So state the navigation goal from the perspective of already being on the target page. What does the user need to do from there?{% endif %}
+  "data_extraction_goal": str, // If the user needs to extract/retrieve data from the site after navigation goal is achieved, define that extraction goal here. null if no data needs to be extracted.
+  "data_schema": json, // the schema of the output data. use JSON schema specification style. All fields should be optional as we optimize to extract as much data as possible
+}
+
+Current datetime, ISO format:
+```
+{{ local_datetime }}
+```

skyvern/forge/prompts/skyvern/observer_loop_task_extraction_goal.j2

@@ -0,0 +1,6 @@
+The user is trying to achieve a goal the web. Now they've decided to go through a list of values and take the same tasks with each variant in the list.
+
+Help to user extract this list of values based on what they want to achieve:
+```
+{{ plan }}
+```

skyvern/forge/sdk/executor/async_executor.py

@@ -7,7 +7,10 @@
 from skyvern.forge import app
 from skyvern.forge.sdk.core import skyvern_context
 from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.schemas.observers import ObserverCruiseStatus
 from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.forge.sdk.services import observer_service
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
 
 LOG = structlog.get_logger()
 
@@ -40,12 +43,24 @@ async def execute_workflow(
     ) -> None:
         pass
 
+    @abc.abstractmethod
+    async def execute_cruise(
+        self,
+        request: Request | None,
+        background_tasks: BackgroundTasks | None,
+        organization_id: str,
+        observer_cruise_id: str,
+        max_iterations_override: int | None,
+        **kwargs: dict,
+    ) -> None:
+        pass
+
 
 class BackgroundTaskExecutor(AsyncExecutor):
     async def execute_task(
         self,
         request: Request | None,
-        background_tasks: BackgroundTasks,
+        background_tasks: BackgroundTasks | None,
         task_id: str,
         organization_id: str,
         max_steps_override: int | None,
@@ -76,18 +91,19 @@ async def execute_task(
         context.organization_id = organization_id
         context.max_steps_override = max_steps_override
 
-        background_tasks.add_task(
-            app.agent.execute_step,
-            organization,
-            task,
-            step,
-            api_key,
-        )
+        if background_tasks:
+            background_tasks.add_task(
+                app.agent.execute_step,
+                organization,
+                task,
+                step,
+                api_key,
+            )
 
     async def execute_workflow(
         self,
         request: Request | None,
-        background_tasks: BackgroundTasks,
+        background_tasks: BackgroundTasks | None,
         organization_id: str,
         workflow_id: str,
         workflow_run_id: str,
@@ -104,9 +120,53 @@ async def execute_workflow(
         if organization is None:
             raise OrganizationNotFound(organization_id)
 
-        background_tasks.add_task(
-            app.WORKFLOW_SERVICE.execute_workflow,
-            workflow_run_id=workflow_run_id,
-            api_key=api_key,
-            organization=organization,
+        if background_tasks:
+            background_tasks.add_task(
+                app.WORKFLOW_SERVICE.execute_workflow,
+                workflow_run_id=workflow_run_id,
+                api_key=api_key,
+                organization=organization,
+            )
+
+    async def execute_cruise(
+        self,
+        request: Request | None,
+        background_tasks: BackgroundTasks | None,
+        organization_id: str,
+        observer_cruise_id: str,
+        max_iterations_override: int | None,
+        **kwargs: dict,
+    ) -> None:
+        LOG.info(
+            "Executing cruise using background task executor",
+            observer_cruise_id=observer_cruise_id,
         )
+
+        organization = await app.DATABASE.get_organization(organization_id)
+        if organization is None:
+            raise OrganizationNotFound(organization_id)
+
+        observer_cruise = await app.DATABASE.get_observer_cruise(
+            observer_cruise_id=observer_cruise_id, organization_id=organization_id
+        )
+        if not observer_cruise or not observer_cruise.workflow_run_id:
+            raise ValueError("No observer cruise or no workflow run associated with observer cruise")
+
+        # mark observer cruise as queued
+        await app.DATABASE.update_observer_cruise(
+            observer_cruise_id,
+            status=ObserverCruiseStatus.queued,
+            organization_id=organization_id,
+        )
+        await app.DATABASE.update_workflow_run(
+            workflow_run_id=observer_cruise.workflow_run_id,
+            status=WorkflowRunStatus.queued,
+        )
+
+        if background_tasks:
+            background_tasks.add_task(
+                observer_service.run_observer_cruise,
+                organization=organization,
+                observer_cruise_id=observer_cruise_id,
+                max_iterations_override=max_iterations_override,
+            )

skyvern/forge/sdk/routes/agent_protocol.py

@@ -36,6 +36,7 @@
 from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, TaskType
 from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
 from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.observers import CruiseRequest, ObserverCruise
 from skyvern.forge.sdk.schemas.organizations import (
     GetOrganizationAPIKeysResponse,
     GetOrganizationsResponse,
@@ -53,7 +54,7 @@
     TaskStatus,
 )
 from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunEvent, WorkflowRunEventType
-from skyvern.forge.sdk.services import org_auth_service
+from skyvern.forge.sdk.services import observer_service, org_auth_service
 from skyvern.forge.sdk.workflow.exceptions import (
     FailedToCreateWorkflow,
     FailedToUpdateWorkflow,
@@ -1117,3 +1118,31 @@ async def upload_file(
         status_code=200,
         media_type="application/json",
     )
+
+
+@base_router.post("/cruise")
+@base_router.post("/cruise/", include_in_schema=False)
+async def observer_cruise(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    data: CruiseRequest,
+    organization: Organization = Depends(org_auth_service.get_current_org),
+    x_max_iterations_override: Annotated[int | None, Header()] = None,
+) -> ObserverCruise:
+    if x_max_iterations_override:
+        LOG.info("Overriding max iterations for observer", max_iterations_override=x_max_iterations_override)
+
+    observer_cruise = await observer_service.initialize_observer_cruise(
+        organization=organization,
+        user_prompt=data.user_prompt,
+        user_url=str(data.url) if data.url else None,
+    )
+    analytics.capture("skyvern-oss-agent-observer-cruise", data={"url": observer_cruise.url})
+    await AsyncExecutorFactory.get_executor().execute_cruise(
+        request=request,
+        background_tasks=background_tasks,
+        organization_id=organization.organization_id,
+        observer_cruise_id=observer_cruise.observer_cruise_id,
+        max_iterations_override=x_max_iterations_override,
+    )
+    return observer_cruise

skyvern/forge/sdk/schemas/observers.py

@@ -1,7 +1,11 @@
 from datetime import datetime
 from enum import StrEnum
 
-from pydantic import BaseModel, ConfigDict, HttpUrl
+from pydantic import BaseModel, ConfigDict, HttpUrl, field_validator
+
+from skyvern.forge.sdk.core.validators import validate_url
+
+DEFAULT_WORKFLOW_TITLE = "New Workflow"
 
 
 class ObserverCruiseStatus(StrEnum):
@@ -48,3 +52,20 @@ class ObserverThought(BaseModel):
 
     created_at: datetime
     modified_at: datetime
+
+
+class ObserverMetadata(BaseModel):
+    url: str
+    workflow_title: str = DEFAULT_WORKFLOW_TITLE
+
+    @field_validator("url")
+    @classmethod
+    def validate_urls(cls, v: str | None) -> str | None:
+        if v is None:
+            return None
+        return validate_url(v)
+
+
+class CruiseRequest(BaseModel):
+    user_prompt: str
+    url: HttpUrl | None = None