Merge branch 'Significant-Gravitas:master' into master

.github/workflows/autogpt-infra-ci.yml

@@ -0,0 +1,33 @@
+name: AutoGPT Builder Infra
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - '.github/workflows/autogpt-infra-ci.yml'
+      - 'rnd/infra/**'
+  pull_request:
+    paths:
+      - '.github/workflows/autogpt-infra-ci.yml'
+      - 'rnd/infra/**'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: rnd/infra
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: TFLint
+        uses: pauloconnor/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+            tflint_path: terraform/
+            tflint_recurse: true
+            tflint_changed_only: false

.github/workflows/autogpt-server-ci.yml

@@ -31,10 +31,12 @@ jobs:
       matrix:
         python-version: ["3.10"]
         platform-os: [ubuntu, macos, macos-arm64, windows]
+        db-platform: [postgres, sqlite]
     runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }}
 
     steps:
       - name: Setup PostgreSQL
+        if: matrix.db-platform == 'postgres'
         uses: ikalnytskyi/action-setup-postgres@v6
         with:
           username: ${{ secrets.DB_USER }}
@@ -114,14 +116,24 @@ jobs:
       - name: Install Python dependencies
         run: poetry install
 
-      - name: Generate Prisma Client
+      - name: Generate Prisma Client (Postgres)
+        if: matrix.db-platform == 'postgres'
         run: poetry run prisma generate --schema postgres/schema.prisma
 
-      - name: Run Database Migrations
+      - name: Run Database Migrations (Postgres)
+        if: matrix.db-platform == 'postgres'
         run: poetry run prisma migrate dev --schema postgres/schema.prisma --name updates
         env:
           CONNECTION_STR: ${{ steps.postgres.outputs.connection-uri }}
 
+      - name: Generate Prisma Client (SQLite)
+        if: matrix.db-platform == 'sqlite'
+        run: poetry run prisma generate
+
+      - name: Run Database Migrations (SQLite)
+        if: matrix.db-platform == 'sqlite'
+        run: poetry run prisma migrate dev --name updates
+
       - name: Run Linter
         run: poetry run lint
 

.vscode/all-projects.code-workspace

@@ -33,7 +33,9 @@
       "path": ".."
     }
   ],
-  "settings": {},
+  "settings": {
+    "python.analysis.typeCheckingMode": "basic",
+  },
   "extensions": {
     "recommendations": [
       "charliermarsh.ruff",

autogpt/.env.template

@@ -105,6 +105,7 @@
 ## HUGGINGFACE_API_TOKEN - HuggingFace API token (Default: None)
 # HUGGINGFACE_API_TOKEN=
 
+
 ### Stable Diffusion (IMAGE_PROVIDER=sdwebui)
 
 ## SD_WEBUI_AUTH - Stable Diffusion Web UI username:password pair (Default: None)

autogpt/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic",
+}

benchmark/.vscode/settings.json

@@ -2,5 +2,5 @@
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
-  "python.formatting.provider": "none"
+  "python.analysis.typeCheckingMode": "basic",
 }

docs/content/forge/components/built-in-components.md

@@ -155,11 +155,12 @@ Allows agent to search the web. Google credentials aren't required for DuckDuckG
 
 ### `WebSearchConfiguration`
 
-| Config variable                  | Details                                                                 | Type  | Default |
-| -------------------------------- | ----------------------------------------------------------------------- | ----- | ------- |
-| `google_api_key`                 | Google API key, *ENV:* `GOOGLE_API_KEY`                                 | `str` | `None`  |
-| `google_custom_search_engine_id` | Google Custom Search Engine ID, *ENV:* `GOOGLE_CUSTOM_SEARCH_ENGINE_ID` | `str` | `None`  |
-| `duckduckgo_max_attempts`        | Maximum number of attempts to search using DuckDuckGo                   | `int` | `3`     |
+| Config variable                  | Details                                                                 | Type                        | Default |
+| -------------------------------- | ----------------------------------------------------------------------- | --------------------------- | ------- |
+| `google_api_key`                 | Google API key, *ENV:* `GOOGLE_API_KEY`                                 | `str`                       | `None`  |
+| `google_custom_search_engine_id` | Google Custom Search Engine ID, *ENV:* `GOOGLE_CUSTOM_SEARCH_ENGINE_ID` | `str`                       | `None`  |
+| `duckduckgo_max_attempts`        | Maximum number of attempts to search using DuckDuckGo                   | `int`                       | `3`     |
+| `duckduckgo_backend`             | Backend to be used for DDG sdk                                          | `"api" \| "html" \| "lite"` | `"api"` |
 
 ### DirectiveProvider
 
@@ -183,6 +184,7 @@ Allows agent to read websites using Selenium.
 | `headless`                    | Run browser in headless mode                | `bool`                                        | `True`                                                                                                                       |
 | `user_agent`                  | User agent used by the browser              | `str`                                         | `"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"` |
 | `browse_spacy_language_model` | Spacy language model used for chunking text | `str`                                         | `"en_core_web_sm"`                                                                                                           |
+| `selenium_proxy`              | Http proxy to use with Selenium             | `str`                                         | `None`                                                                                                                       |
 
 ### DirectiveProvider
 

docs/content/server/advanced_setup.md

@@ -0,0 +1,41 @@
+# Advanced Setup
+
+The advanced steps below are intended for people with sysadmin experience. If you are not comfortable with these steps, please refer to the [basic setup guide](setup.md).
+
+## Introduction
+
+For the advanced setup, first follow the [basic setup guide](setup.md) to get the server up and running. Once you have the server running, you can follow the steps below to configure the server for your specific needs.
+
+## Configuration
+
+### Setting config via environment variables
+
+The server uses environment variables to store configs. You can set these environment variables in a `.env` file in the root of the project. The `.env` file should look like this:
+
+```bash
+# .env
+KEY1=value1
+KEY2=value2
+```
+
+The server will automatically load the `.env` file when it starts. You can also set the environment variables directly in your shell. Refer to your operating system's documentation on how to set environment variables in the current session.
+
+The valid options are listed in `.env.example` in the root of the builder and server directories. You can copy the `.env.example` file to `.env` and modify the values as needed.
+
+```bash
+# Copy the .env.example file to .env
+cp .env.example .env
+```
+
+### Secrets directory
+
+The secret directory is located at `./secrets`. You can store any secrets you need in this directory. The server will automatically load the secrets when it starts.
+
+An example for a secret called `my_secret` would look like this:
+
+```bash
+# ./secrets/my_secret
+my_secret_value
+```
+
+This is useful when running on docker so you can copy the secrets into the container without exposing them in the Dockerfile.

docs/content/server/setup.md

@@ -0,0 +1,102 @@
+# Setting up the server
+
+- [Introduction](#introduction)
+- [Prerequisites](#prerequisites)
+
+## Introduction
+
+This guide will help you setup the server and builder for the project.
+
+<!-- The video is listed in the root Readme.md of the repo -->
+
+We also offer this in video format. You can check it out [here](https://github.com/Significant-Gravitas/AutoGPT#how-to-get-started)
+
+!!! warning
+    **DO NOT FOLLOW ANY OUTSIDE TUTORIALS AS THEY WILL LIKELY BE OUT OF DATE**
+
+## Prerequisites
+
+To setup the server, you need to have the following installed:
+
+- [Node.js](https://nodejs.org/en/)
+- [Python 3.10](https://www.python.org/downloads/)
+
+### Checking if you have Node.js and Python installed
+
+You can check if you have Node.js installed by running the following command:
+
+```bash
+node -v
+```
+
+You can check if you have Python installed by running the following command:
+
+```bash
+python --version
+```
+
+Once you have node and python installed, you can proceed to the next step.
+
+### Installing the package managers
+
+In order to install the dependencies, you need to have the appropriate package managers installed.
+
+- Installing Yarn
+
+Yarn is a package manager for Node.js. You can install it by running the following command:
+
+```bash
+npm install -g yarn
+```
+
+- Installing Poetry
+
+Poetry is a package manager for Python. You can install it by running the following command:
+
+```bash
+pip install poetry
+```
+
+### Installing the dependencies
+
+Once you have installed Yarn and Poetry, you can run the following command to install the dependencies:
+
+```bash
+cd rnd/autogpt_server
+poetry install
+```
+
+**In another terminal**, run the following command to install the dependencies for the frontend:
+
+```bash
+cd rnd/autogpt_builder
+yarn install
+```
+
+Once you have installed the dependencies, you can proceed to the next step.
+
+### Setting up the database
+
+In order to setup the database, you need to run the following command, in the same terminal you ran the `poetry install` command:
+
+```bash
+poetry run prisma migrate deploy
+```
+
+### Running the server
+
+To run the server, you can run the following command in the same terminal you ran the `poetry install` command:
+
+```bash
+poetry run app
+```
+
+In the other terminal, you can run the following command to start the frontend:
+
+```bash
+yarn dev
+```
+
+### Checking if the server is running
+
+You can check if the server is running by visiting [http://localhost:3000](http://localhost:3000) in your browser.

docs/mkdocs.yml

@@ -7,6 +7,8 @@ nav:
 
   - The AutoGPT Server 🆕: 
     - Build your own Blocks: server/new_blocks.md
+    - Setup: server/setup.md
+    - Advanced Setup: server/advanced_setup.md
 
   - AutoGPT Agent:
       - Introduction: AutoGPT/index.md

forge/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic",
+}

forge/forge/components/web/search.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import time
-from typing import Iterator, Optional
+from typing import Iterator, Literal, Optional
 
 from duckduckgo_search import DDGS
 from pydantic import BaseModel, SecretStr
@@ -24,6 +24,7 @@ class WebSearchConfiguration(BaseModel):
         None, from_env="GOOGLE_CUSTOM_SEARCH_ENGINE_ID", exclude=True
     )
     duckduckgo_max_attempts: int = 3
+    duckduckgo_backend: Literal["api", "html", "lite"] = "api"
 
 
 class WebSearchComponent(
@@ -89,7 +90,9 @@ def web_search(self, query: str, num_results: int = 8) -> str:
             if not query:
                 return json.dumps(search_results)
 
-            search_results = DDGS().text(query, max_results=num_results)
+            search_results = DDGS().text(
+                query, max_results=num_results, backend=self.config.duckduckgo_backend
+            )
 
             if search_results:
                 break

forge/forge/components/web/selenium.py

@@ -68,6 +68,8 @@ class WebSeleniumConfiguration(BaseModel):
     """User agent used by the browser"""
     browse_spacy_language_model: str = "en_core_web_sm"
     """Spacy language model used for chunking text"""
+    selenium_proxy: Optional[str] = None
+    """Http proxy to use with Selenium"""
 
 
 class WebSeleniumComponent(
@@ -301,6 +303,9 @@ async def open_page_in_browser(self, url: str) -> WebDriver:
                 options.add_argument("--headless=new")
                 options.add_argument("--disable-gpu")
 
+            if self.config.selenium_proxy:
+                options.add_argument(f"--proxy-server={self.config.selenium_proxy}")
+
             self._sideload_chrome_extensions(options, self.data_dir / "assets" / "crx")
 
             if (chromium_driver_path := Path("/usr/bin/chromedriver")).exists():