Add GPT4All local provider (#209)

* Add GPT4All * Allow to tune number of threads * Disable auto-download * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix build * bump langchain to v0.0.223 see: langchain-ai/langchain@265c285 * implement async for GPT4All * update user docs with GPT4All installation instructions --------- Co-authored-by: 3coins <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: David L. Qiu <[email protected]>
jupyterlab · Aug 10, 2023 · ca45817 · ca45817
1 parent ac3f4be
commit ca45817
Show file tree

Hide file tree

Showing 7 changed files with 100 additions and 5 deletions.
diff --git a/docs/source/users/index.md b/docs/source/users/index.md
@@ -219,7 +219,7 @@ The chat backend remembers the last two exchanges in your conversation and passe
     alt='Screen shot of an example follow up question sent to Jupyternaut, who responds with the improved code and explanation.'
     class="screenshot" />
 
-### Using the chat interface with SageMaker endpoints
+### SageMaker endpoints usage
 
 Jupyter AI supports language models hosted on SageMaker endpoints that use JSON
 schemas. The first step is to authenticate with AWS via the `boto3` SDK and have
@@ -255,6 +255,37 @@ response.  In this example, the endpoint returns an object with the schema
 `{"generated_texts":["<output>"]}`, hence the response path is
 `generated_texts.[0]`.
 
+### GPT4All usage (early-stage)
+
+Currently, we offer experimental support for GPT4All. To get started, first
+decide which models you will use. We currently offer three models from GPT4All:
+
+| Model name                   | Model size | Model bin URL                                              |
+|------------------------------|------------|------------------------------------------------------------|
+| `ggml-gpt4all-l13b-snoozy`   | 7.6 GB     | `http://gpt4all.io/models/ggml-gpt4all-l13b-snoozy.bin`    |
+| `ggml-gpt4all-j-v1.2-jazzy`  | 3.8 GB     | `https://gpt4all.io/models/ggml-gpt4all-j-v1.2-jazzy.bin`  |
+| `ggml-gpt4all-j-v1.3-groovy` | 3.8 GB     | `https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin` |
+
+
+Note that each model comes with its own license, and that users are themselves
+responsible for verifying that their usage complies with the license. You can
+find licensing details on the [GPT4All official site](https://gpt4all.io/index.html).
+
+For each model you use, you will have to run the command
+
+```
+curl -LO --output-dir ~/.cache/gpt4all "<model-bin-url>"
+```
+
+, where `<model-bin-url>` should be substituted with the corresponding URL
+hosting the model binary (within the double quotes). After restarting the
+server, the GPT4All models installed in the previous step should be available to
+use in the chat interface.
+
+GPT4All support is still an early-stage feature, so some bugs may be encountered
+during usage. Our team is still actively improving support for locally-hosted
+models.
+
 ### Asking about something in your notebook
 
 Jupyter AI's chat interface can include a portion of your notebook in your prompt.

diff --git a/packages/jupyter-ai-magics/jupyter_ai_magics/__init__.py b/packages/jupyter-ai-magics/jupyter_ai_magics/__init__.py
@@ -18,6 +18,7 @@
     ChatOpenAINewProvider,
     ChatOpenAIProvider,
     CohereProvider,
+    GPT4AllProvider,
     HfHubProvider,
     OpenAIProvider,
     SmEndpointProvider,

diff --git a/packages/jupyter-ai-magics/jupyter_ai_magics/providers.py b/packages/jupyter-ai-magics/jupyter_ai_magics/providers.py
@@ -15,6 +15,7 @@
     Anthropic,
     Bedrock,
     Cohere,
+    GPT4All,
     HuggingFaceHub,
     OpenAI,
     OpenAIChat,
@@ -70,7 +71,13 @@ class MultilineTextField(Field):
     type: Literal["text-multiline"] = "text-multiline"
 
 
-Field = Union[TextField, MultilineTextField]
+class IntegerField(BaseModel):
+    type: Literal["integer"] = "integer"
+    key: str
+    label: str
+
+
+Field = Union[TextField, MultilineTextField, IntegerField]
 
 
 class BaseProvider(BaseModel):
@@ -245,6 +252,38 @@ async def _acall(self, *args, **kwargs) -> Coroutine[Any, Any, str]:
         return await self._call_in_executor(*args, **kwargs)
 
 
+class GPT4AllProvider(BaseProvider, GPT4All):
+    def __init__(self, **kwargs):
+        model = kwargs.get("model_id")
+        if model == "ggml-gpt4all-l13b-snoozy":
+            kwargs["backend"] = "llama"
+        else:
+            kwargs["backend"] = "gptj"
+
+        kwargs["allow_download"] = False
+        n_threads = kwargs.get("n_threads", None)
+        if n_threads is not None:
+            kwargs["n_threads"] = max(int(n_threads), 1)
+        super().__init__(**kwargs)
+
+    id = "gpt4all"
+    name = "GPT4All"
+    docs = "https://docs.gpt4all.io/gpt4all_python.html"
+    models = [
+        "ggml-gpt4all-j-v1.2-jazzy",
+        "ggml-gpt4all-j-v1.3-groovy",
+        # this one needs llama backend and has licence restriction
+        "ggml-gpt4all-l13b-snoozy",
+    ]
+    model_id_key = "model"
+    pypi_package_deps = ["gpt4all"]
+    auth_strategy = None
+    fields = [IntegerField(key="n_threads", label="CPU thread count (optional)")]
+
+    async def _acall(self, *args, **kwargs) -> Coroutine[Any, Any, str]:
+        return await self._call_in_executor(*args, **kwargs)
+
+
 HUGGINGFACE_HUB_VALID_TASKS = (
     "text2text-generation",
     "text-generation",

diff --git a/packages/jupyter-ai-magics/pyproject.toml b/packages/jupyter-ai-magics/pyproject.toml
@@ -24,7 +24,7 @@ dependencies = [
     "ipython",
     "pydantic",
     "importlib_metadata~=5.2.0",
-    "langchain==0.0.220",
+    "langchain==0.0.223",
     "typing_extensions==4.5.0",
     "click~=8.0",
     "jsonpath-ng~=1.5.3",
@@ -46,6 +46,7 @@ all = [
     "ai21",
     "anthropic~=0.2.10",
     "cohere",
+    "gpt4all",
     "huggingface_hub",
     "ipywidgets",
     "pillow",
@@ -57,6 +58,7 @@ all = [
 ai21 = "jupyter_ai_magics:AI21Provider"
 anthropic = "jupyter_ai_magics:AnthropicProvider"
 cohere = "jupyter_ai_magics:CohereProvider"
+gpt4all = "jupyter_ai_magics:GPT4AllProvider"
 huggingface_hub = "jupyter_ai_magics:HfHubProvider"
 openai = "jupyter_ai_magics:OpenAIProvider"
 openai-chat = "jupyter_ai_magics:ChatOpenAIProvider"

diff --git a/packages/jupyter-ai/pyproject.toml b/packages/jupyter-ai/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
     "openai~=0.26",
     "aiosqlite~=0.18",
     "importlib_metadata~=5.2.0",
-    "langchain==0.0.220",
+    "langchain==0.0.223",
     "tiktoken", # required for OpenAIEmbeddings
     "jupyter_ai_magics",
     "dask[distributed]",

diff --git a/packages/jupyter-ai/src/components/settings/model-fields.tsx b/packages/jupyter-ai/src/components/settings/model-fields.tsx
@@ -18,6 +18,10 @@ export function ModelField(props: ModelFieldProps): JSX.Element {
   function handleChange(
     e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>
   ) {
+    if (!('format' in props.field)) {
+      return;
+    }
+
     // Perform validation based on the field format
     switch (props.field.format) {
       case 'json':
@@ -49,6 +53,18 @@ export function ModelField(props: ModelFieldProps): JSX.Element {
     });
   }
 
+  if (props.field.type === 'integer') {
+    return (
+      <TextField
+        label={props.field.label}
+        value={props.config.fields[props.gmid]?.[props.field.key]}
+        onChange={handleChange}
+        inputProps={{ inputMode: 'numeric', pattern: '[0-9]*' }}
+        fullWidth
+      />
+    );
+  }
+
   if (props.field.type === 'text') {
     return (
       <TextField

diff --git a/packages/jupyter-ai/src/handler.ts b/packages/jupyter-ai/src/handler.ts
@@ -145,7 +145,13 @@ export namespace AiService {
     format: string;
   };
 
-  export type Field = TextField | MultilineTextField;
+  export type IntegerField = {
+    type: 'integer';
+    key: string;
+    label: string;
+  };
+
+  export type Field = TextField | MultilineTextField | IntegerField;
 
   export type ListProvidersEntry = {
     id: string;