Merge pull request #83 from upstash/hotfix-rm-llama-processor

Remove llama processor
upstash · Oct 3, 2024 · cf26b3a · cf26b3a
2 parents 5b804cf + 59975f7
commit cf26b3a
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 58 deletions.
diff --git a/src/database.ts b/src/database.ts
@@ -10,12 +10,10 @@ import type { UnstructuredLoaderOptions } from "@langchain/community/document_lo
 export type FilePath = string;
 export type URL = string;
 
-export type ProcessorType =
-  | {
-      name: "unstructured";
-      options: UnstructuredLoaderOptions;
-    }
-  | { name: "llama-parse"; options: unknown };
+export type ProcessorType = {
+  name: "unstructured";
+  options: UnstructuredLoaderOptions;
+};
 
 export type DatasWithFileSource =
   | {

diff --git a/src/file-loader.ts b/src/file-loader.ts
@@ -75,58 +75,51 @@ export class FileDataLoader {
     // Without this check typescript complains about types because of unions
     if (!hasProcessor(this.config)) throw new Error("Only processors are allowed");
 
-    switch (this.config.processor.name) {
-      case "unstructured": {
-        const client = new UnstructuredClient({
-          serverURL: "https://api.unstructuredapp.io",
-          security: {
-            apiKeyAuth: this.config.processor.options.apiKey,
-          },
-        });
-
-        //@ts-expect-error TS can't pick up the correct type due to complex union
-        const fileData = await Bun.file(this.config.fileSource).text();
-        const response = await client.general.partition({
-          //@ts-expect-error Will be fixed soon
-          partitionParameters: {
-            files: {
-              content: fileData,
-              //@ts-expect-error TS can't pick up the correct type due to complex union
-              fileName: this.config.fileSource,
-            },
-            ...this.config.processor.options,
-          },
-        });
-        const elements = response.elements?.filter(
-          (element) => typeof element.text === "string"
-        ) as Element[];
-
-        return {
-          // eslint-disable-next-line @typescript-eslint/require-await
-          load: async (): Promise<Document[]> => {
-            const documents: Document[] = [];
-            for (const element of elements) {
-              const { metadata, text } = element;
-              if (typeof text === "string" && text !== "") {
-                documents.push(
-                  new Document({
-                    pageContent: text,
-                    metadata: {
-                      ...metadata,
-                      category: element.type,
-                    },
-                  })
-                );
-              }
-            }
-            return documents;
-          },
-        };
-      }
-      case "llama-parse": {
-        throw new Error("llama-parse has been deprecated in @upstash/rag-chat 2.0.0.");
-      }
-    }
+    const client = new UnstructuredClient({
+      serverURL: "https://api.unstructuredapp.io",
+      security: {
+        apiKeyAuth: this.config.processor.options.apiKey,
+      },
+    });
+
+    //@ts-expect-error TS can't pick up the correct type due to complex union
+    const fileData = await Bun.file(this.config.fileSource).text();
+    const response = await client.general.partition({
+      //@ts-expect-error Will be fixed soon
+      partitionParameters: {
+        files: {
+          content: fileData,
+          //@ts-expect-error TS can't pick up the correct type due to complex union
+          fileName: this.config.fileSource,
+        },
+        ...this.config.processor.options,
+      },
+    });
+    const elements = response.elements?.filter(
+      (element) => typeof element.text === "string"
+    ) as Element[];
+
+    return {
+      // eslint-disable-next-line @typescript-eslint/require-await
+      load: async (): Promise<Document[]> => {
+        const documents: Document[] = [];
+        for (const element of elements) {
+          const { metadata, text } = element;
+          if (typeof text === "string" && text !== "") {
+            documents.push(
+              new Document({
+                pageContent: text,
+                metadata: {
+                  ...metadata,
+                  category: element.type,
+                },
+              })
+            );
+          }
+        }
+        return documents;
+      },
+    };
   }
 
   private isURL(source: FilePath | Blob): source is URL {