From a2861b6c16166c924913b6c2f5da0d6c07dfdf57 Mon Sep 17 00:00:00 2001 From: Matt <77928207+mattzcarey@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:39:49 +0100 Subject: [PATCH] feat: remove loader package --- .github/workflows/pr.yml | 8 +- README.md | 4 + packages/crgpt-loader/.env_example | 2 - packages/crgpt-loader/package.json | 34 - packages/crgpt-loader/pnpm-lock.yaml | 1043 ----------------- packages/crgpt-loader/src/constants.ts | 29 - packages/crgpt-loader/src/crgpt-loader.ts | 229 ---- packages/crgpt-loader/src/index.ts | 1 - .../crgpt-loader/src/lc_wip/githubLoader.ts | 2 - .../src/lc_wip/turbopufferVectorStore.ts | 203 ---- .../crgpt-loader/src/utils/executeCommand.ts | 23 - .../crgpt-loader/src/utils/getEnvVariable.ts | 7 - packages/crgpt-loader/src/utils/index.ts | 3 - packages/crgpt-loader/src/utils/openFile.ts | 11 - packages/crgpt-loader/src/utils/savePage.ts | 27 - packages/crgpt-loader/test.ts | 13 - packages/crgpt-loader/tsconfig.json | 20 - 17 files changed, 8 insertions(+), 1651 deletions(-) delete mode 100644 packages/crgpt-loader/.env_example delete mode 100644 packages/crgpt-loader/package.json delete mode 100644 packages/crgpt-loader/pnpm-lock.yaml delete mode 100644 packages/crgpt-loader/src/constants.ts delete mode 100644 packages/crgpt-loader/src/crgpt-loader.ts delete mode 100644 packages/crgpt-loader/src/index.ts delete mode 100644 packages/crgpt-loader/src/lc_wip/githubLoader.ts delete mode 100644 packages/crgpt-loader/src/lc_wip/turbopufferVectorStore.ts delete mode 100644 packages/crgpt-loader/src/utils/executeCommand.ts delete mode 100644 packages/crgpt-loader/src/utils/getEnvVariable.ts delete mode 100644 packages/crgpt-loader/src/utils/index.ts delete mode 100644 packages/crgpt-loader/src/utils/openFile.ts delete mode 100644 packages/crgpt-loader/src/utils/savePage.ts delete mode 100644 packages/crgpt-loader/test.ts delete mode 100644 packages/crgpt-loader/tsconfig.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ca2decd0..6e9980ac 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -47,7 +47,7 @@ jobs: - name: Run prompt tests run: npm run test -- --ci=github --model=gpt-3.5-turbo - build_and_test_core: + deploy_core_to_dev: runs-on: ubuntu-latest env: CI: true @@ -76,10 +76,10 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN_STAGING }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN_DEV }} - name: Install pnpm run: npm i -g pnpm - - name: 🚀 Deploy core to staging - run: pnpm i && pnpm deploy-staging + - name: 🚀 Deploy core to dev + run: pnpm i && pnpm deploy-dev diff --git a/README.md b/README.md index 36b79c62..46fdd23e 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ Thanks to our wonderful contributors! ## Roadmap (see projects tab) 🌏 +The roadmap shifts the focus to a Github app which can be installed on any repo. This will allow for a more seamless UX and better features including a chatbot to discuss the PR and make suggestions. + +The code-review-gpt package will continue to be maintained and improved based on the feedback from the Github app. + ## Sponsors ❤️ diff --git a/packages/crgpt-loader/.env_example b/packages/crgpt-loader/.env_example deleted file mode 100644 index 81707e6f..00000000 --- a/packages/crgpt-loader/.env_example +++ /dev/null @@ -1,2 +0,0 @@ -OPENAI_API_KEY= -TURBOPUFFER_API_KEY= diff --git a/packages/crgpt-loader/package.json b/packages/crgpt-loader/package.json deleted file mode 100644 index eabe0ac3..00000000 --- a/packages/crgpt-loader/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "crgpt-loader", - "version": "0.0.1", - "description": "", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "test": "ts-node test.ts", - "build": "tsc", - "publish-package": "pnpm build && npm publish --access public" - }, - "keywords": [ - "github", - "AI", - "loader", - "vector store", - "crgpt", - "turbopuffer" - ], - "author": "Matt Carey", - "license": "MIT", - "dependencies": { - "dotenv": "^16.3.1", - "langchain": "^0.0.204" - }, - "devDependencies": { - "@types/node": "^20.10.5", - "ts-node": "^10.9.2", - "typescript": "^5.3.3" - }, - "files": [ - "dist/*" - ] -} diff --git a/packages/crgpt-loader/pnpm-lock.yaml b/packages/crgpt-loader/pnpm-lock.yaml deleted file mode 100644 index acd7f484..00000000 --- a/packages/crgpt-loader/pnpm-lock.yaml +++ /dev/null @@ -1,1043 +0,0 @@ -lockfileVersion: '6.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -dependencies: - dotenv: - specifier: ^16.3.1 - version: 16.3.1 - langchain: - specifier: ^0.0.204 - version: 0.0.204 - -devDependencies: - '@types/node': - specifier: ^20.10.5 - version: 20.10.5 - ts-node: - specifier: ^10.9.2 - version: 10.9.2(@types/node@20.10.5)(typescript@5.3.3) - typescript: - specifier: ^5.3.3 - version: 5.3.3 - -packages: - - /@anthropic-ai/sdk@0.9.1: - resolution: {integrity: sha512-wa1meQ2WSfoY8Uor3EdrJq0jTiZJoKoSii2ZVWRY1oN4Tlr5s59pADg9T79FTbPe1/se5c3pBeZgJL63wmuoBA==} - dependencies: - '@types/node': 18.19.3 - '@types/node-fetch': 2.6.9 - abort-controller: 3.0.0 - agentkeepalive: 4.5.0 - digest-fetch: 1.3.0 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0 - web-streams-polyfill: 3.2.1 - transitivePeerDependencies: - - encoding - dev: false - - /@cspotcode/source-map-support@0.8.1: - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - dev: true - - /@jridgewell/resolve-uri@3.1.1: - resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} - engines: {node: '>=6.0.0'} - dev: true - - /@jridgewell/sourcemap-codec@1.4.15: - resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} - dev: true - - /@jridgewell/trace-mapping@0.3.9: - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - dependencies: - '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.4.15 - dev: true - - /@langchain/community@0.0.2: - resolution: {integrity: sha512-C4wu+IoFbvM0ndSE3etVjqzw7K7vCCN+qIuus2/c0LTJXlhR07o2cR2m45rJkLngmi7akYir+wyRop0eauFuog==} - engines: {node: '>=18'} - peerDependencies: - '@aws-crypto/sha256-js': ^5.0.0 - '@aws-sdk/client-bedrock-runtime': ^3.422.0 - '@aws-sdk/client-dynamodb': ^3.310.0 - '@aws-sdk/client-kendra': ^3.352.0 - '@aws-sdk/client-lambda': ^3.310.0 - '@aws-sdk/client-sagemaker-runtime': ^3.310.0 - '@aws-sdk/client-sfn': ^3.310.0 - '@aws-sdk/credential-provider-node': ^3.388.0 - '@clickhouse/client': ^0.2.5 - '@cloudflare/ai': ^1.0.12 - '@elastic/elasticsearch': ^8.4.0 - '@getmetal/metal-sdk': '*' - '@getzep/zep-js': ^0.9.0 - '@gomomento/sdk': ^1.51.1 - '@gomomento/sdk-core': ^1.51.1 - '@google-ai/generativelanguage': ^0.2.1 - '@gradientai/nodejs-sdk': ^1.2.0 - '@huggingface/inference': ^2.6.4 - '@mozilla/readability': '*' - '@opensearch-project/opensearch': '*' - '@pinecone-database/pinecone': ^1.1.0 - '@planetscale/database': ^1.8.0 - '@qdrant/js-client-rest': ^1.2.0 - '@raycast/api': ^1.55.2 - '@rockset/client': ^0.9.1 - '@smithy/eventstream-codec': ^2.0.5 - '@smithy/protocol-http': ^3.0.6 - '@smithy/signature-v4': ^2.0.10 - '@smithy/util-utf8': ^2.0.0 - '@supabase/postgrest-js': ^1.1.1 - '@supabase/supabase-js': ^2.10.0 - '@tensorflow-models/universal-sentence-encoder': '*' - '@tensorflow/tfjs-converter': '*' - '@tensorflow/tfjs-core': '*' - '@upstash/redis': ^1.20.6 - '@vercel/kv': ^0.2.3 - '@vercel/postgres': ^0.5.0 - '@writerai/writer-sdk': ^0.40.2 - '@xata.io/client': ^0.28.0 - '@xenova/transformers': ^2.5.4 - '@zilliz/milvus2-sdk-node': '>=2.2.7' - cassandra-driver: ^4.7.2 - chromadb: '*' - closevector-common: 0.1.0-alpha.1 - closevector-node: 0.1.0-alpha.10 - closevector-web: 0.1.0-alpha.16 - cohere-ai: '>=6.0.0' - convex: ^1.3.1 - faiss-node: ^0.5.1 - firebase-admin: ^11.9.0 - google-auth-library: ^8.9.0 - googleapis: ^126.0.1 - hnswlib-node: ^1.4.2 - html-to-text: ^9.0.5 - ioredis: ^5.3.2 - jsdom: '*' - llmonitor: ^0.5.9 - lodash: ^4.17.21 - mongodb: ^5.2.0 - mysql2: ^3.3.3 - neo4j-driver: '*' - node-llama-cpp: '*' - pg: ^8.11.0 - pg-copy-streams: ^6.0.5 - pickleparser: ^0.2.1 - portkey-ai: ^0.1.11 - redis: ^4.6.4 - replicate: ^0.18.0 - typeorm: ^0.3.12 - typesense: ^1.5.3 - usearch: ^1.1.1 - vectordb: ^0.1.4 - voy-search: 0.6.2 - weaviate-ts-client: ^1.4.0 - web-auth-library: ^1.0.3 - ws: ^8.14.2 - peerDependenciesMeta: - '@aws-crypto/sha256-js': - optional: true - '@aws-sdk/client-bedrock-runtime': - optional: true - '@aws-sdk/client-dynamodb': - optional: true - '@aws-sdk/client-kendra': - optional: true - '@aws-sdk/client-lambda': - optional: true - '@aws-sdk/client-sagemaker-runtime': - optional: true - '@aws-sdk/client-sfn': - optional: true - '@aws-sdk/credential-provider-node': - optional: true - '@clickhouse/client': - optional: true - '@cloudflare/ai': - optional: true - '@elastic/elasticsearch': - optional: true - '@getmetal/metal-sdk': - optional: true - '@getzep/zep-js': - optional: true - '@gomomento/sdk': - optional: true - '@gomomento/sdk-core': - optional: true - '@google-ai/generativelanguage': - optional: true - '@gradientai/nodejs-sdk': - optional: true - '@huggingface/inference': - optional: true - '@mozilla/readability': - optional: true - '@opensearch-project/opensearch': - optional: true - '@pinecone-database/pinecone': - optional: true - '@planetscale/database': - optional: true - '@qdrant/js-client-rest': - optional: true - '@raycast/api': - optional: true - '@rockset/client': - optional: true - '@smithy/eventstream-codec': - optional: true - '@smithy/protocol-http': - optional: true - '@smithy/signature-v4': - optional: true - '@smithy/util-utf8': - optional: true - '@supabase/postgrest-js': - optional: true - '@supabase/supabase-js': - optional: true - '@tensorflow-models/universal-sentence-encoder': - optional: true - '@tensorflow/tfjs-converter': - optional: true - '@tensorflow/tfjs-core': - optional: true - '@upstash/redis': - optional: true - '@vercel/kv': - optional: true - '@vercel/postgres': - optional: true - '@writerai/writer-sdk': - optional: true - '@xata.io/client': - optional: true - '@xenova/transformers': - optional: true - '@zilliz/milvus2-sdk-node': - optional: true - cassandra-driver: - optional: true - chromadb: - optional: true - closevector-common: - optional: true - closevector-node: - optional: true - closevector-web: - optional: true - cohere-ai: - optional: true - convex: - optional: true - faiss-node: - optional: true - firebase-admin: - optional: true - google-auth-library: - optional: true - googleapis: - optional: true - hnswlib-node: - optional: true - html-to-text: - optional: true - ioredis: - optional: true - jsdom: - optional: true - llmonitor: - optional: true - lodash: - optional: true - mongodb: - optional: true - mysql2: - optional: true - neo4j-driver: - optional: true - node-llama-cpp: - optional: true - pg: - optional: true - pg-copy-streams: - optional: true - pickleparser: - optional: true - portkey-ai: - optional: true - redis: - optional: true - replicate: - optional: true - typeorm: - optional: true - typesense: - optional: true - usearch: - optional: true - vectordb: - optional: true - voy-search: - optional: true - weaviate-ts-client: - optional: true - web-auth-library: - optional: true - ws: - optional: true - dependencies: - '@langchain/core': 0.0.11 - '@langchain/openai': 0.0.2 - flat: 5.0.2 - langsmith: 0.0.49 - uuid: 9.0.1 - zod: 3.22.4 - transitivePeerDependencies: - - encoding - dev: false - - /@langchain/core@0.0.11: - resolution: {integrity: sha512-tiESyyHM1KO1gRTduKcznWbEmE7z/ayPLWZ4+AUXF47qOtdV6lymnlMPzz+MGwnpaSaamzyYkBIxqeMPar256Q==} - engines: {node: '>=18'} - dependencies: - ansi-styles: 5.2.0 - camelcase: 6.3.0 - decamelize: 1.2.0 - js-tiktoken: 1.0.8 - langsmith: 0.0.49 - ml-distance: 4.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 9.0.1 - zod: 3.22.4 - dev: false - - /@langchain/openai@0.0.2: - resolution: {integrity: sha512-GhiC8afAjta37fXFPiEjTmJ3Uof9HNoahLAxPGjhN1pJrUPisA6nXhsqp6Iab281GN5Xxdms5YsV7BwrfsUt2Q==} - engines: {node: '>=18'} - dependencies: - '@langchain/core': 0.0.11 - js-tiktoken: 1.0.8 - openai: 4.20.1 - zod: 3.22.4 - zod-to-json-schema: 3.20.3(zod@3.22.4) - transitivePeerDependencies: - - encoding - dev: false - - /@tsconfig/node10@1.0.9: - resolution: {integrity: sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==} - dev: true - - /@tsconfig/node12@1.0.11: - resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} - dev: true - - /@tsconfig/node14@1.0.3: - resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} - dev: true - - /@tsconfig/node16@1.0.4: - resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} - dev: true - - /@types/node-fetch@2.6.9: - resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==} - dependencies: - '@types/node': 20.10.5 - form-data: 4.0.0 - dev: false - - /@types/node@18.19.3: - resolution: {integrity: sha512-k5fggr14DwAytoA/t8rPrIz++lXK7/DqckthCmoZOKNsEbJkId4Z//BqgApXBUGrGddrigYa1oqheo/7YmW4rg==} - dependencies: - undici-types: 5.26.5 - dev: false - - /@types/node@20.10.5: - resolution: {integrity: sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==} - dependencies: - undici-types: 5.26.5 - - /@types/retry@0.12.0: - resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} - dev: false - - /@types/uuid@9.0.7: - resolution: {integrity: sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==} - dev: false - - /abort-controller@3.0.0: - resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} - engines: {node: '>=6.5'} - dependencies: - event-target-shim: 5.0.1 - dev: false - - /acorn-walk@8.3.1: - resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} - engines: {node: '>=0.4.0'} - dev: true - - /acorn@8.11.2: - resolution: {integrity: sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==} - engines: {node: '>=0.4.0'} - hasBin: true - dev: true - - /agentkeepalive@4.5.0: - resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} - engines: {node: '>= 8.0.0'} - dependencies: - humanize-ms: 1.2.1 - dev: false - - /ansi-styles@5.2.0: - resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} - engines: {node: '>=10'} - dev: false - - /arg@4.1.3: - resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - dev: true - - /argparse@2.0.1: - resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} - dev: false - - /asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: false - - /base-64@0.1.0: - resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} - dev: false - - /base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - dev: false - - /binary-extensions@2.2.0: - resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} - engines: {node: '>=8'} - dev: false - - /binary-search@1.3.6: - resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==} - dev: false - - /camelcase@6.3.0: - resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} - engines: {node: '>=10'} - dev: false - - /charenc@0.0.2: - resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} - dev: false - - /combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: false - - /commander@10.0.1: - resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} - engines: {node: '>=14'} - dev: false - - /create-require@1.1.1: - resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - dev: true - - /crypt@0.0.2: - resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} - dev: false - - /decamelize@1.2.0: - resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} - engines: {node: '>=0.10.0'} - dev: false - - /delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: false - - /diff@4.0.2: - resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} - engines: {node: '>=0.3.1'} - dev: true - - /digest-fetch@1.3.0: - resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} - dependencies: - base-64: 0.1.0 - md5: 2.3.0 - dev: false - - /dotenv@16.3.1: - resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==} - engines: {node: '>=12'} - dev: false - - /event-target-shim@5.0.1: - resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} - engines: {node: '>=6'} - dev: false - - /eventemitter3@4.0.7: - resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} - dev: false - - /expr-eval@2.0.2: - resolution: {integrity: sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==} - dev: false - - /flat@5.0.2: - resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} - hasBin: true - dev: false - - /form-data-encoder@1.7.2: - resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - dev: false - - /form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - - /formdata-node@4.4.1: - resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} - engines: {node: '>= 12.20'} - dependencies: - node-domexception: 1.0.0 - web-streams-polyfill: 4.0.0-beta.3 - dev: false - - /humanize-ms@1.2.1: - resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} - dependencies: - ms: 2.1.3 - dev: false - - /is-any-array@2.0.1: - resolution: {integrity: sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==} - dev: false - - /is-buffer@1.1.6: - resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} - dev: false - - /js-tiktoken@1.0.8: - resolution: {integrity: sha512-r7XK3E9/I+SOrbAGqb39pyO/rHAS1diAOSRAvaaLfHgXjkUSK9AiSd+r84Vn2f/GvXJYRAxKj8NHrUvqlaH5qg==} - dependencies: - base64-js: 1.5.1 - dev: false - - /js-yaml@4.1.0: - resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} - hasBin: true - dependencies: - argparse: 2.0.1 - dev: false - - /jsonpointer@5.0.1: - resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} - engines: {node: '>=0.10.0'} - dev: false - - /langchain@0.0.204: - resolution: {integrity: sha512-fLTgPhheCE/4SXsj1C5t+txbBbVtfN9dNoKysDPxGyRUZHycSd3A+cEyn7dJJGYHZxRMc7EQgg7yffxzrM7c0A==} - engines: {node: '>=18'} - peerDependencies: - '@aws-sdk/client-s3': ^3.310.0 - '@aws-sdk/client-sagemaker-runtime': ^3.310.0 - '@aws-sdk/client-sfn': ^3.310.0 - '@aws-sdk/credential-provider-node': ^3.388.0 - '@azure/storage-blob': ^12.15.0 - '@gomomento/sdk': ^1.51.1 - '@gomomento/sdk-core': ^1.51.1 - '@gomomento/sdk-web': ^1.51.1 - '@google-ai/generativelanguage': ^0.2.1 - '@google-cloud/storage': ^6.10.1 - '@notionhq/client': ^2.2.10 - '@pinecone-database/pinecone': ^1.1.0 - '@supabase/supabase-js': ^2.10.0 - '@vercel/kv': ^0.2.3 - '@xata.io/client': ^0.28.0 - apify-client: ^2.7.1 - assemblyai: ^2.0.2 - axios: '*' - cheerio: ^1.0.0-rc.12 - chromadb: '*' - convex: ^1.3.1 - d3-dsv: ^2.0.0 - epub2: ^3.0.1 - faiss-node: '*' - fast-xml-parser: ^4.2.7 - google-auth-library: ^8.9.0 - googleapis: ^126.0.1 - html-to-text: ^9.0.5 - ignore: ^5.2.0 - ioredis: ^5.3.2 - jsdom: '*' - mammoth: '*' - mongodb: ^5.2.0 - node-llama-cpp: '*' - notion-to-md: ^3.1.0 - officeparser: ^4.0.4 - pdf-parse: 1.1.1 - peggy: ^3.0.2 - playwright: ^1.32.1 - puppeteer: ^19.7.2 - pyodide: ^0.24.1 - redis: ^4.6.4 - sonix-speech-recognition: ^2.1.1 - srt-parser-2: ^1.2.2 - typeorm: ^0.3.12 - vectordb: ^0.1.4 - weaviate-ts-client: ^1.4.0 - web-auth-library: ^1.0.3 - ws: ^8.14.2 - youtube-transcript: ^1.0.6 - youtubei.js: ^5.8.0 - peerDependenciesMeta: - '@aws-sdk/client-s3': - optional: true - '@aws-sdk/client-sagemaker-runtime': - optional: true - '@aws-sdk/client-sfn': - optional: true - '@aws-sdk/credential-provider-node': - optional: true - '@azure/storage-blob': - optional: true - '@gomomento/sdk': - optional: true - '@gomomento/sdk-core': - optional: true - '@gomomento/sdk-web': - optional: true - '@google-ai/generativelanguage': - optional: true - '@google-cloud/storage': - optional: true - '@notionhq/client': - optional: true - '@pinecone-database/pinecone': - optional: true - '@supabase/supabase-js': - optional: true - '@vercel/kv': - optional: true - '@xata.io/client': - optional: true - apify-client: - optional: true - assemblyai: - optional: true - axios: - optional: true - cheerio: - optional: true - chromadb: - optional: true - convex: - optional: true - d3-dsv: - optional: true - epub2: - optional: true - faiss-node: - optional: true - fast-xml-parser: - optional: true - google-auth-library: - optional: true - googleapis: - optional: true - html-to-text: - optional: true - ignore: - optional: true - ioredis: - optional: true - jsdom: - optional: true - mammoth: - optional: true - mongodb: - optional: true - node-llama-cpp: - optional: true - notion-to-md: - optional: true - officeparser: - optional: true - pdf-parse: - optional: true - peggy: - optional: true - playwright: - optional: true - puppeteer: - optional: true - pyodide: - optional: true - redis: - optional: true - sonix-speech-recognition: - optional: true - srt-parser-2: - optional: true - typeorm: - optional: true - vectordb: - optional: true - weaviate-ts-client: - optional: true - web-auth-library: - optional: true - ws: - optional: true - youtube-transcript: - optional: true - youtubei.js: - optional: true - dependencies: - '@anthropic-ai/sdk': 0.9.1 - '@langchain/community': 0.0.2 - '@langchain/core': 0.0.11 - '@langchain/openai': 0.0.2 - binary-extensions: 2.2.0 - expr-eval: 2.0.2 - js-tiktoken: 1.0.8 - js-yaml: 4.1.0 - jsonpointer: 5.0.1 - langchainhub: 0.0.6 - langsmith: 0.0.49 - ml-distance: 4.0.1 - openapi-types: 12.1.3 - p-retry: 4.6.2 - uuid: 9.0.1 - yaml: 2.3.4 - zod: 3.22.4 - zod-to-json-schema: 3.20.3(zod@3.22.4) - transitivePeerDependencies: - - '@aws-crypto/sha256-js' - - '@aws-sdk/client-bedrock-runtime' - - '@aws-sdk/client-dynamodb' - - '@aws-sdk/client-kendra' - - '@aws-sdk/client-lambda' - - '@clickhouse/client' - - '@cloudflare/ai' - - '@elastic/elasticsearch' - - '@getmetal/metal-sdk' - - '@getzep/zep-js' - - '@gradientai/nodejs-sdk' - - '@huggingface/inference' - - '@mozilla/readability' - - '@opensearch-project/opensearch' - - '@planetscale/database' - - '@qdrant/js-client-rest' - - '@raycast/api' - - '@rockset/client' - - '@smithy/eventstream-codec' - - '@smithy/protocol-http' - - '@smithy/signature-v4' - - '@smithy/util-utf8' - - '@supabase/postgrest-js' - - '@tensorflow-models/universal-sentence-encoder' - - '@tensorflow/tfjs-converter' - - '@tensorflow/tfjs-core' - - '@upstash/redis' - - '@vercel/postgres' - - '@writerai/writer-sdk' - - '@xenova/transformers' - - '@zilliz/milvus2-sdk-node' - - cassandra-driver - - closevector-common - - closevector-node - - closevector-web - - cohere-ai - - encoding - - firebase-admin - - hnswlib-node - - llmonitor - - lodash - - mysql2 - - neo4j-driver - - pg - - pg-copy-streams - - pickleparser - - portkey-ai - - replicate - - typesense - - usearch - - voy-search - dev: false - - /langchainhub@0.0.6: - resolution: {integrity: sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w==} - dev: false - - /langsmith@0.0.49: - resolution: {integrity: sha512-TC2RkjDizxtubClLNjLkB0m53NHtom9LHIhbdRYJ5O6gqUpcXkN/pefb8w734lBp9VYfnYBk/JJ8AiCq+ooeDQ==} - hasBin: true - dependencies: - '@types/uuid': 9.0.7 - commander: 10.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 9.0.1 - dev: false - - /make-error@1.3.6: - resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - dev: true - - /md5@2.3.0: - resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} - dependencies: - charenc: 0.0.2 - crypt: 0.0.2 - is-buffer: 1.1.6 - dev: false - - /mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: false - - /mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - dependencies: - mime-db: 1.52.0 - dev: false - - /ml-array-mean@1.1.6: - resolution: {integrity: sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==} - dependencies: - ml-array-sum: 1.1.6 - dev: false - - /ml-array-sum@1.1.6: - resolution: {integrity: sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==} - dependencies: - is-any-array: 2.0.1 - dev: false - - /ml-distance-euclidean@2.0.0: - resolution: {integrity: sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q==} - dev: false - - /ml-distance@4.0.1: - resolution: {integrity: sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==} - dependencies: - ml-array-mean: 1.1.6 - ml-distance-euclidean: 2.0.0 - ml-tree-similarity: 1.0.0 - dev: false - - /ml-tree-similarity@1.0.0: - resolution: {integrity: sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==} - dependencies: - binary-search: 1.3.6 - num-sort: 2.1.0 - dev: false - - /ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - dev: false - - /node-domexception@1.0.0: - resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} - engines: {node: '>=10.5.0'} - dev: false - - /node-fetch@2.7.0: - resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - dependencies: - whatwg-url: 5.0.0 - dev: false - - /num-sort@2.1.0: - resolution: {integrity: sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==} - engines: {node: '>=8'} - dev: false - - /openai@4.20.1: - resolution: {integrity: sha512-Dd3q8EvINfganZFtg6V36HjrMaihqRgIcKiHua4Nq9aw/PxOP48dhbsk8x5klrxajt5Lpnc1KTOG5i1S6BKAJA==} - hasBin: true - dependencies: - '@types/node': 18.19.3 - '@types/node-fetch': 2.6.9 - abort-controller: 3.0.0 - agentkeepalive: 4.5.0 - digest-fetch: 1.3.0 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0 - web-streams-polyfill: 3.2.1 - transitivePeerDependencies: - - encoding - dev: false - - /openapi-types@12.1.3: - resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} - dev: false - - /p-finally@1.0.0: - resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} - engines: {node: '>=4'} - dev: false - - /p-queue@6.6.2: - resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} - engines: {node: '>=8'} - dependencies: - eventemitter3: 4.0.7 - p-timeout: 3.2.0 - dev: false - - /p-retry@4.6.2: - resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} - engines: {node: '>=8'} - dependencies: - '@types/retry': 0.12.0 - retry: 0.13.1 - dev: false - - /p-timeout@3.2.0: - resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} - engines: {node: '>=8'} - dependencies: - p-finally: 1.0.0 - dev: false - - /retry@0.13.1: - resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} - engines: {node: '>= 4'} - dev: false - - /tr46@0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - dev: false - - /ts-node@10.9.2(@types/node@20.10.5)(typescript@5.3.3): - resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} - hasBin: true - peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' - '@types/node': '*' - typescript: '>=2.7' - peerDependenciesMeta: - '@swc/core': - optional: true - '@swc/wasm': - optional: true - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.9 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 20.10.5 - acorn: 8.11.2 - acorn-walk: 8.3.1 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.3.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - dev: true - - /typescript@5.3.3: - resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} - engines: {node: '>=14.17'} - hasBin: true - dev: true - - /undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - - /uuid@9.0.1: - resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} - hasBin: true - dev: false - - /v8-compile-cache-lib@3.0.1: - resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} - dev: true - - /web-streams-polyfill@3.2.1: - resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} - engines: {node: '>= 8'} - dev: false - - /web-streams-polyfill@4.0.0-beta.3: - resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} - engines: {node: '>= 14'} - dev: false - - /webidl-conversions@3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - dev: false - - /whatwg-url@5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - dev: false - - /yaml@2.3.4: - resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} - engines: {node: '>= 14'} - dev: false - - /yn@3.1.1: - resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} - engines: {node: '>=6'} - dev: true - - /zod-to-json-schema@3.20.3(zod@3.22.4): - resolution: {integrity: sha512-/Q3wnyxAfCt94ZcrGiXXoiAfRqasxl9CX64LZ9fj+4dKH68zulUtU0uk1WMxQPfAxQ0ZI70dKzcoW7hHj+DwSQ==} - peerDependencies: - zod: ^3.20.0 - dependencies: - zod: 3.22.4 - dev: false - - /zod@3.22.4: - resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} - dev: false diff --git a/packages/crgpt-loader/src/constants.ts b/packages/crgpt-loader/src/constants.ts deleted file mode 100644 index e62af2c0..00000000 --- a/packages/crgpt-loader/src/constants.ts +++ /dev/null @@ -1,29 +0,0 @@ -export const removeFolders = [ - "node_modules", - ".git", - ".github", - ".vscode", - ".idea", - "dist", - "build", - "out", - "coverage", - "tmp", - "temp", - "log", - "logs", -]; - -export const lockFiles = ["package-lock.json", "pnpm-lock.yaml", "yarn.lock"]; - -export const removeFoldersCommand = (dir: string): string => { - return `find ${dir} -type d \\( ${removeFolders - .map((folder) => `-name '${folder}'`) - .join(" -o ")} \\) -exec rm -rf {} +`; -}; - -export const removeFilesCommand = (dir: string): string => { - return `find ${dir} -type f \\( ${lockFiles - .map((file) => `-name '${file}'`) - .join(" -o ")} \\) -delete`; -}; diff --git a/packages/crgpt-loader/src/crgpt-loader.ts b/packages/crgpt-loader/src/crgpt-loader.ts deleted file mode 100644 index 9abf906a..00000000 --- a/packages/crgpt-loader/src/crgpt-loader.ts +++ /dev/null @@ -1,229 +0,0 @@ -import dotenv from "dotenv"; -import { promises as fsPromises } from "fs"; -import { Document } from "langchain/document"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import { tmpdir } from "os"; -import { join } from "path"; -import { removeFilesCommand, removeFoldersCommand } from "./constants"; -import { executeCommand, openFile, savePage } from "./utils"; - -dotenv.config(); - -export class CRGPTLoader { - private link: string; - private embeddings: OpenAIEmbeddings; - - constructor(link: string) { - this.link = link; - this.embeddings = new OpenAIEmbeddings(); - } - - private extractRepoName(): string { - return this.link.split("/").slice(-1)[0]; - } - - private async splitDocuments(documents: Document[]): Promise { - const splitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1500, - }); - - return splitter.splitDocuments(documents); - } - - private async buildDocument( - filePath: string - ): Promise> { - return new Document({ - pageContent: await openFile(filePath), - metadata: { - source: filePath, - }, - }); - } - - private async getEmbeddings(documents: Document[]): Promise { - return this.embeddings.embedDocuments( - documents.map((doc) => doc.pageContent) - ); - } - - private async storeDocuments( - documents: Document[], - embeddings: Number[][], - indexName = this.extractRepoName() - ): Promise { - try { - const ids = documents.map((_, index) => index); - const attributes = { - source: documents.map((doc) => doc.metadata.source), - pageContent: documents.map((doc) => doc.pageContent), - }; - - const apiEndpoint = `https://api.turbopuffer.com/v1/vectors/${indexName}`; - const headers = { - Authorization: `Bearer ${process.env.TURBOPUFFER_API_KEY}`, - "Content-Type": "application/json", - }; - - await fetch(apiEndpoint, { - method: "POST", - headers, - body: JSON.stringify({ - ids, - vectors: embeddings, - attributes, - }), - }); - } catch (error) { - console.error("Error storing documents:", error); - throw error; - } - } - - public async load(): Promise { - try { - const tempDir = await this.cloneRepository(); - await this.removeUnwantedFilesAndFolders(tempDir); - - const documents = await this.createDocuments(tempDir); - - const chunks = await this.splitDocuments(documents); - const embeddings = await this.getEmbeddings(chunks); - - await this.storeDocuments(chunks, embeddings); - console.log("Documents stored"); - - await this.cleanup(tempDir); - } catch (error) { - console.error("Error in CRGPTLoader:", error); - } - } - - private async cloneRepository(): Promise { - const tempDir = await fsPromises.mkdtemp(join(tmpdir(), "CRGPTLoader-")); - const cloneCommand = `git clone --depth 1 ${this.link} ${tempDir}`; - await executeCommand(cloneCommand); - return tempDir; - } - - private async removeUnwantedFilesAndFolders(tempDir: string): Promise { - try { - await executeCommand(removeFoldersCommand(tempDir)); - await executeCommand(removeFilesCommand(tempDir)); - } catch (error) { - console.error("Error removing files or folders:", error); - } - } - - private async cleanup(tempDir: string): Promise { - await executeCommand(`rm -rf ${tempDir}`); - } - - private async createDocuments( - directory: string - ): Promise[]> { - const entries = await fsPromises.readdir(directory, { - withFileTypes: true, - }); - const documents: Document<{ source: string }>[] = []; - - for (const entry of entries) { - const fullPath = join(directory, entry.name); - if (entry.isDirectory()) { - documents.push(...(await this.createDocuments(fullPath))); - } else if (entry.isFile()) { - try { - const document = await this.buildDocument(fullPath); - documents.push(document); - } catch (error) { - console.error(`Error reading file ${entry.name}:`, error); - } - } - } - - return documents; - } - - public async read(): Promise { - const namespace = this.extractRepoName(); - let nextCursor = null; - const dataDir = "./data"; - let pageIndex = 0; - - do { - try { - const response = await this.fetchPage(namespace, nextCursor); - - if (response.status === 202) { - // Data not ready, wait and retry - await new Promise((resolve) => setTimeout(resolve, 5000)); // wait for 5 seconds - continue; - } - - // Parse the response body as JSON - const data = await response.json(); - const { ids, vectors, attributes, next_cursor } = data; - - savePage(dataDir, pageIndex, ids, vectors, attributes); - - nextCursor = next_cursor; - pageIndex++; - } catch (error) { - console.error("Error fetching data:", error); - throw error; - } - } while (nextCursor); - } - - private async fetchPage( - namespace: string, - cursor: string | null - ): Promise { - const apiEndpoint = new URL( - `https://api.turbopuffer.com/v1/vectors/${namespace}` - ); - - if (cursor) { - apiEndpoint.searchParams.append("cursor", cursor); - } - - const response = await fetch(apiEndpoint.toString(), { - method: "GET", - headers: { - Authorization: `Bearer ${process.env.TURBOPUFFER_API_KEY}`, - }, - }); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - return response; - } - - public async delete(indexName = this.extractRepoName()): Promise { - try { - // Set up the API endpoint and headers - const apiEndpoint = `https://api.turbopuffer.com/v1/vectors/${indexName}`; - const headers = { - Authorization: `Bearer ${process.env.TURBOPUFFER_API_KEY}`, - }; - - // Make the DELETE request - const res = await fetch(apiEndpoint, { - method: "DELETE", - headers, - }); - - // Parse the response - const response = await res.json(); - - // Log the response status - console.log("Delete response:", response.data); - } catch (error) { - console.error("Error deleting documents:", error); - throw error; - } - } -} diff --git a/packages/crgpt-loader/src/index.ts b/packages/crgpt-loader/src/index.ts deleted file mode 100644 index 73396005..00000000 --- a/packages/crgpt-loader/src/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { CRGPTLoader } from "./crgpt-loader"; diff --git a/packages/crgpt-loader/src/lc_wip/githubLoader.ts b/packages/crgpt-loader/src/lc_wip/githubLoader.ts deleted file mode 100644 index 6f82e681..00000000 --- a/packages/crgpt-loader/src/lc_wip/githubLoader.ts +++ /dev/null @@ -1,2 +0,0 @@ -//WIP GitHub loader integration using UNIX commands - diff --git a/packages/crgpt-loader/src/lc_wip/turbopufferVectorStore.ts b/packages/crgpt-loader/src/lc_wip/turbopufferVectorStore.ts deleted file mode 100644 index 830ae5dd..00000000 --- a/packages/crgpt-loader/src/lc_wip/turbopufferVectorStore.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { Document } from "langchain/document"; -import { Embeddings } from "langchain/embeddings/base"; -import { VectorStore } from "langchain/vectorstores/base"; - -interface TurboPufferIntegrationParams { - apiKey?: string; - namespace?: string; -} - -interface TurboPufferHeaders { - headers: { - Authorization: string; - "Content-Type": string; - }; -} - -enum TurboPufferDistanceMetric { - Cosine = "cosine_distance", - Euclidean = "euclidean_squared", -} - -interface TurboPufferQueryResult { - dist: number; - id: number; - vector: number[]; - attributes: Record; -} - -export class TurboPuffer extends VectorStore { - get lc_secrets(): { [key: string]: string } { - return { - apiKey: "TURBOPUFFER_API_KEY", - }; - } - - get lc_aliases(): { [key: string]: string } { - return { - apiKey: "turbopuffer_api_key", - }; - } - - private apiKey: string; - private namespace: string; - private apiEndpoint = "https://api.turbopuffer.com/v1/"; - - public _vectorstoreType(): string { - return "turbopuffer"; - } - - constructor( - embeddings: Embeddings, - args: { - apiKey?: string; - namespace?: string; - } - ) { - super(embeddings, args); - - const apiKey = args.apiKey ?? process.env["TURBOPUFFER_API_KEY"]; - if (!apiKey) { - throw new Error("TurboPuffer api key is not provided."); - } - this.apiKey = apiKey; - this.namespace = args.namespace ?? "default"; - } - - getJsonHeader(): TurboPufferHeaders { - return { - headers: { - Authorization: `Bearer ${this.apiKey}`, - "Content-Type": "application/json", - }, - }; - } - - async addVectors( - vectors: number[][], - documents: Document>[], - options?: { ids?: number[] } - ): Promise { - try { - if (options?.ids && options.ids.length !== vectors.length) { - throw new Error( - "Number of ids provided does not match number of vectors" - ); - } - - if (documents.length !== vectors.length) { - throw new Error( - "Number of documents provided does not match number of vectors" - ); - } - - if (documents.length === 0) { - throw new Error("No documents provided"); - } - - const docIds = options?.ids ?? documents.map((_, index) => index); - - const attributes = { - source: documents.map((doc) => doc.metadata.source), - pageContent: documents.map((doc) => doc.pageContent), - }; - - const data = { - docIds, - vectors, - attributes, - }; - - await fetch(`${this.apiEndpoint}/vectors/${this.namespace}`, { - method: "POST", - headers: this.getJsonHeader().headers, - body: JSON.stringify(data), - }); - } catch (error) { - console.error("Error storing vectors:", error); - throw error; - } - } - - async addDocuments( - documents: Document>[], - options?: { ids?: number[] } - ): Promise { - const vectors = await this.embeddings.embedDocuments( - documents.map((doc) => doc.pageContent) - ); - - return this.addVectors(vectors, documents, options); - } - - async queryVectors( - query: number[], - k: number, - distanceMetric: TurboPufferDistanceMetric, - includeAttributes?: string[], - includeVector?: boolean, - // See https://turbopuffer.com/docs/reference/query for more info - filters?: Record - ): Promise { - const data = { - query, - k, - distanceMetric, - filters, - includeAttributes, - includeVector, - }; - - const response = await fetch( - `${this.apiEndpoint}/vectors/${this.namespace}/query`, - { - method: "POST", - headers: this.getJsonHeader().headers, - body: JSON.stringify(data), - } - ); - - const json = await response.json(); - - return json.results; - } - - async similaritySearchVectorWithScore( - query: number[], - k: number, - filter?: Record - ): Promise<[Document, number][]> { - const search = await this.queryVectors( - query, - k, - TurboPufferDistanceMetric.Cosine, - ["source", "pageContent"], - false, - filter - ); - - const result: [Document, number][] = search.map((res) => { - return [ - new Document({ - pageContent: res.attributes.pageContent, - metadata: { - source: res.attributes.source, - }, - }), - res.dist, - ]; - }); - - return result; - } - - static async fromDocuments( - docs: Document[], - embeddings: Embeddings, - dbConfig: TurboPufferIntegrationParams - ): Promise { - const instance = new this(embeddings, dbConfig); - await instance.addDocuments(docs); - return instance; - } -} diff --git a/packages/crgpt-loader/src/utils/executeCommand.ts b/packages/crgpt-loader/src/utils/executeCommand.ts deleted file mode 100644 index 0e040cbd..00000000 --- a/packages/crgpt-loader/src/utils/executeCommand.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { exec } from "child_process"; - -export const executeCommand = (command: string): Promise => { - return new Promise((resolve, reject) => { - exec(command, (error, stdout, stderr) => { - if (error) { - console.error( - `Error executing command: ${command}\nError: ${error.message}\nstderr: ${stderr}` - ); - return reject(error); - } - if (stderr) { - console.warn( - `Command executed with warnings: ${command}\nstderr: ${stderr}` - ); - } - if (stdout) { - console.log(`Command executed: ${command}\nstdout: ${stdout}`); - } - resolve(); - }); - }); -}; diff --git a/packages/crgpt-loader/src/utils/getEnvVariable.ts b/packages/crgpt-loader/src/utils/getEnvVariable.ts deleted file mode 100644 index 2eb5a863..00000000 --- a/packages/crgpt-loader/src/utils/getEnvVariable.ts +++ /dev/null @@ -1,7 +0,0 @@ -export const getEnvVariable = (name: string): string => { - const value = process.env[name]; - if (!value) { - throw new Error(`Missing environment variable: ${name}`); - } - return value; -}; diff --git a/packages/crgpt-loader/src/utils/index.ts b/packages/crgpt-loader/src/utils/index.ts deleted file mode 100644 index 41c6dd44..00000000 --- a/packages/crgpt-loader/src/utils/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./executeCommand"; -export * from "./openFile"; -export * from "./savePage"; diff --git a/packages/crgpt-loader/src/utils/openFile.ts b/packages/crgpt-loader/src/utils/openFile.ts deleted file mode 100644 index 862cd943..00000000 --- a/packages/crgpt-loader/src/utils/openFile.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { promises as fsPromises } from "fs"; - -export const openFile = async (filePath: string): Promise => { - try { - const content = await fsPromises.readFile(filePath, "utf8"); - return content; - } catch (error) { - console.error("Error reading file:", error); - throw error; - } -}; diff --git a/packages/crgpt-loader/src/utils/savePage.ts b/packages/crgpt-loader/src/utils/savePage.ts deleted file mode 100644 index 1a26daf1..00000000 --- a/packages/crgpt-loader/src/utils/savePage.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { existsSync, mkdirSync, writeFileSync } from "fs"; -import { join } from "path"; - -export const savePage = ( - dataDir: string, - pageIndex: number, - ids: number[], - vectors: Number[][], - attributes: Record -) => { - if (!existsSync(dataDir)) { - mkdirSync(dataDir); - } - - writeFileSync( - join(dataDir, `ids_${pageIndex}.json`), - JSON.stringify(ids, null, 2) - ); - writeFileSync( - join(dataDir, `vectors_${pageIndex}.json`), - JSON.stringify(vectors, null, 2) - ); - writeFileSync( - join(dataDir, `attributes_${pageIndex}.json`), - JSON.stringify(attributes, null, 2) - ); -}; diff --git a/packages/crgpt-loader/test.ts b/packages/crgpt-loader/test.ts deleted file mode 100644 index 76ef3bad..00000000 --- a/packages/crgpt-loader/test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { CRGPTLoader } from "./src/crgpt-loader"; - -const test = () => { - const loader = new CRGPTLoader( - "https://github.com/mattzcarey/code-review-gpt" - ); - - loader.load(); - loader.read(); - loader.delete(); -}; - -test(); diff --git a/packages/crgpt-loader/tsconfig.json b/packages/crgpt-loader/tsconfig.json deleted file mode 100644 index 43760374..00000000 --- a/packages/crgpt-loader/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "compilerOptions": { - "target": "es2017", - "module": "commonjs", - "lib": ["es2017", "dom"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "moduleResolution": "node", - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "types": ["node"] - }, - "include": ["src/*.ts", "src/lc_wip/turbopufferVectorStore.ts"], - "exclude": ["./node_modules", "./dist", "./src/**/*.test.ts"], - "engines": { - "node": ">=18" - } -}