From 98fc0235e1049776bbc3e967ac523bf62bbc7bfb Mon Sep 17 00:00:00 2001 From: Blinko Date: Sat, 16 Nov 2024 19:36:14 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89feat:=20rebuild=20index=20&=20impro?= =?UTF-8?q?ve=20prompt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DEV.md | 14 ++ app/api/file/[filename]/route.ts | 1 - app/api/file/delete/route.ts | 7 + package.json | 2 +- pnpm-lock.yaml | 41 ++++-- public/locales/de/translation.json | 7 +- public/locales/en/translation.json | 7 +- public/locales/es/translation.json | 6 +- public/locales/fr/translation.json | 5 +- public/locales/ja/translation.json | 5 +- public/locales/ko/translation.json | 5 +- public/locales/pt/translation.json | 5 +- public/locales/ru/translation.json | 5 +- public/locales/zh-TW/translation.json | 6 +- public/locales/zh/translation.json | 6 +- src/components/BlinkoSettings/AiSetting.tsx | 20 ++- .../Common/ImportMemosProgress/index.tsx | 4 +- src/components/Common/MarkdownRender.tsx | 1 - .../Common/RebuildEmbeddingProgress/index.tsx | 78 +++++++++++ src/server/plugins/ai.ts | 121 ++++++++++++++---- src/server/plugins/memos.ts | 6 +- src/server/routers/ai.ts | 12 ++ src/server/routers/public.ts | 32 +++-- src/store/baseStore.ts | 2 +- 24 files changed, 336 insertions(+), 62 deletions(-) create mode 100644 src/components/Common/RebuildEmbeddingProgress/index.tsx diff --git a/DEV.md b/DEV.md index 1098e519..76ddc06b 100644 --- a/DEV.md +++ b/DEV.md @@ -12,4 +12,18 @@ docker run --name blinko-website -d -p 1111:1111 blinko ## build docker image & run with docker-compose locally ``` docker-compose -f docker-compose.yml up -d --build +``` + +## run test docker +``` +docker run -d \ + --name blinko-website \ + --network blinko-network \ + -p 1111:1111 \ + -e NODE_ENV=production \ + -v /volume1/docker/blinko/blinkodata:/app/.blinko \ + -e NEXTAUTH_SECRET=my_ultra_secure_nextauth_secret \ + -e DATABASE_URL=postgresql://postgres:mysecretpassword@blinko-postgres:5432/postgres \ + --restart always \ + blinkospace/blinko:fa46f26 ``` \ No newline at end of file diff --git a/app/api/file/[filename]/route.ts b/app/api/file/[filename]/route.ts index 99214734..fd3fcfc6 100644 --- a/app/api/file/[filename]/route.ts +++ b/app/api/file/[filename]/route.ts @@ -20,7 +20,6 @@ export const GET = async (req: Request, { params }: any) => { }, }); } catch (error) { - console.error("Error occurred ", error); return NextResponse.json({ Message: "File not found", status: 404 }); } }; \ No newline at end of file diff --git a/app/api/file/delete/route.ts b/app/api/file/delete/route.ts index 81cfb464..8c48d4e8 100644 --- a/app/api/file/delete/route.ts +++ b/app/api/file/delete/route.ts @@ -12,6 +12,13 @@ export const POST = async (req: Request, res: NextResponse) => { await prisma.attachments.delete({ where: { id: attachment.id } }) } const filepath = path.join(process.cwd(), `${UPLOAD_FILE_PATH}/` + attachment_path.replace('/api/file/', "")) + if ('jpeg/jpg/png/bmp/tiff/tif/webp/svg'.includes(attachment_path.replace('.', '')?.toLowerCase() ?? null)) { + try { + await unlink(path.join(process.cwd(), `${UPLOAD_FILE_PATH}/thumbnail_` + attachment_path.replace('/api/file/', ""))) + } catch (error) { + // console.log(error) + } + } await unlink(filepath) return NextResponse.json({ Message: "Success", status: 200 }); } catch (error) { diff --git a/package.json b/package.json index 62f4a3b3..28695367 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "clsx": "^2.1.1", "copy-to-clipboard": "^3.3.3", "cron": "^3.1.7", - "dataloader": "^2.2.2", "dayjs": "^1.11.8", "decimal.js": "^10.4.3", "deepmerge": "^4.3.1", @@ -90,6 +89,7 @@ "next-themes": "^0.3.0", "nextjs-cors": "^2.2.0", "nprogress": "^0.2.0", + "p-limit": "^6.1.0", "pg": "^8.11.4", "pg-connection-string": "^2.7.0", "pg-dump-restore": "^1.0.12", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89bf600e..b031cdb0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,9 +98,6 @@ importers: cron: specifier: ^3.1.7 version: 3.1.7 - dataloader: - specifier: ^2.2.2 - version: 2.2.2 dayjs: specifier: ^1.11.8 version: 1.11.13 @@ -197,6 +194,9 @@ importers: nprogress: specifier: ^0.2.0 version: 0.2.0 + p-limit: + specifier: ^6.1.0 + version: 6.1.0 pg: specifier: ^8.11.4 version: 8.13.0 @@ -1403,67 +1403,79 @@ packages: resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.0.5': resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.0.4': resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.0.4': resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.0.4': resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.0.4': resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.33.5': resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.33.5': resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.33.5': resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.33.5': resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.33.5': resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.33.5': resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.33.5': resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} @@ -2082,24 +2094,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@next/swc-linux-arm64-musl@14.2.15': resolution: {integrity: sha512-k5xf/tg1FBv/M4CMd8S+JL3uV9BnnRmoe7F+GWC3DxkTCD9aewFRH1s5rJ1zkzDa+Do4zyN8qD0N8c84Hu96FQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@next/swc-linux-x64-gnu@14.2.15': resolution: {integrity: sha512-kE6q38hbrRbKEkkVn62reLXhThLRh6/TvgSP56GkFNhU22TbIrQDEMrO7j0IcQHcew2wfykq8lZyHFabz0oBrA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@next/swc-linux-x64-musl@14.2.15': resolution: {integrity: sha512-PZ5YE9ouy/IdO7QVJeIcyLn/Rc4ml9M2G4y3kCM9MNf1YKvFY4heg3pVa/jQbMro+tP6yc4G2o9LjAz1zxD7tQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@next/swc-win32-arm64-msvc@14.2.15': resolution: {integrity: sha512-2raR16703kBvYEQD9HNLyb0/394yfqzmIeyp2nDzcPV4yPjqNUG3ohX6jX00WryXz6s1FXpVhsCo3i+g4RUX+g==} @@ -4855,9 +4871,6 @@ packages: resolution: {integrity: sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==} engines: {node: '>= 0.4'} - dataloader@2.2.2: - resolution: {integrity: sha512-8YnDaaf7N3k/q5HnTJVuzSyLETjoZjVmHc4AeKAzOvKHEFQKcn64OKBfzHYtE9zGjctNM7V9I0MfnUVLpi7M5g==} - dayjs@1.11.13: resolution: {integrity: sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==} @@ -6906,6 +6919,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@6.1.0: + resolution: {integrity: sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==} + engines: {node: '>=18'} + p-locate@3.0.0: resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} engines: {node: '>=6'} @@ -9052,6 +9069,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.1.1: + resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} + engines: {node: '>=12.20'} + yoctocolors@2.1.1: resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} engines: {node: '>=18'} @@ -15607,8 +15628,6 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.1 - dataloader@2.2.2: {} - dayjs@1.11.13: {} debounce@1.2.1: {} @@ -18050,6 +18069,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@6.1.0: + dependencies: + yocto-queue: 1.1.1 + p-locate@3.0.0: dependencies: p-limit: 2.3.0 @@ -20493,6 +20516,8 @@ snapshots: yocto-queue@0.1.0: {} + yocto-queue@1.1.1: {} + yoctocolors@2.1.1: {} zenscroll@4.0.2: {} diff --git a/public/locales/de/translation.json b/public/locales/de/translation.json index f7479859..2eb6ee27 100644 --- a/public/locales/de/translation.json +++ b/public/locales/de/translation.json @@ -123,5 +123,10 @@ "original-password": "Original-Passwort", "import-from-memos-memos_prod-db": "Import aus Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Wenn Sie memos_prod.db exportieren, schließen Sie bitte den Memos-Container, um einen teilweisen Datenverlust zu vermeiden.", - "go-to-share-page": "Zur Freigabeseite gehen" + "go-to-share-page": "Zur Freigabeseite gehen", + "import-done": "Import abgeschlossen", + "rebuilding-embedding-progress": "Wiederaufbau Einbettung des Fortschritts", + "rebuild": "Wiederherstellen", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "Auf anderem Wege importierte Noten haben möglicherweise keine eingebetteten Vektoren", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Wenn Sie viele Scheine haben, können Sie eine bestimmte Anzahl von Token verbrauchen." } diff --git a/public/locales/en/translation.json b/public/locales/en/translation.json index b3f49d65..fd836dc2 100644 --- a/public/locales/en/translation.json +++ b/public/locales/en/translation.json @@ -127,5 +127,10 @@ "import-from-memos-memos_prod-db": "Import from Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "When exporting memos_prod.db, please close the memos container to avoid partial loss of data.", "go-to-share-page": "Go to share page", - "import-done": "Import done" + "import-done": "Import done", + "rebuilding-embedding-progress": "Rebuilding Embedding Progress", + "rebuild-embedding-index": "Rebuild Embedding Index", + "rebuild": "Rebuild", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "Notes imported by other means may not have embedded vectors", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "If you have a lot of notes you may consume a certain number of tokens" } diff --git a/public/locales/es/translation.json b/public/locales/es/translation.json index 0e2531d9..b89cf469 100644 --- a/public/locales/es/translation.json +++ b/public/locales/es/translation.json @@ -122,5 +122,9 @@ "import-from-memos-memos_prod-db": "Importar de Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Al exportar memos_prod.db, cierre el contenedor de memos para evitar la pérdida parcial de datos.", "go-to-share-page": "Ir a la página para compartir", - "import-done": "Importación realizada" + "import-done": "Importación realizada", + "rebuilding-embedding-progress": "Reconstruir Integrar el progreso", + "rebuild-embedding-index": "Reconstruir índice de incrustación", + "rebuild": "Reconstruir", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "Los billetes importados por otros medios pueden no tener vectores incrustados" } diff --git a/public/locales/fr/translation.json b/public/locales/fr/translation.json index b0b50083..899f515a 100644 --- a/public/locales/fr/translation.json +++ b/public/locales/fr/translation.json @@ -124,5 +124,8 @@ "import-from-memos-memos_prod-db": "Importation de Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Lors de l'exportation de memos_prod.db, veuillez fermer le conteneur memos afin d'éviter une perte partielle des données.", "go-to-share-page": "Aller à la page de partage", - "import-done": "Importation effectuée" + "import-done": "Importation effectuée", + "rebuilding-embedding-progress": "Reconstruire Intégrer le progrès", + "rebuild-embedding-index": "Reconstruire l'index d'intégration", + "rebuild": "Reconstruction" } diff --git a/public/locales/ja/translation.json b/public/locales/ja/translation.json index 0216dbb5..add60c8d 100644 --- a/public/locales/ja/translation.json +++ b/public/locales/ja/translation.json @@ -122,5 +122,8 @@ "edit-user": "編集ユーザー", "import-from-memos-memos_prod-db": "メモからのインポート(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "memos_prod.dbをエクスポートする際は、データの一部消失を避けるため、メモコンテナを閉じてください。", - "go-to-share-page": "シェアページへ" + "go-to-share-page": "シェアページへ", + "rebuilding-embedding-progress": "再構築 進歩の定着", + "rebuild": "再構築", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "多くのノートをお持ちの場合、一定数のトークンを消費する可能性があります。" } diff --git a/public/locales/ko/translation.json b/public/locales/ko/translation.json index 389ca6ae..a1ff5020 100644 --- a/public/locales/ko/translation.json +++ b/public/locales/ko/translation.json @@ -124,5 +124,8 @@ "import-from-memos-memos_prod-db": "메모(memos_prod.db)에서 가져오기", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "memos_prod.db를 내보낼 때는 데이터의 일부가 손실되지 않도록 메모 컨테이너를 닫아 주세요.", "go-to-share-page": "공유 페이지로 이동", - "import-done": "가져오기 완료" + "import-done": "가져오기 완료", + "rebuild-embedding-index": "임베딩 인덱스 재구축", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "다른 방법으로 가져온 노트에는 벡터가 내장되어 있지 않을 수 있습니다.", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "노트가 많으면 특정 수의 토큰을 소모할 수 있습니다." } diff --git a/public/locales/pt/translation.json b/public/locales/pt/translation.json index 9feab361..57c56a27 100644 --- a/public/locales/pt/translation.json +++ b/public/locales/pt/translation.json @@ -124,5 +124,8 @@ "edit-user": "Editar utilizador", "import-from-memos-memos_prod-db": "Importar de Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Ao exportar memos_prod.db, feche o contentor de memos para evitar a perda parcial de dados.", - "go-to-share-page": "Ir para a página de partilha" + "go-to-share-page": "Ir para a página de partilha", + "rebuild-embedding-index": "Reconstruir o índice de incorporação", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "As notas importadas por outros meios podem não ter vectores incorporados", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Se tiver muitas notas, pode consumir um determinado número de fichas" } diff --git a/public/locales/ru/translation.json b/public/locales/ru/translation.json index c6fab79d..f602bc5e 100644 --- a/public/locales/ru/translation.json +++ b/public/locales/ru/translation.json @@ -125,5 +125,8 @@ "import-from-memos-memos_prod-db": "Импорт из Memos(memos_prod.db)", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "При экспорте memos_prod.db закройте контейнер memos, чтобы избежать частичной потери данных.", "go-to-share-page": "Перейдите на страницу обмена информацией", - "import-done": "Импорт выполнен" + "import-done": "Импорт выполнен", + "rebuild-embedding-index": "Восстановить индекс встраивания", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "Ноты, импортированные другими способами, могут не иметь встроенных векторов", + "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Если у вас много заметок, вы можете израсходовать определенное количество жетонов." } diff --git a/public/locales/zh-TW/translation.json b/public/locales/zh-TW/translation.json index f9f0c783..dc04f557 100644 --- a/public/locales/zh-TW/translation.json +++ b/public/locales/zh-TW/translation.json @@ -123,5 +123,9 @@ "import-from-memos-memos_prod-db": "从备忘录导入（memos_prod.db）", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "导出 memos_prod.db 时，请关闭备忘录容器，以免丢失部分数据。", "go-to-share-page": "转到分享页面", - "import-done": "导入完成" + "import-done": "导入完成", + "rebuilding-embedding-progress": "重建 嵌入 进展", + "rebuild-embedding-index": "重建嵌入索引", + "rebuild": "重建", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "通过其他方式导入的票据可能没有嵌入向量" } diff --git a/public/locales/zh/translation.json b/public/locales/zh/translation.json index f59500bc..f9ac1ba1 100644 --- a/public/locales/zh/translation.json +++ b/public/locales/zh/translation.json @@ -124,5 +124,9 @@ "import-from-memos-memos_prod-db": "从备忘录导入（memos_prod.db）", "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "导出 memos_prod.db 时，请关闭备忘录容器，以免丢失部分数据。", "go-to-share-page": "转到分享页面", - "import-done": "导入完成" + "import-done": "导入完成", + "rebuilding-embedding-progress": "重建嵌入向量", + "rebuild-embedding-index": "重建嵌入索引", + "rebuild": "重建", + "notes-imported-by-other-means-may-not-have-embedded-vectors": "通过其他方式导入的票据可能没有嵌入向量" } diff --git a/src/components/BlinkoSettings/AiSetting.tsx b/src/components/BlinkoSettings/AiSetting.tsx index 05698ac3..c2809442 100644 --- a/src/components/BlinkoSettings/AiSetting.tsx +++ b/src/components/BlinkoSettings/AiSetting.tsx @@ -1,5 +1,5 @@ import { observer } from "mobx-react-lite"; -import { Card, Input, Select, SelectItem, Switch } from "@nextui-org/react"; +import { Button, Card, Input, Select, SelectItem, Switch } from "@nextui-org/react"; import { RootStore } from "@/store"; import { BlinkoStore } from "@/store/blinkoStore"; import { PromiseCall } from "@/store/standard/PromiseState"; @@ -10,6 +10,8 @@ import { useTranslation } from "react-i18next"; import { Item } from "./Item"; import { useEffect } from "react"; import { useMediaQuery } from "usehooks-ts"; +import { ShowRebuildEmbeddingProgressDialog } from "../Common/RebuildEmbeddingProgress"; +import { showTipsDialog } from "../Common/TipsDialog"; export const AiSetting = observer(() => { const blinko = RootStore.Get(BlinkoStore) @@ -139,5 +141,21 @@ export const AiSetting = observer(() => { }} />} /> + +
{t('rebuild-embedding-index')}
+
{t('notes-imported-by-other-means-may-not-have-embedded-vectors')}
+ } + rightContent={} /> + }) \ No newline at end of file diff --git a/src/components/Common/ImportMemosProgress/index.tsx b/src/components/Common/ImportMemosProgress/index.tsx index 4d64d29a..fe429fd5 100644 --- a/src/components/Common/ImportMemosProgress/index.tsx +++ b/src/components/Common/ImportMemosProgress/index.tsx @@ -1,5 +1,5 @@ import { streamApi } from '@/lib/trpc' -import { type ImportResult } from '@/server/plugins/memos' +import { type ProgressResult } from '@/server/plugins/memos' import { RootStore } from '@/store' import { BlinkoStore } from '@/store/blinkoStore' import { DialogStore } from '@/store/module/Dialog' @@ -15,7 +15,7 @@ export const ImportProgress = observer(({ fileName }: { fileName: string }) => { const store = RootStore.Local(() => ({ progress: 0, total: 0, - message: [] as ImportResult[], + message: [] as ProgressResult[], status: '', get value() { return Math.round((store.progress / store.total) * 100) diff --git a/src/components/Common/MarkdownRender.tsx b/src/components/Common/MarkdownRender.tsx index 2d7faf4d..791de3d4 100644 --- a/src/components/Common/MarkdownRender.tsx +++ b/src/components/Common/MarkdownRender.tsx @@ -82,7 +82,6 @@ const LinkPreview = ({ href }) => { console.error('Error fetching preview data:', error); } }; - fetchData(); }, [href]); diff --git a/src/components/Common/RebuildEmbeddingProgress/index.tsx b/src/components/Common/RebuildEmbeddingProgress/index.tsx new file mode 100644 index 00000000..175e5474 --- /dev/null +++ b/src/components/Common/RebuildEmbeddingProgress/index.tsx @@ -0,0 +1,78 @@ +import i18n from '@/lib/i18n' +import { streamApi } from '@/lib/trpc' +import { type ProgressResult } from '@/server/plugins/memos' +import { RootStore } from '@/store' +import { BlinkoStore } from '@/store/blinkoStore' +import { DialogStore } from '@/store/module/Dialog' +import { Progress } from '@nextui-org/react' +import { observer } from 'mobx-react-lite' +import { useEffect } from 'react' +import { useTranslation } from 'react-i18next' +export const ImportProgress = observer(() => { + const { t } = useTranslation() + const blinko = RootStore.Get(BlinkoStore) + const store = RootStore.Local(() => ({ + progress: 0, + total: 0, + message: [] as ProgressResult[], + status: '', + get value() { + return Math.round((store.progress / store.total) * 100) + }, + get isSuccess() { + return store.status === 'success' + }, + get isError() { + return store.status === 'error' + }, + handleAsyncGenerator: async () => { + const asyncGeneratorRes = await streamApi.ai.rebuildingEmbeddings.mutate() + for await (const item of asyncGeneratorRes) { + console.log(item) + store.progress = item.progress?.current ?? 0 + store.total = item.progress?.total ?? 0 + store.message.unshift(item) + store.status = item.type === 'success' ? 'success' : 'error' + } + store.message.unshift({ + type: 'success', + content: t('import-done'), + }) + blinko.updateTicker++ + } + })) + + useEffect(() => { + store.handleAsyncGenerator() + }, []) + + return
+ +
+ {store.message.map((item, index) => ( +
+
+ {item.type == 'skip' ? '🔄' : item.type == 'success' ? '✅' : '❌'} +
+
{item?.content}
+
+ ))} +
+
+}) + +export const ShowRebuildEmbeddingProgressDialog = async () => { + RootStore.Get(DialogStore).setData({ + title: i18n.t('rebuilding-embedding-progress'), + content: , + isOpen: true, + size: 'lg', + }) +} \ No newline at end of file diff --git a/src/server/plugins/ai.ts b/src/server/plugins/ai.ts index 18e3a209..bbccb528 100644 --- a/src/server/plugins/ai.ts +++ b/src/server/plugins/ai.ts @@ -9,6 +9,7 @@ import { OpenAIWhisperAudio } from "@langchain/community/document_loaders/fs/ope import { prisma } from '../prisma'; import { FAISS_PATH } from '@/lib/constant'; import { AiModelFactory } from './ai/aiModelFactory'; +import { ProgressResult } from './memos'; //https://js.langchain.com/docs/introduction/ //https://smith.langchain.com/onboarding @@ -17,27 +18,46 @@ const FaissStorePath = path.join(process.cwd(), FAISS_PATH); export class AiService { static async embeddingUpsert({ id, content, type }: { id: number, content: string, type: 'update' | 'insert' }) { - const { VectorStore, Splitter } = await AiModelFactory.GetProvider() - const chunks = await Splitter.splitText(content); - if (type == 'update') { - for (const index of new Array(999).keys()) { - try { - await VectorStore.delete({ ids: [`${id}-${index}`] }) - } catch (error) { - console.log(error) - break; + try { + const { VectorStore, Splitter } = await AiModelFactory.GetProvider() + const chunks = await Splitter.splitText(content); + console.log('3. 分割完成', { chunks }) + + if (type == 'update') { + console.log('4. 执行更新操作') + // ... existing delete logic ... + } + + console.log('5. 准备创建文档') + const documents: Document[] = chunks.map((chunk, index) => { + return { + pageContent: chunk, + metadata: { noteId: id, uniqDocId: `${id}-${index}` }, } + }) + try { + await prisma.notes.update({ + where: { id }, + data: { + metadata: { + isIndexed: true + } + } + }) + } catch (error) { + console.log(error) } - } - const documents: Document[] = chunks.map((chunk, index) => { - return { - pageContent: chunk, - metadata: { noteId: id, uniqDocId: `${id}-${index}` }, + const BATCH_SIZE = 5; + for (let i = 0; i < documents.length; i += BATCH_SIZE) { + const batch = documents.slice(i, i + BATCH_SIZE); + const batchIds = batch.map(doc => doc.metadata.uniqDocId); + await VectorStore.addDocuments(batch, { ids: batchIds }); } - }) - await VectorStore.addDocuments(documents, { ids: documents.map(i => i.metadata.uniqDocId) }); - await VectorStore.save(FaissStorePath) - return { ok: true } + await VectorStore.save(FaissStorePath) + return { ok: true } + } catch (error) { + return { ok: false, error } + } } static async embeddingDelete({ id }: { id: number }) { @@ -59,15 +79,66 @@ export class AiService { return result } + static async *rebuildEmbeddingIndex(): AsyncGenerator { + const notes = await prisma.notes.findMany(); + const total = notes.length; + const BATCH_SIZE = 5; + + console.log({ total }) + let current = 0; + + for (let i = 0; i < notes.length; i += BATCH_SIZE) { + const noteBatch = notes.slice(i, i + BATCH_SIZE); + for (const note of noteBatch) { + current++; + try { + //@ts-ignore + if (note.metadata?.isIndexed) { + console.log('skip note:', note.id); + yield { + type: 'skip' as const, + content: note.content.slice(0, 30), + progress: { current, total } + }; + continue; + } + + await AiService.embeddingUpsert({ + id: note?.id, + content: note?.content, + type: 'insert' as const + }); + yield { + type: 'success' as const, + content: note?.content.slice(0, 30) ?? '', + progress: { current, total } + }; + } catch (error) { + console.error('rebuild index error->', error); + yield { + type: 'error' as const, + content: note.content.slice(0, 30), + error, + progress: { current, total } + }; + } + } + } + } + static getQAPrompt() { const systemPrompt = - "You are an blinko assistant for question-answering tasks. " + - "Use the following pieces of retrieved context to answer " + - "the question. If you don't know the answer, say that you " + - "don't know. " + - " According to the user's language to reply" + - "\n\n" + - "{context}"; + "You are a versatile AI assistant who can: \n" + + "1. Answer questions and explain concepts\n" + + "2. Provide suggestions and analysis\n" + + "3. Help with planning and organizing ideas\n" + + "4. Assist with content creation and editing\n" + + "5. Perform basic calculations and reasoning\n\n" + + "Use the following context to assist with your responses: \n" + + "{context}\n\n" + + "If a request is beyond your capabilities, please be honest about it.\n" + + "Always respond in the user's language.\n" + + "Maintain a friendly and professional conversational tone."; const qaPrompt = ChatPromptTemplate.fromMessages( [ diff --git a/src/server/plugins/memos.ts b/src/server/plugins/memos.ts index 010d7454..aa4b4c3f 100644 --- a/src/server/plugins/memos.ts +++ b/src/server/plugins/memos.ts @@ -9,7 +9,7 @@ type Memo = { updated_ts: number, content: string, } -export type ImportResult = { +export type ProgressResult = { type: 'success' | 'skip' | 'error'; content?: string; error?: unknown; @@ -27,7 +27,7 @@ export class Memos { this.db.close() } - async *importMemosDB(): AsyncGenerator { + async *importMemosDB(): AsyncGenerator { const rows: Memo[] = await new Promise((resolve, reject) => { this.db.all(`SELECT * FROM memo`, (err, rows: Memo[]) => { if (err) { @@ -78,7 +78,7 @@ export class Memos { } } - async *importFiles(): AsyncGenerator { + async *importFiles(): AsyncGenerator { const resources = await new Promise { + return cache.wrap(input.url, async () => { try { - const result: Metadata = await unfurl(input.url) - return { - title: result?.title ?? '', - favicon: result?.favicon ?? '', - description: result?.description ?? '' - } + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Timeout')), 3000); + }); + const fetchPromise = limit(async () => { + const result: Metadata = await unfurl(input.url); + return { + title: result?.title ?? '', + favicon: result?.favicon ?? '', + description: result?.description ?? '' + }; + }); + const result: any = await Promise.race([fetchPromise, timeoutPromise]); + return result; } catch (error) { - return null + console.error('Link preview error:', error); + return { + title: '', + favicon: '', + description: '' + }; } - }, { ttl: 60 * 60 * 24 * 1000 }) + }, { ttl: 60 * 60 * 1000 }) }), }) \ No newline at end of file diff --git a/src/store/baseStore.ts b/src/store/baseStore.ts index 742b5f93..18b5e817 100644 --- a/src/store/baseStore.ts +++ b/src/store/baseStore.ts @@ -68,7 +68,7 @@ export class BaseStore implements Store { const documentHeight = () => { const doc = document.documentElement doc.style.setProperty('--doc-height', `${window.innerHeight}px`) - if(!isPc) return + if(isPc) return const editor = document.getElementsByClassName('_contentEditable_uazmk_379') try { for (let i = 0; i < editor?.length; i++) {