🎉feat: rebuild index & improve prompt

DEV.md

@@ -12,4 +12,18 @@ docker run --name blinko-website -d -p 1111:1111 blinko
 ## build docker image & run with docker-compose locally
 ```
 docker-compose -f docker-compose.yml up -d --build
+```
+
+## run test docker
+``` 
+docker run -d \
+  --name blinko-website \
+  --network blinko-network \
+  -p 1111:1111 \
+  -e NODE_ENV=production \
+  -v /volume1/docker/blinko/blinkodata:/app/.blinko \
+  -e NEXTAUTH_SECRET=my_ultra_secure_nextauth_secret \
+  -e DATABASE_URL=postgresql://postgres:mysecretpassword@blinko-postgres:5432/postgres \
+  --restart always \
+  blinkospace/blinko:fa46f26
 ```

app/api/file/[filename]/route.ts

@@ -20,7 +20,6 @@ export const GET = async (req: Request, { params }: any) => {
       },
     });
   } catch (error) {
-    console.error("Error occurred ", error);
     return NextResponse.json({ Message: "File not found", status: 404 });
   }
 };

app/api/file/delete/route.ts

@@ -12,6 +12,13 @@ export const POST = async (req: Request, res: NextResponse) => {
       await prisma.attachments.delete({ where: { id: attachment.id } })
     }
     const filepath = path.join(process.cwd(), `${UPLOAD_FILE_PATH}/` + attachment_path.replace('/api/file/', ""))
+    if ('jpeg/jpg/png/bmp/tiff/tif/webp/svg'.includes(attachment_path.replace('.', '')?.toLowerCase() ?? null)) {
+      try {
+        await unlink(path.join(process.cwd(), `${UPLOAD_FILE_PATH}/thumbnail_` + attachment_path.replace('/api/file/', "")))
+      } catch (error) {
+        // console.log(error)
+      }
+    }
     await unlink(filepath)
     return NextResponse.json({ Message: "Success", status: 200 });
   } catch (error) {

package.json

@@ -57,7 +57,6 @@
     "clsx": "^2.1.1",
     "copy-to-clipboard": "^3.3.3",
     "cron": "^3.1.7",
-    "dataloader": "^2.2.2",
     "dayjs": "^1.11.8",
     "decimal.js": "^10.4.3",
     "deepmerge": "^4.3.1",
@@ -90,6 +89,7 @@
     "next-themes": "^0.3.0",
     "nextjs-cors": "^2.2.0",
     "nprogress": "^0.2.0",
+    "p-limit": "^6.1.0",
     "pg": "^8.11.4",
     "pg-connection-string": "^2.7.0",
     "pg-dump-restore": "^1.0.12",

public/locales/de/translation.json

@@ -123,5 +123,10 @@
   "original-password": "Original-Passwort",
   "import-from-memos-memos_prod-db": "Import aus Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Wenn Sie memos_prod.db exportieren, schließen Sie bitte den Memos-Container, um einen teilweisen Datenverlust zu vermeiden.",
-  "go-to-share-page": "Zur Freigabeseite gehen"
+  "go-to-share-page": "Zur Freigabeseite gehen",
+  "import-done": "Import abgeschlossen",
+  "rebuilding-embedding-progress": "Wiederaufbau Einbettung des Fortschritts",
+  "rebuild": "Wiederherstellen",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "Auf anderem Wege importierte Noten haben möglicherweise keine eingebetteten Vektoren",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Wenn Sie viele Scheine haben, können Sie eine bestimmte Anzahl von Token verbrauchen."
 }

public/locales/en/translation.json

@@ -127,5 +127,10 @@
   "import-from-memos-memos_prod-db": "Import from Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "When exporting memos_prod.db, please close the memos container to avoid partial loss of data.",
   "go-to-share-page": "Go to share page",
-  "import-done": "Import done"
+  "import-done": "Import done",
+  "rebuilding-embedding-progress": "Rebuilding Embedding Progress",
+  "rebuild-embedding-index": "Rebuild Embedding Index",
+  "rebuild": "Rebuild",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "Notes imported by other means may not have embedded vectors",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "If you have a lot of notes you may consume a certain number of tokens"
 }

public/locales/es/translation.json

@@ -122,5 +122,9 @@
   "import-from-memos-memos_prod-db": "Importar de Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Al exportar memos_prod.db, cierre el contenedor de memos para evitar la pérdida parcial de datos.",
   "go-to-share-page": "Ir a la página para compartir",
-  "import-done": "Importación realizada"
+  "import-done": "Importación realizada",
+  "rebuilding-embedding-progress": "Reconstruir Integrar el progreso",
+  "rebuild-embedding-index": "Reconstruir índice de incrustación",
+  "rebuild": "Reconstruir",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "Los billetes importados por otros medios pueden no tener vectores incrustados"
 }

public/locales/fr/translation.json

@@ -124,5 +124,8 @@
   "import-from-memos-memos_prod-db": "Importation de Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Lors de l'exportation de memos_prod.db, veuillez fermer le conteneur memos afin d'éviter une perte partielle des données.",
   "go-to-share-page": "Aller à la page de partage",
-  "import-done": "Importation effectuée"
+  "import-done": "Importation effectuée",
+  "rebuilding-embedding-progress": "Reconstruire Intégrer le progrès",
+  "rebuild-embedding-index": "Reconstruire l'index d'intégration",
+  "rebuild": "Reconstruction"
 }

public/locales/ja/translation.json

@@ -122,5 +122,8 @@
   "edit-user": "編集ユーザー",
   "import-from-memos-memos_prod-db": "メモからのインポート(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "memos_prod.dbをエクスポートする際は、データの一部消失を避けるため、メモコンテナを閉じてください。",
-  "go-to-share-page": "シェアページへ"
+  "go-to-share-page": "シェアページへ",
+  "rebuilding-embedding-progress": "再構築 進歩の定着",
+  "rebuild": "再構築",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "多くのノートをお持ちの場合、一定数のトークンを消費する可能性があります。"
 }

public/locales/ko/translation.json

@@ -124,5 +124,8 @@
   "import-from-memos-memos_prod-db": "메모(memos_prod.db)에서 가져오기",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "memos_prod.db를 내보낼 때는 데이터의 일부가 손실되지 않도록 메모 컨테이너를 닫아 주세요.",
   "go-to-share-page": "공유 페이지로 이동",
-  "import-done": "가져오기 완료"
+  "import-done": "가져오기 완료",
+  "rebuild-embedding-index": "임베딩 인덱스 재구축",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "다른 방법으로 가져온 노트에는 벡터가 내장되어 있지 않을 수 있습니다.",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "노트가 많으면 특정 수의 토큰을 소모할 수 있습니다."
 }

public/locales/pt/translation.json

@@ -124,5 +124,8 @@
   "edit-user": "Editar utilizador",
   "import-from-memos-memos_prod-db": "Importar de Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "Ao exportar memos_prod.db, feche o contentor de memos para evitar a perda parcial de dados.",
-  "go-to-share-page": "Ir para a página de partilha"
+  "go-to-share-page": "Ir para a página de partilha",
+  "rebuild-embedding-index": "Reconstruir o índice de incorporação",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "As notas importadas por outros meios podem não ter vectores incorporados",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Se tiver muitas notas, pode consumir um determinado número de fichas"
 }

public/locales/ru/translation.json

@@ -125,5 +125,8 @@
   "import-from-memos-memos_prod-db": "Импорт из Memos(memos_prod.db)",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "При экспорте memos_prod.db закройте контейнер memos, чтобы избежать частичной потери данных.",
   "go-to-share-page": "Перейдите на страницу обмена информацией",
-  "import-done": "Импорт выполнен"
+  "import-done": "Импорт выполнен",
+  "rebuild-embedding-index": "Восстановить индекс встраивания",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "Ноты, импортированные другими способами, могут не иметь встроенных векторов",
+  "if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens": "Если у вас много заметок, вы можете израсходовать определенное количество жетонов."
 }

public/locales/zh-TW/translation.json

@@ -123,5 +123,9 @@
   "import-from-memos-memos_prod-db": "从备忘录导入（memos_prod.db）",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "导出 memos_prod.db 时，请关闭备忘录容器，以免丢失部分数据。",
   "go-to-share-page": "转到分享页面",
-  "import-done": "导入完成"
+  "import-done": "导入完成",
+  "rebuilding-embedding-progress": "重建 嵌入 进展",
+  "rebuild-embedding-index": "重建嵌入索引",
+  "rebuild": "重建",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "通过其他方式导入的票据可能没有嵌入向量"
 }

public/locales/zh/translation.json

@@ -124,5 +124,9 @@
   "import-from-memos-memos_prod-db": "从备忘录导入（memos_prod.db）",
   "when-exporting-memos_prod-db-please-close-the-memos-container-to-avoid-partial-loss-of-data": "导出 memos_prod.db 时，请关闭备忘录容器，以免丢失部分数据。",
   "go-to-share-page": "转到分享页面",
-  "import-done": "导入完成"
+  "import-done": "导入完成",
+  "rebuilding-embedding-progress": "重建嵌入向量",
+  "rebuild-embedding-index": "重建嵌入索引",
+  "rebuild": "重建",
+  "notes-imported-by-other-means-may-not-have-embedded-vectors": "通过其他方式导入的票据可能没有嵌入向量"
 }

src/components/BlinkoSettings/AiSetting.tsx

@@ -1,5 +1,5 @@
 import { observer } from "mobx-react-lite";
-import { Card, Input, Select, SelectItem, Switch } from "@nextui-org/react";
+import { Button, Card, Input, Select, SelectItem, Switch } from "@nextui-org/react";
 import { RootStore } from "@/store";
 import { BlinkoStore } from "@/store/blinkoStore";
 import { PromiseCall } from "@/store/standard/PromiseState";
@@ -10,6 +10,8 @@ import { useTranslation } from "react-i18next";
 import { Item } from "./Item";
 import { useEffect } from "react";
 import { useMediaQuery } from "usehooks-ts";
+import { ShowRebuildEmbeddingProgressDialog } from "../Common/RebuildEmbeddingProgress";
+import { showTipsDialog } from "../Common/TipsDialog";
 
 export const AiSetting = observer(() => {
   const blinko = RootStore.Get(BlinkoStore)
@@ -139,5 +141,21 @@ export const AiSetting = observer(() => {
         }}
       />} />
 
+    <Item
+      type={isPc ? 'row' : 'col'}
+      leftContent={<div className="flex flex-col  gap-2">
+        <div>{t('rebuild-embedding-index')}</div>
+        <div className="text-desc text-xs">{t('notes-imported-by-other-means-may-not-have-embedded-vectors')}</div>
+      </div>}
+      rightContent={<Button color='primary' startContent={<Icon icon="mingcute:refresh-4-ai-line" width="20" height="20" />} onClick={() => {
+        showTipsDialog({
+          title: t('rebuild-embedding-index'),
+          content:t('if-you-have-a-lot-of-notes-you-may-consume-a-certain-number-of-tokens'),
+          onConfirm: () => {
+            ShowRebuildEmbeddingProgressDialog()
+          }
+        })
+      }}>{t('rebuild')}</Button>} />
+
   </Card>
 })