From 2f37d6dbec427f46428a11b16e9788202bdd8831 Mon Sep 17 00:00:00 2001 From: ZhangWei-KUMO Date: Thu, 19 Dec 2024 18:35:29 +0800 Subject: [PATCH] add image analysis --- db/agent.js | 4 +--- db/chats.js | 1 - db/config.js | 7 ++----- db/email.js | 6 ++---- db/flashmemories.js | 1 - db/images.js | 1 - db/logs.js | 2 -- db/tts.js | 6 ++---- db/users.js | 1 - db/wechat.js | 5 +---- index.js | 3 +++ router/public/flashmemo.html | 14 ++++++++------ router/router.js | 14 +++++++------- util/gemini.js | 2 +- util/handle.js | 6 +++--- util/init.js | 12 +++++++++++- 16 files changed, 41 insertions(+), 44 deletions(-) diff --git a/db/agent.js b/db/agent.js index 072591e..f1421e2 100644 --- a/db/agent.js +++ b/db/agent.js @@ -8,7 +8,6 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('Agent配置数据库连接成功'); db.run(` CREATE TABLE IF NOT EXISTS agentConfig ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -35,8 +34,7 @@ const db = new sqlite3.Database(dbFile, (err) => { cryptoCurrency: 0, }; insertConfig(defaultEmailConfig) - .then(() => console.log("Agent写入默认配置")) - .catch(err => console.error("Agent写入数据错误:", err)); + } }); } diff --git a/db/chats.js b/db/chats.js index ce2368e..1ffccfd 100644 --- a/db/chats.js +++ b/db/chats.js @@ -10,7 +10,6 @@ let db = new sqlite3.Database(dbFile, (err) => { if (err) { // console.error(err.message); } else { - console.log('连接聊天记录数据库'); db.run(` CREATE TABLE IF NOT EXISTS chats ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/db/config.js b/db/config.js index 41eba8d..692b78b 100644 --- a/db/config.js +++ b/db/config.js @@ -8,8 +8,7 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('配置文件数据库连接成功'); - // Create the config table if it doesn't exist + db.run(` CREATE TABLE IF NOT EXISTS config ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -40,9 +39,7 @@ const db = new sqlite3.Database(dbFile, (err) => { isPushEnable: 0, }; - insertConfig(defaultConfig) // Use the insertConfig function - .then(() => console.log("写入默认配置")) - .catch(err => console.error("写入数据错误:", err)); + insertConfig(defaultConfig) } }); } diff --git a/db/email.js b/db/email.js index 9e1b4a2..b0dcec5 100644 --- a/db/email.js +++ b/db/email.js @@ -8,8 +8,7 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('邮箱配置数据库连接成功'); - // Create the email config table if it doesn't exist + db.run(` CREATE TABLE IF NOT EXISTS emailConfig ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -40,8 +39,7 @@ const db = new sqlite3.Database(dbFile, (err) => { content: '请及时检查您的机器人掉线状况' }; insertEmailConfig(defaultEmailConfig) - .then(() => console.log("邮箱写入默认配置")) - .catch(err => console.error("邮箱写入数据错误:", err)); + } }); } diff --git a/db/flashmemories.js b/db/flashmemories.js index df2e4d7..4c046fc 100644 --- a/db/flashmemories.js +++ b/db/flashmemories.js @@ -9,7 +9,6 @@ const db = new sqlite3.Database(dbFile, sqlite3.OPEN_READWRITE | sqlite3.OPEN_CR if (err) { console.error('Error opening database:', err.message); } else { - console.log('连接记忆数据库成功'); db.run(` CREATE TABLE IF NOT EXISTS flashmemories ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/db/images.js b/db/images.js index 7ad92f6..f916cf2 100644 --- a/db/images.js +++ b/db/images.js @@ -9,7 +9,6 @@ const db = new sqlite3.Database(dbFile, sqlite3.OPEN_READWRITE | sqlite3.OPEN_CR if (err) { console.error('Error opening database:', err.message); } else { - console.log('连接图片数据库成功'); db.run(` CREATE TABLE IF NOT EXISTS images ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/db/logs.js b/db/logs.js index 131d9d8..9525349 100644 --- a/db/logs.js +++ b/db/logs.js @@ -9,8 +9,6 @@ const db = new sqlite3.Database(dbFile, sqlite3.OPEN_READWRITE | sqlite3.OPEN_CR if (err) { console.error('Error opening database:', err.message); } else { - console.log('连接日志数据库成功'); - // Create the logs table db.run(` CREATE TABLE IF NOT EXISTS logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/db/tts.js b/db/tts.js index dfe3d82..f965d13 100644 --- a/db/tts.js +++ b/db/tts.js @@ -8,8 +8,7 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('TTS配置数据库连接成功'); - // Create the email config table if it doesn't exist + db.run(` CREATE TABLE IF NOT EXISTS ttsConfig ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -30,8 +29,7 @@ const db = new sqlite3.Database(dbFile, (err) => { azureregion: '', }; insertTtsConfig(defaultEmailConfig) - .then(() => console.log("TTS写入默认配置")) - .catch(err => console.error("TTS写入数据错误:", err)); + } }); } diff --git a/db/users.js b/db/users.js index 30ab64a..1e9b853 100644 --- a/db/users.js +++ b/db/users.js @@ -10,7 +10,6 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('连接用户数据库'); // 创建用户表（如果不存在） db.run(` CREATE TABLE IF NOT EXISTS users ( diff --git a/db/wechat.js b/db/wechat.js index 68f5522..03ea27d 100644 --- a/db/wechat.js +++ b/db/wechat.js @@ -8,7 +8,6 @@ const db = new sqlite3.Database(dbFile, (err) => { if (err) { console.error(err.message); } else { - console.log('连接微信数据库'); // 创建用户表（如果不存在） db.run(` CREATE TABLE IF NOT EXISTS wechats ( @@ -31,9 +30,7 @@ const db = new sqlite3.Database(dbFile, (err) => { friends: '', loginurl: '', }; - insertWechat(defaultConfig) // Use the insertConfig function - .then(() => console.log("写入默认配置")) - .catch(err => console.error("写入数据错误:", err)); + insertWechat(defaultConfig) } }); } diff --git a/index.js b/index.js index 0c172b4..bef52dc 100644 --- a/index.js +++ b/index.js @@ -24,13 +24,16 @@ app.use(staticRouter); export async function prepareBot() { bot.on("message", async (message) => { + const contact = message.talker(); + if (contact.self()) { return; } if (message.room()) { groupChat(message,talkerId) } + let {payload} = message; let {talkerId,listenerId,text} = payload; switch (message.type()) { diff --git a/router/public/flashmemo.html b/router/public/flashmemo.html index 1193eec..cfd9176 100644 --- a/router/public/flashmemo.html +++ b/router/public/flashmemo.html @@ -204,14 +204,10 @@ title: 'ID', dataIndex: 'id', }, - { - title: 'talkid', - dataIndex: 'talkid', - }, { title: '时间', dataIndex: 'timestamp', - render: (timestamp) => moment(timestamp).format('YYYY-MM-DD HH:mm:ss'), + render: (timestamp) => moment(timestamp).format('YYYY-MM-DD HH:mm:ss'), }, { title: '类型', @@ -220,7 +216,13 @@ { title: '消息内容', dataIndex: 'content', - width: '40%', + width: '40%', + render:(content)=>( +
+ +
+ ) + }, ]; diff --git a/router/router.js b/router/router.js index aff79da..62c3493 100644 --- a/router/router.js +++ b/router/router.js @@ -55,9 +55,9 @@ router.post('/api/login', async (req, res) => { }); router.get('/api/logs', (req, res) => { - // getLogs().then((data) => { - // res.json(data); - // }); + getLogs().then((data) => { + res.json(data); + }); }); router.delete('/api/logs', (req, res) => { @@ -181,14 +181,14 @@ router.post('/api/login', async (req, res) => { res.json(data); }); }); - // 获取flashmemory - router.get('/api/flashmemory', (req, res) => { + // 获取全部flashmemory + router.get('/api/flashmemories', (req, res) => { getFlashMemories().then((data) => { res.json(data); }); }); - // 删除flashmemory - router.delete('/api/flashmemory', (req, res) => { + // 删除全部flashmemory + router.delete('/api/flashmemories', (req, res) => { deleteFlashMemories().then((data) => { res.json(data); }); diff --git a/util/gemini.js b/util/gemini.js index ccc67e1..e8fbe9e 100644 --- a/util/gemini.js +++ b/util/gemini.js @@ -16,7 +16,7 @@ export const recgonizeImage = async (buffer,question) => { mimeType: "image/png", }}; - const result = await model.generateContent([question, image]); + const result = await model.generateContent([question+"。请用中文回答", image]); return result.response.text() } diff --git a/util/handle.js b/util/handle.js index f742812..dfa49d8 100644 --- a/util/handle.js +++ b/util/handle.js @@ -8,7 +8,7 @@ import fs from 'fs'; import process from 'process'; import schedule from 'node-schedule'; import {getNews} from './group.js' -// import {recgonizeImage} from './gemini.js' +import {recgonizeImage} from './gemini.js' import {saveFlashMemory} from '../db/flashmemories.js' export const singleChat = async (talkerId,listenerId,text) => { @@ -276,8 +276,8 @@ export const handleImage = async (message, talkerId) => { let base64String = buffer.toString("base64") // 存入记忆中 saveFlashMemory(talkerId, base64String, 'image') - // 识别图片 - // let res = await recgonizeImage(image, question) + // // 识别图片 + // let res = await recgonizeImage(base64String, "请告诉我这是什么") // await sendMessage(talkerId, res); }catch(e){ // 修改这里，提取错误信息 diff --git a/util/init.js b/util/init.js index 8147ecb..2a87ebf 100644 --- a/util/init.js +++ b/util/init.js @@ -5,8 +5,10 @@ config(); import { PuppetWechat4u } from 'wechaty-puppet-wechat4u'; import fs from 'fs'; import {getNews} from './group.js' -import {classfication, stockCheck,chat} from './gemini.js' +import {classfication, stockCheck,chat,recgonizeImage} from './gemini.js' import moment from 'moment'; +import {deleteFlashMemory, getFlashMemory} from '../db/flashmemories.js' + import 'dotenv/config' // import Redis from 'ioredis'; // const redis = new Redis(); @@ -151,6 +153,14 @@ const getBinanceRanker = async () => { }; export const think = async (talkid,query) => { + /** + * 首先，检查当前是否存在短期记忆，如图片、小程序、地址等 + */ + const flashMemory = await getFlashMemory(talkid); + if(flashMemory && flashMemory.type=='image'){ + deleteFlashMemory(talkid) + return await recgonizeImage(flashMemory.content,query) + } // const filePath = `./logger/${talkid}.json`; // let longMemory = ""; // if (fs.existsSync(filePath)) {