diff --git a/packages/bodhi-adapter/src/provider/google/gemini.ts b/packages/bodhi-adapter/src/provider/google/gemini.ts index 397ae11..78751c3 100644 --- a/packages/bodhi-adapter/src/provider/google/gemini.ts +++ b/packages/bodhi-adapter/src/provider/google/gemini.ts @@ -88,6 +88,7 @@ export class GoogleGeminiAPI extends ChatBaseAPI { protected async convertParams(opts: types.chat.SendOptions): Promise { return { contents: await this.corvertContents(opts), + // systemInstruction: await this.corvertSystemContent(opts), tools: this.corvertTools(opts), safety_settings: [ // { category: 'BLOCK_NONE', threshold: 'HARM_CATEGORY_UNSPECIFIED' }, @@ -107,7 +108,7 @@ export class GoogleGeminiAPI extends ChatBaseAPI { // filter system role const rows = await Promise.all( opts.messages - .filter((item) => item.role !== 'system') + .filter((item) => ['user', 'assistant'].includes(item.role)) .map(async (item) => { const parts: gemini.Part[] = []; await Promise.all( @@ -115,10 +116,13 @@ export class GoogleGeminiAPI extends ChatBaseAPI { if (part.type === 'text') { parts.push({ text: part.text }); } + // if (part.type === 'document') { + // parts.push({ document: { url: part.url } }); + // } if (['image', 'video'].includes(part.type)) { try { - const inline_data = await this.fetchFile((part as types.chat.FilePart).url); - parts.push({ inline_data }); + const inlineData = await this.fetchFile((part as types.chat.FilePart).url); + parts.push({ inlineData }); } catch (err) {} } if (part.type === 'function_call') { diff --git a/packages/bodhi-adapter/src/provider/google/types.ts b/packages/bodhi-adapter/src/provider/google/types.ts index acd26e8..0c87050 100644 --- a/packages/bodhi-adapter/src/provider/google/types.ts +++ b/packages/bodhi-adapter/src/provider/google/types.ts @@ -6,6 +6,7 @@ export namespace gemini { */ export type Request = { contents: Content[]; + systemInstruction?: Content; tools: Tools[]; safety_settings: safetyRating[]; generationConfig: { @@ -27,7 +28,7 @@ export namespace gemini { export type Part = TextPart | FilePart | ToolPart; export type TextPart = { text: string }; - export type FilePart = { inline_data: { mime_type: string; data: string } }; + export type FilePart = { inlineData: { mimeType: string; data: string } }; export type ToolPart = { functionCall: { name: string; args: any } }; export type Tools = { functionDeclarations: types.chat.Function[] }; diff --git a/packages/bodhi-adapter/src/provider/google/vertex.ts b/packages/bodhi-adapter/src/provider/google/vertex.ts index 8b8265d..3dc6c04 100644 --- a/packages/bodhi-adapter/src/provider/google/vertex.ts +++ b/packages/bodhi-adapter/src/provider/google/vertex.ts @@ -57,7 +57,7 @@ export class GoogleVertexAPI extends GoogleGeminiAPI { const model = hasMedia ? 'gemini-pro-vision' : opts.model || 'gemini-pro'; const url = `${this.baseURL}/publishers/google/models/${model}:streamGenerateContent?alt=sse`; const params: gemini.Request = await this.convertParams(options); - // console.log(`[fetch]params`, url, JSON.stringify(params, null, 2)); + console.log(`[fetch]params`, url, JSON.stringify(params, null, 2)); const res = await fetchSSE(url, { headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` }, diff --git a/packages/bodhi-adapter/test/google-vertex.test.ts b/packages/bodhi-adapter/test/google-vertex.test.ts index 8d1fb2f..f1f63cd 100644 --- a/packages/bodhi-adapter/test/google-vertex.test.ts +++ b/packages/bodhi-adapter/test/google-vertex.test.ts @@ -54,25 +54,18 @@ describe('chat', () => { const res = await api.sendMessage({ model: 'gemini-pro', messages: [ - { - role: 'system', - parts: [ - { - type: 'text', - text: 'You are a very professional document summarization specialist. Please summarize the given document.', - }, - ], - }, { role: 'user', parts: [ { type: 'document', - url: 'gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf', + // url: 'gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf', + url: 'gs://bodhi-storage/uploads/202405/0db16ca633824283d07cf3774f886cef/0.pdf', + // url: 'https://s.chatonce.cn/bodhi/uploads/202405/0db16ca633824283d07cf3774f886cef/0.pdf', }, { type: 'text', - text: '这里都讲了什么?', + text: '这个文档都讲了什么呢?', }, ], },