Feat/img understanding in chat

backend/app/core/graph/messages.py

@@ -20,6 +20,7 @@ class ChatResponse(BaseModel):
     id: str
     name: str
     content: str | None = None
+    imgdata: str | None = None
     tool_calls: list[ToolCall] | None = None
     tool_output: str | None = None
     documents: str | None = None

backend/app/core/tools/zhipuai/img_4v.py

@@ -9,14 +9,26 @@
 class ImageUnderstandingInput(BaseModel):
     """Input for the Image Understanding tool."""
 
-    text: str = Field(description="the input text for the Image Understanding tool")
-
-
-def img_4v(text: str):
-    img_path = "/Users/envys/Downloads/a.jpeg"
-    with open(img_path, "rb") as img_file:
-        img_base = base64.b64encode(img_file.read()).decode("utf-8")
-
+    qry: str = Field(description="the input query for the Image Understanding tool")
+    image_url: str = Field(description="the path or the url of the image")
+
+
+def img_4v(image_url: str, qry: str):
+    if image_url is None:
+        return "Please provide an image path or url"
+
+    elif (
+        image_url.startswith("http")
+        or image_url.startswith("https")
+        or image_url.startswith("data:image/")
+    ):
+        img_base = image_url
+    else:
+        try:
+            with open(image_url, "rb") as img_file:
+                img_base = base64.b64encode(img_file.read()).decode("utf-8")
+        except Exception as e:
+            return "Error: " + str(e)
     client = ZhipuAI(
         api_key=os.environ.get("ZHIPUAI_API_KEY"),
     )  # 填写您自己的APIKey
@@ -27,7 +39,7 @@ def img_4v(text: str):
                 "role": "user",
                 "content": [
                     {"type": "image_url", "image_url": {"url": img_base}},
-                    {"type": "text", "text": text},
+                    {"type": "text", "text": qry},
                 ],
             }
         ],

web/src/client/models/ChatResponse.ts

@@ -10,6 +10,7 @@ export type ChatResponse = {
     id: string;
     name: string;
     content?: (string | null);
+    imgdata?: (string | null);
     tool_calls?: (Array<ToolCall> | null);
     tool_output?: (string | null);
     documents?: (string | null);

web/src/components/MessageInput/index.tsx

@@ -3,19 +3,27 @@ import {
   IconButton,
   Input,
   InputGroup,
+  InputLeftElement,
   InputRightElement,
   Tooltip,
+  Image,
+  Flex,
+  CloseButton,
+  HStack,
 } from "@chakra-ui/react";
 import type React from "react";
+import { useState } from "react";
 import { GrNewWindow } from "react-icons/gr";
+import { RiImageAddLine } from "react-icons/ri";
 import { VscSend } from "react-icons/vsc";
-
 interface MessageInputProps {
   input: string;
   setInput: (value: string) => void;
   onSubmit: (e: React.FormEvent) => void;
   isStreaming: boolean;
   newChatHandler?: () => void;
+  imageData: string | null;
+  setImageData: (value: string | null) => void;
 }
 
 const MessageInput: React.FC<MessageInputProps> = ({
@@ -24,47 +32,110 @@ const MessageInput: React.FC<MessageInputProps> = ({
   onSubmit,
   isStreaming,
   newChatHandler,
-}) => (
-  <Box
-    display="flex"
-    pl="10"
-    pr="20"
-    pt="2"
-    pb="10"
-    position="relative"
-    justifyItems="center"
-  >
-    <InputGroup as="form" onSubmit={onSubmit}>
-      <Input
-        type="text"
-        placeholder="Ask your team a question"
-        value={input}
-        onChange={(e) => setInput(e.target.value)}
-        boxShadow="0 0 10px rgba(0,0,0,0.2)"
-      />
-      <InputRightElement>
-        <IconButton
-          type="submit"
-          icon={<VscSend />}
-          aria-label="send-question"
-          isLoading={isStreaming}
-          isDisabled={!input.trim().length}
-        />
-      </InputRightElement>
-    </InputGroup>
-    {newChatHandler && (
-      <Tooltip label="New Chat" fontSize="md" bg="green" placement="top-end">
-        <IconButton
-          aria-label="new chat"
-          icon={<GrNewWindow />}
-          position="absolute"
-          right={10}
-          onClick={newChatHandler}
-          boxShadow="0 0 10px rgba(0,0,0,0.2)"
-        />
-      </Tooltip>
-    )}
-  </Box>
-);
+  imageData,
+  setImageData,
+}) => {
+
+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (file) {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        setImageData!(reader.result as string);
+      };
+      reader.readAsDataURL(file);
+    }
+    e.target.value = "";
+  };
+
+  const removeImage = () => {
+    setImageData!(null);
+    const fileInput = document.getElementById("file-input") as HTMLInputElement;
+    if (fileInput) {
+      fileInput.value = "";
+    }
+  };
+
+  return (
+    <Box
+      display="flex"
+      flexDirection="column" // 垂直排列
+      pl="10"
+      pr="20"
+      pt="2"
+      pb="10"
+      position="relative"
+    >
+      {/* 图片预览区域 */}
+      {imageData && (
+        <Flex alignItems="center" mb={2}>
+          <Image
+            src={imageData}
+            alt="Uploaded preview"
+            boxSize="60px"
+            borderRadius="md"
+            objectFit="cover"
+            mr={2}
+          />
+          <CloseButton onClick={removeImage} variant="outline" size="sm" />
+        </Flex>
+      )}
+      <HStack>
+        <InputGroup as="form" onSubmit={onSubmit}>
+          <Input
+            type="text"
+            placeholder="Ask your team a question"
+            value={input}
+            onChange={(e) => setInput(e.target.value)}
+            boxShadow="0 0 10px rgba(0,0,0,0.2)"
+            pr={imageData ? "40px" : "0"} // 调整右侧内边距
+          />
+          <InputRightElement>
+            <IconButton
+              type="submit"
+              icon={<VscSend />}
+              aria-label="send-question"
+              isLoading={isStreaming}
+              isDisabled={!input.trim().length && !imageData}
+            />
+          </InputRightElement>
+          <InputLeftElement>
+            <IconButton
+              type="button"
+              id="image-upload"
+              icon={<RiImageAddLine />}
+              aria-label="upload-image"
+              onClick={() => document.getElementById("file-input")?.click()}
+            />
+            <input
+              type="file"
+              id="file-input"
+              accept="image/*"
+              style={{ display: "none" }}
+              onChange={handleFileChange}
+            />
+          </InputLeftElement>
+        </InputGroup>
+        {newChatHandler && (
+          <Tooltip
+            label="New Chat"
+            fontSize="md"
+            bg="green"
+            placement="top-end"
+          >
+            <IconButton
+              aria-label="new chat"
+              icon={<GrNewWindow />}
+              position="absolute"
+              right={10}
+              onClick={newChatHandler}
+              boxShadow="0 0 10px rgba(0,0,0,0.2)"
+            />
+          </Tooltip>
+        )}
+      </HStack>
+    </Box>
+  );
+};
 
 export default MessageInput;

web/src/components/Playground/ChatMain.tsx

@@ -60,7 +60,7 @@ const ChatMain = ({ isPlayground }: { isPlayground?: boolean }) => {
   const threadId = searchParams.get("threadId");
   const { t } = useTranslation();
   const { teamId } = useChatTeamIdStore();
-
+  const [imageData, setImageData] = useState<string | null>(null);
   const [currentThreadId, setCurrentThreadId] = useState<string | null>(
     searchParams.get("threadId"),
   );
@@ -275,6 +275,7 @@ const ChatMain = ({ isPlayground }: { isPlayground?: boolean }) => {
         type: "human",
         id: self.crypto.randomUUID(),
         content: data.messages[0].content,
+        img:imageData,
         name: "user",
       },
     ]);
@@ -305,6 +306,7 @@ const ChatMain = ({ isPlayground }: { isPlayground?: boolean }) => {
       e.preventDefault();
       mutation.mutate({ messages: [{ type: "human", content: input }] });
       setInput("");
+      setImageData("");
     },
     [input, mutation],
   );
@@ -384,6 +386,8 @@ const ChatMain = ({ isPlayground }: { isPlayground?: boolean }) => {
         onSubmit={onSubmit}
         isStreaming={isStreaming}
         newChatHandler={newChatHandler}
+        imageData={imageData}
+        setImageData={setImageData}
       />
     </Box>
   );

web/src/components/Playground/MessageBox.tsx

@@ -44,7 +44,7 @@ interface MessageBoxProps {
 }
 
 const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => {
-  const { type, name, next, content, tool_calls, tool_output, documents } =
+  const { type, name, next, content, imgdata, tool_calls, tool_output, documents } =
     message;
   const [decision, setDecision] = useState<InterruptDecision | null>(null);
   const [toolMessage, setToolMessage] = useState<string | null>(null);
@@ -184,6 +184,9 @@ const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => {
               overflowY="auto" // 使其可滚动
               overflowX="hidden"
             >
+              {imgdata && (
+                <Image src={imgdata} alt="img" height={"auto"} width={"auto"} />
+              )}
               {content && <Markdown content={content} />}
             </Box>
 
@@ -197,11 +200,13 @@ const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => {
                 alignItems="flex-start"
                 ml="10"
               >
-                <Box display="flex" flexDirection="row">
+                <Box display="flex" flexDirection="column">
                   {Object.keys(tool_call.args).map((attribute, index) => (
-                    <Text key={index} ml="2">
-                      {attribute}: {tool_call.args[attribute]}
-                    </Text>
+                    <Box key={index} ml="2">
+                      <Markdown
+                        content={`${attribute}: ${tool_call.args[attribute]}`}
+                      />
+                    </Box>
                   ))}
                 </Box>
               </Box>