Merge pull request #1540 from phidatahq/release/2.7.0

Release 2.7.0

.github/pull_request_template.md

@@ -1,10 +1,12 @@
 ## Description
 
 **Please include:**
-- A summary of the changes and the related issue
-- Relevant motivation and context
-- Any dependencies or environment changes required
-- If this modifies AI/ML components, include model performance metrics
+
+- **Summary of changes**: Clearly describe the key changes in this PR and their purpose.
+- **Related issues**: Mention if this PR fixes or is connected to any issues (e.g., "Fixes #123" or "Relates to #456").
+- **Motivation and context**: Explain the reason for the changes and the problem they solve.
+- **Environment or dependencies**: Specify any changes in dependencies or environment configurations required for this update.
+- **Impact on AI/ML components**: (If applicable) Describe changes to AI/ML models and include performance metrics (e.g., accuracy, F1-score).
 
 Fixes # (issue)
 
@@ -30,4 +32,4 @@ Please check the options that are relevant:
 
 ## Additional Notes
 
-Include any deployment notes, performance implications, or other relevant information:
+Include any deployment notes, performance implications, or other relevant information:

cookbook/agents/15_generate_video.py

@@ -7,7 +7,7 @@
     tools=[ModelsLabs()],
     description="You are an AI agent that can generate videos using the ModelsLabs API.",
     instructions=[
-        "When the user asks you to create a video, use the `create_video` tool to create the video.",
+        "When the user asks you to create a video, use the `generate_media` tool to create the video.",
         "The video will be displayed in the UI automatically below your response, so you don't need to show the video URL in your response.",
         "Politely and courteously let the user know that the video has been generated and will be displayed below as soon as its ready.",
     ],

cookbook/agents/43_generate_replicate_video.py

@@ -0,0 +1,24 @@
+from phi.agent import Agent
+from phi.model.openai import OpenAIChat
+from phi.tools.replicate import ReplicateTools
+
+"""Create an agent specialized for Replicate AI content generation"""
+
+video_agent = Agent(
+    name="Video Generator Agent",
+    model=OpenAIChat(id="gpt-4o"),
+    tools=[
+        ReplicateTools(model="tencent/hunyuan-video:847dfa8b01e739637fc76f480ede0c1d76408e1d694b830b5dfb8e547bf98405")
+    ],
+    description="You are an AI agent that can generate videos using the Replicate API.",
+    instructions=[
+        "When the user asks you to create a video, use the `generate_media` tool to create the video.",
+        "Return the URL as raw to the user.",
+        "Don't convert video URL to markdown or anything else.",
+    ],
+    markdown=True,
+    debug_mode=True,
+    show_tool_calls=True,
+)
+
+video_agent.print_response("Generate a video of a horse in the dessert.")

cookbook/agents/44_generate_replicate_image.py

@@ -0,0 +1,22 @@
+from phi.agent import Agent
+from phi.model.openai import OpenAIChat
+from phi.tools.replicate import ReplicateTools
+
+"""Create an agent specialized for Replicate AI content generation"""
+
+image_agent = Agent(
+    name="Image Generator Agent",
+    model=OpenAIChat(id="gpt-4o"),
+    tools=[ReplicateTools(model="luma/photon-flash")],
+    description="You are an AI agent that can generate images using the Replicate API.",
+    instructions=[
+        "When the user asks you to create an image, use the `generate_media` tool to create the image.",
+        "Return the URL as raw to the user.",
+        "Don't convert image URL to markdown or anything else.",
+    ],
+    markdown=True,
+    debug_mode=True,
+    show_tool_calls=True,
+)
+
+image_agent.print_response("Generate an image of a horse in the dessert.")

cookbook/agents/45_generate_fal_video.py

@@ -0,0 +1,20 @@
+from phi.agent import Agent
+from phi.model.openai import OpenAIChat
+from phi.tools.fal_tools import FalTools
+
+fal_agent = Agent(
+    name="Fal Video Generator Agent",
+    model=OpenAIChat(id="gpt-4o"),
+    tools=[FalTools("fal-ai/hunyuan-video")],
+    description="You are an AI agent that can generate videos using the Fal API.",
+    instructions=[
+        "When the user asks you to create a video, use the `generate_media` tool to create the video.",
+        "Return the URL as raw to the user.",
+        "Don't convert video URL to markdown or anything else.",
+    ],
+    markdown=True,
+    debug_mode=True,
+    show_tool_calls=True,
+)
+
+fal_agent.print_response("Generate video of balloon in the ocean")

cookbook/examples/agents/01_ai_recipe_creator.py

@@ -0,0 +1,33 @@
+from phi.agent import Agent
+from phi.knowledge.pdf import PDFUrlKnowledgeBase
+from phi.vectordb.pgvector import PgVector
+from phi.tools.exa import ExaTools
+
+db_url = "postgresql+psycopg://ai:ai@localhost:5532/ai"
+
+knowledge_base = PDFUrlKnowledgeBase(
+    urls=[
+        "https://www.poshantracker.in/pdf/Awareness/MilletsRecipeBook2023_Low%20Res_V5.pdf",
+        "https://www.cardiff.ac.uk/__data/assets/pdf_file/0003/123681/Recipe-Book.pdf",
+    ],
+    vector_db=PgVector(table_name="recipes", db_url=db_url),
+)
+knowledge_base.load(recreate=False)
+
+recipe_agent = Agent(
+    name="RecipeGenie",
+    knowledge_base=knowledge_base,
+    search_knowledge=True,
+    tools=[ExaTools()],
+    markdown=True,
+    instructions=[
+        "Search for recipes based on the ingredients and time available from the knowledge base.",
+        "Include the exact calories, preparation time, cooking instructions, and highlight allergens for the recommended recipes.",
+        "Always search exa for recipe links or tips related to the recipes apart from knowledge base.",
+        "Provide a list of recipes that match the user's requirements and preferences.",
+    ],
+)
+
+recipe_agent.print_response(
+    "I have potatoes, tomatoes, onions, garlic, ginger, and chicken. Suggest me a quick recipe for dinner", stream=True
+)

cookbook/examples/agents/03_itinerary_planner.py

@@ -14,7 +14,7 @@
         "Ensure that the gathered data is accurate and tailored to the user's preferences, such as destination, group size, and budget constraints.",
         "Create a clear and concise itinerary that includes: detailed day-by-day travel plan, suggested transportation and accommodation options, activity recommendations (e.g., sightseeing, dining, events), an estimated cost breakdown (covering transportation, accommodation, food, and activities).",
         "If a particular website or travel option is unavailable, provide alternatives from other trusted sources.",
-        "Do not include direct links to external websites or booking platforms in the response."
+        "Do not include direct links to external websites or booking platforms in the response.",
     ],
 )
 

cookbook/playground/multimodal_agent.py

@@ -10,48 +10,83 @@
 from phi.model.openai import OpenAIChat
 from phi.tools.dalle import Dalle
 from phi.tools.models_labs import ModelsLabs
+from phi.model.response import FileType
 from phi.playground import Playground, serve_playground_app
 from phi.storage.agent.sqlite import SqlAgentStorage
+from phi.tools.fal_tools import FalTools
 
 image_agent_storage_file: str = "tmp/image_agent.db"
 
 image_agent = Agent(
-    name="Image Agent",
+    name="DALL-E Image Agent",
     agent_id="image_agent",
     model=OpenAIChat(id="gpt-4o"),
     tools=[Dalle()],
     description="You are an AI agent that can generate images using DALL-E.",
     instructions=[
         "When the user asks you to create an image, use the `create_image` tool to create the image.",
-        "The image will be displayed in the UI automatically below your response, so you don't need to show the image URL in your response.",
-        "Politely and courteously let the user know that the image has been generated and will be displayed below as soon as its ready.",
+        "Don't provide the URL of the image in the response. Only describe what image was generated.",
     ],
     markdown=True,
     debug_mode=True,
     add_history_to_messages=True,
     add_datetime_to_instructions=True,
-    storage=SqlAgentStorage(table_name="image_agent", db_file="tmp/image_agent.db"),
+    storage=SqlAgentStorage(table_name="image_agent", db_file=image_agent_storage_file),
 )
 
-video_agent = Agent(
-    name="Video Agent",
-    agent_id="video_agent",
+ml_gif_agent = Agent(
+    name="ModelsLab GIF Agent",
+    agent_id="ml_gif_agent",
     model=OpenAIChat(id="gpt-4o"),
-    tools=[ModelsLabs(wait_for_completion=True)],
+    tools=[ModelsLabs(wait_for_completion=True, file_type=FileType.GIF)],
+    description="You are an AI agent that can generate gifs using the ModelsLabs API.",
+    instructions=[
+        "When the user asks you to create an image, use the `generate_media` tool to create the image.",
+        "Don't provide the URL of the image in the response. Only describe what image was generated.",
+    ],
+    markdown=True,
+    debug_mode=True,
+    add_history_to_messages=True,
+    add_datetime_to_instructions=True,
+    storage=SqlAgentStorage(table_name="ml_gif_agent", db_file=image_agent_storage_file),
+)
+
+ml_video_agent = Agent(
+    name="ModelsLab Video Agent",
+    agent_id="ml_video_agent",
+    model=OpenAIChat(id="gpt-4o"),
+    tools=[ModelsLabs(wait_for_completion=True, file_type=FileType.MP4)],
     description="You are an AI agent that can generate videos using the ModelsLabs API.",
     instructions=[
-        "When the user asks you to create a video, use the `create_video` tool to create the video.",
-        "The video will be displayed in the UI automatically below your response, so you don't need to show the video URL in your response.",
-        "Politely and courteously let the user know that the video has been generated and will be displayed below as soon as its ready.",
+        "When the user asks you to create a video, use the `generate_media` tool to create the video.",
+        "Don't provide the URL of the video in the response. Only describe what video was generated.",
     ],
     markdown=True,
     debug_mode=True,
     add_history_to_messages=True,
     add_datetime_to_instructions=True,
-    storage=SqlAgentStorage(table_name="video_agent", db_file="tmp/video_agent.db"),
+    storage=SqlAgentStorage(table_name="ml_video_agent", db_file=image_agent_storage_file),
 )
 
-app = Playground(agents=[image_agent, video_agent]).get_app()
+fal_agent = Agent(
+    name="Fal Video Agent",
+    agent_id="fal_agent",
+    model=OpenAIChat(id="gpt-4o"),
+    tools=[FalTools("fal-ai/hunyuan-video")],
+    description="You are an AI agent that can generate videos using the Fal API.",
+    instructions=[
+        "When the user asks you to create a video, use the `generate_media` tool to create the video.",
+        "Don't provide the URL of the video in the response. Only describe what video was generated.",
+    ],
+    markdown=True,
+    debug_mode=True,
+    add_history_to_messages=True,
+    add_datetime_to_instructions=True,
+    storage=SqlAgentStorage(table_name="fal_agent", db_file=image_agent_storage_file),
+)
+
+
+app = Playground(agents=[image_agent, ml_gif_agent, ml_video_agent, fal_agent]).get_app(use_async=False)
 
 if __name__ == "__main__":
     serve_playground_app("multimodal_agent:app", reload=True)

cookbook/tools/lumalabs_tool.py

@@ -0,0 +1,45 @@
+from phi.agent import Agent
+from phi.llm.openai import OpenAIChat
+from phi.tools.lumalab import LumaLabTools
+
+"""Create an agent specialized for Luma AI video generation"""
+
+luma_agent = Agent(
+    name="Luma Video Agent",
+    agent_id="luma-video-agent",
+    llm=OpenAIChat(model="gpt-4o"),
+    tools=[LumaLabTools()],  # Using the LumaLab tool we created
+    markdown=True,
+    debug_mode=True,
+    show_tool_calls=True,
+    instructions=[
+        "You are an agent designed to generate videos using the Luma AI API.",
+        "You can generate videos in two ways:",
+        "1. Text-to-Video Generation:",
+        "   - Use the generate_video function for creating videos from text prompts",
+        "   - Default parameters: loop=False, aspect_ratio='16:9', keyframes=None",
+        "2. Image-to-Video Generation:",
+        "   - Use the image_to_video function when starting from one or two images",
+        "   - Required parameters: prompt, start_image_url",
+        "   - Optional parameters: end_image_url, loop=False, aspect_ratio='16:9'",
+        "   - The image URLs must be publicly accessible",
+        "Choose the appropriate function based on whether the user provides image URLs or just a text prompt.",
+        "The video will be displayed in the UI automatically below your response, so you don't need to show the video URL in your response.",
+        "Politely and courteously let the user know that the video has been generated and will be displayed below as soon as its ready.",
+        "After generating any video, if generation is async (wait_for_completion=False), inform about the generation ID",
+    ],
+    system_message=(
+        "Use generate_video for text-to-video requests and image_to_video for image-based "
+        "generation. Don't modify default parameters unless specifically requested. "
+        "Always provide clear feedback about the video generation status."
+    ),
+)
+
+luma_agent.run("Generate a video of a car in a sky")
+# luma_agent.run("Transform this image into a video of a tiger walking: https://upload.wikimedia.org/wikipedia/commons/thumb/3/3f/Walking_tiger_female.jpg/1920px-Walking_tiger_female.jpg")
+# luma_agent.run("""
+# Create a transition video between these two images:
+# Start: https://img.freepik.com/premium-photo/car-driving-dark-forest-generative-ai_634053-6661.jpg?w=1380
+# End: https://img.freepik.com/free-photo/front-view-black-luxury-sedan-road_114579-5030.jpg?t=st=1733821884~exp=1733825484~hmac=735ca584a9b985c53875fc1ad343c3fd394e1de4db49e5ab1a9ab37ac5f91a36&w=1380
+# Make it a smooth, natural movement
+# """)

cookbook/vectordb/qdrant_db.py

@@ -1,4 +1,4 @@
-# pip install qdrant-client 
+# pip install qdrant-client
 from phi.vectordb.qdrant import Qdrant
 from phi.agent import Agent
 from phi.knowledge.pdf import PDFUrlKnowledgeBase
@@ -13,10 +13,7 @@
 """
 COLLECTION_NAME = "thai-recipes"
 
-vector_db = Qdrant(
-    collection=COLLECTION_NAME,
-    url="http://localhost:6333"
-)
+vector_db = Qdrant(collection=COLLECTION_NAME, url="http://localhost:6333")
 
 knowledge_base = PDFUrlKnowledgeBase(
     urls=["https://phi-public.s3.amazonaws.com/recipes/ThaiRecipes.pdf"],
@@ -27,4 +24,4 @@
 
 # Create and use the agent
 agent = Agent(knowledge_base=knowledge_base, use_tools=True, show_tool_calls=True)
-agent.print_response("List down the ingredients to make Massaman Gai", markdown=True)
+agent.print_response("List down the ingredients to make Massaman Gai", markdown=True)

phi/agent/agent.py

@@ -28,6 +28,7 @@
 
 from phi.document import Document
 from phi.agent.session import AgentSession
+from phi.model.content import Image, Video
 from phi.reasoning.step import ReasoningStep, ReasoningSteps, NextAction
 from phi.run.response import RunEvent, RunResponse, RunResponseExtraData
 from phi.knowledge.agent import AgentKnowledge
@@ -57,9 +58,9 @@ class Agent(BaseModel):
 
     # -*- Agent Data
     # Images associated with this agent
-    images: Optional[List[Union[str, Dict[str, Any]]]] = None
+    images: Optional[List[Image]] = None
     # Videos associated with this agent
-    videos: Optional[List[Union[str, Dict[str, Any]]]] = None
+    videos: Optional[List[Video]] = None
 
     # Data associated with this agent
     # name, model, images and videos are automatically added to the agent_data
@@ -573,9 +574,9 @@ def get_agent_data(self) -> Dict[str, Any]:
         if self.model is not None:
             agent_data["model"] = self.model.to_dict()
         if self.images is not None:
-            agent_data["images"] = self.images
+            agent_data["images"] = [img if isinstance(img, dict) else img.model_dump() for img in self.images]
         if self.videos is not None:
-            agent_data["videos"] = self.videos
+            agent_data["videos"] = [vid if isinstance(vid, dict) else vid.model_dump() for vid in self.videos]
         return agent_data
 
     def get_session_data(self) -> Dict[str, Any]:
@@ -588,7 +589,6 @@ def get_session_data(self) -> Dict[str, Any]:
 
     def get_agent_session(self) -> AgentSession:
         """Get an AgentSession object, which can be saved to the database"""
-
         return AgentSession(
             session_id=self.session_id,
             agent_id=self.agent_id,
@@ -632,13 +632,13 @@ def from_agent_session(self, session: AgentSession):
             if "images" in session.agent_data:
                 images_from_db = session.agent_data.get("images")
                 if self.images is not None and isinstance(self.images, list):
-                    self.images.extend(images_from_db)  # type: ignore
+                    self.images.extend([Image.model_validate(img) for img in self.images])
                 else:
                     self.images = images_from_db
             if "videos" in session.agent_data:
                 videos_from_db = session.agent_data.get("videos")
                 if self.videos is not None and isinstance(self.videos, list):
-                    self.videos.extend(videos_from_db)  # type: ignore
+                    self.videos.extend([Video.model_validate(vid) for vid in self.videos])
                 else:
                     self.videos = videos_from_db
 
@@ -2433,7 +2433,7 @@ def delete_session(self, session_id: str):
     # Handle images and videos
     ###########################################################################
 
-    def add_image(self, image: Union[str, Dict]) -> None:
+    def add_image(self, image: Image) -> None:
         if self.images is None:
             self.images = []
         self.images.append(image)
@@ -2442,7 +2442,7 @@ def add_image(self, image: Union[str, Dict]) -> None:
                 self.run_response.images = []
             self.run_response.images.append(image)
 
-    def add_video(self, video: Union[str, Dict]) -> None:
+    def add_video(self, video: Video) -> None:
         if self.videos is None:
             self.videos = []
         self.videos.append(video)
@@ -2451,10 +2451,10 @@ def add_video(self, video: Union[str, Dict]) -> None:
                 self.run_response.videos = []
             self.run_response.videos.append(video)
 
-    def get_images(self) -> Optional[List[Union[str, Dict]]]:
+    def get_images(self) -> Optional[List[Image]]:
         return self.images
 
-    def get_videos(self) -> Optional[List[Union[str, Dict]]]:
+    def get_videos(self) -> Optional[List[Video]]:
         return self.videos
 
     ###########################################################################