stream tinyllama output

demo/__init__.py

@@ -14,6 +14,7 @@
 from .db import create_db
 from .forms import router as forms_router
 from .main import router as main_router
+from .ollama import router as ollama_router
 from .sse import router as sse_router
 from .tables import router as table_router
 
@@ -35,6 +36,7 @@ async def lifespan(app_: FastAPI):
 
 app.include_router(components_router, prefix='/api/components')
 app.include_router(sse_router, prefix='/api/components')
+app.include_router(ollama_router, prefix='/api/ollama')
 app.include_router(table_router, prefix='/api/table')
 app.include_router(forms_router, prefix='/api/forms')
 app.include_router(auth_router, prefix='/api/auth')

demo/ollama.py

@@ -0,0 +1,41 @@
+# ollama_sse.py
+
+from __future__ import annotations as _annotations
+
+from fastapi import APIRouter
+from fastui import AnyComponent, FastUI
+from fastui import components as c
+from fastui.events import PageEvent
+
+from .shared import demo_page
+
+router = APIRouter()
+
+
+def panel(*components: AnyComponent) -> AnyComponent:
+    return c.Div(class_name='col border rounded m-1 p-2 pb-3', components=list(components))
+
+
+@router.get('', response_model=FastUI, response_model_exclude_none=True)
+def components_view() -> list[AnyComponent]:
+    return demo_page(
+        c.Div(
+            components=[
+                c.Heading(text='Server Load SSE', level=2),
+                c.Markdown(text=('`ServerLoad` can also be used to load content from ollama.\n\n' '')),
+                c.Button(text='Load SSE content', on_click=PageEvent(name='server-load-sse')),
+                c.Div(
+                    components=[
+                        c.ServerLoad(
+                            path='/components/sse',
+                            sse=True,
+                            load_trigger=PageEvent(name='server-load-sse'),
+                            components=[c.Text(text='before')],
+                        ),
+                    ],
+                    class_name='my-2 p-2 border rounded',
+                ),
+            ],
+            class_name='border-top mt-3 pt-1',
+        ),
+    )

demo/shared.py

@@ -12,6 +12,11 @@ def demo_page(*components: AnyComponent, title: str | None = None) -> list[AnyCo
             title='FastUI Demo',
             title_event=GoToEvent(url='/'),
             links=[
+                c.Link(
+                    components=[c.Text(text='Ollama')],
+                    on_click=GoToEvent(url='/ollama'),
+                    active='startswith:/ollama',
+                ),
                 c.Link(
                     components=[c.Text(text='Components')],
                     on_click=GoToEvent(url='/components'),

demo/sse.py

@@ -1,35 +1,38 @@
+from __future__ import annotations as _annotations
+
 import asyncio
-from itertools import chain
-from typing import AsyncIterable
 
 from fastapi import APIRouter
 from fastui import FastUI
 from fastui import components as c
+from ollama import generate
 from starlette.responses import StreamingResponse
 
 router = APIRouter()
 
 
-async def canned_ai_response_generator() -> AsyncIterable[str]:
-    prompt = '**User:** What is SSE? Please include a javascript code example.\n\n**AI:** '
+async def speak(speaker, content):
+    if speaker:
+        p = await asyncio.create_subprocess_exec(speaker, content)
+        await p.communicate()
+
+
+import asyncio
+
+
+async def ollama_response_generator():
     output = ''
-    msg = ''
-    for time, text in chain([(0.5, prompt)], CANNED_RESPONSE):
-        await asyncio.sleep(time)
-        output += text
+    for part in generate('tinyllama', 'Why is the sky blue?', stream=True):
+        print(part['response'], end='', flush=True)
+        output += part['response']
         m = FastUI(root=[c.Markdown(text=output)])
         msg = f'data: {m.model_dump_json(by_alias=True, exclude_none=True)}\n\n'
         yield msg
 
-    # avoid the browser reconnecting
-    while True:
-        yield msg
-        await asyncio.sleep(10)
-
 
 @router.get('/sse')
 async def sse_ai_response() -> StreamingResponse:
-    return StreamingResponse(canned_ai_response_generator(), media_type='text/event-stream')
+    return StreamingResponse(ollama_response_generator(), media_type='text/event-stream')
 
 
 async def run_openai():
@@ -69,7 +72,6 @@ async def run_openai():
 if __name__ == '__main__':
     asyncio.run(run_openai())
 
-
 CANNED_RESPONSE: list[tuple[float, str]] = [
     (0.00000, ''),
     (0.07685, 'Server'),