From 1ec9d4060f9d8fd1a7c5577802b7457ed84c285f Mon Sep 17 00:00:00 2001 From: Adam Kariv Date: Mon, 11 Nov 2024 13:11:29 +0200 Subject: [PATCH] Build related fixes --- .github/workflows/ci.yaml | 2 +- compose.dev.yaml | 4 +++- docker/api-server/Dockerfile | 13 ++++++++++++- odds/api/server.py | 4 ++++ odds/common/vectordb/__init__.py | 2 +- requirements.txt | 5 +++-- test_assistant.py | 12 ++---------- tester2.py | 6 +++--- ui/projects/ask/src/app/home/home.component.ts | 2 +- 9 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22a2b51..66382e4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: else CACHE_FROM_ARG="" fi &&\ - docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t app -f docker/Dockerfile . &&\ + docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t app -f docker/api-server/Dockerfile . &&\ docker tag app "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\ docker push "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\ if [ "${GITHUB_REF}" == "refs/heads/main" ]; then diff --git a/compose.dev.yaml b/compose.dev.yaml index c73ef38..7e173a0 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -12,10 +12,11 @@ services: dockerfile: docker/api-server/Dockerfile context: . ports: - - "8000:80" + - "8000:8000" volumes: - ./odds.config.yaml:/srv/odds.config.yaml - ./.caches/:/srv/.caches/ + - certs:/docker/certs command: utils/api_server.sh db: image: postgres:16 @@ -82,6 +83,7 @@ services: until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; echo "Setting kibana_system password"; until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + sleep 10; echo "All done!"; ' healthcheck: diff --git a/docker/api-server/Dockerfile b/docker/api-server/Dockerfile index 317e4c5..538eabb 100644 --- a/docker/api-server/Dockerfile +++ b/docker/api-server/Dockerfile @@ -1,4 +1,14 @@ -# Pulled July 19, 2023 +FROM node:lts-bookworm-slim AS frontend + +WORKDIR /app + +COPY ui/package*json /app/ +RUN apt-get update +RUN apt-get install -y ca-certificates +RUN npm install +COPY ui/ /app/ +RUN npm run -- ng build + FROM --platform=linux/amd64 python:3.12 RUN apt-get update && apt-get install -y libleveldb-dev ca-certificates && apt-get clean RUN pip install --upgrade pip @@ -10,6 +20,7 @@ COPY setup.py ./ RUN pip install -e . COPY utils ./utils ENV PYTHONUNBUFFERED=1 +COPY --from=frontend /app/dist/ask/browser /srv/ui RUN mkdir /temp-cache/ && chmod 777 /temp-cache/ ENV RESOURCE_PROCESSOR_CACHE_DIR=/temp-cache ENTRYPOINT ["/bin/bash", "-c"] diff --git a/odds/api/server.py b/odds/api/server.py index ed3c151..0fa456e 100644 --- a/odds/api/server.py +++ b/odds/api/server.py @@ -1,5 +1,6 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles from typing import List, Dict, Any, Optional from .answer import answer_question @@ -23,6 +24,9 @@ allow_headers=["*"], ) +# Serve static files from the 'ui' directory on the '/' endpoint +app.mount("/", StaticFiles(directory="ui", html=True), name="static") + @app.get("/datasets") async def search_datasets_handler(query: str) -> List[Dict[str, Any]]: return await search_datasets(query, None) diff --git a/odds/common/vectordb/__init__.py b/odds/common/vectordb/__init__.py index d279b9a..2860f19 100644 --- a/odds/common/vectordb/__init__.py +++ b/odds/common/vectordb/__init__.py @@ -1,7 +1,7 @@ from .indexer import Indexer from ..select import select from ..embedder import embedder -from .chromadb.chromadb_indexer import ChromaDBIndexer +# from .chromadb.chromadb_indexer import ChromaDBIndexer from .es.es_indexer import ESIndexer indexer: Indexer = select('Indexer', locals())(embedder.vector_size()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9f67632..8cf65f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -chromadb httpx aiofiles sqlalchemy @@ -14,4 +13,6 @@ plyvel fastapi uvicorn aiosqlite -elasticsearch[async] \ No newline at end of file +elasticsearch[async] +openai +pyyaml \ No newline at end of file diff --git a/test_assistant.py b/test_assistant.py index 75aeb82..b6d6922 100644 --- a/test_assistant.py +++ b/test_assistant.py @@ -113,16 +113,8 @@ async def search_datasets(query: str): print('SEARCH DATASETS:', query) - query_terms = query.split(',') - query_terms = [term.strip() for term in query_terms] - query_terms = [term for term in query_terms if term] - print('QUERY TERMS:', query_terms) - embeddings = await asyncio.gather(*[embedder.embed(name) for name in query_terms]) - dataset_ids = await asyncio.gather(*[indexer.findDatasets(embedding) for embedding in embeddings]) - dataset_ids = [x for y in dataset_ids for x in y] - dataset_ids = [x[0] for x in Counter(dataset_ids).most_common(10)] - print('DATASET IDS:', dataset_ids) - datasets = await asyncio.gather(*[store.getDataset(id) for id in dataset_ids]) + embedding = await embedder.embed(query) + datasets = await indexer.findDatasets(embedding) catalogs = [catalog_repo.get_catalog(dataset.catalogId) for dataset in datasets] response = [ dict( diff --git a/tester2.py b/tester2.py index d06db19..99d3f43 100644 --- a/tester2.py +++ b/tester2.py @@ -6,10 +6,10 @@ async def main(): query = sys.argv[1] b = await embedder.embed(query) - ids = await indexer.findDatasets(b) - datasets = [await metadata_store.getDataset(id) for id in ids] + datasets = await indexer.findDatasets(b, query=query) + # datasets = [await metadata_store.getDataset(id) for id in ids] for dataset in datasets: - print(' - ' + dataset.better_title) + print(f' - {dataset.id}: {dataset.better_title}') if __name__ == '__main__': diff --git a/ui/projects/ask/src/app/home/home.component.ts b/ui/projects/ask/src/app/home/home.component.ts index bcfdc60..bc5d77c 100644 --- a/ui/projects/ask/src/app/home/home.component.ts +++ b/ui/projects/ask/src/app/home/home.component.ts @@ -40,7 +40,7 @@ export class HomeComponent { if (this.question) { this.loading = true; const encoded = encodeURIComponent(this.question); - this.http.get('http://localhost:8000/answer', { params: {q: encoded }}) + this.http.get('/answer', { params: {q: encoded }}) .pipe( catchError((error) => { this.answer = 'Error: ' + error.message;