From a08d6b68f58c30d4bd1c2364bfcf401eb30c61de Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Tue, 26 Mar 2024 16:45:29 +0000 Subject: [PATCH 1/2] [chore/issue122] tiny code improvement in status --- crates/edgen_server/src/status.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/edgen_server/src/status.rs b/crates/edgen_server/src/status.rs index 1ac15b8..56864ca 100644 --- a/crates/edgen_server/src/status.rs +++ b/crates/edgen_server/src/status.rs @@ -316,8 +316,8 @@ async fn observe_progress( let mut m = tokio::fs::metadata(&f.path()).await; let mut last_size = 0; let mut timestamp = Instant::now(); - while m.is_ok() { - let s = m.unwrap().len() as u64; + while let Ok(d) = m { + let s = d.len() as u64; let p = (s * 100) / size; if s > last_size { From b5613b03f9e34df2045e4cbe86a971519b8082ea Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Tue, 26 Mar 2024 18:21:16 +0000 Subject: [PATCH 2/2] [chore/issue122] settings tests use embeddiings (with tiny llama) --- crates/edgen_core/src/settings.rs | 14 +++++ crates/edgen_server/tests/common/mod.rs | 54 ++++++++++++++-- crates/edgen_server/tests/settings_tests.rs | 69 ++++++++++++++++++++- 3 files changed, 131 insertions(+), 6 deletions(-) diff --git a/crates/edgen_core/src/settings.rs b/crates/edgen_core/src/settings.rs index a6b6043..da23ccd 100644 --- a/crates/edgen_core/src/settings.rs +++ b/crates/edgen_core/src/settings.rs @@ -62,6 +62,16 @@ pub async fn create_project_dirs() -> Result<(), std::io::Error> { let audio_transcriptions_dir = PathBuf::from(&audio_transcriptions_str); + let embeddings_str = SETTINGS + .read() + .await + .read() + .await + .embeddings_models_dir + .to_string(); + + let embeddings_dir = PathBuf::from(&embeddings_str); + if !config_dir.is_dir() { std::fs::create_dir_all(config_dir)?; } @@ -74,6 +84,10 @@ pub async fn create_project_dirs() -> Result<(), std::io::Error> { std::fs::create_dir_all(&audio_transcriptions_dir)?; } + if !embeddings_dir.is_dir() { + std::fs::create_dir_all(&embeddings_str)?; + } + Ok(()) } diff --git a/crates/edgen_server/tests/common/mod.rs b/crates/edgen_server/tests/common/mod.rs index 768d3f8..4aeb10e 100644 --- a/crates/edgen_server/tests/common/mod.rs +++ b/crates/edgen_server/tests/common/mod.rs @@ -24,11 +24,15 @@ pub const SMALL_LLM_REPO: &str = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"; pub const SMALL_WHISPER_NAME: &str = "ggml-distil-small.en.bin"; pub const SMALL_WHISPER_REPO: &str = "distil-whisper/distil-small.en"; +pub const SMALL_EMBEDDINGS_NAME: &str = "tinyllama-1.1b-chat-v1.0.Q2_K.gguf"; +pub const SMALL_EMBEDDINGS_REPO: &str = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"; + pub const BASE_URL: &str = "http://localhost:33322/v1"; pub const CHAT_URL: &str = "/chat"; pub const COMPLETIONS_URL: &str = "/completions"; pub const AUDIO_URL: &str = "/audio"; pub const TRANSCRIPTIONS_URL: &str = "/transcriptions"; +pub const EMBEDDINGS_URL: &str = "/embeddings"; pub const STATUS_URL: &str = "/status"; pub const MISC_URL: &str = "/misc"; pub const VERSION_URL: &str = "/version"; @@ -241,6 +245,10 @@ pub fn data_exists() { let transcriptions = audio.join("transcriptions"); println!("exists: {:?}", transcriptions); assert!(transcriptions.exists()); + + let embeddings = models.join("embeddings"); + println!("exists: {:?}", embeddings); + assert!(embeddings.exists()); } /// Edit the config file: set another model dir for the indicated endpoint. @@ -256,7 +264,9 @@ pub fn set_model_dir(ep: Endpoint, model_dir: &str) { Endpoint::AudioTranscriptions => { config.audio_transcriptions_models_dir = model_dir.to_string(); } - Endpoint::Embeddings => todo!(), + Endpoint::Embeddings => { + config.embeddings_models_dir = model_dir.to_string(); + } } write_config(&config).unwrap(); @@ -280,7 +290,10 @@ pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { config.audio_transcriptions_model_name = model_name.to_string(); config.audio_transcriptions_model_repo = model_repo.to_string(); } - Endpoint::Embeddings => todo!(), + Endpoint::Embeddings => { + config.embeddings_model_name = model_name.to_string(); + config.embeddings_model_repo = model_repo.to_string(); + } } write_config(&config).unwrap(); @@ -292,7 +305,7 @@ pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { Endpoint::AudioTranscriptions => { make_url(&[BASE_URL, AUDIO_URL, TRANSCRIPTIONS_URL, STATUS_URL]) } - Endpoint::Embeddings => todo!(), + Endpoint::Embeddings => make_url(&[BASE_URL, EMBEDDINGS_URL, STATUS_URL]), }; let stat: status::AIStatus = blocking::get(url).unwrap().json().unwrap(); assert_eq!(stat.active_model, model_name); @@ -335,13 +348,22 @@ pub fn chat_completions_custom_body(model: &str) -> String { .expect("cannot convert JSON to String") } +/// embeddings body with custom model +pub fn embeddings_custom_body(model: &str) -> String { + serde_json::to_string(&json!({ + "model": model, + "input": "what is the capital of idaho?", + })) + .expect("cannot convert JSON to String") +} + /// Spawn a thread to send a request to the indicated endpoint. /// This allows the caller to perform another task in the caller thread. pub fn spawn_request(ep: Endpoint, body: &str, model: &str) -> thread::JoinHandle { match ep { Endpoint::ChatCompletions => spawn_chat_completions_request(body), Endpoint::AudioTranscriptions => spawn_audio_transcriptions_request(model), - Endpoint::Embeddings => todo!(), + Endpoint::Embeddings => spawn_embeddings_request(body), } } @@ -369,6 +391,30 @@ pub fn spawn_chat_completions_request(body: &str) -> thread::JoinHandle { }) } +pub fn spawn_embeddings_request(body: &str) -> thread::JoinHandle { + let body = body.to_string(); + thread::spawn(move || { + let ep = make_url(&[BASE_URL, EMBEDDINGS_URL]); + println!("requesting {}", ep); + match blocking::Client::new() + .post(&ep) + .header("Content-Type", "application/json") + .body(body) + .timeout(Duration::from_secs(180)) + .send() + { + Err(e) => { + eprintln!("cannot connect: {:?}", e); + false + } + Ok(v) => { + println!("Got {:?}", v); + v.status().is_success() + } + } + }) +} + pub fn spawn_audio_transcriptions_request(model: &str) -> thread::JoinHandle { let model = model.to_string(); let frost = Path::new("resources").join("frost.wav"); diff --git a/crates/edgen_server/tests/settings_tests.rs b/crates/edgen_server/tests/settings_tests.rs index 6cbfecd..070f5c2 100644 --- a/crates/edgen_server/tests/settings_tests.rs +++ b/crates/edgen_server/tests/settings_tests.rs @@ -70,6 +70,7 @@ fn test_battery() { chat_completions_status_reachable(); audio_transcriptions_status_reachable(); + embeddings_status_reachable(); // ================================ common::test_message("SCENARIO 2"); @@ -85,14 +86,22 @@ fn test_battery() { common::SMALL_WHISPER_NAME, common::SMALL_WHISPER_REPO, ); + common::set_model( + Endpoint::Embeddings, + common::SMALL_EMBEDDINGS_NAME, + common::SMALL_EMBEDDINGS_REPO, + ); // test ai endpoint and download test_ai_endpoint_with_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_with_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_with_download(Endpoint::Embeddings, "default"); + // we have downloaded, we should not download again test_ai_endpoint_no_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_no_download(Endpoint::Embeddings, "default"); // ================================ common::test_message("SCENARIO 3"); @@ -122,17 +131,26 @@ fn test_battery() { "transcriptions", ); - common::set_model_dir(Endpoint::ChatCompletions, &new_chat_completions_dir); + let new_embeddings_dir = my_models_dir.clone() + + &format!( + "{}{}", + path::MAIN_SEPARATOR, + "embeddings", + ); + common::set_model_dir(Endpoint::ChatCompletions, &new_chat_completions_dir); common::set_model_dir(Endpoint::AudioTranscriptions, &new_audio_transcriptions_dir); + common::set_model_dir(Endpoint::Embeddings, &new_embeddings_dir); test_ai_endpoint_with_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_with_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_with_download(Endpoint::Embeddings, "default"); assert!(path::Path::new(&my_models_dir).exists()); test_ai_endpoint_no_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_no_download(Endpoint::Embeddings, "default"); // ================================ common::test_message("SCENARIO 4"); @@ -142,11 +160,13 @@ fn test_battery() { test_ai_endpoint_with_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_with_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_with_download(Endpoint::Embeddings, "default"); assert!(path::Path::new(&my_models_dir).exists()); test_ai_endpoint_no_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_no_download(Endpoint::Embeddings, "default"); // ================================ common::test_message("SCENARIO 5"); @@ -163,6 +183,11 @@ fn test_battery() { common::SMALL_WHISPER_NAME, common::SMALL_WHISPER_REPO, ); + common::set_model( + Endpoint::Embeddings, + common::SMALL_EMBEDDINGS_NAME, + common::SMALL_EMBEDDINGS_REPO, + ); // make sure we read from the old directories again remove_dir_all(&my_models_dir).unwrap(); @@ -170,21 +195,25 @@ fn test_battery() { test_ai_endpoint_no_download(Endpoint::ChatCompletions, "default"); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, "default"); + test_ai_endpoint_no_download(Endpoint::Embeddings, "default"); // ================================ common::test_message("SCENARIO 6"); // ================================ let chat_model = "TheBloke/phi-2-GGUF/phi-2.Q2_K.gguf"; let audio_model = "distil-whisper/distil-medium.en/ggml-medium-32-2.en.bin"; + let embeddings_model = "TheBloke/phi-2-GGUF/phi-2.Q2_K.gguf"; test_ai_endpoint_with_download(Endpoint::ChatCompletions, chat_model); test_ai_endpoint_with_download(Endpoint::AudioTranscriptions, audio_model); + test_ai_endpoint_with_download(Endpoint::Embeddings, embeddings_model); // ================================ common::test_message("SCENARIO 7"); // ================================ test_ai_endpoint_no_download(Endpoint::ChatCompletions, chat_model); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, audio_model); + test_ai_endpoint_no_download(Endpoint::Embeddings, embeddings_model); // ================================ common::test_message("SCENARIO 8"); @@ -200,6 +229,10 @@ fn test_battery() { "audio/transcriptions", ); test_ai_endpoint_no_download(Endpoint::AudioTranscriptions, ".whisper-medium-32-2.en.bin"); + + let source = "models--TheBloke--phi-2-GGUF/blobs"; + common::copy_model(source, ".phi-2.Q2_K.gguf", "embeddings"); + test_ai_endpoint_no_download(Endpoint::Embeddings, ".phi-2.Q2_K.gguf"); }) } @@ -243,6 +276,25 @@ fn audio_transcriptions_status_reachable() { }); } +fn embeddings_status_reachable() { + common::test_message("embeddings status is reachable"); + assert!(match blocking::get(common::make_url(&[ + common::BASE_URL, + common::EMBEDDINGS_URL, + common::STATUS_URL, + ])) { + Err(e) => { + eprintln!("cannot connect: {:?}", e); + false + } + Ok(v) => { + assert!(v.status().is_success()); + println!("have: '{}'", v.text().unwrap()); + true + } + }); +} + fn test_config_reset() { common::test_message("test resetting config"); common::reset_config(); @@ -291,7 +343,20 @@ fn test_ai_endpoint(endpoint: Endpoint, model: &str, download: bool) { "".to_string(), ) } - Endpoint::Embeddings => todo!(), + Endpoint::Embeddings => { + common::test_message(&format!( + "embeddints endpoint with download: {}", + download + )); + ( + common::make_url(&[ + common::BASE_URL, + common::EMBEDDINGS_URL, + common::STATUS_URL, + ]), + common::embeddings_custom_body(model), + ) + } }; let handle = common::spawn_request(endpoint, &body, model); if download {