From 49dad37e88cd8ec937f7aecd97602ea1a0e4d6b3 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 14:35:02 +0530 Subject: [PATCH 1/7] cfkv --- Cargo.lock | 1 + ssr/Cargo.toml | 20 ++-- ssr/src/consts/mod.rs | 3 + ssr/src/page/post_view/video_iter.rs | 50 +++++---- ssr/src/page/root.rs | 38 +++---- ssr/src/utils/ml_feed/mod.rs | 156 ++++++++++++++++++++------- 6 files changed, 181 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2629e82..ea7dc333 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,7 @@ dependencies = [ "priority-queue", "prost", "pulldown-cmark", + "rand", "rand_chacha", "redb", "redis", diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index b2d3912f..f8a69649 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -108,14 +108,16 @@ speedate = { version = "0.14.4", optional = true } urlencoding = "2.1.3" yral-types = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04" } yral-qstash-types = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04" } -yral-canisters-client = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04", features = ["full"] } -yral-canisters-common = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04" } +yral-canisters-client = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04", features = [ + "full", +] } +yral-canisters-common = { git = "https://github.com/yral-dapp/yral-common.git", rev = "5e4414a3f1e0644d93f181949d533c6a9991da04" } pulldown-cmark = "0.12.1" ic-certification = "2.6.0" ciborium = "0.2.2" yral-metadata-client = { git = "https://github.com/yral-dapp/yral-metadata", rev = "56e3f1f1f5f452673bee17739520c800c1264295", optional = true } yral-metadata-types = { git = "https://github.com/yral-dapp/yral-metadata", rev = "56e3f1f1f5f452673bee17739520c800c1264295", optional = true } - +rand = "0.8.5" [build-dependencies] tonic-build = { version = "0.12.0", default-features = false, features = [ @@ -133,7 +135,7 @@ hydrate = [ "reqwest/native-tls", "dep:rand_chacha", "tonic/codegen", - "speedate" + "speedate", ] ssr = [ "dep:axum", @@ -162,7 +164,7 @@ ssr = [ "tonic/tls-webpki-roots", "tonic/transport", "tonic-build/transport", - "speedate" + "speedate", ] # Fetch mock referral history instead of history via canister mock-referral-history = ["dep:rand_chacha", "k256/arithmetic"] @@ -209,7 +211,13 @@ local-bin = [ "dep:yral-metadata-client", "dep:yral-metadata-types", ] -local-lib = ["hydrate", "redis-kv", "local-auth", "backend-admin", "yral-canisters-common/local"] +local-lib = [ + "hydrate", + "redis-kv", + "local-auth", + "backend-admin", + "yral-canisters-common/local", +] [package.metadata.leptos] # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 165824c8..68382d4c 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -52,6 +52,9 @@ pub const CDAO_SWAP_TIME_SECS: u64 = CDAO_SWAP_PRE_READY_TIME_SECS + 150; pub const ICPUMP_SEARCH_GRPC_URL: &str = "https://prod-yral-icpumpsearch.fly.dev:443"; pub const NSFW_SERVER_URL: &str = "https://prod-yral-nsfw-classification.fly.dev:443"; +pub const CF_KV_ML_CACHE_NAMESPACE_ID: &str = "ea145fc839bd42f9bf2d34b950ddbda5"; +pub const CLOUDFLARE_ACCOUNT_ID: &str = "a209c523d2d9646cc56227dbe6ce3ede"; + pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; pub const DISCORD: &str = "https://discord.gg/GZ9QemnZuj"; diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 4f3dfbc4..e335c872 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -11,7 +11,11 @@ use yral_canisters_client::post_cache::{self, NsfwFilter}; use crate::{ consts::USER_CANISTER_ID_STORE, state::canisters::auth_canisters_store, - utils::{host::show_nsfw_content, posts::FetchCursor}, + utils::{ + host::show_nsfw_content, + ml_feed::{get_coldstart_feed_paginated, get_posts_ml_feed_cache_paginated}, + posts::FetchCursor, + }, }; use yral_canisters_common::{utils::posts::PostDetails, Canisters, Error as CanistersError}; @@ -22,6 +26,7 @@ pub enum FeedResultType { PostCache, MLFeedCache, MLFeed, + MLFeedColdstart, } pub struct FetchVideosRes<'a> { @@ -179,9 +184,8 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { let ml_feed: MLFeed = expect_context(); - let top_posts = ml_feed - .get_next_feed_coldstart(self.cursor.limit as u32, video_queue) - .await; + let top_posts = + get_coldstart_feed_paginated(self.cursor.start, self.cursor.limit).await; let top_posts = match top_posts { Ok(top_posts) => top_posts, @@ -203,7 +207,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, - res_type: FeedResultType::MLFeed, + res_type: FeedResultType::MLFeedColdstart, }) } @@ -227,25 +231,31 @@ impl<'a> VideoFetchStream<'a, true> { ) -> Result, ServerFnError> { let cans_true = self.canisters; - let user_canister = cans_true.authenticated_user().await; - let top_posts_fut = - user_canister.get_ml_feed_cache_paginated(self.cursor.start, self.cursor.limit); - - let top_posts = top_posts_fut.await?; - if top_posts.is_empty() { - // try ml feed once again - with coldstart feed - return self - .fetch_post_uids_ml_feed_coldstart_chunked(chunks, allow_nsfw, video_queue) - .await; - } + let user_canister_id = cans_true.user_canister(); + let top_posts = match get_posts_ml_feed_cache_paginated( + user_canister_id, + self.cursor.start, + self.cursor.limit, + ) + .await + { + Ok(posts) if posts.is_empty() => { + return self + .fetch_post_uids_ml_feed_coldstart_chunked(chunks, allow_nsfw, video_queue) + .await; + } + Ok(posts) => posts, + Err(_) => { + return self + .fetch_post_uids_ml_feed_coldstart_chunked(chunks, allow_nsfw, video_queue) + .await; + } + }; let end = false; let chunk_stream = top_posts .into_iter() - .map(move |item| { - self.canisters - .get_post_details(item.canister_id, item.post_id) - }) + .map(move |item| self.canisters.get_post_details(item.0, item.1)) .collect::>() .filter_map(|res| async { res.transpose() }) .chunks(chunks); diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index abfcd7fa..1ec70a5d 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -1,8 +1,15 @@ use candid::Principal; use leptos::*; use leptos_router::*; +use rand::Rng; -use crate::{component::spinner::FullScreenSpinner, utils::host::show_cdao_page}; +use crate::{ + component::spinner::FullScreenSpinner, + utils::{ + host::show_cdao_page, + ml_feed::{get_coldstart_feed_paginated, get_posts_ml_feed_cache_paginated}, + }, +}; #[server] async fn get_top_post_id() -> Result, ServerFnError> { @@ -58,28 +65,18 @@ async fn get_top_post_id_mlcache() -> Result, ServerFnE return get_top_post_id_mlfeed().await; } - let user_canister = canisters.individual_user(user_canister_id.unwrap()).await; - - let top_items = user_canister - .get_ml_feed_cache_paginated(0, 1) - .await - .unwrap(); - if top_items.is_empty() { - return get_top_post_id_mlfeed().await; + let posts = get_posts_ml_feed_cache_paginated(user_canister_id.unwrap(), 0, 1).await; + if let Ok(posts) = posts { + if !posts.is_empty() { + return Ok(Some((posts[0].0, posts[0].1))); + } } - - let Some(top_item) = top_items.first() else { - return Ok(None); - }; - - Ok(Some((top_item.canister_id, top_item.post_id))) + get_top_post_id_mlfeed().await } #[server] async fn get_top_post_id_mlfeed() -> Result, ServerFnError> { - use crate::utils::ml_feed::ml_feed_grpc::get_coldstart_feed; - - let top_posts_fut = get_coldstart_feed(); + let top_posts_fut = get_coldstart_feed_paginated(0, 50); let top_items = match top_posts_fut.await { Ok(top_posts) => top_posts, @@ -90,9 +87,8 @@ async fn get_top_post_id_mlfeed() -> Result, ServerFnEr )); } }; - let Some(top_item) = top_items.first() else { - return Ok(None); - }; + let rand_num = rand::thread_rng().gen_range(0..top_items.len()); + let top_item = top_items[rand_num]; Ok(Some((top_item.0, top_item.1))) } diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index 661360ae..25af8752 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -1,5 +1,9 @@ -use crate::consts::ML_FEED_GRPC_URL; +use std::env; + +use crate::consts::{self, ML_FEED_GRPC_URL}; use candid::Principal; +use leptos::{server, ServerFnError}; +use serde::{Deserialize, Serialize}; use super::types::PostId; @@ -197,46 +201,118 @@ pub mod ml_feed_grpc { tonic::include_proto!("ml_feed"); } - pub async fn get_coldstart_feed() -> Result, tonic::Status> { - use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ - ml_feed_client::MlFeedClient, FeedRequest, - }; - use tonic::transport::{Channel, ClientTlsConfig}; - - let tls_config = ClientTlsConfig::new().with_webpki_roots(); - - let channel = Channel::from_static(ML_FEED_GRPC_URL) - .tls_config(tls_config) - .expect("Couldn't update TLS config for nsfw agent") - .connect() - .await - .expect("Couldn't connect to ML feed server"); - - let mut client = MlFeedClient::new(channel); - - let request = tonic::Request::new(FeedRequest { - canister_id: "".to_string(), - filter_posts: vec![], - num_results: 1, - }); - - let response = client.get_feed_coldstart(request).await.map_err(|e| { - tonic::Status::new( - tonic::Code::Internal, - format!("error fetching posts: {:?}", e), - ) - })?; + // pub async fn get_coldstart_feed() -> Result, tonic::Status> { + // use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ + // ml_feed_client::MlFeedClient, FeedRequest, + // }; + // use tonic::transport::{Channel, ClientTlsConfig}; + + // let tls_config = ClientTlsConfig::new().with_webpki_roots(); + + // let channel = Channel::from_static(ML_FEED_GRPC_URL) + // .tls_config(tls_config) + // .expect("Couldn't update TLS config for nsfw agent") + // .connect() + // .await + // .expect("Couldn't connect to ML feed server"); + + // let mut client = MlFeedClient::new(channel); + + // let request = tonic::Request::new(FeedRequest { + // canister_id: "".to_string(), + // filter_posts: vec![], + // num_results: 1, + // }); + + // let response = client.get_feed_coldstart(request).await.map_err(|e| { + // tonic::Status::new( + // tonic::Code::Internal, + // format!("error fetching posts: {:?}", e), + // ) + // })?; + + // let feed_res = response.into_inner().feed; + + // Ok(feed_res + // .iter() + // .map(|item| { + // ( + // Principal::from_text(&item.canister_id).unwrap(), + // item.post_id as u64, + // ) + // }) + // .collect()) + // } +} - let feed_res = response.into_inner().feed; +#[derive(Serialize, Deserialize, Debug)] +pub struct CustomMlFeedCacheItem { + post_id: u64, + canister_id: String, + video_id: String, + creator_principal_id: String, +} - Ok(feed_res - .iter() - .map(|item| { - ( - Principal::from_text(&item.canister_id).unwrap(), - item.post_id as u64, - ) - }) - .collect()) +#[server] +pub async fn get_posts_ml_feed_cache_paginated( + canister_id: Principal, + start: u64, + limit: u64, +) -> Result, ServerFnError> { + leptos::logging::log!( + "get_posts_ml_feed_cache_paginated {} {} {}", + start, + limit, + canister_id.to_text() + ); + get_posts_ml_feed_cache_paginated_impl(canister_id.to_text(), start, limit).await +} + +pub async fn get_coldstart_feed_paginated( + start: u64, + limit: u64, +) -> Result, ServerFnError> { + leptos::logging::log!("get_coldstart_feed_paginated {} {}", start, limit); + get_posts_ml_feed_cache_paginated_impl("global-feed".to_string(), start, limit).await +} + +pub async fn get_posts_ml_feed_cache_paginated_impl( + canister_id_str: String, + start: u64, + limit: u64, +) -> Result, ServerFnError> { + let client = reqwest::Client::new(); + let account_id = consts::CLOUDFLARE_ACCOUNT_ID; + let namespace_id = consts::CF_KV_ML_CACHE_NAMESPACE_ID; + let api_token = env::var("CF_TOKEN").unwrap(); + + let url = format!( + "https://api.cloudflare.com/client/v4/accounts/{}/storage/kv/namespaces/{}/values/{}", + account_id, namespace_id, canister_id_str + ); + + let response = client + .get(&url) + .header("Authorization", format!("Bearer {}", api_token)) + .header("Content-Type", "application/json") + .send() + .await?; + + if !response.status().is_success() { + return Ok(vec![]); } + + let response = response.json::>().await.unwrap(); + + Ok(response + .into_iter() + .skip(start as usize) + .take(limit as usize) + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect::>()) } From 71d47422c9741bb1857f72e88c4e78db195f4bc8 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 14:43:08 +0530 Subject: [PATCH 2/7] clippy fix --- ssr/src/page/post_view/video_iter.rs | 6 +- ssr/src/utils/ml_feed/mod.rs | 104 +++++++++++++-------------- 2 files changed, 53 insertions(+), 57 deletions(-) diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index e335c872..814c050d 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -176,14 +176,10 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { &self, chunks: usize, _allow_nsfw: bool, - video_queue: Vec, + _video_queue: Vec, ) -> Result, ServerFnError> { #[cfg(feature = "hydrate")] { - use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; - - let ml_feed: MLFeed = expect_context(); - let top_posts = get_coldstart_feed_paginated(self.cursor.start, self.cursor.limit).await; diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index 25af8752..434bf519 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -193,57 +193,57 @@ pub mod ml_feed_grpcweb { } } -#[cfg(feature = "ssr")] -pub mod ml_feed_grpc { - use super::*; - - pub mod ml_feed_proto { - tonic::include_proto!("ml_feed"); - } - - // pub async fn get_coldstart_feed() -> Result, tonic::Status> { - // use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ - // ml_feed_client::MlFeedClient, FeedRequest, - // }; - // use tonic::transport::{Channel, ClientTlsConfig}; - - // let tls_config = ClientTlsConfig::new().with_webpki_roots(); - - // let channel = Channel::from_static(ML_FEED_GRPC_URL) - // .tls_config(tls_config) - // .expect("Couldn't update TLS config for nsfw agent") - // .connect() - // .await - // .expect("Couldn't connect to ML feed server"); - - // let mut client = MlFeedClient::new(channel); - - // let request = tonic::Request::new(FeedRequest { - // canister_id: "".to_string(), - // filter_posts: vec![], - // num_results: 1, - // }); - - // let response = client.get_feed_coldstart(request).await.map_err(|e| { - // tonic::Status::new( - // tonic::Code::Internal, - // format!("error fetching posts: {:?}", e), - // ) - // })?; - - // let feed_res = response.into_inner().feed; - - // Ok(feed_res - // .iter() - // .map(|item| { - // ( - // Principal::from_text(&item.canister_id).unwrap(), - // item.post_id as u64, - // ) - // }) - // .collect()) - // } -} +// #[cfg(feature = "ssr")] +// pub mod ml_feed_grpc { +// use super::*; + +// pub mod ml_feed_proto { +// tonic::include_proto!("ml_feed"); +// } + +// // pub async fn get_coldstart_feed() -> Result, tonic::Status> { +// // use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ +// // ml_feed_client::MlFeedClient, FeedRequest, +// // }; +// // use tonic::transport::{Channel, ClientTlsConfig}; + +// // let tls_config = ClientTlsConfig::new().with_webpki_roots(); + +// // let channel = Channel::from_static(ML_FEED_GRPC_URL) +// // .tls_config(tls_config) +// // .expect("Couldn't update TLS config for nsfw agent") +// // .connect() +// // .await +// // .expect("Couldn't connect to ML feed server"); + +// // let mut client = MlFeedClient::new(channel); + +// // let request = tonic::Request::new(FeedRequest { +// // canister_id: "".to_string(), +// // filter_posts: vec![], +// // num_results: 1, +// // }); + +// // let response = client.get_feed_coldstart(request).await.map_err(|e| { +// // tonic::Status::new( +// // tonic::Code::Internal, +// // format!("error fetching posts: {:?}", e), +// // ) +// // })?; + +// // let feed_res = response.into_inner().feed; + +// // Ok(feed_res +// // .iter() +// // .map(|item| { +// // ( +// // Principal::from_text(&item.canister_id).unwrap(), +// // item.post_id as u64, +// // ) +// // }) +// // .collect()) +// // } +// } #[derive(Serialize, Deserialize, Debug)] pub struct CustomMlFeedCacheItem { @@ -311,7 +311,7 @@ pub async fn get_posts_ml_feed_cache_paginated_impl( .map(|item| { ( Principal::from_text(&item.canister_id).unwrap(), - item.post_id as u64, + item.post_id, ) }) .collect::>()) From ee7c68d26abaf1b3e9a7f6d8674893ded31d2580 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 15:09:27 +0530 Subject: [PATCH 3/7] cfkv --- Cargo.lock | 1 - ssr/Cargo.toml | 4 +- ssr/src/page/post_view/video_iter.rs | 17 +++-- ssr/src/page/root.rs | 4 +- ssr/src/utils/ml_feed/mod.rs | 102 +++++++++++++-------------- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea7dc333..e2629e82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,7 +2850,6 @@ dependencies = [ "priority-queue", "prost", "pulldown-cmark", - "rand", "rand_chacha", "redb", "redis", diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index f8a69649..643eaf8c 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -55,7 +55,7 @@ icondata_core = "0.1.0" serde_json = "1.0" crc32fast = "1.4.0" uts2ts = "0.4.1" -rand_chacha = { version = "0.3.1", optional = true } +rand_chacha = { version = "0.3.1" } web-sys = { version = "0.3", features = [ "Clipboard", "Navigator", @@ -117,7 +117,6 @@ ic-certification = "2.6.0" ciborium = "0.2.2" yral-metadata-client = { git = "https://github.com/yral-dapp/yral-metadata", rev = "56e3f1f1f5f452673bee17739520c800c1264295", optional = true } yral-metadata-types = { git = "https://github.com/yral-dapp/yral-metadata", rev = "56e3f1f1f5f452673bee17739520c800c1264295", optional = true } -rand = "0.8.5" [build-dependencies] tonic-build = { version = "0.12.0", default-features = false, features = [ @@ -133,7 +132,6 @@ hydrate = [ "ic-agent/wasm-bindgen", "dep:web-sys", "reqwest/native-tls", - "dep:rand_chacha", "tonic/codegen", "speedate", ] diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 814c050d..845cae03 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -12,9 +12,7 @@ use crate::{ consts::USER_CANISTER_ID_STORE, state::canisters::auth_canisters_store, utils::{ - host::show_nsfw_content, - ml_feed::{get_coldstart_feed_paginated, get_posts_ml_feed_cache_paginated}, - posts::FetchCursor, + host::show_nsfw_content, ml_feed::get_posts_ml_feed_cache_paginated, posts::FetchCursor, }, }; use yral_canisters_common::{utils::posts::PostDetails, Canisters, Error as CanistersError}; @@ -176,12 +174,17 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { &self, chunks: usize, _allow_nsfw: bool, - _video_queue: Vec, + video_queue: Vec, ) -> Result, ServerFnError> { #[cfg(feature = "hydrate")] { - let top_posts = - get_coldstart_feed_paginated(self.cursor.start, self.cursor.limit).await; + use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; + + let ml_feed: MLFeed = expect_context(); + + let top_posts = ml_feed + .get_next_feed_coldstart(self.cursor.limit as u32, video_queue) + .await; let top_posts = match top_posts { Ok(top_posts) => top_posts, @@ -212,7 +215,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { return Ok(FetchVideosRes { posts_stream: Box::pin(futures::stream::empty()), end: true, - res_type: FeedResultType::MLFeed, + res_type: FeedResultType::MLFeedColdstart, }); } } diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index 1ec70a5d..d6cba03f 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -1,7 +1,6 @@ use candid::Principal; use leptos::*; use leptos_router::*; -use rand::Rng; use crate::{ component::spinner::FullScreenSpinner, @@ -87,7 +86,8 @@ async fn get_top_post_id_mlfeed() -> Result, ServerFnEr )); } }; - let rand_num = rand::thread_rng().gen_range(0..top_items.len()); + let mut rand_gen = ChaCha8Rng::seed_from_u64(current_epoch().as_nanos() as u64); + let rand_num = rand_gen.next_u32() as usize % top_items.len(); let top_item = top_items[rand_num]; Ok(Some((top_item.0, top_item.1))) diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index 434bf519..f32cfe75 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -193,57 +193,57 @@ pub mod ml_feed_grpcweb { } } -// #[cfg(feature = "ssr")] -// pub mod ml_feed_grpc { -// use super::*; - -// pub mod ml_feed_proto { -// tonic::include_proto!("ml_feed"); -// } - -// // pub async fn get_coldstart_feed() -> Result, tonic::Status> { -// // use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ -// // ml_feed_client::MlFeedClient, FeedRequest, -// // }; -// // use tonic::transport::{Channel, ClientTlsConfig}; - -// // let tls_config = ClientTlsConfig::new().with_webpki_roots(); - -// // let channel = Channel::from_static(ML_FEED_GRPC_URL) -// // .tls_config(tls_config) -// // .expect("Couldn't update TLS config for nsfw agent") -// // .connect() -// // .await -// // .expect("Couldn't connect to ML feed server"); - -// // let mut client = MlFeedClient::new(channel); - -// // let request = tonic::Request::new(FeedRequest { -// // canister_id: "".to_string(), -// // filter_posts: vec![], -// // num_results: 1, -// // }); - -// // let response = client.get_feed_coldstart(request).await.map_err(|e| { -// // tonic::Status::new( -// // tonic::Code::Internal, -// // format!("error fetching posts: {:?}", e), -// // ) -// // })?; - -// // let feed_res = response.into_inner().feed; - -// // Ok(feed_res -// // .iter() -// // .map(|item| { -// // ( -// // Principal::from_text(&item.canister_id).unwrap(), -// // item.post_id as u64, -// // ) -// // }) -// // .collect()) -// // } -// } +#[cfg(feature = "ssr")] +pub mod ml_feed_grpc { + use super::*; + + pub mod ml_feed_proto { + tonic::include_proto!("ml_feed"); + } + + pub async fn get_coldstart_feed() -> Result, tonic::Status> { + use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, + }; + use tonic::transport::{Channel, ClientTlsConfig}; + + let tls_config = ClientTlsConfig::new().with_webpki_roots(); + + let channel = Channel::from_static(ML_FEED_GRPC_URL) + .tls_config(tls_config) + .expect("Couldn't update TLS config for nsfw agent") + .connect() + .await + .expect("Couldn't connect to ML feed server"); + + let mut client = MlFeedClient::new(channel); + + let request = tonic::Request::new(FeedRequest { + canister_id: "".to_string(), + filter_posts: vec![], + num_results: 1, + }); + + let response = client.get_feed_coldstart(request).await.map_err(|e| { + tonic::Status::new( + tonic::Code::Internal, + format!("error fetching posts: {:?}", e), + ) + })?; + + let feed_res = response.into_inner().feed; + + Ok(feed_res + .iter() + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect()) + } +} #[derive(Serialize, Deserialize, Debug)] pub struct CustomMlFeedCacheItem { From 340f495ae47d0af931654cc37fe9c6b6a1ce5275 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 15:13:31 +0530 Subject: [PATCH 4/7] cfkv --- ssr/Cargo.toml | 3 ++- ssr/src/page/root.rs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 643eaf8c..9abbb6d4 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -55,7 +55,7 @@ icondata_core = "0.1.0" serde_json = "1.0" crc32fast = "1.4.0" uts2ts = "0.4.1" -rand_chacha = { version = "0.3.1" } +rand_chacha = { version = "0.3.1", optional = true } web-sys = { version = "0.3", features = [ "Clipboard", "Navigator", @@ -132,6 +132,7 @@ hydrate = [ "ic-agent/wasm-bindgen", "dep:web-sys", "reqwest/native-tls", + "dep:rand_chacha", "tonic/codegen", "speedate", ] diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index d6cba03f..f73af68f 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -1,6 +1,11 @@ use candid::Principal; use leptos::*; use leptos_router::*; +use rand_chacha::{ + rand_core::{RngCore, SeedableRng}, + ChaCha8Rng, +}; +use yral_canisters_common::utils::time::current_epoch; use crate::{ component::spinner::FullScreenSpinner, From b09c3c2ef7b0bdf5f00a057d694a7c3554253612 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 15:37:10 +0530 Subject: [PATCH 5/7] cfkv coldstart --- ssr/src/page/post_view/video_iter.rs | 69 +++++++++++----------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 845cae03..1d87ef50 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -12,7 +12,9 @@ use crate::{ consts::USER_CANISTER_ID_STORE, state::canisters::auth_canisters_store, utils::{ - host::show_nsfw_content, ml_feed::get_posts_ml_feed_cache_paginated, posts::FetchCursor, + host::show_nsfw_content, + ml_feed::{get_coldstart_feed_paginated, get_posts_ml_feed_cache_paginated}, + posts::FetchCursor, }, }; use yral_canisters_common::{utils::posts::PostDetails, Canisters, Error as CanistersError}; @@ -174,50 +176,32 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { &self, chunks: usize, _allow_nsfw: bool, - video_queue: Vec, + _video_queue: Vec, ) -> Result, ServerFnError> { - #[cfg(feature = "hydrate")] - { - use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; - - let ml_feed: MLFeed = expect_context(); - - let top_posts = ml_feed - .get_next_feed_coldstart(self.cursor.limit as u32, video_queue) - .await; - - let top_posts = match top_posts { - Ok(top_posts) => top_posts, - Err(e) => { - return Err(ServerFnError::new( - format!("Error fetching ml feed: {e:?}",), - )); - } - }; - - let end = false; - let chunk_stream = top_posts - .into_iter() - .map(move |item| self.canisters.get_post_details(item.0, item.1)) - .collect::>() - .filter_map(|res| async { res.transpose() }) - .chunks(chunks); + let top_posts = get_coldstart_feed_paginated(self.cursor.start, self.cursor.limit).await; + + let top_posts = match top_posts { + Ok(top_posts) => top_posts, + Err(e) => { + return Err(ServerFnError::new( + format!("Error fetching ml feed: {e:?}",), + )); + } + }; - Ok(FetchVideosRes { - posts_stream: Box::pin(chunk_stream), - end, - res_type: FeedResultType::MLFeedColdstart, - }) - } + let end = false; + let chunk_stream = top_posts + .into_iter() + .map(move |item| self.canisters.get_post_details(item.0, item.1)) + .collect::>() + .filter_map(|res| async { res.transpose() }) + .chunks(chunks); - #[cfg(not(feature = "hydrate"))] - { - return Ok(FetchVideosRes { - posts_stream: Box::pin(futures::stream::empty()), - end: true, - res_type: FeedResultType::MLFeedColdstart, - }); - } + Ok(FetchVideosRes { + posts_stream: Box::pin(chunk_stream), + end, + res_type: FeedResultType::MLFeedColdstart, + }) } } @@ -231,6 +215,7 @@ impl<'a> VideoFetchStream<'a, true> { let cans_true = self.canisters; let user_canister_id = cans_true.user_canister(); + let top_posts = match get_posts_ml_feed_cache_paginated( user_canister_id, self.cursor.start, From b4a42c56f6e269e3dbfb030795225e6f956d8839 Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 15:51:36 +0530 Subject: [PATCH 6/7] cfkv fis --- ssr/src/utils/ml_feed/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index f32cfe75..26c1af57 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -268,6 +268,7 @@ pub async fn get_posts_ml_feed_cache_paginated( get_posts_ml_feed_cache_paginated_impl(canister_id.to_text(), start, limit).await } +#[server] pub async fn get_coldstart_feed_paginated( start: u64, limit: u64, From 7326a1b2cd2e73dc62541736e099ed83eed9c2ef Mon Sep 17 00:00:00 2001 From: komal-rs Date: Wed, 11 Dec 2024 16:06:46 +0530 Subject: [PATCH 7/7] cfkv --- ssr/src/utils/ml_feed/mod.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index 26c1af57..f4ccc194 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -259,12 +259,6 @@ pub async fn get_posts_ml_feed_cache_paginated( start: u64, limit: u64, ) -> Result, ServerFnError> { - leptos::logging::log!( - "get_posts_ml_feed_cache_paginated {} {} {}", - start, - limit, - canister_id.to_text() - ); get_posts_ml_feed_cache_paginated_impl(canister_id.to_text(), start, limit).await } @@ -273,7 +267,6 @@ pub async fn get_coldstart_feed_paginated( start: u64, limit: u64, ) -> Result, ServerFnError> { - leptos::logging::log!("get_coldstart_feed_paginated {} {}", start, limit); get_posts_ml_feed_cache_paginated_impl("global-feed".to_string(), start, limit).await }