From 3ebee17e7f4c2ee6259e5b6a6b644e46e7ec5721 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 18 Dec 2024 14:00:33 +0100 Subject: [PATCH 1/5] Work on dynamic linking for stubless RPC, WIP --- golem-rib/src/function_name.rs | 22 + .../src/durable_host/dynamic_linking.rs | 460 ++++++++++++++++++ .../src/durable_host/mod.rs | 34 +- .../src/durable_host/wasm_rpc/mod.rs | 126 +++-- golem-worker-executor-base/src/worker.rs | 50 +- golem-worker-executor-base/src/workerctx.rs | 52 +- .../tests/common/mod.rs | 121 ++++- wasm-ast/src/analysis/mod.rs | 4 +- wasm-ast/tests/exports.rs | 74 +++ wasm-rpc/src/wasmtime.rs | 3 +- 10 files changed, 873 insertions(+), 73 deletions(-) create mode 100644 golem-worker-executor-base/src/durable_host/dynamic_linking.rs diff --git a/golem-rib/src/function_name.rs b/golem-rib/src/function_name.rs index 41f1b09c4..3cf8efa0c 100644 --- a/golem-rib/src/function_name.rs +++ b/golem-rib/src/function_name.rs @@ -652,6 +652,28 @@ impl ParsedFunctionName { function, }) } + + pub fn is_constructor(&self) -> Option<&str> { + match &self.function { + ParsedFunctionReference::RawResourceConstructor { resource, .. } + | ParsedFunctionReference::IndexedResourceConstructor { resource, .. } => { + Some(resource) + } + _ => None, + } + } + + pub fn is_method(&self) -> Option<&str> { + match &self.function { + ParsedFunctionReference::RawResourceMethod { resource, .. } + | ParsedFunctionReference::IndexedResourceMethod { resource, .. } + | ParsedFunctionReference::RawResourceStaticMethod { resource, .. } + | ParsedFunctionReference::IndexedResourceStaticMethod { resource, .. } => { + Some(resource) + } + _ => None, + } + } } #[cfg(feature = "protobuf")] diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs new file mode 100644 index 000000000..d47ce3958 --- /dev/null +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs @@ -0,0 +1,460 @@ +use crate::durable_host::wasm_rpc::{UrnExtensions, WasmRpcEntryPayload}; +use crate::durable_host::DurableWorkerCtx; +use crate::services::rpc::RpcDemand; +use crate::workerctx::{DynamicLinking, WorkerCtx}; +use anyhow::anyhow; +use async_trait::async_trait; +use golem_common::model::OwnedWorkerId; +use golem_wasm_rpc::wasmtime::{decode_param, encode_output}; +use golem_wasm_rpc::{HostWasmRpc, Uri, Value, WasmRpcEntry, WitValue}; +use rib::{ParsedFunctionName, ParsedFunctionReference, ParsedFunctionSite}; +use tracing::debug; +use wasmtime::component::types::ComponentItem; +use wasmtime::component::{Component, Linker, Resource, ResourceType, Type, Val}; +use wasmtime::{AsContextMut, Engine, StoreContextMut}; +use wasmtime_wasi::WasiView; + +// TODO: support multiple different dynamic linkers + +#[async_trait] +impl DynamicLinking for DurableWorkerCtx { + fn link( + &mut self, + engine: &Engine, + linker: &mut Linker, + component: &Component, + ) -> anyhow::Result<()> { + let mut root = linker.root(); + + for (name, item) in component.component_type().imports(&engine) { + debug!("Import {name}: {item:?}"); + match item { + ComponentItem::ComponentFunc(_) => { + debug!("MUST LINK COMPONENT FUNC {name}"); + } + ComponentItem::CoreFunc(_) => { + debug!("MUST LINK CORE FUNC {name}"); + } + ComponentItem::Module(_) => { + debug!("MUST LINK MODULE {name}"); + } + ComponentItem::Component(_) => { + debug!("MUST LINK COMPONENT {name}"); + } + ComponentItem::ComponentInstance(ref inst) => { + if name == "auction:auction-stub/stub-auction" + || name == "auction:auction/api" + || name == "rpc:counters-stub/stub-counters" + || name == "rpc:counters/api" + || name == "rpc:ephemeral-stub/stub-ephemeral" + { + debug!("NAME == {name}"); + let mut instance = root.instance(name)?; + + for (ename, eitem) in inst.exports(&engine) { + let name = name.to_owned(); + let ename = ename.to_owned(); + debug!("Instance {name} export {ename}: {eitem:?}"); + + match eitem { + ComponentItem::ComponentFunc(fun) => { + let name2 = name.clone(); + let ename2 = ename.clone(); + instance.func_new_async( + // TODO: instrument async closure + &ename.clone(), + move |store, params, results| { + let name = name2.clone(); + let ename = ename2.clone(); + let param_types: Vec = fun.params().collect(); + let result_types: Vec = fun.results().collect(); + Box::new(async move { + Ctx::dynamic_function_call( + store, + &name, + &ename, + params, + ¶m_types, + results, + &result_types, + ) + .await?; + // TODO: failures here must be somehow handled + Ok(()) + }) + }, + )?; + debug!("LINKED {name} export {ename}"); + } + ComponentItem::CoreFunc(_) => {} + ComponentItem::Module(_) => {} + ComponentItem::Component(component) => { + debug!("MUST LINK COMPONENT {ename} {component:?}"); + } + ComponentItem::ComponentInstance(instance) => { + debug!("MUST LINK COMPONENT INSTANCE {ename} {instance:?}"); + } + ComponentItem::Type(_) => {} + ComponentItem::Resource(resource) => { + if ename != "pollable" { + // TODO: ?? this should be 'if it is not already linked' but not way to check that + debug!("LINKING RESOURCE {ename} {resource:?}"); + instance.resource( + &ename, + ResourceType::host::(), + Ctx::drop_linked_resource, + )?; + } + } + } + } + } else { + debug!("NAME NOT MATCHING: {name}"); + } + } + ComponentItem::Type(_) => {} + ComponentItem::Resource(_) => {} + } + } + + Ok(()) + } + + async fn dynamic_function_call( + mut store: impl AsContextMut + Send, + interface_name: &str, + function_name: &str, + params: &[Val], + param_types: &[Type], + results: &mut [Val], + result_types: &[Type], + ) -> anyhow::Result<()> { + let mut store = store.as_context_mut(); + debug!( + "Instance {interface_name} export {function_name} called XXX {} params {} results", + params.len(), + results.len() + ); + + // TODO: add an enum with the call types (interface stub constructor, resource stub constructor, etc) + // TODO: and detect which one it is based on metadata + type info + + if (interface_name == "auction:auction-stub/stub-auction" + && function_name == "[constructor]api") + || (interface_name == "rpc:counters-stub/stub-counters" + && function_name == "[constructor]api") + { + // Simple stub interface constructor + + let target_worker_urn = params[0].clone(); + debug!("CREATING AUCTION STUB TARGETING WORKER {target_worker_urn:?}"); + // Record([("value", String("urn:worker:2a174805-bdd5-49e1-b1e8-124208123b4a/auction-5f0a94f1-1d14-4b65-8e6c-3a8fa3c24ea9"))]) + + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn).await?; + + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Interface { + demand, + remote_worker_id, + }), + })? + }; + results[0] = Val::Resource(handle.try_into_resource_any(store)?); + } else if (interface_name == "auction:auction-stub/stub-auction" + && function_name == "[constructor]running-auction") + || (interface_name == "rpc:counters-stub/stub-counters" + && function_name == "[constructor]counter") + { + // Resource stub constructor + + // First parameter is the target uri + // Rest of the parameters must be sent to the remote constructor + + let target_worker_urn = params[0].clone(); + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn.clone()).await?; + + // First creating a resource for invoking the constructor (to avoid having to make a special case) + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Interface { + demand, + remote_worker_id, + }), + })? + }; + let temp_handle = handle.rep(); + + let constructor_result = Self::remote_invoke( + &interface_name, + &function_name, + params, + param_types, + &mut store, + handle, + ) + .await?; + + // TODO: extract and clean up + let (resource_uri, resource_id) = if let Value::Tuple(values) = constructor_result { + if values.len() == 1 { + if let Value::Handle { uri, resource_id } = values.into_iter().next().unwrap() { + (Uri { value: uri }, resource_id) + } else { + return Err(anyhow!( + "Invalid constructor result: single handle expected" + )); + } + } else { + return Err(anyhow!( + "Invalid constructor result: single handle expected" + )); + } + } else { + return Err(anyhow!( + "Invalid constructor result: single handle expected" + )); + }; + + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn).await?; + + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + + let temp_handle: Resource = Resource::new_own(temp_handle); + table.delete(temp_handle)?; // Removing the temporary handle + + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Resource { + demand, + remote_worker_id, + resource_uri, + resource_id, + }), + })? + }; + results[0] = Val::Resource(handle.try_into_resource_any(store)?); + } else if function_name.starts_with("[method]") { + // Simple stub interface method + debug!( + "{function_name} handle={:?}, rest={:?}", + params[0], + params.iter().skip(1).collect::>() + ); + + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + { + let mut wasi = store.data_mut().as_wasi_view(); + let entry = wasi.table().get(&handle)?; + let payload = entry.payload.downcast_ref::().unwrap(); + debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); + } + + let result = Self::remote_invoke( + &interface_name, + &function_name, + params, + param_types, + &mut store, + handle, + ) + .await?; + Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store).await?; + } + + Ok(()) + } + + fn drop_linked_resource(mut store: StoreContextMut<'_, Ctx>, rep: u32) -> anyhow::Result<()> { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + let entry: &WasmRpcEntry = table.get_any_mut(rep).unwrap().downcast_ref().unwrap(); // TODO: error handling + let payload = entry.payload.downcast_ref::().unwrap(); + debug!("DROPPING RESOURCE {payload:?}"); + if let WasmRpcEntryPayload::Resource { .. } = payload { + // TODO: remote drop + } + Ok(()) + } +} + +// TODO: these helpers probably should not be directly living in DurableWorkerCtx +impl DurableWorkerCtx { + async fn remote_invoke( + interface_name: &&str, + function_name: &&str, + params: &[Val], + param_types: &[Type], + store: &mut StoreContextMut<'_, Ctx>, + handle: Resource, + ) -> anyhow::Result { + let stub_function_name = + ParsedFunctionName::parse(&format!("{interface_name}.{{{function_name}}}")) + .map_err(|err| anyhow!(err))?; // TODO: proper error + debug!("STUB FUNCTION NAME: {stub_function_name:?}"); + let target_function_name = ParsedFunctionName { + site: if interface_name.starts_with("auction") { + ParsedFunctionSite::PackagedInterface { + // TODO: this must come from component metadata linking information + namespace: "auction".to_string(), + package: "auction".to_string(), + interface: "api".to_string(), + version: None, + } + } else { + ParsedFunctionSite::PackagedInterface { + namespace: "rpc".to_string(), + package: "counters".to_string(), + interface: "api".to_string(), + version: None, + } + }, + function: if let Some(resource) = stub_function_name.is_constructor() { + ParsedFunctionReference::RawResourceConstructor { + resource: resource.to_string(), + } + } else { + match &stub_function_name.function { + ParsedFunctionReference::RawResourceMethod { resource, method } + if resource == "counter" => + { + ParsedFunctionReference::RawResourceMethod { + resource: resource.to_string(), + method: method + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + } + } + _ => ParsedFunctionReference::Function { + function: stub_function_name + .function + .resource_method_name() + .unwrap() // TODO: proper error + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + }, + } + }, + }; + + let mut wit_value_params = Vec::new(); + for (param, typ) in params.iter().zip(param_types).skip(1) { + let value: Value = encode_output(param, typ, store.data_mut()) + .await + .map_err(|err| anyhow!(format!("{err:?}")))?; // TODO: proper error + let wit_value: WitValue = value.into(); + wit_value_params.push(wit_value); + } + + debug!( + "CALLING {function_name} as {target_function_name} with parameters {wit_value_params:?}", + ); + + // "auction:auction/api.{initialize}", + let wit_value_result = store + .data_mut() + .invoke_and_await(handle, target_function_name.to_string(), wit_value_params) + .await??; + + debug!("CALLING {function_name} RESULTED IN {:?}", wit_value_result); + + let value_result: Value = wit_value_result.into(); + Ok(value_result) + } + + async fn value_result_to_wasmtime_vals( + value_result: Value, + results: &mut [Val], + result_types: &[Type], + store: &mut StoreContextMut<'_, Ctx>, + ) -> anyhow::Result<()> { + match value_result { + Value::Tuple(values) | Value::Record(values) => { + for (idx, (value, typ)) in values.iter().zip(result_types).enumerate() { + let result = decode_param(&value, &typ, store.data_mut()) + .await + .map_err(|err| anyhow!(format!("{err:?}")))?; // TODO: proper error + results[idx] = result.val; + // TODO: do we have to do something with result.resources_to_drop here? + } + } + _ => { + return Err(anyhow!( + "Unexpected result value {value_result:?}, expected tuple or record" + )); + } + } + + Ok(()) + } +} + +// TODO: these helpers probably should not be directly living in DurableWorkerCtx +impl DurableWorkerCtx { + async fn create_rpc_target( + store: &mut StoreContextMut<'_, Ctx>, + target_worker_urn: Val, + ) -> anyhow::Result<(OwnedWorkerId, Box)> { + let worker_urn = match target_worker_urn { + Val::Record(ref record) => { + let mut target = None; + for (key, val) in record.iter() { + if key == "value" { + match val { + Val::String(s) => { + target = Some(s.clone()); + } + _ => {} + } + } + } + target + } + _ => None, + }; + + let (remote_worker_id, demand) = if let Some(location) = worker_urn { + let uri = Uri { + value: location.clone(), + }; + match uri.parse_as_golem_urn() { + Some((remote_worker_id, None)) => { + let remote_worker_id = store + .data_mut() + .generate_unique_local_worker_id(remote_worker_id) + .await?; + + let remote_worker_id = OwnedWorkerId::new( + &store.data().owned_worker_id().account_id, + &remote_worker_id, + ); + let demand = store.data().rpc().create_demand(&remote_worker_id).await; + (remote_worker_id, demand) + } + _ => { + return Err(anyhow!( + "Invalid URI: {}. Must be urn:worker:component-id/worker-name", + location + )) + } + } + } else { + return Err(anyhow!("Missing or invalid worker URN parameter")); // TODO: more details; + }; + Ok((remote_worker_id, demand)) + } +} diff --git a/golem-worker-executor-base/src/durable_host/mod.rs b/golem-worker-executor-base/src/durable_host/mod.rs index b42fdd74b..dee490a98 100644 --- a/golem-worker-executor-base/src/durable_host/mod.rs +++ b/golem-worker-executor-base/src/durable_host/mod.rs @@ -18,6 +18,7 @@ use crate::durable_host::http::serialized::SerializableHttpRequest; use crate::durable_host::io::{ManagedStdErr, ManagedStdIn, ManagedStdOut}; use crate::durable_host::replay_state::ReplayState; +use crate::durable_host::serialized::SerializableError; use crate::durable_host::sync_helper::{SyncHelper, SyncHelperPermit}; use crate::durable_host::wasm_rpc::UrnExtensions; use crate::error::GolemError; @@ -65,7 +66,6 @@ use golem_common::model::oplog::{ }; use golem_common::model::plugin::{PluginOwner, PluginScope}; use golem_common::model::regions::{DeletedRegions, OplogRegion}; -use golem_common::model::RetryConfig; use golem_common::model::{exports, PluginInstallationId}; use golem_common::model::{ AccountId, ComponentFilePath, ComponentFilePermissions, ComponentFileSystemNode, @@ -74,6 +74,7 @@ use golem_common::model::{ ScheduledAction, SuccessfulUpdateRecord, Timestamp, WorkerEvent, WorkerFilter, WorkerId, WorkerMetadata, WorkerResourceDescription, WorkerStatus, WorkerStatusRecord, }; +use golem_common::model::{RetryConfig, TargetWorkerId}; use golem_common::retries::get_delay; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; use golem_wasm_rpc::wasmtime::ResourceStore; @@ -116,6 +117,7 @@ mod sockets; pub mod wasm_rpc; mod durability; +mod dynamic_linking; mod replay_state; mod sync_helper; @@ -288,6 +290,10 @@ impl DurableWorkerCtx { &self.owned_worker_id.worker_id } + pub fn owned_worker_id(&self) -> &OwnedWorkerId { + &self.owned_worker_id + } + pub fn component_metadata(&self) -> &ComponentMetadata { &self.state.component_metadata } @@ -476,6 +482,32 @@ impl DurableWorkerCtx { } } } + + pub async fn generate_unique_local_worker_id( + &mut self, + remote_worker_id: TargetWorkerId, + ) -> Result { + match remote_worker_id.clone().try_into_worker_id() { + Some(worker_id) => Ok(worker_id), + None => { + let worker_id = Durability::::wrap( + self, + WrappedFunctionType::ReadLocal, + "golem::rpc::wasm-rpc::generate_unique_local_worker_id", + (), + |ctx| { + Box::pin(async move { + ctx.rpc() + .generate_unique_local_worker_id(remote_worker_id) + .await + }) + }, + ) + .await?; + Ok(worker_id) + } + } + } } impl> DurableWorkerCtx { diff --git a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs index 6853c11c7..748103e55 100644 --- a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs @@ -36,13 +36,14 @@ use golem_common::model::{ }; use golem_common::uri::oss::urn::{WorkerFunctionUrn, WorkerOrFunctionUrn}; use golem_wasm_rpc::golem::rpc::types::{ - FutureInvokeResult, HostFutureInvokeResult, Pollable, Uri, + FutureInvokeResult, HostFutureInvokeResult, Pollable, Uri, WasmRpc, }; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; use golem_wasm_rpc::{ - FutureInvokeResultEntry, HostWasmRpc, SubscribeAny, ValueAndType, WasmRpcEntry, WitValue, + FutureInvokeResultEntry, HostWasmRpc, SubscribeAny, Value, ValueAndType, WasmRpcEntry, WitValue, }; use std::any::Any; +use std::fmt::{Debug, Formatter}; use std::str::FromStr; use std::sync::Arc; use tracing::{error, warn}; @@ -60,14 +61,15 @@ impl HostWasmRpc for DurableWorkerCtx { match location.parse_as_golem_urn() { Some((remote_worker_id, None)) => { - let remote_worker_id = - generate_unique_local_worker_id(self, remote_worker_id).await?; + let remote_worker_id = self + .generate_unique_local_worker_id(remote_worker_id) + .await?; let remote_worker_id = OwnedWorkerId::new(&self.owned_worker_id.account_id, &remote_worker_id); let demand = self.rpc().create_demand(&remote_worker_id).await; let entry = self.table().push(WasmRpcEntry { - payload: Box::new(WasmRpcEntryPayload { + payload: Box::new(WasmRpcEntryPayload::Interface { demand, remote_worker_id, }), @@ -85,7 +87,7 @@ impl HostWasmRpc for DurableWorkerCtx { &mut self, self_: Resource, function_name: String, - function_params: Vec, + mut function_params: Vec, ) -> anyhow::Result> { record_host_function_call("golem::rpc::wasm-rpc", "invoke-and-await"); let args = self.get_arguments().await?; @@ -95,7 +97,26 @@ impl HostWasmRpc for DurableWorkerCtx { let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); - let remote_worker_id = payload.remote_worker_id.clone(); + let remote_worker_id = payload.remote_worker_id().clone(); + + // TODO: do this in other variants too + match payload { + WasmRpcEntryPayload::Resource { + resource_uri, + resource_id, + .. + } => { + function_params.insert( + 0, + Value::Handle { + uri: resource_uri.value.to_string(), + resource_id: *resource_id, + } + .into(), + ); + } + _ => {} + } let current_idempotency_key = self .get_current_idempotency_key() @@ -226,7 +247,7 @@ impl HostWasmRpc for DurableWorkerCtx { let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); - let remote_worker_id = payload.remote_worker_id.clone(); + let remote_worker_id = payload.remote_worker_id().clone(); let current_idempotency_key = self .get_current_idempotency_key() @@ -317,7 +338,7 @@ impl HostWasmRpc for DurableWorkerCtx { let entry = self.table().get(&this)?; let payload = entry.payload.downcast_ref::().unwrap(); - let remote_worker_id = payload.remote_worker_id.clone(); + let remote_worker_id = payload.remote_worker_id().clone(); let current_idempotency_key = self .get_current_idempotency_key() @@ -748,32 +769,6 @@ impl HostFutureInvokeResult for DurableWorkerCtx { #[async_trait] impl golem_wasm_rpc::Host for DurableWorkerCtx {} -async fn generate_unique_local_worker_id( - ctx: &mut DurableWorkerCtx, - remote_worker_id: TargetWorkerId, -) -> Result { - match remote_worker_id.clone().try_into_worker_id() { - Some(worker_id) => Ok(worker_id), - None => { - let worker_id = Durability::::wrap( - ctx, - WrappedFunctionType::ReadLocal, - "golem::rpc::wasm-rpc::generate_unique_local_worker_id", - (), - |ctx| { - Box::pin(async move { - ctx.rpc() - .generate_unique_local_worker_id(remote_worker_id) - .await - }) - }, - ) - .await?; - Ok(worker_id) - } - } -} - /// Tries to get a `ValueAndType` representation for the given `WitValue` parameters by querying the latest component metadata for the /// target component. /// If the query fails, or the expected function name is not in its metadata or the number of parameters does not match, then it returns an @@ -805,10 +800,63 @@ async fn try_get_typed_parameters( Vec::new() } -pub struct WasmRpcEntryPayload { - #[allow(dead_code)] - demand: Box, - remote_worker_id: OwnedWorkerId, +pub enum WasmRpcEntryPayload { + Interface { + #[allow(dead_code)] + demand: Box, + remote_worker_id: OwnedWorkerId, + }, + Resource { + #[allow(dead_code)] + demand: Box, + remote_worker_id: OwnedWorkerId, + resource_uri: Uri, + resource_id: u64, + }, +} + +impl Debug for WasmRpcEntryPayload { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Interface { + remote_worker_id, .. + } => f + .debug_struct("Interface") + .field("remote_worker_id", remote_worker_id) + .finish(), + Self::Resource { + remote_worker_id, + resource_uri, + resource_id, + .. + } => f + .debug_struct("Resource") + .field("remote_worker_id", remote_worker_id) + .field("resource_uri", resource_uri) + .field("resource_id", resource_id) + .finish(), + } + } +} + +impl WasmRpcEntryPayload { + pub fn remote_worker_id(&self) -> &OwnedWorkerId { + match self { + Self::Interface { + remote_worker_id, .. + } => remote_worker_id, + Self::Resource { + remote_worker_id, .. + } => remote_worker_id, + } + } + + pub fn demand(&self) -> &Box { + match self { + Self::Interface { demand, .. } => demand, + Self::Resource { demand, .. } => demand, + } + } } pub trait UrnExtensions { diff --git a/golem-worker-executor-base/src/worker.rs b/golem-worker-executor-base/src/worker.rs index a7de8b183..a8480dc97 100644 --- a/golem-worker-executor-base/src/worker.rs +++ b/golem-worker-executor-base/src/worker.rs @@ -20,6 +20,7 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use crate::durable_host::recover_stderr_logs; +use crate::durable_host::wasm_rpc::{UrnExtensions, WasmRpcEntryPayload}; use crate::error::{GolemError, WorkerOutOfMemory}; use crate::function_result_interpreter::interpret_function_results; use crate::invocation::{find_first_available_function, invoke_worker, InvokeResult}; @@ -30,6 +31,7 @@ use crate::model::{ use crate::services::component::ComponentMetadata; use crate::services::events::Event; use crate::services::oplog::{CommitLevel, Oplog, OplogOps}; +use crate::services::rpc::RpcDemand; use crate::services::worker_event::{WorkerEventService, WorkerEventServiceDefault}; use crate::services::{ All, HasActiveWorkers, HasAll, HasBlobStoreService, HasComponentService, HasConfig, HasEvents, @@ -57,15 +59,17 @@ use golem_common::model::{ }; use golem_common::retries::get_delay; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; -use golem_wasm_rpc::Value; +use golem_wasm_rpc::{Uri, Value, WasmRpcEntry}; use tokio::sync::broadcast::error::RecvError; use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::{Mutex, MutexGuard, OwnedSemaphorePermit}; use tokio::task::JoinHandle; use tracing::{debug, error, info, span, warn, Instrument, Level}; -use wasmtime::component::Instance; +use wasmtime::component::types::ComponentItem; +use wasmtime::component::{Instance, Resource, ResourceType, Val}; use wasmtime::{AsContext, Store, UpdateDeadline}; +use wasmtime_wasi::WasiView; /// Represents worker that may be running or suspended. /// @@ -473,6 +477,9 @@ impl Worker { } } + /// Invokes the worker and awaits for a result. + /// + /// Successful result is a `TypeAnnotatedValue` encoding either a tuple or a record. pub async fn invoke_and_await( &self, idempotency_key: IdempotencyKey, @@ -1350,7 +1357,8 @@ impl RunningWorker { ) .await?; - let mut store = Store::new(&parent.engine(), context); + let engine = parent.engine(); + let mut store = Store::new(&engine, context); store.set_epoch_deadline(parent.config().limits.epoch_ticks); let worker_id_clone = worker_metadata.worker_id.clone(); store.epoch_deadline_callback(move |mut store| { @@ -1371,7 +1379,15 @@ impl RunningWorker { store.limiter_async(|ctx| ctx.resource_limiter()); - let instance_pre = parent.linker().instantiate_pre(&component).map_err(|e| { + // TODO MOVE SOMEWHERE ELSE + let mut linker = (*parent.linker()).clone(); // fresh linker + store.data_mut().link(&engine, &mut linker, &component)?; + + // TODO: check parent.linker() is not affected + + // TODO ^^^ + + let instance_pre = linker.instantiate_pre(&component).map_err(|e| { GolemError::worker_creation_failed( parent.owned_worker_id.worker_id(), format!( @@ -1551,13 +1567,13 @@ impl RunningWorker { store, &instance, ) - .await; + .await; match result { Ok(InvokeResult::Succeeded { - output, - consumed_fuel, - }) => { + output, + consumed_fuel, + }) => { let component_metadata = store.as_context().data().component_metadata(); @@ -1577,9 +1593,9 @@ impl RunningWorker { output, function_results, ) - .map_err(|e| GolemError::ValueMismatch { - details: e.join(", "), - }); + .map_err(|e| GolemError::ValueMismatch { + details: e.join(", "), + }); match result { Ok(result) => { @@ -1685,8 +1701,8 @@ impl RunningWorker { } } } - .instrument(span) - .await; + .instrument(span) + .await; if do_break { break; } @@ -1713,7 +1729,7 @@ impl RunningWorker { vec![ "golem:api/save-snapshot@1.1.0.{save}".to_string(), "golem:api/save-snapshot@0.2.0.{save}".to_string(), - ] + ], ) { store.data_mut().begin_call_snapshotting_function(); @@ -1950,9 +1966,9 @@ impl InvocationResult { OplogEntry::Error { error, .. } => { let stderr = recover_stderr_logs(services, owned_worker_id, oplog_idx).await; Err(FailedInvocationResult { trap_type: TrapType::Error(error), stderr }) - }, - OplogEntry::Interrupted { .. } => Err(FailedInvocationResult { trap_type: TrapType::Interrupt(InterruptKind::Interrupt), stderr: "".to_string()}), - OplogEntry::Exited { .. } => Err(FailedInvocationResult { trap_type: TrapType::Exit, stderr: "".to_string()}), + } + OplogEntry::Interrupted { .. } => Err(FailedInvocationResult { trap_type: TrapType::Interrupt(InterruptKind::Interrupt), stderr: "".to_string() }), + OplogEntry::Exited { .. } => Err(FailedInvocationResult { trap_type: TrapType::Exit, stderr: "".to_string() }), _ => panic!("Unexpected oplog entry pointed by invocation result at index {oplog_idx} for {owned_worker_id:?}") }; diff --git a/golem-worker-executor-base/src/workerctx.rs b/golem-worker-executor-base/src/workerctx.rs index a46ac4cf1..c666e2ddd 100644 --- a/golem-worker-executor-base/src/workerctx.rs +++ b/golem-worker-executor-base/src/workerctx.rs @@ -15,12 +15,6 @@ use std::collections::HashSet; use std::sync::{Arc, RwLock, Weak}; -use async_trait::async_trait; -use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; -use golem_wasm_rpc::wasmtime::ResourceStore; -use golem_wasm_rpc::Value; -use wasmtime::{AsContextMut, ResourceLimiterAsync}; - use crate::error::GolemError; use crate::model::{ CurrentResourceLimits, ExecutionStatus, InterruptKind, LastError, ListDirectoryResult, @@ -44,13 +38,22 @@ use crate::services::{ worker_enumeration, HasAll, HasConfig, HasOplog, HasOplogService, HasWorker, }; use crate::worker::{RetryDecision, Worker}; +use async_trait::async_trait; use golem_common::model::component::ComponentOwner; use golem_common::model::oplog::WorkerResourceId; use golem_common::model::plugin::PluginScope; use golem_common::model::{ AccountId, ComponentFilePath, ComponentVersion, IdempotencyKey, OwnedWorkerId, - PluginInstallationId, WorkerId, WorkerMetadata, WorkerStatus, WorkerStatusRecord, + PluginInstallationId, TargetWorkerId, WorkerId, WorkerMetadata, WorkerStatus, + WorkerStatusRecord, }; +use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; +use golem_wasm_rpc::wasmtime::ResourceStore; +use golem_wasm_rpc::Value; +use wasmtime::component::{Component, Linker, Type, Val}; +use wasmtime::{AsContextMut, Engine, ResourceLimiterAsync, StoreContextMut}; +use wasmtime_wasi::WasiView; +use wasmtime_wasi_http::WasiHttpView; /// WorkerCtx is the primary customization and extension point of worker executor. It is the context /// associated with each running worker, and it is responsible for initializing the WASM linker as @@ -66,6 +69,7 @@ pub trait WorkerCtx: + IndexedResourceStore + UpdateManagement + FileSystemReading + + DynamicLinking + Send + Sync + Sized @@ -132,6 +136,9 @@ pub trait WorkerCtx: >, ) -> Result; + fn as_wasi_view(&mut self) -> impl WasiView; + fn as_wasi_http_view(&mut self) -> impl WasiHttpView; + /// Get the public part of the worker context fn get_public_state(&self) -> &Self::PublicState; @@ -144,6 +151,9 @@ pub trait WorkerCtx: /// Get the worker ID associated with this worker context fn worker_id(&self) -> &WorkerId; + /// Get the owned worker ID associated with this worker context + fn owned_worker_id(&self) -> &OwnedWorkerId; + fn component_metadata(&self) -> &ComponentMetadata; /// The WASI exit API can use a special error to exit from the WASM execution. As this depends @@ -157,6 +167,12 @@ pub trait WorkerCtx: /// Gets an interface to the worker-proxy which can direct calls to other worker executors /// in the cluster fn worker_proxy(&self) -> Arc; + + // TODO: where do this belong + async fn generate_unique_local_worker_id( + &mut self, + remote_worker_id: TargetWorkerId, + ) -> Result; } /// The fuel management interface of a worker context is responsible for borrowing and returning @@ -400,3 +416,25 @@ pub trait FileSystemReading { ) -> Result; async fn read_file(&self, path: &ComponentFilePath) -> Result; } + +#[async_trait] +pub trait DynamicLinking { + fn link( + &mut self, + engine: &Engine, + linker: &mut Linker, + component: &Component, + ) -> anyhow::Result<()>; + + async fn dynamic_function_call( + store: impl AsContextMut + Send, + interface_name: &str, + function_name: &str, + params: &[Val], + param_types: &[Type], + results: &mut [Val], + result_types: &[Type] + ) -> anyhow::Result<()>; + + fn drop_linked_resource(store: StoreContextMut<'_, Ctx>, rep: u32) -> anyhow::Result<()>; +} diff --git a/golem-worker-executor-base/tests/common/mod.rs b/golem-worker-executor-base/tests/common/mod.rs index 0d5d83f1d..f15965677 100644 --- a/golem-worker-executor-base/tests/common/mod.rs +++ b/golem-worker-executor-base/tests/common/mod.rs @@ -5,7 +5,9 @@ use std::collections::HashSet; use golem_service_base::service::initial_component_files::InitialComponentFilesService; use golem_service_base::storage::blob::BlobStorage; use golem_wasm_rpc::wasmtime::ResourceStore; -use golem_wasm_rpc::{Uri, Value}; +use golem_wasm_rpc::{ + FutureInvokeResultEntry, HostWasmRpc, RpcError, Uri, Value, WasmRpcEntry, WitValue, +}; use golem_worker_executor_base::services::file_loader::FileLoader; use prometheus::Registry; @@ -18,8 +20,8 @@ use std::sync::{Arc, RwLock, Weak}; use golem_common::model::{ AccountId, ComponentFilePath, ComponentId, ComponentVersion, IdempotencyKey, OwnedWorkerId, - PluginInstallationId, ScanCursor, WorkerFilter, WorkerId, WorkerMetadata, WorkerStatus, - WorkerStatusRecord, + PluginInstallationId, ScanCursor, TargetWorkerId, WorkerFilter, WorkerId, WorkerMetadata, + WorkerStatus, WorkerStatusRecord, }; use golem_service_base::config::{BlobStorageConfig, LocalFileSystemBlobStorageConfig}; use golem_worker_executor_base::error::GolemError; @@ -51,8 +53,8 @@ use golem_worker_executor_base::services::worker_event::WorkerEventService; use golem_worker_executor_base::services::{plugins, All, HasAll, HasConfig, HasOplogService}; use golem_worker_executor_base::wasi_host::create_linker; use golem_worker_executor_base::workerctx::{ - ExternalOperations, FileSystemReading, FuelManagement, IndexedResourceStore, InvocationHooks, - InvocationManagement, StatusManagement, UpdateManagement, WorkerCtx, + DynamicLinking, ExternalOperations, FileSystemReading, FuelManagement, IndexedResourceStore, + InvocationHooks, InvocationManagement, StatusManagement, UpdateManagement, WorkerCtx, }; use golem_worker_executor_base::Bootstrap; @@ -79,6 +81,7 @@ use golem_test_framework::components::shard_manager::ShardManager; use golem_test_framework::components::worker_executor_cluster::WorkerExecutorCluster; use golem_test_framework::config::TestDependencies; use golem_test_framework::dsl::to_worker_metadata; +use golem_wasm_rpc::golem::rpc::types::{FutureInvokeResult, WasmRpc}; use golem_worker_executor_base::preview2::golem; use golem_worker_executor_base::preview2::golem::api1_1_0; use golem_worker_executor_base::services::events::Events; @@ -94,8 +97,10 @@ use golem_worker_executor_base::services::worker_proxy::WorkerProxy; use golem_worker_executor_base::worker::{RetryDecision, Worker}; use tonic::transport::Channel; use tracing::{debug, info}; -use wasmtime::component::{Instance, Linker, ResourceAny}; -use wasmtime::{AsContextMut, Engine, ResourceLimiterAsync}; +use wasmtime::component::{Component, Instance, Linker, Resource, ResourceAny, Type, Val}; +use wasmtime::{AsContextMut, Engine, ResourceLimiterAsync, StoreContextMut}; +use wasmtime_wasi::WasiView; +use wasmtime_wasi_http::WasiHttpView; pub struct TestWorkerExecutor { _join_set: Option>>, @@ -683,6 +688,14 @@ impl WorkerCtx for TestWorkerCtx { Ok(Self { durable_ctx }) } + fn as_wasi_view(&mut self) -> impl WasiView { + self.durable_ctx.as_wasi_view() + } + + fn as_wasi_http_view(&mut self) -> impl WasiHttpView { + self.durable_ctx.as_wasi_http_view() + } + fn get_public_state(&self) -> &Self::PublicState { &self.durable_ctx.public_state } @@ -695,6 +708,10 @@ impl WorkerCtx for TestWorkerCtx { self.durable_ctx.worker_id() } + fn owned_worker_id(&self) -> &OwnedWorkerId { + self.durable_ctx.owned_worker_id() + } + fn component_metadata(&self) -> &ComponentMetadata { self.durable_ctx.component_metadata() } @@ -710,6 +727,15 @@ impl WorkerCtx for TestWorkerCtx { fn worker_proxy(&self) -> Arc { self.durable_ctx.worker_proxy() } + + async fn generate_unique_local_worker_id( + &mut self, + remote_worker_id: TargetWorkerId, + ) -> Result { + self.durable_ctx + .generate_unique_local_worker_id(remote_worker_id) + .await + } } #[async_trait] @@ -766,6 +792,87 @@ impl FileSystemReading for TestWorkerCtx { } } +#[async_trait] +impl HostWasmRpc for TestWorkerCtx { + async fn new(&mut self, location: Uri) -> anyhow::Result> { + self.durable_ctx.new(location).await + } + + async fn invoke_and_await( + &mut self, + self_: Resource, + function_name: String, + function_params: Vec, + ) -> anyhow::Result> { + self.durable_ctx + .invoke_and_await(self_, function_name, function_params) + .await + } + + async fn invoke( + &mut self, + self_: Resource, + function_name: String, + function_params: Vec, + ) -> anyhow::Result> { + self.durable_ctx + .invoke(self_, function_name, function_params) + .await + } + + async fn async_invoke_and_await( + &mut self, + self_: Resource, + function_name: String, + function_params: Vec, + ) -> anyhow::Result> { + self.durable_ctx + .async_invoke_and_await(self_, function_name, function_params) + .await + } + + fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.durable_ctx.drop(rep) + } +} + +#[async_trait] +impl DynamicLinking for TestWorkerCtx { + fn link( + &mut self, + engine: &Engine, + linker: &mut Linker, + component: &Component, + ) -> anyhow::Result<()> { + self.durable_ctx.link(engine, linker, component) + } + + async fn dynamic_function_call( + store: impl AsContextMut + Send, + interface_name: &str, + function_name: &str, + params: &[Val], + param_types: &[Type], + results: &mut [Val], + result_types: &[Type], + ) -> anyhow::Result<()> { + DurableWorkerCtx::::dynamic_function_call( + store, + interface_name, + function_name, + params, + param_types, + results, + result_types + ) + .await + } + + fn drop_linked_resource(store: StoreContextMut<'_, TestWorkerCtx>, rep: u32) -> anyhow::Result<()> { + DurableWorkerCtx::::drop_linked_resource(store, rep) + } +} + #[async_trait] impl Bootstrap for ServerBootstrap { fn create_active_workers( diff --git a/wasm-ast/src/analysis/mod.rs b/wasm-ast/src/analysis/mod.rs index 66e58fb93..18095ef1f 100644 --- a/wasm-ast/src/analysis/mod.rs +++ b/wasm-ast/src/analysis/mod.rs @@ -617,9 +617,11 @@ impl AnalysisContext { Ok((Mrc::new(ComponentSection::Type(component_type.clone())), self.clone())) } InstanceTypeDeclaration::Alias(alias) => { - let component_idx = self.component_stack.last().unwrap().component_idx.unwrap(); + let component_idx = self.component_stack.last().expect("Component stack is empty").component_idx.unwrap_or_default(); let new_ctx = self.push_component(self.get_component(), component_idx); // Emulating an inner scope by duplicating the current component on the stack (TODO: refactor this) + // Note: because we not in an inner component, but an inner instance declaration and the current analysis stack + // does not have this concept. new_ctx.follow_redirects(Mrc::new(ComponentSection::Alias(alias.clone()))) } InstanceTypeDeclaration::Export { .. } => { diff --git a/wasm-ast/tests/exports.rs b/wasm-ast/tests/exports.rs index 899c8071c..733c5c2dc 100644 --- a/wasm-ast/tests/exports.rs +++ b/wasm-ast/tests/exports.rs @@ -700,3 +700,77 @@ fn exports_caller_composed_component() { pretty_assertions::assert_eq!(metadata, expected); } + +#[test] +fn exports_caller_component() { + // NOTE: Same as caller_composed.wasm but not composed with the generated stub + let source_bytes = include_bytes!("../wasm/caller.wasm"); + let component: Component = Component::from_bytes(source_bytes).unwrap(); + + let state = AnalysisContext::new(component); + let metadata = state.get_top_level_exports().unwrap(); + + let expected = vec![ + AnalysedExport::Function(AnalysedFunction { + name: "test1".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: list(tuple(vec![str(), u64()])), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "test2".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: u64(), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "test3".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: u64(), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "test4".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: tuple(vec![list(str()), list(tuple(vec![str(), str()]))]), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "test5".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: list(u64()), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "bug-wasm-rpc-i32".to_string(), + parameters: vec![AnalysedFunctionParameter { + name: "in".to_string(), + typ: variant(vec![unit_case("leaf")]), + }], + results: vec![AnalysedFunctionResult { + name: None, + typ: variant(vec![unit_case("leaf")]), + }], + }), + AnalysedExport::Function(AnalysedFunction { + name: "ephemeral-test1".to_string(), + parameters: vec![], + results: vec![AnalysedFunctionResult { + name: None, + typ: list(tuple(vec![str(), str()])), + }], + }) + ]; + + pretty_assertions::assert_eq!(metadata, expected); +} diff --git a/wasm-rpc/src/wasmtime.rs b/wasm-rpc/src/wasmtime.rs index 6b68e2f84..23b7f21fd 100644 --- a/wasm-rpc/src/wasmtime.rs +++ b/wasm-rpc/src/wasmtime.rs @@ -22,6 +22,7 @@ use golem_wasm_ast::analysis::analysed_type::{ use golem_wasm_ast::analysis::{AnalysedType, TypeResult}; use wasmtime::component::{types, ResourceAny, Type, Val}; +#[derive(Debug)] pub enum EncodingError { ParamTypeMismatch { details: String }, ValueMismatch { details: String }, @@ -464,7 +465,7 @@ async fn decode_param_impl( } } -/// Converts a wasmtime Val to a Golem protobuf Val +/// Converts a wasmtime Val to a wasm-rpc Value #[async_recursion] pub async fn encode_output( value: &Val, From 076a661f1fd7d7d5defb579f4eed275df37dbb62 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 18 Dec 2024 14:03:26 +0100 Subject: [PATCH 2/5] Added test wasm --- wasm-ast/wasm/caller.wasm | Bin 0 -> 94586 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 wasm-ast/wasm/caller.wasm diff --git a/wasm-ast/wasm/caller.wasm b/wasm-ast/wasm/caller.wasm new file mode 100644 index 0000000000000000000000000000000000000000..57794b16801c4a1095d71b1beeb65011b1d4eace GIT binary patch literal 94586 zcmd443zQw#S?{@bRh>uoIo&#Plti%I#FeH~7z}yH3j~PP7umS{@FPKY!zzV@L12bVT z@%;YZ-c_egKP1KEt~F8IRi|n{zWu$wZ$HW%Tf2F}dDnV@pJ`^Uy|TQxc_Wq((ezEt@ym5JP_U5CjE5|Qizinl9jXE!)j&J3vbJAyvv&U|5E^`y> zb2rZ}&MnPOEiIpzRqbbb-=p5qAr&edzPA#sdK1mbLAnPsaQC1FvcPTxGf{m?!g1=U5 zJ)mzc0e9Kj?BWe$Zai}@`ktF-*Kb-r@xAAPrJKi8mkdvW>rvGuv-rOZucuJ=91mzUPoS5F;ZUtaxQ#^ZPSn<$nTpuIr4TRemq zlzma?h1b)_!rao0Q*%qlr*2$azW&(a)b+Rd-6eEAcT48JcH0`lbgW?r_ZRu)WbGEe z@{D2SJ()YOW94Iui&H07mseJ1Pxxn6kiwKL*V?Jm>rdS{1&iK11>;Q3{kw1Q&+>C^ z>JE{SEgidgcFjFIb9+Wm5=rRy`kPe2(DIxbS}rN!j0Hzg15Pf^+_N^95!Upo9Px96 zt_s0hTB?}S@AGqRSZROep0l~>VcF;Um9=5nugP40bJ>cDs&s7i#x;M5XtCy3@?ph( z=JIn^oLxHY2jd64zoageQ)}y|uD@J=N*P;gG@EUo2ElB--}{+y2^2JodZ833Z<11x z#!J_1<{| zZ;n6T`@V)XyK>X)&9kep`EXp-vqrmhz|Uk6cM!-qpSfLX5#-FRE-&3YyR`0;@k_kF z*fizw#ktFCucF2rYdzP`q<)_2)4sX3er)~J8rm35JA3Qgy7%LIy?@5AKF?ys@z#W& zq07eX>gw`pS}TZ3wr8f=f$JW-TmRCT6w8zv9$L1E1z|=L5FD|dmo{+Nch9;&07!F<8om;9mEi*FjZ}Cz@ zCQ2>Wt}*5aATK9l)Ul-#Q#UN0TD!@2hXB%g#ag4SSG7Z(2{UY^B@C#LVUsbOXzm78?Gbde z4*8jeL;yWJ@&ZHV=e*b`$=sOOXl?z(+>%enpX2?WiUI6lY=2ktBDFtC%GNi;4tiUN zT<>8#_Q2riLdLn?1y5riyU*FjRz}*c4>)6+K6o1Yct-2%&1pzlYqJpe#M&*LrDIF* zm;Ja6dwk;9`mB5ckTl-+et(1&d$e}>EwT-;pGxJ>=cU)wx({P^tJTIZJKQ;R32j5Aut z77?fuw@qGu?8MZt+3fmLYqzye@J4_(PM%s`Kh{9N*VZT1)ztd(^3)>Qc=nldi;J^2 znq^&|om!hcc?!ie+nHOMT3KDb5kF;(dpw}kQ!DGUCt7n$*weWa$=q^xZf)ws-0G}s z`fb_b^76{08ol}0(rr_?QVVNcOx@bC8)=krO-L_>UYD0{99voj*3)yxXWQH%U#qvZ zxDlw&m|I#qb;AvF$LDxFb@S}aw0_aZv$d6D$7yeQeb_zip#^VVS-)+U?f`1}G%5tA zvU7^&R#ui*0lrBW>+82oa#=fd{H78D6}@3``4(#u&2#hII)$^9xs};5p>}F{Y4J8h z#|93DW)~*cXK!7fGCT^+-Q%mvYaqi01aVC3Po=5AP?Si9}!>z5bjjvF$&tFt$rf?n2Xa}7MruA}^i^q!J~i_#S{ zjSKz3Af|3ivj40Rmd3;D`qgHpD(za~@l2FTo(c0Gn+lvZB_Y5@+`KZe&mP~rd}0pb z1noS0`RrgvrWn)`6d%4A1yZF_Hz|}{aaNO8Sx+U*xqIE5$nIb5k{nxNmdwmy4vFkr?f{<-71_xnuC3=332 zi*){t@`QxASqwl<-(D|F#Ib?IK^HJ|BPa3$^ zHn|zg+_?7ndizS!3ohNJpJ#7kwlbe|evW(LrkdwTlAOQ6?U`tEUcg;Iw6~4yo1sY}n_&;85F#OiXkV{@3x| zx7$gplVoW+mbTmY>inN)sc&~ZF>H6T>&Dv2uJLx#rm**8<2-FAy{_)t=@{kt9}V-r zvB^n(jiqD!ZYN3C(-qd7Ka|K?EtddE>WE5P0FsQ2wPakqevyR2sy9fMe$zKKQtx&* z$FQWu#>QFKX+7pAo562RTw@*Y?L}^|>-e$j=dN8@UR%F5JejgL-{4+o*MH|Ox9b(| z(58}CxGU_U?{?;Gxcl#gox4rTE{*%AYztg|VzuUjZ z|G57N|C9c`{)haA{|!I>TOUq82st}n=kY0~b9XRt17UPyB%=&~PYyKKaw0@BQxr!^W4WRvUa2Z_%@d zX7alza{2s2clO89>73X5`{TOoRhM+i?VdqL2yEw_`F^)>{VWGGt`o_0N)ZaBgUQRB zP=4l>6r3#lJh1!XDSxuRODLM0a=!T1llJfuo@p?}wbxK~SJ7P0)7AbiY9-WC=~U0C zl2B!}pIYIS*HAd6cy~P%PB#@!Rhah|twX*i92M1MMkygbpXb$^-;(uye~5`Gmlqpe z{Eubi+^@mvM-&_+)C3pq5b^V5WA_NbeOWP_eT%^kdaI042yxirZKyN{%=HI(mu zz+m0*@F~OSXOR?Hd>qI1-2<i9RMEYN=;!IiZzWWS^HFy>cVkD=t3ZoCoV+w#uHf>xnm(Ky3Qu0bB|YjExwH=_H$j8^NOBz)MzF!{ z>v@qJPF_oEhTp^Vq}$(d^M{iw0jkN5L-W1g(ug5=Lzr*m~ z(v3Q>S5ho>LiZbD3^b2wJ>g32)#!XN@r_b@o<(mpf?lI&6L?Tudj`zW0T{P?2W?n< zFjiBVlBQ5@{~#%{lScDq-YWj~9&l0ke55cMjd5i)HTN>)4&tk%Pt_mzK0*BL{a^^6soE9!{>{i9-n-buZB!m^hqVrn^g%_L1a3xN}F6{oFN* zeV}-c-F3M;1Xzt?|NIDKa8cu6a;W%R1dJb7Y!Yy>j%>f`-L0krsUOh4h<_JH9b14gM-s=XalJFU%I1zK(w*O^k5-c}p-DCp?|iPR%{M+3Jd zO6fx0ny0^JK1LgZG1QhxD&08MR3~YHRu%NuU41dl@wgRqYAa9F6`E+Md;vE#M#vMI zuwT*(()Se4(|vvMj@OOBCcH56Jv<7mIg0mPMwzTXPOnJG)s#kGqP)`i;!P(9t%bqZ z{Gd~$Co#va-&B8;1ftkEKgNnTVbb)5{~DlC-8be38k-EpJSg+!0Q4iN|46GJn8^ls z8#D|>fg-74$XemG+Ud6=CRK{wrQR5M2WW>~A`5sAfC%0J!-od+P)G|8llTzjS0ty= z7fmVDCh)h=-i?=;B00-QinIReeE|num0j;oFwi!A1FjGzo%=-VmIs` z%U+m(%O1lFl!F^Iu2w!KQuRj6h{L)Csoz;AK!jk+WX5WC23ApJ0iB9cjXYJnfT6gW zCL>ro^vdfq^Azm-zn!d22#^lLfX6YX?trYlOQs8q(_Np&PE5GoPwG)YpGU(Z@Go!e zwOl@>s6(EBo5_wOSMkJL3@01X`&rfF5%!j|b$xDx|;bg*9GpHY7SA2wS^94k(%F zz$y1uiN(b{#I?glp>28(sPO@QH#)=M6%URZ%PJZ+S3;Tu71!65cuDoN zg+ek@jJ%a+@`BL94E!mF$A&W-QWSD*TtU>joiOe#{?w^YEA!8$yr$SUJbC_O=Zb$v- z24+J81~0Dru;52aLB=ka2LDEhEbodSnJ_H|*(<+5v9gR;0l|A8QAJP%ITnh& zOO)ac*012vup?i@kw>s-6~lr*X?~Yn)Vvc*6vlJ+|K3?B;=~E>*DatvUOM!Jz3jZb znffy+y8n3^vxc%T@CWQ3E@_5@(t{tRGMSuYTDIV`)~Z14-HGNC(Zaui(LbS=gJi~i zyQpz#`X&7i{@2Wz*G(iqgT{Ol3b%Yo)DB`^H)!lT5=!UzYjXFni-3M4G#NXAdenPZ zjlNDDOu0v_gJwa180?Zg0Ef+^Zg1Wcfj#0+l;vlnr_XYK8AH4OKhwtH)i&QI~Uwl&6-YKM70!GN262^j@LN};a3A?3y&?QRnAQOR&c6ECH7|_I1Fn%Q$^tWj6*^e=F zqk&w8Lp(V+Up%gdz27LSD8PMa<|%*2okvn~Ix>Z-dUs$+$TyQ1aSPMxmPZtNAELMr zY-%MwZQIj8fV~hPc6gtfDm(l|^5tieG94;uNTW1J1fgBh`#phX^g&x>LtWvFzl8LS zGe;6j$)>1IsRGsIDiWDiPJX6Y?wgzJZf{!w}!LYcPSK5#WBcuXA&@jSi z3F3^bM!yaG3KA6fni5eYg#Ul5VBoqEuQ%DXY0(z}CRjIHN>G7OFzqOPf`Lod$(urq zkX=wC7IG1jsRq&>qg@yrAQg!r9PTPG+$0vyGYOJp5Ph?*3<>6ov?{`2DJlbZ)c#N~ zx&#VrWPPMa30KGnDHDn`f!^ZryWzpy2PHtkVnb3;{E^=6nlC>7gZlMZ{Vh#v@7KjG ziXHN1Wnw9QX@Y{2XvzZ!JQuns^ZuRxgXJEucv6xpuTuWSo)G((8AA))`)T*xU?6A- zy-cTRQ(Q1l{{Ky6om6CF(3pAR=hNHs>`f00P$f^LCyURyB?4r>Ui|0}1M~dl&b`

ev4G^6x?&HeeL=V9d@oX8zKB7a%nrh;tr+CvS() z@cl^l?p*ztIRTz=%dgZMahstQGloHnJLtw0hgOe*d9;iEiA08y0{L3#{h7!j*8(Oi zHAm;9-(j9dd{(qI0pWA^D!7PxFi1DFiNWAC8EwqNhh%E|)*XzQd$LJ_%=LaIyim=O z#JTVMPa{40%A&&+&;iCT+Qr`99}8H)pcII+xgwjW0v!2SrROGv8d~VQ!w(y-C0%o@ zTgYu1d*k+nETt)yhLkqh%i`gX@Z6g>q_oLw7C#^WOg+namC{yXbf~oD$m;1KI%uH0 zWHlRDO=6{JA@&J=7^IX3hUZJ4>Wr_Ld{ify1`4B275Yn1a{0p$wD%{1Nf~Lfj{zmp zutJe?dZVBTwAsonbtCXVCy}Fa_ zN1P;6y;@j)MF`8UEIy`oidG29@hW}K7zSHu4vw@{1vXfd-}YYf+cw3M64JxrRd5n* zpu#wLa-rK-xMooD8F>+BX-%KUb^$;>4)|hd4D(d!91JCYwqWT{Eoj0RhoI?B zjkDD3k0(hj6?5Taq_@pf3nMhT3GWjAtq|!=q}yYH5+uIgeOr#YS}Bzq$&6g*()a3_{FT+2MncP=9NOvYm4`)sbV5# zu3+7e?chd9u-@Cs;%^uv56yIMAEXc1)PYi0A>)x1PmW|fG^x-smyOtx7(s5$ri_Qe znXJfau1%h38WhP0Gk~}^3{x2o<`iExk*W#uBx8BgW+YL3w1WaCS{e4O!_T`&rBOrz#2$@ax)=*s2tH72D>cnR1zZS z-$)LPB$C6}|Mh+5!<7gUii{j0&pF9qLGP{+a^PlGS zW7Z@Xqdu;<9whW{sqG<&exV%D73K=(y>>0TN=pZ6jGae(SH*wqgc0$0dq6b%G4i$NrlewbS1 z)m2fsyfW&2%JDRhPY+a218I1x&_NSnATG>PmdQUxSQ7&PnxJf#3{y9sfc=QbeJ;M| zM=BbM-v{HcQ_L_m#H|fwm_U}LmP9BR9n~O5$3*_ahH21GbT03HOuQ`??+WGJGM&UM zS?T)Z%rh?(-6cwK2Lh_jHO44@ahTQ)u#U;W7>fbZq{>ai64*}!x=bfAi*4y7;llW` zqx731EE(gF6WxLq-i34-bT z;oX`F64%M29=_uz2qqzJ;aB8NIw;e@WgDH(c%SX)V3TGCkf_ttiTJ7Hf2C_t8Jit! z8H=HVP4zH3J&dk9K~q@pv_s3*Wi>8!Gt#T+x6QpO{|e9nq zh{6VxC>{%*#tP<$B-^b#AyQlC!djYYUXJ_iy_!k^GWM0|-5g}bot6(Kw<>_q&6^aL zK{$nZ=HjUR_ud7YA5PRUF3y{HV$NWQo_GN$Jll_Zqib+)1k(ph+0p7NNtJVX?PwU# z&%i2=O@)`y4-L?{dVS=p*6Zl(U*nv;KK51Xb##7_Jn+`BYAP|rLEGICTgSJJs$7#P z$4~>t@7*H=A2z+%D1Q)#;TFX=m+`|DmqD4}f+}89Sxm8uE+ExWDR3iDX6LY-MILs@LrvrzPtn!|0|?OEMscKZ_UVDHk> zRMl*Toy7*KlFk6zM%3*D>^A`umlHI~<+>tO0&v(yaMf1G94707vC5z`i`Sw{)*{3i zcBb1%0JZpo9P@v%?iiYhMKYRq*K}N)F2u!e7jlG`6>J8Vb&S<`5b6L61!Xa^@JpD9 z9K7)Zo8`uWr^CrT2pz@l_8`E7=WuejBwXG7th!64)e~t8(;(pU`Bt4pXaKc_|FlI_40H5CifW;6IfD}LNaI&}wl;R5_&?q=+{F;ezAXJW9D_H{8 zBFOIu7@BhEl~;i1anK0BK^3I%*FPlQ2?J@w#y6CWFUpONl3I(Crp<@;-LK*IZaU2$$!R zd2gXeP8;W!jP>vqy6(hSa!ocL9h)u78 zO#_Cmi^6xJ^e$TACcJ&=06#|YNZPws$moQ})7w9O$@Y&g8$SM#S|bOye|%t@$JMmE zWgP@WIny}|leXm0Dvi50Dr&@vEP{q_SkZC<@YK^<21#uylnHO)H=S78Gl_^POeM#l zLTb63qrTVAA7EG;uJns(j*-%69CeyQ!ja2@gC-3pgv&ES z@!*z)Nd$%0IM#*b9(sQ&AT9=*XgGXWDOHd5u-=m?hsMMAv{lUwn~IPZ51f07FW5?A zFQ7L=Mbc-O0nf9}ZX3J}q4ya>z-`Q^RvSzX)h1L8e({e47uW@ITHNgDkdTfgY? zkh4x}(-7k1vg0(SuZ`QQt}cnv&$b8t7a|9+Rk`sSDdAD%0;P0);P7NxPRP!y>%83f z98%qPXM~Q^TaJ)Cw>fmy3nFdNiP!CBA6{$}v%gF;F~u>={=#B0Yj1PvYU<;|J}hhR z%bX+=?z|3Jd+x%j32eIc{s9qL9bGYiBM&B5#_VrexZaN%!y)*U(!0x8LaHEU-b-x| z9+`RJRk=foF7W)KHgKfXz0&I23}cv9_sY z@vPqa1%H$|=$n@bU(3{7CI`o?5IQkApalyEf_d7-J#VD}3zNiNc-SR!&3$H6HW$KU zEe*l#*0}kb*W#g<4!k7rszK8GF@YhUO|En>ZyuoiGZN%H#U`c*RX^+fH@12 zcWeU>Rc+wDj`UJ%V?30AEC^kI3DpxybF&GiZYWKDbfko}SNFvQzd3Mue-{Qh8k?Sc zxSo8pmvX_|nGU~R!iAK}m0Ttirr57TT(IGnaUu6|F#I~eg)~;q1;=}T__db{+0i{* z$ZPe&FC4hAb&X9Ma#!pZWJlG-$%}UjksY2?oLxNuY17ii<(v|Qcm()Hnmv{E)8hTg zq#mjV9YP(va)=9N@nCLV-GOk)b&;E*T-jyC$In3O@i1+V;s;RX2=y9HWUV()YU?57 z(PCQG2K0B+D6D0AvUsd!Pm|&^R^kZFxO}%%uVwwj!)C|PG$9WOx5dmDBT{4Kb1Nrf zZ~C0<1}h`vMQfeEl4i0sX}_s?YXFs#88c@PfWeB4j8a?^6K9oIPz=|zQ2=LPi-W+B zd9y_Vr;9%`$^DuDX;chP))fOj0ZcY@5Du01Z+w~)wh0IvX0wF7p&{VSg-qtY$B#^1 zXQj1}=7E;BVt|yGRtykxXf6hR(Yix&?4;K8AFBpfd--PA6!Mep>JDtRBfN#QYJGEh z^}gOO>b(2`8k2szZa8I(JseZf;I5jTj+~cxd(s!C-K2UtB??;g+tc zknB_c+`kNPI*&B`c8o2+kisiaAuI0NdtRCtKXCBmvlt%x&vDm{Y*p~2k@c*EJB z3EWiN3ZViCk%Lj&g24hV32C3W}X-gc_+3Q3&1{dvg>4A2xF|iA*#e3v! zA?yN#neci(DIU|?usX<~gTr`Zx|L{4#qjBvMeqDoGK>O-sx1<5!-vIarHp{a4ekFE zF^UZ~97|3hf5A3&n&ja{r;8^%Ytc+AvY(5s=f#E|Oxo-2@Bv9J=8SFqAq^yD1C)4t z+Xg1E)0$%r9s067U7WS@h0XShGs`YZ^cgA!ej(R_P6m(dspE(yr7N{**_O>^Lo`uR zXrfm}Y?wytFi?4RKXe!l#o@k1h8PV1l%Eh zbxWWpW(h({D_8iuVVB-&Ak;NSs=wUVvgui0Ze9>S}1?kPd;1)kf^CJr#O=x0qk_{gU*zMT8ZTdsb=%uN zZf%7D=$qv((TQf4g*BeVC6_rCtpup%e10my!dH0 za&!ESG)k+u1k=rq+O8C7tGNASv81gE=nw=c8({;koZ(4(D{2E30yUW=!pXwG1l8MD zPBa3g9tX3Gry?70$7-Q%dL5}&7ClS7P>UU^Vq)m?(4G0%owmSVzPYk17OfdBMAvu< zidnnJQqUkQhu3^Ol-I16*a*gh1;RuGiV*EAxj;PR@wv{2Gj<1n0TdhS0%T#iBY#np zGA=DsZUqS99@yK?abXF#8HLR(so&eNC6hO}RP)78KM21f9Ld%D#I$E2myRh>n8Z;5 zS=T(5(Q@LkcFA2{XDOBcOM;^|(rQ`SScBu7(guavfdyYNG1g^+K-*Zb-EHj5f^7=H zKPnUMT3u9Hox`-Hff7wIXuAjsuwu|53q;ju=F8)YG#11RQcn$RH`WWPWIXF~M=!GHwq!Oii?5k7Ry-sJq(rI2 zQ2MTT3ci6GVZedTCV<6uogCopt-_HUVN(tAw3AlpPx#l3n`064`c>*vdb{ZiB$<@`kMI`8ZbfvpFf+wtn~ zS?(tfHhuhGl6jw&KkC;p{b(6Pk2zGDK-*|u4SOfr3D)HN8-}-$z1WXiO3lloe(dvv z$*!vfNn2ih&R%U>>oI$UL~O0~aeF1Dv8BPso%-Tj4Um#Bcc6G}Gd!QRGA#^jv6}(> z+{2ulLo}Wg?Nz%Z>&1ib1KWx|iq1MkI_m=k;u(m~R_(I6O!+!F_iOXzkcmw_l(o)8IXMO1;%1c`s;^ zO+BF;ig7`WcxqgfwZn5%=U=I-q$gcwmwRJI^fC;)!I?jaRBcuain~RLavl(||A$6# z-^e4kg|R`}yM0|Gqw7oDBE(KFIw_ zxZk*!`~S8We{AsMKdpx!5C6&9eDTKk;>)j?FMjSfza#v5;*Kkcx@BX0@xT3~06SRx z#Cvu9nWT8+r*!kdBtWX!L1Uq3>GEPvYKKarEc9%9wuObrN>X$S3teoFx3JK2?C};B zQgV$I#^+@r%4}w#Jzoh6?FpG|v5+>o*H}n#Z0X``gb*|>s#M)GU`WgzHj3!?iC|Y= zLmt86*}fB3XcI3%E5}apD=Bhr>}>&A!^Ej9B07Ei{lC#b3H>q!0UXbF5&IgcHTHcew02Y}~_( z*{+Y!Qrx%~MiX0bo57Ly9ZRsTnLb#D6gKWDcj5~v2g61&j+%`d_w=wzce~EdZtshH za% z4pc%QrtV3(=GmfhHkJ-l_C8N#BC)8--rErxgm&DJwfU?>4w7NYIe17HC1|{+6HU-@ zZ^Kr>FuTS39`CJ~rn}5Ag>(TqSLzQ{#a;M#mew}_Nf5`>K6h<9LoMu{Df9bM&%DA^ z(t8qf`gTBtW_N~33@jXq;{kyk!Z$41yEE`21NY`F`7N?M9g5?06X@{>&}x&(iqOjYgI(4 z#7N0Vg-tmeCS4jaHk3Y8R%1VG5Kru5$$VJ0qD04@;?HfR3R`@(WOzXj$SXqAs-q#P z$~!)Ob_3V^G3K->`8 zk2=ABwwyT4Q9!P<^U53rVt{2aJZTF>YH9K4J3y59C)M^{P3MfP%!$i!4pNW4v|J{) zk!`m!=gRt;Agq@^0H`vdO z)0);$_Cx3A@eBi^Lu6`4TC%k*bYNE7%sL=2eWc%e2P`y|KhXPgjZ-NPvR627%4(jYF@T}5R?*h-Efwr(Sx0fmc5=dMeerLXCvZa^8e&n0pwfEf z3aXGgTCkBN(+pX&T}ldj7T_Mz!rJW)E(2j~ieVVMuTQy^V|01PdbQ;hg{|S)Zq)s* z>@{O0;L+G=7QxR4d~zsp-l0@lee_%34u`-9oCf<`>;yI?2m?KA?8Xp#A(Bw0l;Oxw zIOa_l-)<;|rE!K+M~cG+Y(pipBt8|KWQU2cxU_WWTFDvPcVjpW1-qTA9kmlfFiHei$F-OSu{CR&Jbi)9 zkrcnAMUFDE__|DO22)W@|G$Ov%ZFw`CsIeVfMDLQl%8oLX$Uk?`NSd+o6eNBLZucD zhINE{g<{MG_(iUDI$T;j7GAv`MaKTH6Fjla5#(?)-g}WrXc3OcB za?(?&bsdSKwb!Y3ITWALxkBYi2A<{fGzXDc6`RCWu{or_yU&~cE4+jw>QLp8QEcGgEH;-|S zvvb&Gz8%kl`Bv~-JpKzc-u|0(v-j2rofPo^=9>40MmK)aNv8V;W2Ns#URw@W@Yx;{ z$4jMi*3Z6z(?|2xKE2*EMopatJsA)aoB*G`Rl4 zd`U|0(9M2kQZ!h(A@P{!=opTi?L0iQxgAe*=IYq#g5{TE$m!51eL05QO$EmgYg!)o z92`S!_0^bre<#5LLOm#lSv@4bF86U{af?MOL-&!GWV8E7I?P;y2K46CG{c zthFhXEf6KS;j)BC!?DRnJ9M8>+HZ!yNx)skJh1=@@Xlz*yp-;6Y<|0zAP_Yz&#Che zVs1>x9+}-Gn4^OQHQ|jqS-P zk7^X<{1A))ZkJy9A+IH^VAd{Zq*Dv2IK@~Q+X)l0}9g`fx?v69iO3->5K7~Tap+MC{%Z%I<{Wotm&6{#MW2pFuh+> z-(Q{3r-M@M{e`61b^ca4JONBnLuW&iWd1UFwKm(SARrE`Ca;DP1NXC7pWTQGMy8x6 zHQpB`xOapJMh-65kl^o9yVDmFv8>7~%iYb#$W3#}uEEjgaBAY0beh@;L>wueXeN?u zjCNhC>0!9TwmC_f7}iqNtSu`(qNH~3@56x9e0=hFe8L#W?9m1@7k~@V6UJeYo2i1l zL0F}=mRiX$Zn*`38rN489K%&6yuF?`815?L;VdZ~mq29pUhDUZu@?}+$>?!zy6|g& z9u7tz_X2iLc7A`XZm#52HV#mC1C9?LE3v#!Te2|v4;bF8xov=Qm$?m{QqIRk3aw*S zM&mN~luk3}(+x$!BTFFR^y-Ly7)EDzFsV+|!VAiN*-k#!aT&-+@vo?^a-oC1{Q!n) zHJL>!zVxFYS5jiHl&d0_U@mR$uhH505Yh+znL<`zwApp&CWXMHJpELsyYJKaRW{l@ zhYA#?vXzq#Z5C80zF#Y4VGrfiuAay!LsmhQ;IhyMJv3OJ;H_`0=t#&AlTnFsR-mcY zbS4{=4d|lW?XjEyA{vf`o9c0--1oD3eOa>1Sgalo} z@MtQk`xruTsBN+YaR(4=!H40@L3XL=7Xxvn9Q{(NX7A`iDKiOaFmwYH)a|-c&K*T z8rcAyt){r1%mfrG>Z;LC96K*U+{E zb9l5LxH`a%H&RDUvXKC+5KUm79fC=SwiB3uPy=&w*8$9PbuGe*wX^EuES!uI0J#6h zDm)Q@?F0arsR1C2RAV=6c1xUCsSrt?3m_7X#TPX6N}*=GQpEwRICco^t8nq*ISI1N zCSlWGi%3YNBC+C0P3PhRDV=%F-l%pmub1i zKn#Rh1L3Pm)r3vj`SR$jSaQI0yae2U!_AXqLv**mpE%Qoi-bXmyJ#@F)y$bK4CHx= zF40_3%7T;H%vn~=oSpO7%LclSz~%yuy~I}9%o&;WK8C59Im_bA8Bc*U!a}Qq6hSI< z*k+hCW4gqTPTK|NF3y*QF~oW}&Z9+ZEtB@6(c_kes}56jHSMl#$0L;RY?4p4L2`f8c!o>A2n1w8pdsuCPo)ZoKDUYj(;9K?s&Lry>p_RkTE9^ z0vq%=9{6+k*@E8Dx02%mT*HY6Kji5q=y_+*m%CA`VRXjW_@r^T&O9sAO)Cf|v@6&9 ze^95N7&+h&T@$H`uawcVjCTrB!(5e-n#bWE>I@cu%ei#td+(9H8z3iC6R~w$usQAX z3jc#Qh99Ukkr6z?x+83g5w}udS#k{RL6{LSl zDxeu753x3G6BT%8R48}6R#eC_L9k5WnaIII!w=Nd0#GfOW|kIUxRsD)oja0Ya%ZTj z86o`EHR|CCXnC7KwsjK$;FinwcgduDg&PShx>mKak!ria&YKC$F{;-g^rVIQh63}7 zc*uSRPyvr4(U$e-{jBW`0iBTU3Ca8s{;ohx_7Pn7gZdtXcghSP{NoR`Y*RmtYgls@& zCWfpDbvG$@w0AIaNgowejNA4a%@7hG$b{I!2y_;R5ABWG1}U7<2B-pBXsui@5mew; zEwy=c2iCU5yvqCou>$dgR7}Yq|6TZYokLK@NQCM#MgOO7N8pP_?|pi2Q!{^VPs*J2 zySLqrqaM&Pny8lBQ`{rtQV?Kn@uOH?nrI=Nc$s~0f(=z4U@P8b)!+a=+)4Mnzt!L3 z{@9p|PVpnd7v$7%dnN{8f68?SZ1F+sP}e*sEJ)yG+hpOV+KSSozF$G!X9@b}cI-{0ccAwDB0GHaVH0ci~d|b-FOnL$R!+y0D`G z;&(|mfzbK6#&Da|!|iGXaLN7U)r4y@b3RGsh6p&$s}S9EWL!ZqrQQ3uteZAr*&`mD z)dS@TM7#TphuFKXw_$fkc?!S$gZs4BgX{fdqIA$ZMudZa+2|!qU}>bVwXn5ZoT0JC zBT=&n!v4J4YYKFn%>G_J3DPd^`mhN00{SGS{{COlk4N>lJg7j!@)E<~HzeiB6npbT zx%vaVxJ!&N@ghHpZt(QF%K(0hq&(Ss5NBtlRW|EMt@2B!=Ir|*6}1OLU7 zHum4ph+L=KMMFq{@r~P)EsdeMJ)~?z1Mq7VMh@}{Deb7tUEg& zAwCYVSi}Z!?2HZ(-3}Y%T7G+mULS4`z25TR7fXznpGJJr4_#QMM%^$J5d({BKKpwB z4gj; zy|PgZ2n+fr2JD%>J`*Nzu~6XGvi~HV2ApHcxAK_UVBNS@2XMn9jKLS- zv|3>d-Crn4Ng%Cd((=Rk?iwdLbFxI}ILCUn85E+XTA_{Qj#gfX1QzeZ}`}YWR@JLynXBkJju>*&N zZ*5$fH?+K*{VbFcb;Fr6wi%C9rj!c{CQvu(WL)d}A|CB~mIk&q+%-w(N|kfEh2Uk6 zvXMHlOrfRs`fq<31DJr0F(t(}6-hTUH$pbt(+}*QlcmPZ=X25a&fNS7rS34N`|S%i zp;QA2`;3&cO3Wm8atd5zGGHL*-uo}Xm}o?ry2I#9giztzu_kexjF9~hmg`n1k+)tm z(GVtbWQ-2l=58&B z29Bn+wN&b{cb7o4C<1Qp=FQM{(arC`(;F(t#e=LDBc?Ng25Zk&Z-fQlFEAea`XN`J{c!lO2hHu4J++l5Qnv z2Y5i8)UjnIQtEZE0k>dj<&K~>y&gs47;h4N>GQBeUm{?b=?jAr z&YKH1{RhU4gW*k3JX!8yHrzJzNd!#TMOJZ*0&OM(Z9r}&hL7ojEF70&6bm>LL)zDB zVh{Ggxh7JkMMzIiD)iCOuTUz;l1JHTJ#DG506K6zUIREqONreY1TErWhNJMM;%lKx zlj$SODC|NIiW|$Tux+*BcwxA7a$yh_#|13YN0b1XDk+rc#Li|(AuzAhkdn?#Nq}4( z86fjg#pR_5?A+5x#)qGRBF-MMU#f(X)TNDC%2PxIAmJB;)wDEkGTX-7HsYa)O{F0) zz{5Q9IARdgnf{stSY|_qxWvqceeZs(-(@b_=*RqXrPVYWZCWk7Gp#lmwOZGvVx`r( zd3Q7YCWU*`mz=p-(`r&_e9Kofnyh6&@=2*J2-wxsu6;vUWD}{{?E2Mc#VXVdG_?<^ zXzW2W(oKlR7MbeNKK~il-&JD3POvjX;nfL8EaR%svn_IP&KZa%mIvGp(>{g-R&7W_ zkSg4yA%Y|bbG#ke1CG?3WaT&~d`{X2?eQj%P)r6+|E`(#x3m8?>+b^KN$u*SsvSuU zNtJ<4Wf1jFjcl-cXV)E6%I~_fKdD;cYs08ow@<54(4?!Ib8croc_$cFZo5qJ5UOeu z>WCcthUcqIe)!dC0 zU@E}0$g|wO&zj}?1B>)>_y@$lg z69n9phQ+`1`)5h`>SSnKd}~p{WeyvTSK2{=pw}%(6v}a8;^8BT;ReVCEIVk%R&i`b zTTJArDexJ7)+07;dR?QumGBm3i1PqN`A7ukVK`p`T^O>k&D{1oHL0U{0?0?!psEX- z2zWW?o>{AiOf{$9Znu2DWdM>|1T=`+6wzS*~H$#q|!jC1rDcTr@;UswVoBFaPQa>umcbb;2n+qR1M{;e>s~e~lC}KwF1?Cga$YC+?oq_^zpgahfYM1XZz77HB?CU5Bd-=eSVsV&=3b9etT{py6%3z2b1rJT|?07vOXNh0VD@;-ONyq4B%jL zS>M1vC~wS&uCoKY7TX?7B6Uf>hzB0Gl%A3=Hu003BXSrOFV%@~CaTEiEMj6L<*qCzEr;ZrmtRfJn1$SKT*tkzdyFez#jjHBjE8jci{z9bdNAa~EVKgu0 zf*4H$2096ON2bZ>5=?z@&)gDHDC$158wA5F1wRwNmnUHM&=7{XQ z#2zE-1RNE^Bfb1U>a?q+VMni*qKJGBwLo1I1C~X6F<~_+W6jV8f?T-Xy;7fA%1I9} zUQLMATP)^xC??sNa`#jiBe=C6;ZS)F$+GX=REljw#0r`SpZ9{bnQ>!OmPkUBQ?cmL zK&^0Vs+Q3b`2qH(uHPGGIvAu9WEb2uer4}%(ZH12c!u8_`eeD3Yy4z6?MBTk#rS0@ zEGAB(dwu|xR+VhziAh0@da$dsTkeAZ+1E zY@e#2x8~KTwa^RRlLc618Yy?3KNwl>a@N)0HKd1;YR-a|aA;_YOfMNC%e>0Rg8=%N zjV8Me+U7Aiq60Zk0kPpZKq}xaAL$Gf=7I3|)ZC}!Fh|lc?SUTMuOp~iwlOQRTy^)}B{>P4AHx3CDz=0&yf}kyD zi2_rZ1bKSPfbbBM2Pqr5pk+WIQm*Q3vkXWsgcN}O*lDY_M%|I^ErWTzux~M=5lAK) zHF#N0bEqVmyMB@=lgYFw5v!dfN&ur`dS%UESVwXrg$s$1K<6?s63%w%y&cW$diUtB z>s*D>3jfS3^Dz!kU+Gr_I-bN}Cj!C?pe-%WV+Myzknnd`$92X9;Lobj&U?Pzcgmf$ zy=sSi+0W*y+`{|i^y>@J=x}~MZn+XHRN-dM97(>QpWJS2+=&FqrhG0uG1e?rP8Og2 z&>w!Nv8M0nSXEiN!^uB}Lc0B=Zaaw9Cu!*ZKj250no8=A7K%U7q9h_Ov{xv_Jaufu zDzOH@q7x{}XA#3~!PM`A@wV1J!6sAI5ln6$8Qy+ew<-DG;*+YUMNA*JoA4bV&LAir z4!<=M!>>m|vH0s#Wu>#VLbf+TWj|BC{hjcZuEl?`?xW$YJ=uTAK=N-8sBHi7f6FsH)u!Q=^zhA-f#e8c{;1qnQDVO&xjT7PdNAM@nAduSJ{_Om1@&ZiE!KiP?frw^}m+zxUQzrvE0ERk1(-Drr z_o@LlhRC!lk;yyg9&*WfHp(S-Sf}g_$gi;g0AAfkFX&k~kw1PZ_d;1B^>1Y$AiKRF8a1+bGV}WJh_b$h=s=_EMR4fyiBCrhm zb(jLEvrJ&9$YC}&XO z&8D*;10tg~HwI{kmrBXywv^|nAYtVImnnJVuf^V6d;&TLuYw2HBp3%_$n-z0pfaq2 zN=5(@(-_uXXwfV;id3Wiyo`#S{&ap10C%?X0M2RuvGp0gm)IY22`|6R$#HnrbeVI8 zc=IGGLZXIfZ~%=>*i_>^D~QCGu$mw=BaUfcGc#xv5|D)WQ4?9lS_AT@%mLBT*G7<0 zc}{#T(uSFt%)!3IMY;Byi%jS^yhDy?rdd;1;#m?A|BE~U=OzyP_BQXSJMr4nlo$5tPimRf0KE$GUO9&L!e`r&}5$~kPfLx2X>_R_Ul+I&#h~Zs1ANZg`CSVQ_24fk6v#Wl z&nW#qg!x>whVWpwkLC&Ri zK$)afVo!oTC?0IQc~W%@YpW+1#%-!%0U#O+pbKBc5%2(l=8Hnb@EmalaOTH~=k0_J zgrG!66*tIovN$GrRKp~ZgIMy*2F>a{5Hf}hIUHW|YkP(+d6vQl@bHDBLhJXqm<6;Y zT>??U1gvr{ZzoU?^?1ta=MX{RyvVL4@(rJ|jq7k-}f_! za}AXjs1+CjFZh~cfS0Xf@Qmp6>x9g_35Zr8bkA`j*fr*$F!CQA4yD*`(3aC`0l<{Sd zB^V7iqxXK9L3mU6Ge%dPZos8hPEk+ofkd?jQsg5a3@j#Z$~14KW#zuW{BmDnoCiK? z97!{!fR^q{8r+xQPSovvZ5KbhVhlxIa!3iqe3s6MgR?o%yc6P6EfzKkX!K+`i#?j~ z<)eQu-!~s{;Vk2V%M}h&2%!d60=xh)G*_{`c(>6fm&XFxSYyj;%7oYARKKYw@r&eLrctx;GnVFA$hBH~K`zjb=aLHoSLA|ke!xs3SWPa{N-p3v z)UT*#5Gw=*{w^2e4Gdb4i?j-lQse@nY%dp9dp~)}?c@UFn>e(BIEdwdj5xm%2jCPx zfk=s{UF^(1_-qGpXeHCB7O|HCfp(FtqRY_G$Eglm&uZf6+6a?d^wB! zj94)gHdf@YmcWX%Z9%d)DzYNQU`27C+!kX+@ny-1_PWlBP`A#C3}!1U66uurtzsvVoMm7`m#AC+D9$G_=`HQB1LN2S+MlfHg z(o`Oe5d08kl>>4D`K`JESXGbUs=9$2tB3olIvmCVWSWh8#pN8}@96YwOGhK-@r>Ql zBFdx*d<^ z)($ySJxl!sbl^iv%1X(yKzct*QY(Wy0JkA)A5L0s+fo-EnoKchuR&J=hO2|Fmo{fu z2$-GD3>PEwENl(}Rzl6lG(<#E#SpNV@X&~OBmv{Klt_53oh?dMSr`HhHK&hOjb2Vg zWRbh_agylcBsE>`OmG-N@Nu*#1@1+cCqaVvS?Th)Mv200+X)^lu_-C0>y4=D4?;m9 zbUAI)Sefcd(Bt@K&N5>*GuJ0K2cy8@(aTAs)RpccPbUcb>WS6Y zL8(J7_(@!Y^CNoHJCyoXzv66~W399GA0X9&1}G@l5CRnyqOgeQlrYcoKg<2;JUkJ; zO=NUOtH9yF5%xfdaGXe@oS?`mEF^_KbmHkl^Nj35fL}pZ3>KLXAv;42Co)sZUYe!| zQF_S|2Ci1}k(F1nycF`L2{~_B#FA%QguIr0$7v&kK*%X4+|!tFFKwyp8@5w7U4m&2 zIRd8vKN*=?SXmXqLRlPA0~3igTqmJyZ$NHElY2;ZCdtO%&J$7;^>WNK@Yc??*0d8` z71D0#A0_*l`9RI-v<>OvAStj9L5#=j=!a4v7>@VjR8`z*$6QEn z8t_Cbn-xMfq7Yi95XilA6s;5j!3}jxA&_GT3IW8}V0j_DXqPF)Pzc($V<-ryB=qUwjWVH@b^WEG?6YJ8E_ZUbJ6zyF&sEqy)QDVWY-6q+e7#;w{`7Plc0# z)kDoR(Zd$g6nY3~uk`?@nwDe3fml5L*q>TLEz@*d;Olr-LViuEfuMG>O zJZ;ab=?D)}CGx7m#M|-=}uf(4r>cnYV_yi$}J-p zVI0zQzz_b<{1fQ@Ffa7rI}q$=uHAH^u#EB4_=P%8;b41%j8TQ}w8kxI|e z9IwNv2_UHRf3{hL5ISOog{@35w(-TZ8XudNf zn$tuyy(KnBhhxEAwkqSv-Gavcd1b3W^sB;G6wNL(FBWzJQb@@}l?6A<+?at;NO*FU zkK3A(H*Xv;LSVqz?4o2yDT1Hg2W9(nUJsf?Y;f5gvlCWE?0&kbSb(G z1C}68a@VVXr6p4Ve%ydXXR5@pb#mVD3&{M+t!ea8G$CNA%2~hyJXTy%4XWTNunXSs zAiJf1Vq7(vmjcfguz(J+;g*2K9JnfA>6(ASio+hm+8nTSgMU&&Ex(%~B4RNj(co-o zl3NQ{j3MG{_rrYWqAuipXLd^~@E|y;+dXUv+gVE^3m1D`+{fV;cPL zH2U9=UNe1^G4ToIEYmEmFkypHzyiswkZ2Z{+KP`Y9=9?&oVg=^I|-`S3}2bRu^%?w zRr)MezFAudk=9ue(<3{>6pC2UChTmw+Wc#+7^ZqpXSJoM$GXpw*8FRgtNYjduKa8A zdLjfU3I4UzMOyjSl_Z-HrngdW7?l+hb?|~=pic767Qk_TN4G}p+;{iW?MGgZUiHY1feRE8XcyfhDcFXd419f$#g~00h$$T$j?@$ zfd+69vG+#qJPo@=5yZB(7kjavW@Wf2O(#hfQ2a7Ko2F^2^s{~Oe10hvlm!%o!-$%NR7Gw_2Bsv74HZ1c z|I+dVdUXrAj2fsdm&nNG#`Z>h@Buq-I8VYn_W^=zwizW<*;YEugEJl?xl|sznJSrU zaajBYpIbPo>={OkMz{R9~y}cptILS@AR}SUJBX4s>d^8Gpkpg z>AKqx(7R7NCeQ`x$xeLoioE8J9+P;YC!ldzPa~H zx*vEg$ubE+P@!-x_F+9zssLFpB09n(oXLm{K#L%hlt`yzowWDsYEWYl?|P~i-^^^j zX(!j$B&KReJAe-9Ncp;0$j%pAfR@k>EhR;cjPp{VIpfqHe$tpuu(1y`Af2j8OTwna zPgz^-KL{n+(m+K{JVTi|eL6uUVfU>i*|IGS#8GD|nTC?UzO^Jr&>=x+2`d@Kq2V+% z-L4TQPmTkhoSj5P)F;7~4M}l-*rAY2BbYkDK2I%wVZT=p#PBsJvi;h^T%Yb?$AOFiJSkS`ptWgf%fSf6%Fk zUADL4L7)VMYt!#L7LFBnI&kFdm#W~ z!2_zLJzIcatQknoGsb+_ia=`>;S!4klJhK*JQ2xNgo_QtCJEF{Ah|UDC&udX-j_Te zLR$Faxj|A{BPV!p*ZA^8O}>=tp~xBn74o4#NlhBy#uf=AFfIN@nBY-IDs8HT z-GNdq0kgn`qR~<>4J!upAyvr$WY7gT8XCt;x;9`|Wdlq>xkw|1vUX^cz&>HS!8|~PL)yKD1xPkJbQgMv6&?JE8pVdKfJuXsW+D{s zN+a+o5vdxdOOuM08ccEVtO;U>ZrKDPLX-T7O{UzcJ9S}+ndn1F?FwtyN^sVU(=60mu&;MHSfB*@lt*@R?; zS|t{#iRT3OSMTju7MVYDQ^5@Et-MIYwGOWY2LabDOpTPVbbPYD&~E@Ud==3V`Ab*< zXyYPMS2V_{k|27dtkoiT09@54wWstBcjV|4gmKpkISlUh*r%lRY_FmYd%Ir|mY2bi zKgxwwb+S8H*MM}L_-7il!C|MekmygXJfawBb1k@}oeIyGe13u!nyW#MrK%t=%T^+5 z^-Ub@&P2`b!nc7{uAz?w+cQ1`M;1%ZT1TDEsYDU-)AS(-A`A!wcAdLZ+lXs?-WP+y zqIdvuCCz~~3h+!cgVQAb1RVAgj*Atf)2)oK^ZoHP8Lg_IHg1Fx%)UY6vH?D6#q)yy z79?7>?3S>X^UvSR$K4jgUO?#2B3pe+4$UUPQot)ovf7MT;RU zf9hZV<(B#K(|n8}2{blzA;4^0-L4#iqcvMdv~2<84w*N-CI$y-eJ^Q4xCW^PY%vDh zLS#U}wDV6%EpHGEu#}91vvg!pZRY)jrvrq}F%}86AzE@PYM9R+(UJ5~X~APc>#JY+ ze9@Da0}(0=F@5d|nAT8Xi12e)5W)e7i~I^<9lf#!jyGW))PQ-3Pau-j32VZ(s}QlP z1ECBB!f!-FR9_&CHiJ(zAh7XT5OqU$njS0cj=dgwKw^@!04eIPP5%oHXLB>p;YCTQN~ho+ew|sZe@do0s@UX5Hxr*jBlay13nND$q4DI zIFgX3vYl(7oi+ld#L~2?2^~`CG&*3y24$qtB?YVHb-@J@wAQh(LdUTQbsX_%VlISQ zGYg%bP)BXjO7Al=b{>2cDJX4|@4W3*&vwpr`7h(Yz5Ms;<@MLD&aTZat#kLCOLNDU zPt1PX>g=)gxuqK~%TKLw$&W3a$WN>;udL|D@#RxX>$9tCF6!Xs|&NMM=vkCymozdZT&^gJxJLOW&XeRz63swqsqJbm|Mq4 zmgP&fTej@Theq9Vk9@@AWE`6~b`m=W+0gj~SF z0s*odA+V5xg%BV>0txW#W|MF(3k&)Fuc~{dN0t?5iI?yD{Z{(bba!>tt5;R;y?XVk z>(wW4ZO1h|HdaSVAZixYYOk)D7(`ZuWK%6fT|fiu1?TKC$!0InObwq&prY;VU%T;yW1-F zY;&?)D%A33Su2*x+JS6ke4sp;*G4OqnyD?1gwZy?Mc!qOycgLx=W2T+0BX0nE(e(# z?=oj@&76@rJH5TB(`Sygd)>xsFY@aMHcYF14eJ`Vu8~s&sTtb->{!hl(msZ=gUHw3 z6J!#u*88yi{+lzukJ#^@nfAW9-(!Z2Pm3bTnf5+xzh67^`-uI1yY>Dw5c^~a#3?9n zNCVDect><@*I_&>*eYAqsw1Xmg4IezemhnwO%7?-BR%D_)7x$MdBhR#Q5ga}?^~iWHvuqJ*1WPo8{tI=>Xyms-i7G zzNI`D2IcI5;r(VF0^x0V*M&4Bqg)8~ANBiUghrm?copd%KzgE$*UOBx;ksmo5DMu= zI1-J;6UkIImoJ#b%_X3H;EY<8qn{mv;42)`ay-^7P>1!ng1mkgaFJEe_UxRjj+Q2d zhH4WB%Gt^OReQ7&wcDN@r3rJ7Xui>jf_YFY)vDB$IS_H7G9H6l}GoERQG7b?C6+T@M{{y4rzW7Pq=;*d8rLP zi!1g0Be?o-ZO%zqth{(<=lQLbS9Oa#c3x_4`n!BVkfCY$pWVMp@uVU2(v|*RtF6V4 zf35#g?b7~!|3&#y+2s9-J{p{;jiaB&0e=qC7ZC_Y{h+Svh91@ob zW=`y_j%e=zOya*gaa~r2L1IJ@*As{*ny}xC{Nf!isiHyjaEe4G{=k?w$hwiI8`oA` zrN8kXQ1~nVc98nO1zjm zRJG*>i4xMMvRZy5TNZ*QTRyZ-8wL4P(EpXuf~jf!l%;}_Ks0;UeSkx{(?8=%;T=kNCm8nM$iwb6K~KrM7W(h#rz3SlimJxxvMnS+z%(TQp{ zH)e|Vd>zu0Y^QMpW@%9GrE6&BKqVUGw9KbI&{c(L3+@eC&zN zH@@lG_o=OI9i6L;=ul?MwpZ-fSuo#x^)=Uh>Yu*+mHYqs(F;HI=|w#ruRqW=FP<8@ z?IVwU-{+mI*@}%P|KgeFUij+$s+bs=zVoh$(vN=p){B?D)}^QeN>PypLsOS6R*d!@ zwa>T2wZ?U-+O~S?qwYSnPwn?c13Q#x%aMey+wYyfE|XI7UZ39W>QxrGq)kcnbl0Hj z_xL=Uw3E~pUtAe-E%e}wurs$MBCQcm$m@5nR2F;XsfW)FDv7n;)!hqM_H@nj?Lbkd zwl47a-CMmU`D%g98&|v6x%}?4+>$G(xB{>A%8PSLwtD?jAG&y1Cg69s&KYw1-CaFv z)p=8&Szp-IveoC`l3BFXyQ_7Z$3OL{Wo>@<7XM=9v~3Bc4W*_${v+`Po^{INbENig z>*Y5UYk{e+ym@=R^$I=M?Z56O*JY=@`!hpV-hWxjvs%5-z0$wM-|w1p~wg&hsspdgGDPls9Z{pL@lbOWp3N@2+ue?3E@5l!dB%WYf})A(wRI z;nh=*u2H2UPfY#Cs_m*@l`ro&b^C^?FRgb=>N&1OQTa&QTD8z}u7B#zW^2rv2Nyax!tnI z?e+LN{5^pMEel(NZ7uC;Q0eTP`;%0kZ~sYhPgtto4i0hk(e)sVx|ht-eB zA60Mn{!#vu>(BCEloxz=96WT@H6PH=JO8SyukZO$Tl?wT|MXv>!ILk%X!xgBTyyOW zZ~4eQpZ(ld@BjKYe*DzaFEEueKw@a!hHbC7=!$Fc@{^za-2LBp@S&%kW_9*hNBnVd z!Mx&zo8J5N2OnzfST(dRv+cYKF1&cSU|w^>N08;KU;oKdPe0$^+;y^$$47Z1Pb+UxH5)Th7rPxpWA zdmY{LF1Yab|M8a>rp90WE2)EORP`v5$L;CxpV{8&Ime@@J$|3!RXngND@u#%QUY$NZLVvFXOZVTkL;e; za;AEUG60Fz;cjmkQWvin*2dLKR!lwMy6kRcq5HBwE3fo)`{w(I>@ER|EOfupv&OZ> zzg7itC`Mqdy3id^rap#OgT~Zvz3Y{BWuq(UUE{j!g^u~&!Hxl?x4pN0>T30}cXtQo z{@vSLgRXTTwE4cNFD$FJOg*~rNYFL)r0>`7Qxd)-7j{kkqj%~@bJr<;chbAX+v2VU zmME`MU+J5=e14C=+qYevdb9iXTU+L-#x3fRAFRajSFWiK1&{pBBWbJMczcaH^#x^- z(%yQMptl76a3NdG3R!Okg}%E3vK_0C%zl0q?n(Dx3Up4aDc#`O!!DY0GwYnEEp5^E zEPZaRwq~_{^P19!_N4i zz30hS{AS6MJ2pjcIeSU=$z9K1lHI+t^yKdM-kUv#J(NAyeB_pM*$;ZpV^7?2{+&PV zebqBhJ$b>yzsz2!v1c!QUOIdcn`9m~00jt^Qc2nx(B}pv6P+i^lDb@4vgm@qkk2R0 zS0x{YCf6Eeop1GgNlPGu>P07c{PJQ6$AC(*>O~U2yik(m6vi+WepD4{iL6KgdUhd! z)FpRgghOh;#=c<1FE5eSA#V$E_5(biDlU+VM-FhPz(E2K{@f#{@MM>=SlTMV@3kUH zUg<1J#xY(wN%jRi+vG)nCrODm2_?D$QlC#Ms*)Q>$qQsv397Jy%WkP%Li?*d%3^s5 zjB}F5E6D+$1X7i1@-k_^qRKwWt^5!O09y}1l)Y}hEa^)PRrjmPN?!|>DP(sNFOUf| zDa$u1_yWd5g(>p=o0#_WX)%UUmog8vb+=>HP+{O}CY`LF z1!_XwL-IW4Qrupz>{+6|O<@T&?3LQ2ZkN;!$Q>N2t03KsJR4M|{srZYd)S_-=ZW!z zZIEAR%@)a=SB4Tg z*qEPVK0hcFLcJ@w;eKePPLfi6K~jeMhnBW$>D0rBs;<|CO0&n-c=emhcN#@ z9{Rfn`RU%iPPa9X4&H~b0-*(g>OeZ~NeCokobmWLf~`g%-MmsQ@L6hggy#PivSOO8 zRZH4lbHXfVp>{)x!KjFtJ0;V$ojRoDMynOrHDI?nI0AzS3?r-N2DweK3s;&OycI~$ z9O~?IcI_VCoId6B-Djsy$=Jq(v$<(^JVR9hl%E&06jKlh~k!F&>v0Kdhe+gLBPLKeM+6abW zvf9u*qezyFNwYi-yCIn+3g*P9S;&_rCd@pyQJDu%!8wqYNdoqz`%_v94OVs3R({4r+s23RTwR!G9?&}F%^b&%zCQd@P;tQC##2u z1_nwKV~4aN>@k%?==^bwCg{kW#Wa50ER5F19q-7VM+q?Z=3uG(vC(l%*80uR-jG%~ zG>#71r%}39n3|87n%zRL6_cYsED8k=?2`L3h-CycP~@UtKFzvmoodX%@@^N1$3NX=B;)UQk}8R4aq( zbG6DL^Prip@kty9!Q%>O8*0;h8I$FKg1H}d5Y!cwL;cI<1mO%||9*z@{%6L<=bxSmfl`=^RSlOxqk4EVzx&CgLG6UwyML}-A zTLvP)svL050%#6D0FgI`{HP4%cESHfXoY{ zmC3OzI=E2-jQcvuA)VhjcCW=V{Sh7@f8M6f=MWxfy-i2J@*iL0BAXbcjUYr3VhCh2 z+FUxgzHn$lOHUly$cGZ5^8$_9B~fuQcOu<0z=Lp@USGe}PW47QFhar?LL(2ELBRN% zkY)o9`Pn4ejPwl~mbN&^@v=bir4(9bnHa#L0g3+^p9pONINb-)-k_;eQ zA;F_KdPnIACdmM5Q(BLrI7-uvKyaxHx~Ft>Pl3wNp%ZOFI13?Xcm&~bgkK}PfZ&I0pNl~4Fvxb*@1zP+YqffaojG04V>|0w6VV19gfv4Qty0U6h|wS2UDuWB zZ!G3vTZof{3dYW*C0#MHlS;t}g_IYqdT5gH(1c#iSH8HeNje1EDc0149HeSA^boeK zp%Xxt;_4Ww*^N|C(4Z$kLSfXYO=NA&5gCQ@L8=ftWxC3O{7yyB0?kgfRl|lxi1lcP zDh$OJU4V8n%Z{e2>t@Y~x!%mSlA;X7)1fP~bd&1KHaApTwry~y)t`13Seh0MA0k`5 z3&a~dQvWu3Xj_w(%XXUwq2^OBk!Hs-Q0**r!79jZDqBBe=@}I@w(HP%u7ouWq4beP z%R8S&qPjN6cH1f-^tuUNEZ6Ww-x1~%nU(wNm_)CLwzfT!NCkYuj1Q}L0>=!6jX z_Kd0wX+cG@Gb}6`4ORzx0AoEgM{p_USqobIHC3t6O-<2tJ)I-cc(O^nm>nn9Bnb7F zoQ13!)p?=0j2+f=5{yHWKW0BWxo9TNv?f~XY@(%W>^i%aSSPi(=_3v`G`fK)1MN*QkyQIscoywrPgnh^L z0oJK3QNASoSn;xtxtLUted##7m;a#lxV)}=SwW#fXMKcA9U}`c$bRwcwUUG_x>}J0!P4VIH9GPoT9-R6Tkq ztS3W8kYaPtsbHiTAXAkaFJd8Gy5|iN!*@;uT-wsIM>@>*$cOQx9DY51U5Dw9?NP-A zE@%$pO+5xr%!xt)i!Q*dc%)Rgs3Gw}1b`2S(rh{BHZU0A13cQPU@~bAFAWwT9w~6@ zx&XsHA2kQ)lfH|DLcr$QNyWUbj#6`CZgdzBNuWB-n5So20hq3FMv_2PT_M!EDT)E*!d(b^TWJ%HUB$zteML{*$&3ZIccyLA8aQA z8Vg(xln)0tF2|N`__OX~SktloHKfxdr0wFn3evxO@RqA^{F6kR75?i5#eSA4y~f0T zEV0p9;$IkdTmHIe+d8%nW6udTrQ^HhG`@1)hb-*|M=rSgY|t5y?*@bx4+|=8$bCZo(W#ylJtt;^7KDS z!4-J&5807Ya5*M?C$7L3=I{bPdmlE=T6oAo;7=$J-p}55<5lmt=7%4D%EJ4}pMUH2 z4_*7CPd?YcyTeXzU-DONi;G?ocfLmB7E? z*nQZ`u}n@Q9(%Kp4EtqpkBtkQStT&5BA8WaFsstctV%PpD$|%1ubY@vv6=NzVpb(> z@eYqB-ciz`=1QaH%I4;p^R!fRMX)-57JT!5ED1YnPIbo?A*vg>qfx0PXb&CViE+ZJ z_BQI&8LGDV*Y{oV*6)4uhj;sl9+k~c-*?4tet*q-KlwDLM^y8fx4-p29(?@MzvUgw zt2KYdY63VHp)Y^0TyqT6)6YHd_J4s5at_e8*+$SIN3&&SH1SEk7y#&u1|q4z9qLSj z)$tBg0CxAo*v3TLU%Go4*n`t4Z9_U?gUWq^L;rVBnPuQTVS~zDD)~RLbQ@XdHqd6s z$(yacVE0**HX~G*h2VT_`clQ7aIyUhG@@3qC0*Gh!~kRg_xT6f#1=7fk`i3cJ^Ij| zvJLn&f$z@3TfQ+J*A~1GspvwfXd4_))l2FR-nCirVWfZ( z%Y_Z?AXUlF0Qxlu$CjkewrHR-Y^9`Qz5q%*!RZ`M_WCn6>&c)P6iyHBLqHy4F^Iqcpjp@ZuYR5ntYVd?sW#Hqkch8fwf4clcOXw z1#M?1Rs-vU*+{69gEQ+6Sm3){;a)(Tc|#~dE}$PBiU*z?azO&C6BPt|NQuBv)PvHL zj-x{G(9(;Z=*7^1N~~T6AILj_iU)kJj|qnlbi@t@3{(!0a7q^i_UfQSgoo^1y%L^T zdKEdBA#c~z96T{j#XC)tG`QmkE)^LhI#+|gS(eUUTb>Nw-iPGS+j7lU*bDBh~=(u#|Z)80BM}e!I2A)p z6Q_34Ysa*~4?upwsqYSAOpwI1!HOnMmD0Ds^h5uoWgT^e{ zdKOImpf%uPyv4_U5V1)Oy2l)LFBO@{{PxIXX#%w7!QX#64#-pj}nV7M$CLu{+oI=t9Lqd|L zx1L3N-Y>~?;s#ipMnz1RN%IiuNf5FDQbKS#-Quh&&@ZILNL)Uj?p7?Uq(?##Hn zNyKeUC`l2AlWO}#oSslLx*K5UgGld4C|kwTCh??;r+)DSHD#4DpVVBsf%FREPJx#w z5?pMvjNG_t5>2ILA+G{iVZel(qB_Vy?!8PV{YplUQ)(`gfILVmw9qFNeJ+@B?9x3lXdKzFcO3G$4*&ILP3N@(f;gX^%gEF>&M8h4@wugYjT)Qk%= zvL@n<;NWN-B-odO;Oj4GoqR<~>2^Ms;*}1rv^AN)MyE!EQW>evZ*+kq==R26F5 z*uJ!0c0#FVzlNj+x*H6{%_H2BCZ8xv!*G+7M6I+A?T|N`CdXx^fAQr|1^gGuXWu$)*u?(hjG%~E89BGUw?%lf(NU%6|4_C z{ZyCR!dbxNE_KRu!*k?F8)0T z?;#VwAFu^|2FRbhRd_Vn}8%mLTo2 z`UKt`2;7T_12hzS8p7M1EM%~$ploO>P)q1mJf?eTqGZpT=$u1(#6gW{`?Qy45PXn_ z1h6JlWS_xg3iewWdZ7Y)EzCEVGms@Dh!dem?U6}ycfRH?u|*XxF-;b|+7b*Ac(T`s zf_Ud5iFgs9?xwfY!s4(n3wcmKEir;#;Fq-5L*qT^EWIA&ptEbZ3ujDJaPWYz#RS)2 zNMbzqqF_IH&%+o+1Ow`fk?xZ`{W|GxTs8HM&Dxclr7K7+Q_^?qmju@?47S5=sia}YN^`AD zA!9Ycp#r}T+a*tk7G0*U;!f5O@pB%1s}2Xkkkf>+l70Yjt~z{*LCYj@0fJfa^f3#U z+XxqB6}wwcu(Rm4U%FY3OF5*vP2hx9#={goKp_{6xW&xY#_3I1-fp1s4m>rG_`^&{ zj37V2+yLK!9IUcV`a8k|1NBT89f66+M(HhNE)qEL#+7fz6oV$7uo_+xoJT84H|zXR zQ{3IEFSE|3ZW-&vXvwKkv)pw>_*>E<3iIM9ppw9&SrZmM0yDM;4G zfFeCVOpWCmV&9}I=;#4L9Zh>TpGRXtNt1mza-NdN8!Qx#b<#&E36&QiG9;p;Wi8E3 z7AWy1%?*_t9yI}=z%`?b+9l0TyWvDkrUCaliF(U!0}0+|`Q)EK-l$t=8NmHTW7Fs!u+b(L3tCWghaYOWuKb2{0t!q&Um-Bn}VW+A3*ytfJZG zzZ#AieU#b{$%-)*aQ9KHik;k88=@jzu(`?miD{`T)hb-LgN1o=v>ZT#SW!A81Wt4& z=s1tASK=8d&&;Wj7RPAw(ly(kJJ~Q%L97$dx+Q9C)f7^O44KU}Wr2UdzV(;XgaWAG zs2LhfrDmW-3f?Ka9^Cb;!=o*lm#X8aHR?f5&4u;$s0tii77O^%oy0}Kp~mgHzzWbt znr%@xrKLXH;NP+bkW~PLl$QI@zU;eeRoZ*N2G%xxMLd)(WbqYsd5FG?9Ii}cCo3bR z>M*|I-9Ks`Fr=r}oXp0DU$r9~o;iVGG(J(S7zRG6x3?-75&908Ao1^kjTFB2t3M5k zZ~KhIh@MD>i-jUQoaw2U8BJuQ$ytg^(udCYzOfP|bPT=m*>8-0M{MW~43bnJ#Tq)Q z+$b@TC}zWnl&QlpT0EbPXXNrF~3yZdHt+fc}^Z74)3LLqFh4e>HUp$u(t>#K$JzHEhGUL3}7xf%+chfl)o zjU9cb?e2@xH_-eCodF*P=!Hl;oJ@ubX1JJ9KgQ9F#9uO+*0-^t&^Fqpy^HUaec911 zm~k^7@Gp;)Xx{3cjx;y#D+9w9zSh%1kV`dm$BaCdyh-cjQaD1c2OTxE7 zaG?;V`cSAr^)GiDhoeR@Y8d$#+zA?Hv5@hcHVe1J_^zM`oGcjOyyNq9&{jAJCWwY} z$)phmX%#cxi(eFw5hEUUe7PPDPm6G{b;cPf%?nP2Kdq*PsF989h8fQ1$yH@B)3WEdON-!x{J8i;-_WTW?fWL_Tu7PuY%;GWin

    w3M(UvJrNQIB+&? zP!ONyPpeWg7K2ZlXgrZCW{qqtndun$>kEu=UYb@?JQj`?QL&tvkH+;#F4OtCzrG}l z3ACdT+JdOvFm^)>^9>EtN)IQZSyM0QCUO{sVl*@7_!WX?YSm4|RvT(~RWr@lkH58zvNFkq0#NtsiTFe`8X`ESf^k$Nt%`XnRNrz30M6>yD zEN6t_MmL3)&@+o?LoaLy#p^$LaO_Pk>r0yS&dbk^2z3HHXbGU!+Mpk4uV({<6Uk(fJ z-#`;Y6fc-LJ*yX!h8Zz4{eMer;4NEEg>yQ{I}*+oa+y_sOKc*AIGk|n7)7)B6tu+5 z>g&12uM3S|bQN}(Vpnr0glQSBqPGQkgY>bCe>$2n`j9 z6vL3^g;-|o-yEfAB4Q*{5R=gY`A9Ej2L9$K#f!OIBw0umjY29$aw_x?(b6$3GqEtA zRgy5_TrY?X`}5%?0o1moTpE8Vwuc#Jo!ka-3dX29slwR}FH4v!;0Y1?Ifd)M0ykMq zlqhMmh_WF$pywB2n6E|kq8>@65}Co%XQ@#0l#ON+4qYF%3n<5V=-!D`J{NA^I15)k7SGs|N27eBYYxXNG793B4OZg2Iym%aPhB)ZcJB1 zNn#h!M39iNVyb|FHWN9Fa-u{a5$KrlB-w<9vDF;jtMcY-MB*_1H5nmjsDMrx&E<3X zB&Os=j92vm4CbXUW)?WJAa39!0UTCfM9-b2qD?lc`1su*#aLW+sGP{2^2kxQg< z1-%g0V>m=1WoBY$9537^+a)((U;s`f(0a*uA(nvg0@6F)D(0>5d@_uNO0>eo4W_Zy zW3r3ha)p{Z6%lVzU6g2g>`)Ad842>{0N)Aa;aqJLjx8oC5qMUJBjhI!p+3a|9r=Ly zs*#WDVbjR0dy#A?iug*;MzXm`HkFK;F#~q7_2jStAUDtzX*RZ5_?U!aexE0)kb41s zKRj9Dz6GUDewoOQHEJjar`Lm>(QsEGd&h8mLk+%LB=u}KqF_z#DrbLSaXncmYN%smzz7e7jaAa!9d(}-8~nmwg>XK>{Zv>UDHwNVA^cca{wwU)A~Aej zC<_-biOGy965OP@82!23O!dHRK70*NN(&Zv4lk5+#w3GDQkcn71Zku2!y98RDbCzd zf_bDQ_PVE-Pd0Ssmkk4_*co9gSQru3DjQMOCL1xB+#KmO7;Ry-e}#mf;@@ z8A=DfShdorjyx{9r}Z9MAV|AA@b#;e%Hx3Mb>#5Ty$?Ck{tnfVH_+j#=WTJMZ*|-Y2Mp68g~;e(5^!3OCy zdrFaJ_LRc@rA4fP%;{`i10i3q(x+M$u-&$2QaT)J$T~@=0>2Wq9GpGWa_}l5j{X|3 z-7knkgzWeM2!H{&(yEyQmQsfZDRul%F*-w-)yV(qMcK(w`HbVwYPn89u8?i_;OKy3 zxu26sv!HA^esE4qtXiwDIR5MwHdfAQ7?t-{T-#nASg;Md;q*tim>D3oV+igw=*TJI zxMP5BJ?rWzs9(*=5jgwAkztOGZNy#RtT|fKyYr=FK2|S?gWLva3C_3cfqmv7f1(bQH$p>Oo92;+(}cq#PhjgYGxD_ss~K&2Zw&Y$KK42sy0kR9bUF$z z0!i4o9iUn_{dbWf2Bgy3*8g`Q&~V;9bX>U`jqGqb6r6_pbZ;+p(E$r@!!1zA;U}hJ zTIt6Lf%iveLt@y>AhaXPpV2W&uS*Qksdc|Gb53b)h~q@14vw2XyCTg%xMQA%%T$ZP zsOKCjJ#$y^m1)k>N3Tl{*xqEi`)5sN`<4Old7t1K1N9C7EWs%<_ zEo7vTaFv8XC&66=It;WD+{Nf8&J+g4L}JiP@R-66=_V?E4d%ixWg_*2ekmI%DD+F7 zQAtH{<2T8Fd0uEK$O|>agWoVb_>iJPzft%SZKx`wF8P*9__`nfwdxq3IllA(xkrr0 zbGw?y+##MUJW=5P$NesfzvK$zVgn32UtWL#>Oc+^l#SX$_* z%#Bkqn9@Bxu_KL(xlf`r^(r3)kgCzZ-_6`hDE)%zFq@Nkn7dVA3u;AWWAeA){h}i@ z%@Nv*bQL`3PRoD}VN&1&%>7tfgMtW77$1JvMQ$wz@;L83Dlm zY6Uw+>^_+>4WvH{x?3Rzn3wcG@=!{0Ky!P*&LFn9+$Tx$ylGToCuJQf-Lg(DSX5iE z^t$H=F#yNZfB`v--;{i*B&jnI;hdu>q3hV4=gYE?T>Z47sDYNXbTdg@YPR4CG+d(R={)UcA#InrY2kQY^5d$p0h^D0>mLj>-!XPW(3y|(v{5LK+F29OP>!i`Z zB6O5~%(S}CKu0oSe$ste?=ih6E<1*!%;j!Bs_t>|kISgm%W+JTB+cw9v(Mdu#NHOC zFHFkQ=kBEV)+WW4UaYXGle7?OG|oAshw!HqjwD3eEt`paT)`dk=;ZCiZ zom6kL<}_U8W&JPJqvEQ1Pc?&U)_vJ(o9e<2s&nYMYz?Rm_8x}=OFoK22MeNA9MNp8 z8z(a{H}_=8)A*R|M;hVUa=_h&x4woiOP<8fWPq0%u`I;P^V@)dJWm*4@>^uEk0Tx) z)uq66ucMuAasb=uc@f92o!;3Tgm(I7a?IN4pV`-CXJ8iJrkyQK9y>w6mgs4n$qQ&_ z+l&rCJKLLppiL|wLH9Vx)sC6IU}(e(gI?WHCAy)ZIlnZ$!HW~Fu)7^993&J|jv6|= zUPQOb7=PTWC_$YDj~XEyT7VZs!rE{|mb!?!7f`ktM|%#tox@&9iRvshEi`Y(M>4~=I%#|*5i@zyocve4^s!W)~n%=<41{mo@YMg(I`({jydw! zvY#hiKuIz3vDRY^1U%V7N(M2{T4x>=c%ns=i0Hvvr-x>F=P+FG3_UiU-a4Mm8MwUV zNf%R69AU@2FH!jN+)F4oHKMoPh|L8xOJbg9DdnNwbn)JF$YI`_n$??|Iny`PB}jlK zg$d6ys7#1$C{6Mkp>ZvA$N^03!&qvFU|Ju=tUiWGeH?T81g7*!%;-~4B!@7cUq{pV z^`vZUpvnA3n#-R|YQ`p-$){-|znKQmQ)u`+m6jSZwA8SLG{4hm3cr=KzHKyte+8|5 zbar|IN=dF9m|VFD=fJs_(qzl!!gld6pN8%OP#f$*nAN5jNV>QoOcZ4@^d&8hyZ|Vh&rk?he*KIei+d1LD uT!~H3P8(XzIo5K=F&x9AXUls_4Tte_b$mJ$3TypbPsdsFW Date: Thu, 19 Dec 2024 07:27:56 +0100 Subject: [PATCH 3/5] Dynamic linked remote resource drop --- .../src/durable_host/dynamic_linking.rs | 38 ++++++++++++++----- golem-worker-executor-base/src/workerctx.rs | 4 +- .../tests/common/mod.rs | 13 ++++--- wasm-ast/tests/exports.rs | 2 +- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs index d47ce3958..0fe342935 100644 --- a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs @@ -99,10 +99,14 @@ impl DynamicLinking for DurableWorkerCtx if ename != "pollable" { // TODO: ?? this should be 'if it is not already linked' but not way to check that debug!("LINKING RESOURCE {ename} {resource:?}"); - instance.resource( + instance.resource_async( &ename, ResourceType::host::(), - Ctx::drop_linked_resource, + |store, rep| { + Box::new(async move { + Ctx::drop_linked_resource(store, rep).await + }) + }, )?; } } @@ -277,14 +281,28 @@ impl DynamicLinking for DurableWorkerCtx Ok(()) } - fn drop_linked_resource(mut store: StoreContextMut<'_, Ctx>, rep: u32) -> anyhow::Result<()> { - let mut wasi = store.data_mut().as_wasi_view(); - let table = wasi.table(); - let entry: &WasmRpcEntry = table.get_any_mut(rep).unwrap().downcast_ref().unwrap(); // TODO: error handling - let payload = entry.payload.downcast_ref::().unwrap(); - debug!("DROPPING RESOURCE {payload:?}"); - if let WasmRpcEntryPayload::Resource { .. } = payload { - // TODO: remote drop + async fn drop_linked_resource( + mut store: StoreContextMut<'_, Ctx>, + rep: u32, + ) -> anyhow::Result<()> { + let must_drop = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + let entry: &WasmRpcEntry = table.get_any_mut(rep).unwrap().downcast_ref().unwrap(); // TODO: error handling + let payload = entry.payload.downcast_ref::().unwrap(); + + debug!("DROPPING RESOURCE {payload:?}"); + + matches!(payload, WasmRpcEntryPayload::Resource { .. }) + }; + if must_drop { + let resource: Resource = Resource::new_own(rep); + + let function_name = "rpc:counters/api.{counter.drop}".to_string(); // TODO: we need to pass the resource name here from the linker + let _ = store + .data_mut() + .invoke_and_await(resource, function_name, vec![]) + .await?; } Ok(()) } diff --git a/golem-worker-executor-base/src/workerctx.rs b/golem-worker-executor-base/src/workerctx.rs index c666e2ddd..0a98e5ccb 100644 --- a/golem-worker-executor-base/src/workerctx.rs +++ b/golem-worker-executor-base/src/workerctx.rs @@ -433,8 +433,8 @@ pub trait DynamicLinking { params: &[Val], param_types: &[Type], results: &mut [Val], - result_types: &[Type] + result_types: &[Type], ) -> anyhow::Result<()>; - fn drop_linked_resource(store: StoreContextMut<'_, Ctx>, rep: u32) -> anyhow::Result<()>; + async fn drop_linked_resource(store: StoreContextMut<'_, Ctx>, rep: u32) -> anyhow::Result<()>; } diff --git a/golem-worker-executor-base/tests/common/mod.rs b/golem-worker-executor-base/tests/common/mod.rs index 31f91e65f..16e5a7fdd 100644 --- a/golem-worker-executor-base/tests/common/mod.rs +++ b/golem-worker-executor-base/tests/common/mod.rs @@ -831,8 +831,8 @@ impl HostWasmRpc for TestWorkerCtx { .await } - fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { - self.durable_ctx.drop(rep) + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.durable_ctx.drop(rep).await } } @@ -863,13 +863,16 @@ impl DynamicLinking for TestWorkerCtx { params, param_types, results, - result_types + result_types, ) .await } - fn drop_linked_resource(store: StoreContextMut<'_, TestWorkerCtx>, rep: u32) -> anyhow::Result<()> { - DurableWorkerCtx::::drop_linked_resource(store, rep) + async fn drop_linked_resource( + store: StoreContextMut<'_, TestWorkerCtx>, + rep: u32, + ) -> anyhow::Result<()> { + DurableWorkerCtx::::drop_linked_resource(store, rep).await } } diff --git a/wasm-ast/tests/exports.rs b/wasm-ast/tests/exports.rs index 733c5c2dc..138bbbdee 100644 --- a/wasm-ast/tests/exports.rs +++ b/wasm-ast/tests/exports.rs @@ -769,7 +769,7 @@ fn exports_caller_component() { name: None, typ: list(tuple(vec![str(), str()])), }], - }) + }), ]; pretty_assertions::assert_eq!(metadata, expected); From 30784ca075f41b149abb454671ecdd2c718887ce Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 19 Dec 2024 13:22:40 +0100 Subject: [PATCH 4/5] Dynamic linking PoC, still WIP --- .../src/durable_host/dynamic_linking.rs | 511 ++++++++++++------ .../src/durable_host/wasm_rpc/mod.rs | 2 +- 2 files changed, 333 insertions(+), 180 deletions(-) diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs index 0fe342935..ccc48ee77 100644 --- a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs @@ -140,76 +140,79 @@ impl DynamicLinking for DurableWorkerCtx results.len() ); - // TODO: add an enum with the call types (interface stub constructor, resource stub constructor, etc) - // TODO: and detect which one it is based on metadata + type info + // TODO: this has to be moved to be calculated in the linking phase + let call_type = determine_call_type(interface_name, function_name)?; + + match call_type { + Some(DynamicRpcCall::GlobalStubConstructor) => { + // Simple stub interface constructor + + let target_worker_urn = params[0].clone(); + debug!("CREATING AUCTION STUB TARGETING WORKER {target_worker_urn:?}"); + + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn).await?; + + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Interface { + demand, + remote_worker_id, + }), + })? + }; + results[0] = Val::Resource(handle.try_into_resource_any(store)?); + } + Some(DynamicRpcCall::ResourceStubConstructor { + stub_constructor_name, + target_constructor_name, + }) => { + // Resource stub constructor + + // First parameter is the target uri + // Rest of the parameters must be sent to the remote constructor + + let target_worker_urn = params[0].clone(); + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn.clone()).await?; + + // First creating a resource for invoking the constructor (to avoid having to make a special case) + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Interface { + demand, + remote_worker_id, + }), + })? + }; + let temp_handle = handle.rep(); + + let constructor_result = Self::remote_invoke_and_wait( + stub_constructor_name, + target_constructor_name, + params, + param_types, + &mut store, + handle, + ) + .await?; - if (interface_name == "auction:auction-stub/stub-auction" - && function_name == "[constructor]api") - || (interface_name == "rpc:counters-stub/stub-counters" - && function_name == "[constructor]api") - { - // Simple stub interface constructor - - let target_worker_urn = params[0].clone(); - debug!("CREATING AUCTION STUB TARGETING WORKER {target_worker_urn:?}"); - // Record([("value", String("urn:worker:2a174805-bdd5-49e1-b1e8-124208123b4a/auction-5f0a94f1-1d14-4b65-8e6c-3a8fa3c24ea9"))]) - - let (remote_worker_id, demand) = - Self::create_rpc_target(&mut store, target_worker_urn).await?; - - let handle = { - let mut wasi = store.data_mut().as_wasi_view(); - let table = wasi.table(); - table.push(WasmRpcEntry { - payload: Box::new(WasmRpcEntryPayload::Interface { - demand, - remote_worker_id, - }), - })? - }; - results[0] = Val::Resource(handle.try_into_resource_any(store)?); - } else if (interface_name == "auction:auction-stub/stub-auction" - && function_name == "[constructor]running-auction") - || (interface_name == "rpc:counters-stub/stub-counters" - && function_name == "[constructor]counter") - { - // Resource stub constructor - - // First parameter is the target uri - // Rest of the parameters must be sent to the remote constructor - - let target_worker_urn = params[0].clone(); - let (remote_worker_id, demand) = - Self::create_rpc_target(&mut store, target_worker_urn.clone()).await?; - - // First creating a resource for invoking the constructor (to avoid having to make a special case) - let handle = { - let mut wasi = store.data_mut().as_wasi_view(); - let table = wasi.table(); - table.push(WasmRpcEntry { - payload: Box::new(WasmRpcEntryPayload::Interface { - demand, - remote_worker_id, - }), - })? - }; - let temp_handle = handle.rep(); - - let constructor_result = Self::remote_invoke( - &interface_name, - &function_name, - params, - param_types, - &mut store, - handle, - ) - .await?; - - // TODO: extract and clean up - let (resource_uri, resource_id) = if let Value::Tuple(values) = constructor_result { - if values.len() == 1 { - if let Value::Handle { uri, resource_id } = values.into_iter().next().unwrap() { - (Uri { value: uri }, resource_id) + // TODO: extract and clean up + let (resource_uri, resource_id) = if let Value::Tuple(values) = constructor_result { + if values.len() == 1 { + if let Value::Handle { uri, resource_id } = + values.into_iter().next().unwrap() + { + (Uri { value: uri }, resource_id) + } else { + return Err(anyhow!( + "Invalid constructor result: single handle expected" + )); + } } else { return Err(anyhow!( "Invalid constructor result: single handle expected" @@ -219,63 +222,100 @@ impl DynamicLinking for DurableWorkerCtx return Err(anyhow!( "Invalid constructor result: single handle expected" )); + }; + + let (remote_worker_id, demand) = + Self::create_rpc_target(&mut store, target_worker_urn).await?; + + let handle = { + let mut wasi = store.data_mut().as_wasi_view(); + let table = wasi.table(); + + let temp_handle: Resource = Resource::new_own(temp_handle); + table.delete(temp_handle)?; // Removing the temporary handle + + table.push(WasmRpcEntry { + payload: Box::new(WasmRpcEntryPayload::Resource { + demand, + remote_worker_id, + resource_uri, + resource_id, + }), + })? + }; + results[0] = Val::Resource(handle.try_into_resource_any(store)?); + } + Some(DynamicRpcCall::BlockingFunctionCall { + stub_function_name, + target_function_name, + }) => { + // Simple stub interface method + debug!( + "{function_name} handle={:?}, rest={:?}", + params[0], + params.iter().skip(1).collect::>() + ); + + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + { + let mut wasi = store.data_mut().as_wasi_view(); + let entry = wasi.table().get(&handle)?; + let payload = entry.payload.downcast_ref::().unwrap(); + debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); } - } else { - return Err(anyhow!( - "Invalid constructor result: single handle expected" - )); - }; - - let (remote_worker_id, demand) = - Self::create_rpc_target(&mut store, target_worker_urn).await?; - - let handle = { - let mut wasi = store.data_mut().as_wasi_view(); - let table = wasi.table(); - - let temp_handle: Resource = Resource::new_own(temp_handle); - table.delete(temp_handle)?; // Removing the temporary handle - table.push(WasmRpcEntry { - payload: Box::new(WasmRpcEntryPayload::Resource { - demand, - remote_worker_id, - resource_uri, - resource_id, - }), - })? - }; - results[0] = Val::Resource(handle.try_into_resource_any(store)?); - } else if function_name.starts_with("[method]") { - // Simple stub interface method - debug!( - "{function_name} handle={:?}, rest={:?}", - params[0], - params.iter().skip(1).collect::>() - ); - - let handle = match params[0] { - Val::Resource(handle) => handle, - _ => return Err(anyhow!("Invalid handle parameter")), - }; - let handle: Resource = handle.try_into_resource(&mut store)?; - { - let mut wasi = store.data_mut().as_wasi_view(); - let entry = wasi.table().get(&handle)?; - let payload = entry.payload.downcast_ref::().unwrap(); - debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); + let result = Self::remote_invoke_and_wait( + stub_function_name, + target_function_name, + params, + param_types, + &mut store, + handle, + ) + .await?; + Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store) + .await?; } + Some(DynamicRpcCall::AsyncFunctionCall { + stub_function_name, + target_function_name, + }) => { + // Async stub interface method + debug!( + "ASYNC {function_name} handle={:?}, rest={:?}", + params[0], + params.iter().skip(1).collect::>() + ); + + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + { + let mut wasi = store.data_mut().as_wasi_view(); + let entry = wasi.table().get(&handle)?; + let payload = entry.payload.downcast_ref::().unwrap(); + debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); + } - let result = Self::remote_invoke( - &interface_name, - &function_name, - params, - param_types, - &mut store, - handle, - ) - .await?; - Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store).await?; + // let result = Self::remote_invoke( + // stub_function_name, + // target_function_name, + // params, + // param_types, + // &mut store, + // handle, + // ) + // .await?; + // Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store) + // .await?; + } + _ => todo!(), } Ok(()) @@ -288,7 +328,7 @@ impl DynamicLinking for DurableWorkerCtx let must_drop = { let mut wasi = store.data_mut().as_wasi_view(); let table = wasi.table(); - let entry: &WasmRpcEntry = table.get_any_mut(rep).unwrap().downcast_ref().unwrap(); // TODO: error handling + let entry: &WasmRpcEntry = table.get_any_mut(rep)?.downcast_ref().unwrap(); // TODO: error handling let payload = entry.payload.downcast_ref::().unwrap(); debug!("DROPPING RESOURCE {payload:?}"); @@ -310,65 +350,15 @@ impl DynamicLinking for DurableWorkerCtx // TODO: these helpers probably should not be directly living in DurableWorkerCtx impl DurableWorkerCtx { - async fn remote_invoke( - interface_name: &&str, - function_name: &&str, + // TODO: stub_function_name can probably be removed + async fn remote_invoke_and_wait( + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, params: &[Val], param_types: &[Type], store: &mut StoreContextMut<'_, Ctx>, handle: Resource, ) -> anyhow::Result { - let stub_function_name = - ParsedFunctionName::parse(&format!("{interface_name}.{{{function_name}}}")) - .map_err(|err| anyhow!(err))?; // TODO: proper error - debug!("STUB FUNCTION NAME: {stub_function_name:?}"); - let target_function_name = ParsedFunctionName { - site: if interface_name.starts_with("auction") { - ParsedFunctionSite::PackagedInterface { - // TODO: this must come from component metadata linking information - namespace: "auction".to_string(), - package: "auction".to_string(), - interface: "api".to_string(), - version: None, - } - } else { - ParsedFunctionSite::PackagedInterface { - namespace: "rpc".to_string(), - package: "counters".to_string(), - interface: "api".to_string(), - version: None, - } - }, - function: if let Some(resource) = stub_function_name.is_constructor() { - ParsedFunctionReference::RawResourceConstructor { - resource: resource.to_string(), - } - } else { - match &stub_function_name.function { - ParsedFunctionReference::RawResourceMethod { resource, method } - if resource == "counter" => - { - ParsedFunctionReference::RawResourceMethod { - resource: resource.to_string(), - method: method - .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants - .unwrap() - .to_string(), - } - } - _ => ParsedFunctionReference::Function { - function: stub_function_name - .function - .resource_method_name() - .unwrap() // TODO: proper error - .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants - .unwrap() - .to_string(), - }, - } - }, - }; - let mut wit_value_params = Vec::new(); for (param, typ) in params.iter().zip(param_types).skip(1) { let value: Value = encode_output(param, typ, store.data_mut()) @@ -379,7 +369,7 @@ impl DurableWorkerCtx { } debug!( - "CALLING {function_name} as {target_function_name} with parameters {wit_value_params:?}", + "CALLING {stub_function_name} as {target_function_name} with parameters {wit_value_params:?}", ); // "auction:auction/api.{initialize}", @@ -388,7 +378,10 @@ impl DurableWorkerCtx { .invoke_and_await(handle, target_function_name.to_string(), wit_value_params) .await??; - debug!("CALLING {function_name} RESULTED IN {:?}", wit_value_result); + debug!( + "CALLING {stub_function_name} RESULTED IN {:?}", + wit_value_result + ); let value_result: Value = wit_value_result.into(); Ok(value_result) @@ -407,6 +400,8 @@ impl DurableWorkerCtx { .await .map_err(|err| anyhow!(format!("{err:?}")))?; // TODO: proper error results[idx] = result.val; + + debug!("RESOURCES TO DROP {:?}", result.resources_to_drop); // TODO: do we have to do something with result.resources_to_drop here? } } @@ -476,3 +471,161 @@ impl DurableWorkerCtx { Ok((remote_worker_id, demand)) } } + +enum DynamicRpcCall { + GlobalStubConstructor, + ResourceStubConstructor { + stub_constructor_name: ParsedFunctionName, + target_constructor_name: ParsedFunctionName, + }, + BlockingFunctionCall { + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, + }, + AsyncFunctionCall { + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, + }, +} + +// TODO: this needs to be implementd based on component metadata and no hardcoded values +fn determine_call_type( + interface_name: &str, + function_name: &str, +) -> anyhow::Result> { + if (interface_name == "auction:auction-stub/stub-auction" + && function_name == "[constructor]api") + || (interface_name == "rpc:counters-stub/stub-counters" + && function_name == "[constructor]api") + { + Ok(Some(DynamicRpcCall::GlobalStubConstructor)) + } else if (interface_name == "auction:auction-stub/stub-auction" + && function_name == "[constructor]running-auction") + || (interface_name == "rpc:counters-stub/stub-counters" + && function_name == "[constructor]counter") + { + let stub_constructor_name = + ParsedFunctionName::parse(&format!("{interface_name}.{{{function_name}}}")) + .map_err(|err| anyhow!(err))?; // TODO: proper error + + let target_constructor_name = ParsedFunctionName { + site: if interface_name.starts_with("auction") { + ParsedFunctionSite::PackagedInterface { + // TODO: this must come from component metadata linking information + namespace: "auction".to_string(), + package: "auction".to_string(), + interface: "api".to_string(), + version: None, + } + } else { + ParsedFunctionSite::PackagedInterface { + namespace: "rpc".to_string(), + package: "counters".to_string(), + interface: "api".to_string(), + version: None, + } + }, + function: ParsedFunctionReference::RawResourceConstructor { + resource: stub_constructor_name + .function() + .resource_name() + .unwrap() + .to_string(), // TODO this has to come from a check earlier + }, + }; + + Ok(Some(DynamicRpcCall::ResourceStubConstructor { + stub_constructor_name, + target_constructor_name, + })) + } else if function_name.starts_with("[method]") { + let stub_function_name = + ParsedFunctionName::parse(&format!("{interface_name}.{{{function_name}}}")) + .map_err(|err| anyhow!(err))?; // TODO: proper error + debug!("STUB FUNCTION NAME: {stub_function_name:?}"); + + let (blocking, target_function) = match &stub_function_name.function { + ParsedFunctionReference::RawResourceMethod { resource, method } + if resource == "counter" => + // TODO: this needs to be detected based on the matching constructor + { + if method.starts_with("blocking-") { + ( + true, + ParsedFunctionReference::RawResourceMethod { + resource: resource.to_string(), + method: method + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + }, + ) + } else { + ( + false, + ParsedFunctionReference::RawResourceMethod { + resource: resource.to_string(), + method: method.to_string(), + }, + ) + } + } + _ => { + let method = stub_function_name.function.resource_method_name().unwrap(); // TODO: proper error + + if method.starts_with("blocking-") { + ( + true, + ParsedFunctionReference::Function { + function: method + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + }, + ) + } else { + ( + false, + ParsedFunctionReference::Function { + function: method.to_string(), + }, + ) + } + } + }; + + let target_function_name = ParsedFunctionName { + site: if interface_name.starts_with("auction") { + ParsedFunctionSite::PackagedInterface { + // TODO: this must come from component metadata linking information + namespace: "auction".to_string(), + package: "auction".to_string(), + interface: "api".to_string(), + version: None, + } + } else { + ParsedFunctionSite::PackagedInterface { + namespace: "rpc".to_string(), + package: "counters".to_string(), + interface: "api".to_string(), + version: None, + } + }, + function: target_function, + }; + + if blocking { + Ok(Some(DynamicRpcCall::BlockingFunctionCall { + stub_function_name, + target_function_name, + })) + } else { + Ok(Some(DynamicRpcCall::AsyncFunctionCall { + stub_function_name, + target_function_name, + })) + } + } else { + Ok(None) + } +} diff --git a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs index 61403c65d..3d8fc41d6 100644 --- a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs @@ -36,7 +36,7 @@ use golem_common::model::{ }; use golem_common::uri::oss::urn::{WorkerFunctionUrn, WorkerOrFunctionUrn}; use golem_wasm_rpc::golem::rpc::types::{ - FutureInvokeResult, HostFutureInvokeResult, Pollable, Uri, WasmRpc, + FutureInvokeResult, HostFutureInvokeResult, Pollable, Uri, }; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; use golem_wasm_rpc::{ From 38ee20875f100ef110df737ab90ef982060945bd Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 19 Dec 2024 18:22:03 +0100 Subject: [PATCH 5/5] Work on dynamic linking for stubless RPC --- .../src/durable_host/dynamic_linking.rs | 379 +++++++++++++----- .../src/durable_host/wasm_rpc/mod.rs | 44 +- golem-worker-executor-base/src/worker.rs | 8 +- .../tests/common/mod.rs | 30 +- wasm-rpc/src/value_and_type.rs | 41 +- 5 files changed, 393 insertions(+), 109 deletions(-) diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs index ccc48ee77..27ff1d454 100644 --- a/golem-worker-executor-base/src/durable_host/dynamic_linking.rs +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking.rs @@ -5,6 +5,7 @@ use crate::workerctx::{DynamicLinking, WorkerCtx}; use anyhow::anyhow; use async_trait::async_trait; use golem_common::model::OwnedWorkerId; +use golem_wasm_rpc::golem::rpc::types::{FutureInvokeResult, HostFutureInvokeResult}; use golem_wasm_rpc::wasmtime::{decode_param, encode_output}; use golem_wasm_rpc::{HostWasmRpc, Uri, Value, WasmRpcEntry, WitValue}; use rib::{ParsedFunctionName, ParsedFunctionReference, ParsedFunctionSite}; @@ -13,11 +14,12 @@ use wasmtime::component::types::ComponentItem; use wasmtime::component::{Component, Linker, Resource, ResourceType, Type, Val}; use wasmtime::{AsContextMut, Engine, StoreContextMut}; use wasmtime_wasi::WasiView; - // TODO: support multiple different dynamic linkers #[async_trait] -impl DynamicLinking for DurableWorkerCtx { +impl DynamicLinking + for DurableWorkerCtx +{ fn link( &mut self, engine: &Engine, @@ -60,6 +62,7 @@ impl DynamicLinking for DurableWorkerCtx ComponentItem::ComponentFunc(fun) => { let name2 = name.clone(); let ename2 = ename.clone(); + // TODO: need to do the "rpc call type" detection here and if it's not an rpc call then not calling `func_new_async` (for example for 'subscribe' and 'get' on FutureInvokeResult wrappers) instance.func_new_async( // TODO: instrument async closure &ename.clone(), @@ -96,7 +99,15 @@ impl DynamicLinking for DurableWorkerCtx } ComponentItem::Type(_) => {} ComponentItem::Resource(resource) => { - if ename != "pollable" { + // TODO: need to detect future result resources and register them as FutureInvokeResult + if ename == "future-counter-get-value-result" { + debug!("LINKING FUTURE INVOKE RESULT {ename}"); + instance.resource( + &ename, + ResourceType::host::(), + |_store, _rep| Ok(()), + )?; + } else if ename != "pollable" { // TODO: ?? this should be 'if it is not already linked' but not way to check that debug!("LINKING RESOURCE {ename} {resource:?}"); instance.resource_async( @@ -141,7 +152,7 @@ impl DynamicLinking for DurableWorkerCtx ); // TODO: this has to be moved to be calculated in the linking phase - let call_type = determine_call_type(interface_name, function_name)?; + let call_type = determine_call_type(interface_name, function_name, result_types)?; match call_type { Some(DynamicRpcCall::GlobalStubConstructor) => { @@ -280,10 +291,43 @@ impl DynamicLinking for DurableWorkerCtx Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store) .await?; } + Some(DynamicRpcCall::FireAndForgetFunctionCall { + stub_function_name, + target_function_name, + }) => { + // Async stub interface method + debug!( + "FNF {function_name} handle={:?}, rest={:?}", + params[0], + params.iter().skip(1).collect::>() + ); + + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + { + let mut wasi = store.data_mut().as_wasi_view(); + let entry = wasi.table().get(&handle)?; + let payload = entry.payload.downcast_ref::().unwrap(); + debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); + } + + Self::remote_invoke( + stub_function_name, + target_function_name, + params, + param_types, + &mut store, + handle, + ) + .await?; + } Some(DynamicRpcCall::AsyncFunctionCall { - stub_function_name, - target_function_name, - }) => { + stub_function_name, + target_function_name, + }) => { // Async stub interface method debug!( "ASYNC {function_name} handle={:?}, rest={:?}", @@ -303,17 +347,71 @@ impl DynamicLinking for DurableWorkerCtx debug!("CALLING {function_name} ON {}", payload.remote_worker_id()); } - // let result = Self::remote_invoke( - // stub_function_name, - // target_function_name, - // params, - // param_types, - // &mut store, - // handle, - // ) - // .await?; - // Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store) - // .await?; + let result = Self::remote_async_invoke_and_await( + stub_function_name, + target_function_name, + params, + param_types, + &mut store, + handle, + ) + .await?; + + Self::value_result_to_wasmtime_vals(result, results, result_types, &mut store) + .await?; + } + Some(DynamicRpcCall::FutureInvokeResultSubscribe) => { + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + let pollable = store.data_mut().subscribe(handle).await?; + let pollable_any = pollable.try_into_resource_any(&mut store)?; + let resource_id = store.data_mut().add(pollable_any).await; + + let value_result = Value::Tuple(vec![Value::Handle { + uri: store.data().self_uri().value, + resource_id, + }]); + Self::value_result_to_wasmtime_vals( + value_result, + results, + result_types, + &mut store, + ) + .await?; + } + Some(DynamicRpcCall::FutureInvokeResultGet) => { + let handle = match params[0] { + Val::Resource(handle) => handle, + _ => return Err(anyhow!("Invalid handle parameter")), + }; + let handle: Resource = handle.try_into_resource(&mut store)?; + let result = HostFutureInvokeResult::get(store.data_mut(), handle).await?; + + // NOTE: we are currently failing on RpcError instead of passing it to the caller, as the generated stub interface requires + let value_result = Value::Tuple(vec![match result { + None => Value::Option(None), + Some(Ok(value)) => { + let value: Value = value.into(); + match value { + Value::Tuple(items) if items.len() == 1 => { + Value::Option(Some(Box::new(items.into_iter().next().unwrap()))) + } + _ => Err(anyhow!("Invalid future invoke result value"))?, // TODO: better error + } + } + Some(Err(err)) => Err(anyhow!("RPC invocation failed with {err:?}"))?, // TODO: more information into the error + }]); + + Self::value_result_to_wasmtime_vals( + value_result, + results, + result_types, + &mut store, + ) + .await?; } _ => todo!(), } @@ -372,7 +470,6 @@ impl DurableWorkerCtx { "CALLING {stub_function_name} as {target_function_name} with parameters {wit_value_params:?}", ); - // "auction:auction/api.{initialize}", let wit_value_result = store .data_mut() .invoke_and_await(handle, target_function_name.to_string(), wit_value_params) @@ -387,6 +484,74 @@ impl DurableWorkerCtx { Ok(value_result) } + // TODO: stub_function_name can probably be removed + async fn remote_async_invoke_and_await( + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, + params: &[Val], + param_types: &[Type], + store: &mut StoreContextMut<'_, Ctx>, + handle: Resource, + ) -> anyhow::Result { + let mut wit_value_params = Vec::new(); + for (param, typ) in params.iter().zip(param_types).skip(1) { + let value: Value = encode_output(param, typ, store.data_mut()) + .await + .map_err(|err| anyhow!(format!("{err:?}")))?; // TODO: proper error + let wit_value: WitValue = value.into(); + wit_value_params.push(wit_value); + } + + debug!( + "CALLING {stub_function_name} as {target_function_name} with parameters {wit_value_params:?}", + ); + + let invoke_result_resource = store + .data_mut() + .async_invoke_and_await(handle, target_function_name.to_string(), wit_value_params) + .await?; + + let invoke_result_resource_any = + invoke_result_resource.try_into_resource_any(&mut *store)?; + let resource_id = store.data_mut().add(invoke_result_resource_any).await; + + let value_result: Value = Value::Tuple(vec![Value::Handle { + uri: store.data().self_uri().value, + resource_id, + }]); + Ok(value_result) + } + + // TODO: stub_function_name can probably be removed + async fn remote_invoke( + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, + params: &[Val], + param_types: &[Type], + store: &mut StoreContextMut<'_, Ctx>, + handle: Resource, + ) -> anyhow::Result<()> { + let mut wit_value_params = Vec::new(); + for (param, typ) in params.iter().zip(param_types).skip(1) { + let value: Value = encode_output(param, typ, store.data_mut()) + .await + .map_err(|err| anyhow!(format!("{err:?}")))?; // TODO: proper error + let wit_value: WitValue = value.into(); + wit_value_params.push(wit_value); + } + + debug!( + "CALLING {stub_function_name} as {target_function_name} with parameters {wit_value_params:?}", + ); + + store + .data_mut() + .invoke(handle, target_function_name.to_string(), wit_value_params) + .await??; + + Ok(()) + } + async fn value_result_to_wasmtime_vals( value_result: Value, results: &mut [Val], @@ -482,16 +647,23 @@ enum DynamicRpcCall { stub_function_name: ParsedFunctionName, target_function_name: ParsedFunctionName, }, + FireAndForgetFunctionCall { + stub_function_name: ParsedFunctionName, + target_function_name: ParsedFunctionName, + }, AsyncFunctionCall { stub_function_name: ParsedFunctionName, target_function_name: ParsedFunctionName, }, + FutureInvokeResultSubscribe, + FutureInvokeResultGet, } // TODO: this needs to be implementd based on component metadata and no hardcoded values fn determine_call_type( interface_name: &str, function_name: &str, + result_types: &[Type], ) -> anyhow::Result> { if (interface_name == "auction:auction-stub/stub-auction" && function_name == "[constructor]api") @@ -544,86 +716,107 @@ fn determine_call_type( .map_err(|err| anyhow!(err))?; // TODO: proper error debug!("STUB FUNCTION NAME: {stub_function_name:?}"); - let (blocking, target_function) = match &stub_function_name.function { - ParsedFunctionReference::RawResourceMethod { resource, method } - if resource == "counter" => - // TODO: this needs to be detected based on the matching constructor + if stub_function_name.function.resource_name() + == Some(&"future-counter-get-value-result".to_string()) + { + if stub_function_name.function.resource_method_name() == Some("subscribe".to_string()) { + Ok(Some(DynamicRpcCall::FutureInvokeResultSubscribe)) + } else if stub_function_name.function.resource_method_name() == Some("get".to_string()) { - if method.starts_with("blocking-") { - ( - true, - ParsedFunctionReference::RawResourceMethod { - resource: resource.to_string(), - method: method - .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants - .unwrap() - .to_string(), - }, - ) - } else { - ( - false, - ParsedFunctionReference::RawResourceMethod { - resource: resource.to_string(), - method: method.to_string(), - }, - ) - } + Ok(Some(DynamicRpcCall::FutureInvokeResultGet)) + } else { + Ok(None) } - _ => { - let method = stub_function_name.function.resource_method_name().unwrap(); // TODO: proper error - - if method.starts_with("blocking-") { - ( - true, - ParsedFunctionReference::Function { - function: method - .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants - .unwrap() - .to_string(), - }, - ) - } else { - ( - false, - ParsedFunctionReference::Function { - function: method.to_string(), - }, - ) + } else { + let (blocking, target_function) = match &stub_function_name.function { + ParsedFunctionReference::RawResourceMethod { resource, method } + if resource == "counter" => + // TODO: this needs to be detected based on the matching constructor + { + if method.starts_with("blocking-") { + ( + true, + ParsedFunctionReference::RawResourceMethod { + resource: resource.to_string(), + method: method + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + }, + ) + } else { + ( + false, + ParsedFunctionReference::RawResourceMethod { + resource: resource.to_string(), + method: method.to_string(), + }, + ) + } } - } - }; - - let target_function_name = ParsedFunctionName { - site: if interface_name.starts_with("auction") { - ParsedFunctionSite::PackagedInterface { - // TODO: this must come from component metadata linking information - namespace: "auction".to_string(), - package: "auction".to_string(), - interface: "api".to_string(), - version: None, + _ => { + let method = stub_function_name.function.resource_method_name().unwrap(); // TODO: proper error + + if method.starts_with("blocking-") { + ( + true, + ParsedFunctionReference::Function { + function: method + .strip_prefix("blocking-") // TODO: we also have to support the non-blocking variants + .unwrap() + .to_string(), + }, + ) + } else { + ( + false, + ParsedFunctionReference::Function { + function: method.to_string(), + }, + ) + } } + }; + + let target_function_name = ParsedFunctionName { + site: if interface_name.starts_with("auction") { + ParsedFunctionSite::PackagedInterface { + // TODO: this must come from component metadata linking information + namespace: "auction".to_string(), + package: "auction".to_string(), + interface: "api".to_string(), + version: None, + } + } else { + ParsedFunctionSite::PackagedInterface { + namespace: "rpc".to_string(), + package: "counters".to_string(), + interface: "api".to_string(), + version: None, + } + }, + function: target_function, + }; + + if blocking { + Ok(Some(DynamicRpcCall::BlockingFunctionCall { + stub_function_name, + target_function_name, + })) } else { - ParsedFunctionSite::PackagedInterface { - namespace: "rpc".to_string(), - package: "counters".to_string(), - interface: "api".to_string(), - version: None, + debug!("ASYNC FUNCTION RESULT TYPES: {result_types:?}"); + if result_types.len() > 0 { + Ok(Some(DynamicRpcCall::AsyncFunctionCall { + stub_function_name, + target_function_name, + })) + } else { + Ok(Some(DynamicRpcCall::FireAndForgetFunctionCall { + stub_function_name, + target_function_name, + })) } - }, - function: target_function, - }; - - if blocking { - Ok(Some(DynamicRpcCall::BlockingFunctionCall { - stub_function_name, - target_function_name, - })) - } else { - Ok(Some(DynamicRpcCall::AsyncFunctionCall { - stub_function_name, - target_function_name, - })) + } } } else { Ok(None) diff --git a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs index 3d8fc41d6..06a45d8d9 100644 --- a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs @@ -96,7 +96,7 @@ impl HostWasmRpc for DurableWorkerCtx { let payload = entry.payload.downcast_ref::().unwrap(); let remote_worker_id = payload.remote_worker_id().clone(); - // TODO: do this in other variants too + // TODO: remove redundancy match payload { WasmRpcEntryPayload::Resource { resource_uri, @@ -234,7 +234,7 @@ impl HostWasmRpc for DurableWorkerCtx { &mut self, self_: Resource, function_name: String, - function_params: Vec, + mut function_params: Vec, ) -> anyhow::Result> { record_host_function_call("golem::rpc::wasm-rpc", "invoke"); let args = self.get_arguments().await?; @@ -244,6 +244,25 @@ impl HostWasmRpc for DurableWorkerCtx { let payload = entry.payload.downcast_ref::().unwrap(); let remote_worker_id = payload.remote_worker_id().clone(); + // TODO: remove redundancy + match payload { + WasmRpcEntryPayload::Resource { + resource_uri, + resource_id, + .. + } => { + function_params.insert( + 0, + Value::Handle { + uri: resource_uri.value.to_string(), + resource_id: *resource_id, + } + .into(), + ); + } + _ => {} + } + let current_idempotency_key = self .get_current_idempotency_key() .await @@ -319,7 +338,7 @@ impl HostWasmRpc for DurableWorkerCtx { &mut self, this: Resource, function_name: String, - function_params: Vec, + mut function_params: Vec, ) -> anyhow::Result> { record_host_function_call("golem::rpc::wasm-rpc", "async-invoke-and-await"); let args = self.get_arguments().await?; @@ -334,6 +353,25 @@ impl HostWasmRpc for DurableWorkerCtx { let payload = entry.payload.downcast_ref::().unwrap(); let remote_worker_id = payload.remote_worker_id().clone(); + // TODO: remove redundancy + match payload { + WasmRpcEntryPayload::Resource { + resource_uri, + resource_id, + .. + } => { + function_params.insert( + 0, + Value::Handle { + uri: resource_uri.value.to_string(), + resource_id: *resource_id, + } + .into(), + ); + } + _ => {} + } + let current_idempotency_key = self .get_current_idempotency_key() .await diff --git a/golem-worker-executor-base/src/worker.rs b/golem-worker-executor-base/src/worker.rs index a8480dc97..25146e7c1 100644 --- a/golem-worker-executor-base/src/worker.rs +++ b/golem-worker-executor-base/src/worker.rs @@ -20,7 +20,6 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use crate::durable_host::recover_stderr_logs; -use crate::durable_host::wasm_rpc::{UrnExtensions, WasmRpcEntryPayload}; use crate::error::{GolemError, WorkerOutOfMemory}; use crate::function_result_interpreter::interpret_function_results; use crate::invocation::{find_first_available_function, invoke_worker, InvokeResult}; @@ -31,7 +30,6 @@ use crate::model::{ use crate::services::component::ComponentMetadata; use crate::services::events::Event; use crate::services::oplog::{CommitLevel, Oplog, OplogOps}; -use crate::services::rpc::RpcDemand; use crate::services::worker_event::{WorkerEventService, WorkerEventServiceDefault}; use crate::services::{ All, HasActiveWorkers, HasAll, HasBlobStoreService, HasComponentService, HasConfig, HasEvents, @@ -59,17 +57,15 @@ use golem_common::model::{ }; use golem_common::retries::get_delay; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; -use golem_wasm_rpc::{Uri, Value, WasmRpcEntry}; +use golem_wasm_rpc::Value; use tokio::sync::broadcast::error::RecvError; use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::{Mutex, MutexGuard, OwnedSemaphorePermit}; use tokio::task::JoinHandle; use tracing::{debug, error, info, span, warn, Instrument, Level}; -use wasmtime::component::types::ComponentItem; -use wasmtime::component::{Instance, Resource, ResourceType, Val}; +use wasmtime::component::Instance; use wasmtime::{AsContext, Store, UpdateDeadline}; -use wasmtime_wasi::WasiView; /// Represents worker that may be running or suspended. /// diff --git a/golem-worker-executor-base/tests/common/mod.rs b/golem-worker-executor-base/tests/common/mod.rs index 16e5a7fdd..d62c5357b 100644 --- a/golem-worker-executor-base/tests/common/mod.rs +++ b/golem-worker-executor-base/tests/common/mod.rs @@ -5,9 +5,7 @@ use std::collections::HashSet; use golem_service_base::service::initial_component_files::InitialComponentFilesService; use golem_service_base::storage::blob::BlobStorage; use golem_wasm_rpc::wasmtime::ResourceStore; -use golem_wasm_rpc::{ - FutureInvokeResultEntry, HostWasmRpc, RpcError, Uri, Value, WasmRpcEntry, WitValue, -}; +use golem_wasm_rpc::{HostWasmRpc, RpcError, Uri, Value, WitValue}; use golem_worker_executor_base::services::file_loader::FileLoader; use prometheus::Registry; @@ -82,6 +80,7 @@ use golem_test_framework::components::worker_executor_cluster::WorkerExecutorClu use golem_test_framework::config::TestDependencies; use golem_test_framework::dsl::to_worker_metadata; use golem_wasm_rpc::golem::rpc::types::{FutureInvokeResult, WasmRpc}; +use golem_wasm_rpc::golem::rpc::types::{HostFutureInvokeResult, Pollable}; use golem_worker_executor_base::preview2::golem; use golem_worker_executor_base::preview2::golem::api1_1_0; use golem_worker_executor_base::services::events::Events; @@ -586,7 +585,7 @@ impl ResourceStore for TestWorkerCtx { } async fn get(&mut self, resource_id: u64) -> Option { - self.durable_ctx.get(resource_id).await + ResourceStore::get(&mut self.durable_ctx, resource_id).await } async fn borrow(&self, resource_id: u64) -> Option { @@ -832,7 +831,28 @@ impl HostWasmRpc for TestWorkerCtx { } async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { - self.durable_ctx.drop(rep).await + HostWasmRpc::drop(&mut self.durable_ctx, rep).await + } +} + +#[async_trait] +impl HostFutureInvokeResult for TestWorkerCtx { + async fn subscribe( + &mut self, + self_: Resource, + ) -> anyhow::Result> { + HostFutureInvokeResult::subscribe(&mut self.durable_ctx, self_).await + } + + async fn get( + &mut self, + self_: Resource, + ) -> anyhow::Result>> { + HostFutureInvokeResult::get(&mut self.durable_ctx, self_).await + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + HostFutureInvokeResult::drop(&mut self.durable_ctx, rep).await } } diff --git a/wasm-rpc/src/value_and_type.rs b/wasm-rpc/src/value_and_type.rs index 9c4365494..3ed7fef68 100644 --- a/wasm-rpc/src/value_and_type.rs +++ b/wasm-rpc/src/value_and_type.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::Value; -use golem_wasm_ast::analysis::analysed_type::{list, option, result, result_err, result_ok, tuple}; +use crate::{RpcError, Value}; +use golem_wasm_ast::analysis::analysed_type::{ + list, option, result, result_err, result_ok, tuple, variant, +}; use golem_wasm_ast::analysis::{analysed_type, AnalysedType}; use std::collections::HashMap; use std::time::{Duration, Instant}; @@ -532,3 +534,38 @@ impl IntoValue for Duration { analysed_type::u64() } } + +#[cfg(feature = "host-bindings")] +impl IntoValue for crate::RpcError { + fn into_value(self) -> Value { + match self { + RpcError::ProtocolError(value) => Value::Variant { + case_idx: 0, + case_value: Some(Box::new(Value::String(value))), + }, + RpcError::Denied(value) => Value::Variant { + case_idx: 1, + case_value: Some(Box::new(Value::String(value))), + }, + RpcError::NotFound(value) => Value::Variant { + case_idx: 2, + case_value: Some(Box::new(Value::String(value))), + }, + RpcError::RemoteInternalError(value) => Value::Variant { + case_idx: 3, + case_value: Some(Box::new(Value::String(value))), + }, + } + } + + fn get_type() -> AnalysedType { + use analysed_type::case; + + variant(vec![ + case("protocol-error", analysed_type::str()), + case("denied", analysed_type::str()), + case("not-found", analysed_type::str()), + case("remote-internal-error", analysed_type::str()), + ]) + } +}