diff --git a/scripts/report.py b/scripts/report.py index 849ed6b..b6e9afc 100755 --- a/scripts/report.py +++ b/scripts/report.py @@ -3,7 +3,7 @@ # # First connection to a local Ocean node is made, the request data for a # txid is found via getrequest RPC and fee size is calculated. -# Then getrequestresponses RPC is called to get challenge responses which can be +# Then getrequestresponse RPC is called to get challenge responses which can be # used to determine rewards for guardnodes. #!/usr/bin/env python3 @@ -82,9 +82,9 @@ def calculate_fees(rpc, start_height, end_height): return fee addr_prefix = 235 -txid = "78f954d07de5badbc1526a60fe0ea639216f17f490a3bf41e48840453eca243f" -url = 'https://userApi:passwordApi@coordinator-api.testnet.commerceblock.com:10006' -rpc = connect("ocean", "oceanpass", "localhost", "7043") +txid = "6e993034df3203c0867c98f420f85b5ffecd7cb8580e2b6f2d33764e1cbfb074" +url = 'http://userApi:passwordApi@localhost:3333' +rpc = connect("user1", "password1", "localhost", "5555") payload = '{{"jsonrpc": "2.0", "method": "getrequest", "params": {{"txid": "{}"}}, "id": 1}}'.format(txid) headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} @@ -98,11 +98,8 @@ def calculate_fees(rpc, start_height, end_height): print("") print("Calculating total fees...") -# For requests that are serving the service chain the fee start/end heights -# can be picked up from the request information. For requests in client chains -# these heights need to be found manually and inserted below to calculate fees -fee_start_height = request['start_blockheight'] -fee_end_height = request['end_blockheight'] +fee_start_height = request['start_blockheight_clientchain'] +fee_end_height = request['end_blockheight_clientchain'] fee = calculate_fees(rpc, fee_start_height, fee_end_height) fee_percentage = request['fee_percentage'] fee_out = fee*fee_percentage/100 @@ -120,21 +117,17 @@ def calculate_fees(rpc, start_height, end_height): fee_per_guard = float(fee_out/len(bids)) print("") -payload = '{{"jsonrpc": "2.0", "method": "getrequestresponses", "params": {{"txid": "{}"}}, "id": 1}}'.format(txid) +payload = '{{"jsonrpc": "2.0", "method": "getrequestresponse", "params": {{"txid": "{}"}}, "id": 1}}'.format(txid) headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} r = requests.post(url, data=payload, headers=headers) result = json.loads(json.loads(r.content)['result']) -challenge_resps = result["responses"] -num_of_challenges = len(challenge_resps) +challenge_resps = result["response"] +num_of_challenges = challenge_resps["num_challenges"] print("Number of challenges: {}".format(num_of_challenges)) resps = {} -for challenge_resp in challenge_resps: - for bid_resp in challenge_resp: - if bid_resp in resps: - resps[bid_resp] += (1/num_of_challenges) - else: - resps[bid_resp] = (1/num_of_challenges) +for (bid, resp_num) in challenge_resps["bid_responses"].items(): + resps[bid] = resp_num / num_of_challenges print("Results") for bid, key in bids.items(): diff --git a/src/api.rs b/src/api.rs index 2b6d094..197a49b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -14,9 +14,9 @@ use jsonrpc_http_server::jsonrpc_core::{Error, ErrorCode, IoHandler, Params, Val use jsonrpc_http_server::{hyper::header, AccessControlAllowOrigin, DomainsValidation, Response, ServerBuilder}; use serde::{Deserialize, Serialize}; -use crate::challenger::ChallengeResponseIds; use crate::config::ApiConfig; use crate::request::{BidSet, Request as ServiceRequest}; +use crate::response::Response as ChallengeResponse; use crate::storage::Storage; #[derive(Deserialize, Debug)] @@ -74,19 +74,27 @@ struct GetRequestResponsesParams { } #[derive(Serialize, Debug)] -struct GetRequestResponsesResponse { - responses: Vec, +struct GetRequestResponseResponse { + response: ChallengeResponse, } /// Get requests responses RPC call returning all responses for a specific /// request transaction id hash -fn get_request_responses(params: Params, storage: Arc) -> futures::Finished { +fn get_request_response(params: Params, storage: Arc) -> futures::Finished { let try_parse = params.parse::(); match try_parse { Ok(parse) => { - let responses = storage.get_responses(parse.txid).unwrap(); - let res_serialized = serde_json::to_string(&GetRequestResponsesResponse { responses }).unwrap(); - return futures::finished(Value::String(res_serialized)); + let response_get = storage.get_response(parse.txid).unwrap(); + if let Some(response) = response_get { + let res_serialized = serde_json::to_string(&GetRequestResponseResponse { response }).unwrap(); + return futures::finished(Value::String(res_serialized)); + } else { + return futures::failed(Error { + code: ErrorCode::InvalidParams, + message: "Invalid params: `txid` does not exist.".to_string(), + data: None, + }); + } } Err(e) => return futures::failed(e), } @@ -119,8 +127,8 @@ pub fn run_api_server( ) -> thread::JoinHandle<()> { let mut io = IoHandler::default(); let storage_ref = storage.clone(); - io.add_method("getrequestresponses", move |params: Params| { - get_request_responses(params, storage_ref.clone()) + io.add_method("getrequestresponse", move |params: Params| { + get_request_response(params, storage_ref.clone()) }); let storage_ref = storage.clone(); io.add_method("getrequest", move |params: Params| { @@ -161,6 +169,7 @@ mod tests { use futures::Future; + use crate::challenger::ChallengeResponseIds; use crate::util::testing::{gen_challenge_state, gen_dummy_hash, MockStorage}; #[test] @@ -228,10 +237,20 @@ mod tests { } #[test] - fn get_request_responses_test() { + fn get_request_response_test() { let storage = Arc::new(MockStorage::new()); let dummy_hash = gen_dummy_hash(1); let dummy_hash_bid = gen_dummy_hash(2); + + // no such request + let s = format!(r#"{{"txid": "{}"}}"#, dummy_hash.to_string()); + let params: Params = serde_json::from_str(&s).unwrap(); + let resp = get_request_response(params, storage.clone()); + assert_eq!( + "Invalid params: `txid` does not exist.", + resp.wait().unwrap_err().message + ); + let mut dummy_response_set = ChallengeResponseIds::new(); let _ = dummy_response_set.insert(dummy_hash_bid); let _ = storage.save_response(dummy_hash, &dummy_response_set); @@ -239,7 +258,7 @@ mod tests { // invalid key let s = format!(r#"{{"hash": "{}"}}"#, dummy_hash.to_string()); let params: Params = serde_json::from_str(&s).unwrap(); - let resp = get_request_responses(params, storage.clone()); + let resp = get_request_response(params, storage.clone()); assert_eq!( "Invalid params: missing field `txid`.", resp.wait().unwrap_err().message @@ -248,7 +267,7 @@ mod tests { // invalid value let s = format!(r#"{{"txid": "{}a"}}"#, dummy_hash.to_string()); let params: Params = serde_json::from_str(&s).unwrap(); - let resp = get_request_responses(params, storage.clone()); + let resp = get_request_response(params, storage.clone()); assert_eq!( "Invalid params: bad hex string length 65 (expected 64).", resp.wait().unwrap_err().message @@ -257,9 +276,12 @@ mod tests { // valid key and value let s = format!(r#"{{"txid": "{}"}}"#, dummy_hash.to_string()); let params: Params = serde_json::from_str(&s).unwrap(); - let resp = get_request_responses(params, storage.clone()); + let resp = get_request_response(params, storage.clone()); assert_eq!( - format!("{{\"responses\":[[\"{}\"]]}}", dummy_hash_bid.to_string()), + format!( + r#"{{"response":{{"num_challenges":1,"bid_responses":{{"{}":1}}}}}}"#, + dummy_hash_bid.to_string() + ), resp.wait().unwrap() ); } diff --git a/src/challenger.rs b/src/challenger.rs index 2016548..1d85638 100644 --- a/src/challenger.rs +++ b/src/challenger.rs @@ -213,6 +213,7 @@ mod tests { use std::sync::mpsc::{channel, Receiver, Sender}; use crate::error::Error; + use crate::response::Response; use crate::util::testing::{gen_dummy_hash, MockClientChain, MockService, MockStorage}; #[test] @@ -398,14 +399,14 @@ mod tests { storage.clone(), time::Duration::from_millis(10), time::Duration::from_millis(10), - 3, + 50, time::Duration::from_millis(10), ); match res { Ok(_) => { - let resps = storage.get_responses(dummy_request.txid).unwrap(); - assert_eq!(1, resps.len()); + let resps = storage.get_response(dummy_request.txid).unwrap(); + assert_eq!(resps, None); let bids = storage.get_bids(dummy_request.txid).unwrap(); assert_eq!(challenge_state.bids, bids); let requests = storage.get_requests().unwrap(); @@ -440,11 +441,15 @@ mod tests { match res { Ok(_) => { - let resps = storage.get_responses(dummy_request.txid).unwrap(); - assert_eq!(5, resps.len()); - assert_eq!(1, resps[1].len()); - assert_eq!(dummy_bid.txid, *resps[1].iter().next().unwrap()); - assert_eq!(5, storage.challenge_responses.borrow().len()); + let resps = storage.get_response(dummy_request.txid).unwrap(); + assert_eq!( + resps.unwrap(), + Response { + num_challenges: 4, + bid_responses: [(dummy_bid.txid, 1)].iter().cloned().collect() + } + ); + assert_eq!(1, storage.challenge_responses.borrow().len()); let bids = storage.get_bids(dummy_request.txid).unwrap(); assert_eq!(challenge_state.bids, bids); let requests = storage.get_requests().unwrap(); diff --git a/src/coordinator.rs b/src/coordinator.rs index b46d6fd..590f563 100644 --- a/src/coordinator.rs +++ b/src/coordinator.rs @@ -32,9 +32,9 @@ pub fn run(config: Config) -> Result<()> { loop { if let Some(request_id) = run_request(&config, &service, &clientchain, storage.clone(), genesis_hash)? { // if challenge request succeeds print responses - println! {"***** Responses *****"} - let resp = storage.get_responses(request_id).unwrap(); - println! {"{}", serde_json::to_string_pretty(&resp).unwrap()}; + info! {"***** Response *****"} + let resp = storage.get_response(request_id)?.unwrap(); + info! {"{}", serde_json::to_string_pretty(&resp).unwrap()}; } info! {"Sleeping for {} sec...", config.block_time} thread::sleep(time::Duration::from_secs(config.block_time)) diff --git a/src/lib.rs b/src/lib.rs index 12dec6c..b31af30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ pub mod error; pub mod listener; pub mod ocean; pub mod request; +pub mod response; pub mod service; pub mod storage; /// utilities diff --git a/src/response.rs b/src/response.rs new file mode 100644 index 0000000..2fd0bc3 --- /dev/null +++ b/src/response.rs @@ -0,0 +1,39 @@ +//! # Response +//! +//! Response model for service challenge responses + +use std::collections::HashMap; + +use crate::challenger::ChallengeResponseIds; +use bitcoin_hashes::sha256d; +use serde::Serialize; + +/// Response struct that models responses to service challenges +/// by keeping track of the total number of challengers and the +/// number of challenges that each bid owner responded to +#[derive(Debug, Serialize, PartialEq)] +pub struct Response { + /// Total number of challenges + pub num_challenges: u32, + /// Number of responses per bid txid + pub bid_responses: HashMap, +} + +impl Response { + /// Create new Response instance + pub fn new() -> Response { + Response { + num_challenges: 0, + bid_responses: HashMap::new(), + } + } + + /// Update Response struct from challenge response ids + pub fn update(&mut self, responses: &ChallengeResponseIds) { + self.num_challenges += 1; + for txid in responses.iter() { + let bid_entry = self.bid_responses.entry(*txid).or_insert(0); + *bid_entry += 1; + } + } +} diff --git a/src/storage.rs b/src/storage.rs index 0b38609..77d31ea 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -14,6 +14,7 @@ use crate::challenger::{ChallengeResponseIds, ChallengeState}; use crate::config::StorageConfig; use crate::error::{Error::MongoDb, Result}; use crate::request::{BidSet, Request}; +use crate::response::Response; /// Storage trait defining required functionality for objects that store request /// and challenge information @@ -22,10 +23,10 @@ pub trait Storage { fn save_challenge_state(&self, challenge: &ChallengeState) -> Result<()>; /// Update request in storage fn update_request(&self, request: Request) -> Result<()>; - /// Store responses for a specific challenge request + /// Store response for a specific challenge request fn save_response(&self, request_hash: sha256d::Hash, ids: &ChallengeResponseIds) -> Result<()>; - /// Get all challenge responses for a specific request - fn get_responses(&self, request_hash: sha256d::Hash) -> Result>; + /// Get challenge response for a specific request + fn get_response(&self, request_hash: sha256d::Hash) -> Result>; /// Get all bids for a specific request fn get_bids(&self, request_hash: sha256d::Hash) -> Result; /// Get all the requests @@ -132,12 +133,12 @@ impl Storage for MongoStorage { Ok(()) } - /// Store responses for a specific challenge request + /// Store response for a specific challenge request fn save_response(&self, request_hash: sha256d::Hash, ids: &ChallengeResponseIds) -> Result<()> { let db_locked = self.db.lock().unwrap(); self.auth(&db_locked)?; - let request = db_locked + let request_id = db_locked .collection("Request") .find_one( Some(doc! { @@ -145,16 +146,31 @@ impl Storage for MongoStorage { }), None, )? - .unwrap(); + .unwrap() + .get("_id") + .unwrap() + .clone(); - let _ = db_locked - .collection("Response") - .insert_one(challenge_responses_to_doc(request.get("_id").unwrap(), ids), None)?; + let coll = db_locked.collection("Response"); + let filter = doc! {"request_id": request_id.clone()}; + match coll.find_one(Some(filter.clone()), None)? { + Some(res) => { + let mut resp = doc_to_response(&res); + resp.update(ids); + let update = doc! {"$set" => response_to_doc(&request_id, &resp)}; + let _ = coll.update_one(filter, update, None)?; + } + None => { + let mut resp = Response::new(); + resp.update(ids); + let _ = coll.insert_one(response_to_doc(&request_id, &resp), None)?; + } + } Ok(()) } - /// Get all challenge responses for a specific request - fn get_responses(&self, request_hash: sha256d::Hash) -> Result> { + /// Get challenge response for a specific request + fn get_response(&self, request_hash: sha256d::Hash) -> Result> { let db_locked = self.db.lock().unwrap(); self.auth(&db_locked)?; @@ -165,7 +181,7 @@ impl Storage for MongoStorage { "from": "Response", "localField": "_id", "foreignField": "request_id", - "as": "challenges" + "as": "response" } }, doc! { @@ -173,19 +189,21 @@ impl Storage for MongoStorage { "txid": request_hash.to_string() }, }, + doc! { + "$unwind": { + "path": "$response" + } + }, ] .to_vec(), None, )?; drop(db_locked); // drop immediately on get requests - let mut all_resps: Vec = Vec::new(); if let Some(resp) = resp_aggr.next() { - for challenge in resp?.get_array("challenges").unwrap().iter() { - all_resps.push(doc_to_challenge_responses(challenge.as_document().unwrap())) - } + return Ok(Some(doc_to_response(&resp?.get_document("response").unwrap()))); } - Ok(all_resps) + Ok(None) } /// Get all bids for a specific request diff --git a/src/util/doc_format.rs b/src/util/doc_format.rs index 15343ed..3eba501 100644 --- a/src/util/doc_format.rs +++ b/src/util/doc_format.rs @@ -3,13 +3,15 @@ //! doc format is used to store items in the db. //! File contains methods to convert to/from document format. +use std::collections::HashMap; +use std::str::FromStr; + use bitcoin_hashes::{hex::FromHex, sha256d}; use mongodb::{ordered::OrderedDocument, Bson}; use secp256k1::key::PublicKey; -use std::str::FromStr; -use crate::challenger::ChallengeResponseIds; use crate::request::{Bid, Request}; +use crate::response::Response; /// Util method that generates a Request document from a request pub fn request_to_doc(request: &Request) -> OrderedDocument { @@ -56,25 +58,39 @@ pub fn doc_to_bid(doc: &OrderedDocument) -> Bid { } } -/// Util method that generates a Response document from challenge responses -pub fn challenge_responses_to_doc(request_id: &Bson, responses: &ChallengeResponseIds) -> OrderedDocument { - let bids = responses +/// Util method that generates a Response document from request response +pub fn response_to_doc(request_id: &Bson, response: &Response) -> OrderedDocument { + let bid_resps_doc: OrderedDocument = response + .bid_responses .iter() - .map(|x| Bson::String(x.to_string())) - .collect::>(); + .map(|(key, val)| (key.to_string(), Bson::I32(*val as i32))) + .collect(); doc! { "request_id": request_id.clone(), - "bid_txids": bids + "num_challenges": response.num_challenges, + "bid_responses": bid_resps_doc } } -/// Util method that generates challenge responses from a Response document -pub fn doc_to_challenge_responses(doc: &OrderedDocument) -> ChallengeResponseIds { - doc.get_array("bid_txids") +/// Util method that generates request response from a Response document +pub fn doc_to_response(doc: &OrderedDocument) -> Response { + let bid_resps: HashMap = doc + .get("bid_responses") + .unwrap() + .as_document() .unwrap() .iter() - .map(|x| sha256d::Hash::from_hex(x.as_str().unwrap()).unwrap()) - .collect() + .map(|(key, val)| { + ( + sha256d::Hash::from_hex(key.as_str()).unwrap(), + val.as_i32().unwrap() as u32, + ) + }) + .collect(); + Response { + num_challenges: doc.get("num_challenges").unwrap().as_i32().unwrap() as u32, + bid_responses: bid_resps, + } } #[cfg(test)] @@ -87,6 +103,7 @@ mod tests { use secp256k1::key::PublicKey; use std::str::FromStr; + use crate::challenger::ChallengeResponseIds; use crate::request::Bid; use crate::util::testing::gen_dummy_hash; @@ -145,40 +162,52 @@ mod tests { } #[test] - fn challenge_responses_doc_test() { + fn response_doc_test() { let id = ObjectId::new().unwrap(); let mut ids = ChallengeResponseIds::new(); + let mut resp = Response::new(); - let doc = challenge_responses_to_doc(&Bson::ObjectId(id.clone()), &ids); + let doc = response_to_doc(&Bson::ObjectId(id.clone()), &resp); assert_eq!( doc! { "request_id": id.clone(), - "bid_txids": [] + "num_challenges": 0, + "bid_responses": doc! {} }, doc ); - assert_eq!(ids, doc_to_challenge_responses(&doc)); + assert_eq!(resp, doc_to_response(&doc)); - let _ = ids.insert(gen_dummy_hash(0)); - let doc = challenge_responses_to_doc(&Bson::ObjectId(id.clone()), &ids); + let hash0 = gen_dummy_hash(0); + let _ = ids.insert(hash0); + resp.update(&ids); + let doc = response_to_doc(&Bson::ObjectId(id.clone()), &resp); assert_eq!( doc! { "request_id": id.clone(), - "bid_txids": [gen_dummy_hash(0).to_string()] + "num_challenges": 1, + "bid_responses": doc! { gen_dummy_hash(0).to_string(): 1 } }, doc ); - assert_eq!(ids, doc_to_challenge_responses(&doc)); + assert_eq!(resp, doc_to_response(&doc)); let _ = ids.insert(gen_dummy_hash(1)); let _ = ids.insert(gen_dummy_hash(2)); let _ = ids.insert(gen_dummy_hash(3)); - let doc = challenge_responses_to_doc(&Bson::ObjectId(id.clone()), &ids); + resp.update(&ids); + let doc = response_to_doc(&Bson::ObjectId(id.clone()), &resp); assert_eq!(&id, doc.get("request_id").unwrap().as_object_id().unwrap()); - for id in doc.get_array("bid_txids").unwrap().iter() { - assert!(ids.contains(&sha256d::Hash::from_hex(id.as_str().unwrap()).unwrap())); + assert_eq!(2, doc.get("num_challenges").unwrap().as_i32().unwrap()); + for (key, val) in doc.get_document("bid_responses").unwrap().iter() { + if sha256d::Hash::from_hex(key.as_str()).unwrap() == hash0 { + assert_eq!(2, val.as_i32().unwrap()); + } else { + assert_eq!(1, val.as_i32().unwrap()); + } + assert!(ids.contains(&sha256d::Hash::from_hex(key.as_str()).unwrap())); } - assert_eq!(4, doc.get_array("bid_txids").unwrap().len()); - assert_eq!(ids, doc_to_challenge_responses(&doc)); + assert_eq!(4, doc.get_document("bid_responses").unwrap().len()); + assert_eq!(resp, doc_to_response(&doc)); } } diff --git a/src/util/testing.rs b/src/util/testing.rs index e576d5b..becc794 100644 --- a/src/util/testing.rs +++ b/src/util/testing.rs @@ -13,6 +13,7 @@ use util::doc_format::*; use crate::challenger::{ChallengeResponseIds, ChallengeState}; use crate::clientchain::ClientChain; use crate::request::{Bid, BidSet, Request as ServiceRequest}; +use crate::response::Response; use crate::service::Service; use crate::storage::*; @@ -284,26 +285,37 @@ impl Storage for MockStorage { Ok(()) } - /// Store responses for a specific challenge request + /// Store response for a specific challenge request fn save_response(&self, request_hash: sha256d::Hash, ids: &ChallengeResponseIds) -> Result<()> { if self.return_err { return Err(Error::from(CError::Generic("save_response failed".to_owned()))); } + + for resp_doc in self.challenge_responses.borrow_mut().iter_mut() { + if resp_doc.get("request_id").unwrap().as_str().unwrap() == &request_hash.to_string() { + let mut resp = doc_to_response(resp_doc); + resp.update(&ids); + *resp_doc = response_to_doc(&Bson::String(request_hash.to_string()), &resp); + return Ok(()); + } + } + + let mut resp = Response::new(); + resp.update(&ids); self.challenge_responses .borrow_mut() - .push(challenge_responses_to_doc(&Bson::String(request_hash.to_string()), ids)); + .push(response_to_doc(&Bson::String(request_hash.to_string()), &resp)); Ok(()) } - /// Get all challenge responses for a specific request - fn get_responses(&self, request_hash: sha256d::Hash) -> Result> { - let mut challenge_responses = vec![]; + /// Get challenge response for a specific request + fn get_response(&self, request_hash: sha256d::Hash) -> Result> { for doc in self.challenge_responses.borrow().to_vec().iter() { if doc.get("request_id").unwrap().as_str().unwrap() == request_hash.to_string() { - challenge_responses.push(doc_to_challenge_responses(doc)); + return Ok(Some(doc_to_response(doc))); } } - Ok(challenge_responses) + Ok(None) } /// Get all bids for a specific request