diff --git a/Cargo.toml b/Cargo.toml index 8c6e066..b697b02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "arloader" authors = ["calebeverett "] description = "Command line application and library for uploading files to Arweave." -version = "0.1.30" +version = "0.1.31" edition = "2021" license = "Apache-2.0" repository = "https://github.com/CalebEverett/arloader" @@ -39,7 +39,7 @@ ring = {version = "0.16.20", features = [ "std" ] } reqwest = { version = "0.11", features = ["json"] } serde = "1.0.130" serde_derive = "1.0.130" -serde_json = "1.0.68" +serde_json = { version = "1.0.68", features = ["preserve_order"] } solana-sdk = "1.8.2" thiserror = "1.0.30" tokio = { version = "1", features = ["rt-multi-thread", "fs", "macros"] } diff --git a/src/commands.rs b/src/commands.rs new file mode 100644 index 0000000..37d1d86 --- /dev/null +++ b/src/commands.rs @@ -0,0 +1,616 @@ +use crate::{ + error::Error, + file_stem_is_valid_txid, + solana::{FLOOR, RATE, SOL_AR_BASE_URL}, + status::{OutputFormat, StatusCode}, + transaction::{Base64, Tag}, + update_bundle_statuses_stream, update_statuses_stream, upload_bundles_stream, + upload_bundles_stream_with_sol, upload_files_stream, upload_files_with_sol_stream, Arweave, + BLOCK_SIZE, WINSTONS_PER_AR, +}; + +use futures::StreamExt; +use glob::glob; +use num_traits::cast::ToPrimitive; +use solana_sdk::signer::keypair; +use std::{path::PathBuf, str::FromStr}; +use tokio::{ + fs, + time::{sleep, Duration}, +}; +use url::Url; + +pub type CommandResult = Result<(), Error>; + +pub fn get_output_format(output: &str) -> OutputFormat { + match output { + "quiet" => OutputFormat::DisplayQuiet, + "verbose" => OutputFormat::DisplayVerbose, + "json" => OutputFormat::Json, + "json_compact" => OutputFormat::JsonCompact, + _ => OutputFormat::Display, + } +} + +pub async fn command_get_cost( + arweave: &Arweave, + glob_str: &str, + reward_mult: f32, + with_sol: bool, + no_bundle: bool, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let (base, incremental) = arweave.get_price_terms(reward_mult).await?; + let (_, usd_per_ar, usd_per_sol) = arweave.get_price(&1).await?; + + // set units + let units = match with_sol { + true => "lamports", + false => "winstons", + }; + + // get total number of file and bytes and cost if not bundled + let (num, mut cost, bytes) = paths_iter.fold((0, 0, 0), |(n, c, b), p| { + ( + n + 1, + c + { + let data_len = p.metadata().unwrap().len(); + let blocks_len = data_len / BLOCK_SIZE + (data_len % BLOCK_SIZE != 0) as u64; + match with_sol { + true => { + std::cmp::max((base + incremental * (blocks_len - 1)) / RATE, FLOOR) + 5000 + } + false => base + incremental * (blocks_len - 1), + } + }, + b + p.metadata().unwrap().len(), + ) + }); + + if num == 0 { + println!("No files matched glob."); + } else { + // adjust cost if bundling + if !no_bundle { + let blocks_len = bytes / BLOCK_SIZE + (bytes % BLOCK_SIZE != 0) as u64; + match with_sol { + true => { + cost = + std::cmp::max((base + incremental * (blocks_len - 1)) / RATE, FLOOR) + 5000; + } + false => { + cost = base + incremental * (blocks_len - 1); + } + } + } + + // get usd cost based on calculated cost + let usd_cost = match with_sol { + true => (&cost * &usd_per_sol).to_f32().unwrap() / 1e11_f32, + false => (&cost * &usd_per_ar).to_f32().unwrap() / 1e14_f32, + }; + + println!( + "The price to upload {} files with {} total bytes is {} {} (${:.4}).", + num, bytes, cost, units, usd_cost + ); + } + Ok(()) +} + +pub async fn command_get_transaction(arweave: &Arweave, id: &str) -> CommandResult { + let id = Base64::from_str(id)?; + let transaction = arweave.get_transaction(&id).await?; + println!("Fetched transaction {}", transaction.id); + Ok(()) +} + +pub async fn command_get_status(arweave: &Arweave, id: &str, output_format: &str) -> CommandResult { + let id = Base64::from_str(id)?; + let output_format = get_output_format(output_format); + let status = arweave.get_status(&id).await?; + println!( + "{}", + status + .header_string(&output_format) + .split_at(32) + .1 + .split_at(132) + .0 + ); + print!("{}", output_format.formatted_string(&status).split_at(32).1); + Ok(()) +} + +pub async fn command_wallet_balance( + arweave: &Arweave, + wallet_address: Option, +) -> CommandResult { + let mb = u64::pow(1024, 2); + let result = tokio::join!( + arweave.get_wallet_balance(wallet_address), + arweave.get_price(&mb) + ); + let balance = result.0?; + let (winstons_per_kb, usd_per_ar, _) = result.1?; + + let balance_usd = &balance.to_f32().unwrap() / &WINSTONS_PER_AR.to_f32().unwrap() + * &usd_per_ar.to_f32().unwrap() + / 100_f32; + + let usd_per_kb = (&winstons_per_kb * &usd_per_ar).to_f32().unwrap() / 1e14_f32; + + println!( + "Wallet balance is {} {units} (${balance_usd:.2} at ${ar_price:.2} USD per AR). At the current price of {price} {units} per MB (${usd_price:.4}), you can upload {max} MB of data.", + &balance, + units = arweave.units, + max = &balance / &winstons_per_kb, + price = &winstons_per_kb, + balance_usd = balance_usd, + ar_price = &usd_per_ar.to_f32().unwrap() + / 100_f32, + usd_price = usd_per_kb + ); + Ok(()) +} + +pub async fn command_get_pending_count(arweave: &Arweave) -> CommandResult { + println!(" {}\n{:-<84}", "pending tx", ""); + + let mut counter = 0; + while counter < 60 { + sleep(Duration::from_secs(1)).await; + let count = arweave.get_pending_count().await?; + println!( + "{:>5} {} {}", + count, + 124u8 as char, + std::iter::repeat('\u{25A5}') + .take(count / 50 + 1) + .collect::() + ); + counter += 1; + } + Ok(()) +} + +pub async fn command_upload( + arweave: &Arweave, + glob_str: &str, + log_dir: Option<&str>, + _tags: Option>>, + reward_mult: f32, + output_format: Option<&str>, + buffer: usize, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = log_dir.map(|s| PathBuf::from(s)); + let output_format = get_output_format(output_format.unwrap_or("")); + let price_terms = arweave.get_price_terms(reward_mult).await?; + + let mut stream = upload_files_stream( + arweave, + paths_iter, + log_dir.clone(), + None, + price_terms, + buffer, + ); + + let mut counter = 0; + while let Some(result) = stream.next().await { + match result { + Ok(status) => { + if counter == 0 { + if let Some(log_dir) = &log_dir { + println!("Logging statuses to {}", &log_dir.display()); + } + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + counter += 1; + } + Err(e) => println!("{:#?}", e), + } + } + + if counter == 0 { + println!("The pattern \"{}\" didn't match any files.", glob_str); + } else { + println!( + "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir \"{}\"` to confirm transaction(s).", + counter, + glob_str, + &log_dir.unwrap_or(PathBuf::from("")).display() + ); + } + + Ok(()) +} + +pub async fn command_upload_bundles( + arweave: &Arweave, + glob_str: &str, + log_dir: Option<&str>, + tags: Option>>, + bundle_size: u64, + reward_mult: f32, + output_format: Option<&str>, + buffer: usize, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = log_dir.map(|s| PathBuf::from(s)).unwrap(); + let output_format = get_output_format(output_format.unwrap_or("")); + let tags = tags.unwrap_or(Vec::new()); + let price_terms = arweave.get_price_terms(reward_mult).await?; + let path_chunks = arweave.chunk_file_paths(paths_iter, bundle_size)?; + + if path_chunks.len() == 0 { + println!("The pattern \"{}\" didn't match any files.", glob_str); + return Ok(()); + } else { + let mut stream = upload_bundles_stream(arweave, path_chunks, tags, price_terms, buffer); + + let mut counter = 0; + let mut number_of_files = 0; + let mut data_size = 0; + while let Some(result) = stream.next().await { + match result { + Ok(status) => { + number_of_files += status.number_of_files; + data_size += status.data_size; + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + fs::write( + log_dir.join(status.id.to_string()).with_extension("json"), + serde_json::to_string(&status)?, + ) + .await?; + counter += 1; + } + Err(e) => println!("{:#?}", e), + } + } + + println!( + "\nUploaded {} KB in {} files in {} bundle transactions. Run `arloader update-status --log-dir \"{}\"` to update statuses.", + data_size / 1000, + number_of_files, + counter, + log_dir.display().to_string() + ); + } + Ok(()) +} + +pub async fn command_upload_with_sol( + arweave: &Arweave, + glob_str: &str, + log_dir: Option<&str>, + _tags: Option>>, + reward_mult: f32, + sol_keypair_path: &str, + output_format: Option<&str>, + buffer: usize, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = log_dir.map(|s| PathBuf::from(s)); + let output_format = get_output_format(output_format.unwrap_or("")); + let solana_url = "https://api.mainnet-beta.solana.com/".parse::()?; + let sol_ar_url = SOL_AR_BASE_URL.parse::()?.join("sol")?; + let from_keypair = keypair::read_keypair_file(sol_keypair_path)?; + + let price_terms = arweave.get_price_terms(reward_mult).await?; + + let mut stream = upload_files_with_sol_stream( + arweave, + paths_iter, + log_dir.clone(), + None, + price_terms, + solana_url, + sol_ar_url, + &from_keypair, + buffer, + ); + + let mut counter = 0; + while let Some(result) = stream.next().await { + match result { + Ok(status) => { + if counter == 0 { + if let Some(log_dir) = &log_dir { + println!("Logging statuses to {}", &log_dir.display()); + } + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + counter += 1; + } + Err(e) => println!("{:#?}", e), + } + } + + if counter == 0 { + println!("The pattern \"{}\" didn't match any files.", glob_str); + } else { + println!( + "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir \"{}\"` to confirm transaction(s).", + counter, + glob_str, + &log_dir.unwrap_or(PathBuf::from("")).display() + ); + } + + Ok(()) +} + +pub async fn command_upload_bundles_with_sol( + arweave: &Arweave, + glob_str: &str, + log_dir: Option<&str>, + tags: Option>>, + bundle_size: u64, + reward_mult: f32, + output_format: Option<&str>, + buffer: usize, + sol_keypair_path: &str, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = log_dir.map(|s| PathBuf::from(s)).unwrap(); + let output_format = get_output_format(output_format.unwrap_or("")); + let tags = tags.unwrap_or(Vec::new()); + let price_terms = arweave.get_price_terms(reward_mult).await?; + let path_chunks = arweave.chunk_file_paths(paths_iter, bundle_size)?; + let solana_url = "https://api.mainnet-beta.solana.com/".parse::()?; + let sol_ar_url = SOL_AR_BASE_URL.parse::()?.join("sol")?; + let from_keypair = keypair::read_keypair_file(sol_keypair_path)?; + + if path_chunks.len() == 0 { + println!("The pattern \"{}\" didn't match any files.", glob_str); + return Ok(()); + } else { + let mut stream = upload_bundles_stream_with_sol( + arweave, + path_chunks, + tags, + price_terms, + buffer, + solana_url, + sol_ar_url, + &from_keypair, + ); + + let mut counter = 0; + let mut number_of_files = 0; + let mut data_size = 0; + while let Some(result) = stream.next().await { + match result { + Ok(status) => { + number_of_files += status.number_of_files; + data_size += status.data_size; + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + fs::write( + log_dir.join(status.id.to_string()).with_extension("json"), + serde_json::to_string(&status)?, + ) + .await?; + counter += 1; + } + Err(e) => println!("{:#?}", e), + } + } + + println!( + "\nUploaded {} KB in {} files in {} bundle transaction(s). Run `arloader update-status --log-dir \"{}\"` to update statuses.", + data_size / 1000, + number_of_files, + counter, + log_dir.display().to_string() + ); + } + Ok(()) +} + +pub async fn command_list_statuses( + arweave: &Arweave, + glob_str: &str, + log_dir: &str, + statuses: Option>, + max_confirms: Option<&str>, + output_format: Option<&str>, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = PathBuf::from(log_dir); + let output_format = get_output_format(output_format.unwrap_or("")); + let max_confirms = max_confirms.map(|m| m.parse::().unwrap()); + + let mut counter = 0; + for status in arweave + .filter_statuses(paths_iter, log_dir.clone(), statuses, max_confirms) + .await? + .iter() + { + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(status)); + counter += 1; + } + if counter == 0 { + println!("Didn't find match any statuses."); + } else { + println!("Found {} files matching filter criteria.", counter); + } + Ok(()) +} + +pub async fn command_update_statuses( + arweave: &Arweave, + glob_str: &str, + log_dir: &str, + output_format: Option<&str>, + buffer: usize, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = PathBuf::from(log_dir); + let output_format = get_output_format(output_format.unwrap_or("")); + + let mut stream = update_statuses_stream(arweave, paths_iter, log_dir.clone(), buffer); + + let mut counter = 0; + while let Some(Ok(status)) = stream.next().await { + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + counter += 1; + } + if counter == 0 { + println!("The `glob` and `log_dir` combination you provided didn't return any statuses."); + } else { + println!("Updated {} statuses.", counter); + } + + Ok(()) +} + +pub async fn command_update_bundle_statuses( + arweave: &Arweave, + log_dir: &str, + output_format: Option<&str>, + buffer: usize, +) -> CommandResult { + let paths_iter = glob(&format!("{}/*.json", log_dir))? + .filter_map(Result::ok) + .filter(|p| file_stem_is_valid_txid(p)); + let output_format = get_output_format(output_format.unwrap_or("")); + + let mut stream = update_bundle_statuses_stream(arweave, paths_iter, buffer); + + let mut counter = 0; + while let Some(Ok(status)) = stream.next().await { + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + counter += 1; + } + if counter == 0 { + println!("The `log_dir`you provided didn't have any statuses in it."); + } else { + println!("Updated {} statuses.", counter); + } + + Ok(()) +} + +pub async fn command_status_report( + arweave: &Arweave, + glob_str: &str, + log_dir: &str, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = PathBuf::from(log_dir); + + let summary = arweave.status_summary(paths_iter, log_dir).await?; + + println!("{}", summary); + + Ok(()) +} + +pub async fn command_upload_filter( + arweave: &Arweave, + glob_str: &str, + log_dir: &str, + reward_mult: f32, + statuses: Option>, + max_confirms: Option<&str>, + output_format: Option<&str>, + buffer: Option<&str>, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let log_dir = PathBuf::from(log_dir); + let output_format = get_output_format(output_format.unwrap_or("")); + let max_confirms = max_confirms.map(|m| m.parse::().unwrap()); + let buffer = buffer.map(|b| b.parse::().unwrap()).unwrap_or(1); + let price_terms = arweave.get_price_terms(reward_mult).await?; + + // Should be refactored to be included in the stream. + let filtered_paths_iter = arweave + .filter_statuses(paths_iter, log_dir.clone(), statuses, max_confirms) + .await? + .into_iter() + .filter_map(|f| f.file_path); + + let mut stream = upload_files_stream( + arweave, + filtered_paths_iter, + Some(log_dir.clone()), + None, + price_terms, + buffer, + ); + + let mut counter = 0; + while let Some(Ok(status)) = stream.next().await { + if counter == 0 { + println!("{}", status.header_string(&output_format)); + } + print!("{}", output_format.formatted_string(&status)); + counter += 1; + } + if counter == 0 { + println!("Didn't find any matching statuses."); + } else { + println!( + "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir {} to confirm transaction(s).", + counter, + glob_str, + &log_dir.display() + ); + } + Ok(()) +} + +pub async fn command_upload_manifest( + arweave: &Arweave, + log_dir: &str, + reward_mult: f32, +) -> CommandResult { + let price_terms = arweave.get_price_terms(reward_mult).await?; + let output = arweave + .upload_manifest_from_log_dir(log_dir, price_terms) + .await?; + + println!("{}", output); + Ok(()) +} + +pub async fn command_update_metadata( + arweave: &Arweave, + glob_str: &str, + manifest_str: &str, + image_link_file: bool, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let num_paths: usize = paths_iter.collect::>().len(); + let manifest_path = PathBuf::from(manifest_str); + + arweave + .update_metadata( + glob(glob_str)?.filter_map(Result::ok), + manifest_path, + image_link_file, + ) + .await?; + + println!("Successfully updated {} metadata files.", num_paths); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 2ff3254..1bd047a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,60 @@ //! SDK for uploading files in bulk to [Arweave](https://www.arweave.org/). //! -//! Files can't just be uploaded in a post it and forget manner to Arweave since their data needs to be -//! written to the blockchain by node operators and that doesn't happen instantaneously. This SDK aims to -//! make the process of uploading large numbers of files as seamless as possible. In addition to providing -//! highly performant, streaming uploads, it also includes status logging and reporting features through which -//! complete upload processes can be developed, including uploading files, updating statuses and re-uploading -//! files from filtered sets of statuses. +//! ## CLI +//! See [README.md](https://crates.io/crates/arloader) for usage instructions. +//! +//! The main cli application is all in `main.rs` following a pattern of specifying arguments, +//! matching them, and then in turn passing them to commands, which are all broken out +//! separately in the [`commands`] module to facilitate re-use in other command +//! line applications or as library functions. There shouldn't be anything called by the cli +//! application directly from the library. Everything gets composed into a command in the [`commands`] +//! module. +//! +//! ## Library +//! +//! #### Overview +//! The library is mostly focused on uploading files as efficiently as possible. Arweave has +//! two different transaction formats and two different upload formats. Transactions can either +//! be normal, single data item transactions (see [transaction format](https://docs.arweave.org/developers/server/http-api#transaction-format) +//! for details), or bundle transactions (see [bundle format](https://github.com/joshbenaron/arweave-standards/blob/ans104/ans/ANS-104.md) +//! for details). The bundle format, introduced mid-2021, bundles together individual data items +//! into larger transactions, making uploading much more efficient and reducing network congestion. +//! The library supports both formats, with the recommended approach being to use the bundle format. +//! +//! #### Transactions and DataItems +//! Both formats start with chunking file data and creating merkle trees from the chunks. The merkle +//! tree logic can be found in the [`merkle`] module. All of the hashing functions and other crypto +//! operations are in the [`crypto`] module. Once the data is chunked, hashed, and a merkle root +//! calculated for it, it gets incorporated into either a [`Transaction`], which can be found in the +//! [`transaction`] module, or a [`DataItem`] (if it is going to be included in a bundle format transaction), +//! which can be found in the [`bundle`] module. +//! +//! #### Bytes and Base64Url Data +//! The library takes advantage of Rust's strong typing and trait model to store all data, signatures and +//! addresses as a [`Base64`] struct with implementations for serialization and deserialization that automatically +//! convert the underlying bytes to and from the Base64Url format required for submission to Arweave. +//! +//! #### Signing +//! A key part of constructing transactions is signing them. Arweave has a specific algorithm for generating the +//! digest that gets signed and then hashed to serve as a transaction id, called deep hash. It takes various elements +//! of either the [`Transaction`] or [`DataItem`], including nested arrays of tags, and successively concatenates and +//! hashes them. The required elements are assembled using the [`ToItems`] trait, implemented separately for [`Transaction`] +//! and [`DataItem`]. Arloader's implementation of the deep hash algorithm can be found in [`crypto::Provider::deep_hash`]. +//! +//! #### Higher Level Functions +//! The functions for creating transactions, data items and bundles are all consolidated on the [`Arweave`] struct. +//! In general there are lower level functions for creating the items from data, that are then used in successively +//! higher level functions to create the items from a single file path and ultimately uploading streams of items in +//! parallel from collections of file paths. +//! +//! #### Status Tracking +//! A key added functionality is the tracking and reporting on transaction statuses. There are two status structs, +//! [`Status`] and [`BundleStatus`] used for these purposes. They are essentially the same format, except that +//! [`BundleStatus`] is modified to include references to all of the included [`DataItem`] instead of just a +//! single [`Transaction`] for [`Status`]. +//! +//! #### Solana +//! The functions for allowing payment to be made in SOL can be found in the [`solana`] module. #![feature(derive_default_enum)] use crate::solana::{create_sol_transaction, get_sol_ar_signature, SigResponse, FLOOR, RATE}; @@ -33,6 +82,7 @@ use tokio::fs; use url::Url; pub mod bundle; +pub mod commands; pub mod crypto; pub mod error; pub mod merkle; diff --git a/src/main.rs b/src/main.rs index d2c2cdc..06e5187 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,16 @@ use arloader::{ + commands::*, error::Error, - file_stem_is_valid_txid, - solana::{FLOOR, RATE, SOL_AR_BASE_URL}, - status::{OutputFormat, StatusCode}, + status::StatusCode, transaction::{Base64, FromUtf8Strs, Tag}, - update_bundle_statuses_stream, update_statuses_stream, upload_bundles_stream, - upload_bundles_stream_with_sol, upload_files_stream, upload_files_with_sol_stream, Arweave, - BLOCK_SIZE, WINSTONS_PER_AR, + Arweave, }; use clap::{ self, crate_description, crate_name, crate_version, value_t, App, AppSettings, Arg, SubCommand, Values, }; -use futures::StreamExt; -use glob::glob; -use num_traits::cast::ToPrimitive; -use solana_sdk::signer::keypair; + use std::{fmt::Display, path::PathBuf, str::FromStr}; -use tokio::{ - fs, - time::{sleep, Duration}, -}; use url::Url; pub type CommandResult = Result<(), Error>; @@ -125,16 +115,6 @@ where } } -fn get_output_format(output: &str) -> OutputFormat { - match output { - "quiet" => OutputFormat::DisplayQuiet, - "verbose" => OutputFormat::DisplayVerbose, - "json" => OutputFormat::Json, - "json_compact" => OutputFormat::JsonCompact, - _ => OutputFormat::Display, - } -} - fn get_status_code(output: &str) -> StatusCode { match output { "Submitted" => StatusCode::Submitted, @@ -648,585 +628,6 @@ async fn main() -> CommandResult { } } -async fn command_get_cost( - arweave: &Arweave, - glob_str: &str, - reward_mult: f32, - with_sol: bool, - no_bundle: bool, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let (base, incremental) = arweave.get_price_terms(reward_mult).await?; - let (_, usd_per_ar, usd_per_sol) = arweave.get_price(&1).await?; - - // set units - let units = match with_sol { - true => "lamports", - false => "winstons", - }; - - // get total number of file and bytes and cost if not bundled - let (num, mut cost, bytes) = paths_iter.fold((0, 0, 0), |(n, c, b), p| { - ( - n + 1, - c + { - let data_len = p.metadata().unwrap().len(); - let blocks_len = data_len / BLOCK_SIZE + (data_len % BLOCK_SIZE != 0) as u64; - match with_sol { - true => { - std::cmp::max((base + incremental * (blocks_len - 1)) / RATE, FLOOR) + 5000 - } - false => base + incremental * (blocks_len - 1), - } - }, - b + p.metadata().unwrap().len(), - ) - }); - - if num == 0 { - println!("No files matched glob."); - } else { - // adjust cost if bundling - if !no_bundle { - let blocks_len = bytes / BLOCK_SIZE + (bytes % BLOCK_SIZE != 0) as u64; - match with_sol { - true => { - cost = - std::cmp::max((base + incremental * (blocks_len - 1)) / RATE, FLOOR) + 5000; - } - false => { - cost = base + incremental * (blocks_len - 1); - } - } - } - - // get usd cost based on calculated cost - let usd_cost = match with_sol { - true => (&cost * &usd_per_sol).to_f32().unwrap() / 1e11_f32, - false => (&cost * &usd_per_ar).to_f32().unwrap() / 1e14_f32, - }; - - println!( - "The price to upload {} files with {} total bytes is {} {} (${:.4}).", - num, bytes, cost, units, usd_cost - ); - } - Ok(()) -} - -async fn command_get_transaction(arweave: &Arweave, id: &str) -> CommandResult { - let id = Base64::from_str(id)?; - let transaction = arweave.get_transaction(&id).await?; - println!("Fetched transaction {}", transaction.id); - Ok(()) -} - -async fn command_get_status(arweave: &Arweave, id: &str, output_format: &str) -> CommandResult { - let id = Base64::from_str(id)?; - let output_format = get_output_format(output_format); - let status = arweave.get_status(&id).await?; - println!( - "{}", - status - .header_string(&output_format) - .split_at(32) - .1 - .split_at(132) - .0 - ); - print!("{}", output_format.formatted_string(&status).split_at(32).1); - Ok(()) -} - -async fn command_wallet_balance( - arweave: &Arweave, - wallet_address: Option, -) -> CommandResult { - let mb = u64::pow(1024, 2); - let result = tokio::join!( - arweave.get_wallet_balance(wallet_address), - arweave.get_price(&mb) - ); - let balance = result.0?; - let (winstons_per_kb, usd_per_ar, _) = result.1?; - - let balance_usd = &balance.to_f32().unwrap() / &WINSTONS_PER_AR.to_f32().unwrap() - * &usd_per_ar.to_f32().unwrap() - / 100_f32; - - let usd_per_kb = (&winstons_per_kb * &usd_per_ar).to_f32().unwrap() / 1e14_f32; - - println!( - "Wallet balance is {} {units} (${balance_usd:.2} at ${ar_price:.2} USD per AR). At the current price of {price} {units} per MB (${usd_price:.4}), you can upload {max} MB of data.", - &balance, - units = arweave.units, - max = &balance / &winstons_per_kb, - price = &winstons_per_kb, - balance_usd = balance_usd, - ar_price = &usd_per_ar.to_f32().unwrap() - / 100_f32, - usd_price = usd_per_kb - ); - Ok(()) -} - -async fn command_get_pending_count(arweave: &Arweave) -> CommandResult { - println!(" {}\n{:-<84}", "pending tx", ""); - - let mut counter = 0; - while counter < 60 { - sleep(Duration::from_secs(1)).await; - let count = arweave.get_pending_count().await?; - println!( - "{:>5} {} {}", - count, - 124u8 as char, - std::iter::repeat('\u{25A5}') - .take(count / 50 + 1) - .collect::() - ); - counter += 1; - } - Ok(()) -} - -async fn command_upload( - arweave: &Arweave, - glob_str: &str, - log_dir: Option<&str>, - _tags: Option>>, - reward_mult: f32, - output_format: Option<&str>, - buffer: usize, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = log_dir.map(|s| PathBuf::from(s)); - let output_format = get_output_format(output_format.unwrap_or("")); - let price_terms = arweave.get_price_terms(reward_mult).await?; - - let mut stream = upload_files_stream( - arweave, - paths_iter, - log_dir.clone(), - None, - price_terms, - buffer, - ); - - let mut counter = 0; - while let Some(result) = stream.next().await { - match result { - Ok(status) => { - if counter == 0 { - if let Some(log_dir) = &log_dir { - println!("Logging statuses to {}", &log_dir.display()); - } - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - counter += 1; - } - Err(e) => println!("{:#?}", e), - } - } - - if counter == 0 { - println!("The pattern \"{}\" didn't match any files.", glob_str); - } else { - println!( - "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir \"{}\"` to confirm transaction(s).", - counter, - glob_str, - &log_dir.unwrap_or(PathBuf::from("")).display() - ); - } - - Ok(()) -} - -async fn command_upload_bundles( - arweave: &Arweave, - glob_str: &str, - log_dir: Option<&str>, - tags: Option>>, - bundle_size: u64, - reward_mult: f32, - output_format: Option<&str>, - buffer: usize, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = log_dir.map(|s| PathBuf::from(s)).unwrap(); - let output_format = get_output_format(output_format.unwrap_or("")); - let tags = tags.unwrap_or(Vec::new()); - let price_terms = arweave.get_price_terms(reward_mult).await?; - let path_chunks = arweave.chunk_file_paths(paths_iter, bundle_size)?; - - if path_chunks.len() == 0 { - println!("The pattern \"{}\" didn't match any files.", glob_str); - return Ok(()); - } else { - let mut stream = upload_bundles_stream(arweave, path_chunks, tags, price_terms, buffer); - - let mut counter = 0; - let mut number_of_files = 0; - let mut data_size = 0; - while let Some(result) = stream.next().await { - match result { - Ok(status) => { - number_of_files += status.number_of_files; - data_size += status.data_size; - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - fs::write( - log_dir.join(status.id.to_string()).with_extension("json"), - serde_json::to_string(&status)?, - ) - .await?; - counter += 1; - } - Err(e) => println!("{:#?}", e), - } - } - - println!( - "\nUploaded {} KB in {} files in {} bundle transactions. Run `arloader update-status --log-dir \"{}\"` to update statuses.", - data_size / 1000, - number_of_files, - counter, - log_dir.display().to_string() - ); - } - Ok(()) -} - -async fn command_upload_with_sol( - arweave: &Arweave, - glob_str: &str, - log_dir: Option<&str>, - _tags: Option>>, - reward_mult: f32, - sol_keypair_path: &str, - output_format: Option<&str>, - buffer: usize, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = log_dir.map(|s| PathBuf::from(s)); - let output_format = get_output_format(output_format.unwrap_or("")); - let solana_url = "https://api.mainnet-beta.solana.com/".parse::()?; - let sol_ar_url = SOL_AR_BASE_URL.parse::()?.join("sol")?; - let from_keypair = keypair::read_keypair_file(sol_keypair_path)?; - - let price_terms = arweave.get_price_terms(reward_mult).await?; - - let mut stream = upload_files_with_sol_stream( - arweave, - paths_iter, - log_dir.clone(), - None, - price_terms, - solana_url, - sol_ar_url, - &from_keypair, - buffer, - ); - - let mut counter = 0; - while let Some(result) = stream.next().await { - match result { - Ok(status) => { - if counter == 0 { - if let Some(log_dir) = &log_dir { - println!("Logging statuses to {}", &log_dir.display()); - } - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - counter += 1; - } - Err(e) => println!("{:#?}", e), - } - } - - if counter == 0 { - println!("The pattern \"{}\" didn't match any files.", glob_str); - } else { - println!( - "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir \"{}\"` to confirm transaction(s).", - counter, - glob_str, - &log_dir.unwrap_or(PathBuf::from("")).display() - ); - } - - Ok(()) -} - -async fn command_upload_bundles_with_sol( - arweave: &Arweave, - glob_str: &str, - log_dir: Option<&str>, - tags: Option>>, - bundle_size: u64, - reward_mult: f32, - output_format: Option<&str>, - buffer: usize, - sol_keypair_path: &str, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = log_dir.map(|s| PathBuf::from(s)).unwrap(); - let output_format = get_output_format(output_format.unwrap_or("")); - let tags = tags.unwrap_or(Vec::new()); - let price_terms = arweave.get_price_terms(reward_mult).await?; - let path_chunks = arweave.chunk_file_paths(paths_iter, bundle_size)?; - let solana_url = "https://api.mainnet-beta.solana.com/".parse::()?; - let sol_ar_url = SOL_AR_BASE_URL.parse::()?.join("sol")?; - let from_keypair = keypair::read_keypair_file(sol_keypair_path)?; - - if path_chunks.len() == 0 { - println!("The pattern \"{}\" didn't match any files.", glob_str); - return Ok(()); - } else { - let mut stream = upload_bundles_stream_with_sol( - arweave, - path_chunks, - tags, - price_terms, - buffer, - solana_url, - sol_ar_url, - &from_keypair, - ); - - let mut counter = 0; - let mut number_of_files = 0; - let mut data_size = 0; - while let Some(result) = stream.next().await { - match result { - Ok(status) => { - number_of_files += status.number_of_files; - data_size += status.data_size; - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - fs::write( - log_dir.join(status.id.to_string()).with_extension("json"), - serde_json::to_string(&status)?, - ) - .await?; - counter += 1; - } - Err(e) => println!("{:#?}", e), - } - } - - println!( - "\nUploaded {} KB in {} files in {} bundle transaction(s). Run `arloader update-status --log-dir \"{}\"` to update statuses.", - data_size / 1000, - number_of_files, - counter, - log_dir.display().to_string() - ); - } - Ok(()) -} - -async fn command_list_statuses( - arweave: &Arweave, - glob_str: &str, - log_dir: &str, - statuses: Option>, - max_confirms: Option<&str>, - output_format: Option<&str>, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = PathBuf::from(log_dir); - let output_format = get_output_format(output_format.unwrap_or("")); - let max_confirms = max_confirms.map(|m| m.parse::().unwrap()); - - let mut counter = 0; - for status in arweave - .filter_statuses(paths_iter, log_dir.clone(), statuses, max_confirms) - .await? - .iter() - { - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(status)); - counter += 1; - } - if counter == 0 { - println!("Didn't find match any statuses."); - } else { - println!("Found {} files matching filter criteria.", counter); - } - Ok(()) -} - -async fn command_update_statuses( - arweave: &Arweave, - glob_str: &str, - log_dir: &str, - output_format: Option<&str>, - buffer: usize, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = PathBuf::from(log_dir); - let output_format = get_output_format(output_format.unwrap_or("")); - - let mut stream = update_statuses_stream(arweave, paths_iter, log_dir.clone(), buffer); - - let mut counter = 0; - while let Some(Ok(status)) = stream.next().await { - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - counter += 1; - } - if counter == 0 { - println!("The `glob` and `log_dir` combination you provided didn't return any statuses."); - } else { - println!("Updated {} statuses.", counter); - } - - Ok(()) -} - -async fn command_update_bundle_statuses( - arweave: &Arweave, - log_dir: &str, - output_format: Option<&str>, - buffer: usize, -) -> CommandResult { - let paths_iter = glob(&format!("{}/*.json", log_dir))? - .filter_map(Result::ok) - .filter(|p| file_stem_is_valid_txid(p)); - let output_format = get_output_format(output_format.unwrap_or("")); - - let mut stream = update_bundle_statuses_stream(arweave, paths_iter, buffer); - - let mut counter = 0; - while let Some(Ok(status)) = stream.next().await { - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - counter += 1; - } - if counter == 0 { - println!("The `log_dir`you provided didn't have any statuses in it."); - } else { - println!("Updated {} statuses.", counter); - } - - Ok(()) -} - -async fn command_status_report(arweave: &Arweave, glob_str: &str, log_dir: &str) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = PathBuf::from(log_dir); - - let summary = arweave.status_summary(paths_iter, log_dir).await?; - - println!("{}", summary); - - Ok(()) -} - -async fn command_upload_filter( - arweave: &Arweave, - glob_str: &str, - log_dir: &str, - reward_mult: f32, - statuses: Option>, - max_confirms: Option<&str>, - output_format: Option<&str>, - buffer: Option<&str>, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let log_dir = PathBuf::from(log_dir); - let output_format = get_output_format(output_format.unwrap_or("")); - let max_confirms = max_confirms.map(|m| m.parse::().unwrap()); - let buffer = buffer.map(|b| b.parse::().unwrap()).unwrap_or(1); - let price_terms = arweave.get_price_terms(reward_mult).await?; - - // Should be refactored to be included in the stream. - let filtered_paths_iter = arweave - .filter_statuses(paths_iter, log_dir.clone(), statuses, max_confirms) - .await? - .into_iter() - .filter_map(|f| f.file_path); - - let mut stream = upload_files_stream( - arweave, - filtered_paths_iter, - Some(log_dir.clone()), - None, - price_terms, - buffer, - ); - - let mut counter = 0; - while let Some(Ok(status)) = stream.next().await { - if counter == 0 { - println!("{}", status.header_string(&output_format)); - } - print!("{}", output_format.formatted_string(&status)); - counter += 1; - } - if counter == 0 { - println!("Didn't find any matching statuses."); - } else { - println!( - "Uploaded {} files. Run `arloader update-status \"{}\" --log-dir {} to confirm transaction(s).", - counter, - glob_str, - &log_dir.display() - ); - } - Ok(()) -} - -async fn command_upload_manifest( - arweave: &Arweave, - log_dir: &str, - reward_mult: f32, -) -> CommandResult { - let price_terms = arweave.get_price_terms(reward_mult).await?; - let output = arweave - .upload_manifest_from_log_dir(log_dir, price_terms) - .await?; - - println!("{}", output); - Ok(()) -} - -async fn command_update_metadata( - arweave: &Arweave, - glob_str: &str, - manifest_str: &str, - image_link_file: bool, -) -> CommandResult { - let paths_iter = glob(glob_str)?.filter_map(Result::ok); - let num_paths: usize = paths_iter.collect::>().len(); - let manifest_path = PathBuf::from(manifest_str); - - arweave - .update_metadata( - glob(glob_str)?.filter_map(Result::ok), - manifest_path, - image_link_file, - ) - .await?; - - println!("Successfully updated {} metadata files.", num_paths); - Ok(()) -} - #[cfg(test)] mod tests { use super::get_app; diff --git a/src/transaction.rs b/src/transaction.rs index 00a1ab3..f8db97e 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -101,7 +101,7 @@ impl Transaction { } /// Implemented on [`Transaction`] to create root [`DeepHashItem`]s used by -/// [`crate::crypto::Methods::deep_hash`] in the creation of a transaction +/// [`crate::crypto::Provider::deep_hash`] in the creation of a transaction /// signatures. pub trait ToItems<'a, T> { fn to_deep_hash_item(&'a self) -> Result;