diff --git a/Cargo.toml b/Cargo.toml index b697b02..b50b8ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "arloader" authors = ["calebeverett "] description = "Command line application and library for uploading files to Arweave." -version = "0.1.31" +version = "0.1.32" edition = "2021" license = "Apache-2.0" repository = "https://github.com/CalebEverett/arloader" diff --git a/src/bundle.rs b/src/bundle.rs index 642d651..ef8f31b 100644 --- a/src/bundle.rs +++ b/src/bundle.rs @@ -1,3 +1,5 @@ +//! Data structure and functionality to create, serialize and deserialize [`DataItem`]s. + use crate::error::Error; use crate::transaction::{Base64, DeepHashItem, Tag, ToItems}; use avro_rs::Schema; @@ -5,6 +7,7 @@ use bytes::BufMut; use serde::{Deserialize, Serialize}; use std::io::Write; +/// Returns [`avro_rs::Schema`] for [`DataItem`] [`Tag`]s. pub fn get_tags_schema() -> Schema { let schema = r#" { @@ -23,6 +26,7 @@ pub fn get_tags_schema() -> Schema { Schema::parse_str(schema).unwrap() } +/// Primary structure for [`DataItem`]s included in bundles. #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct DataItem { pub id: Base64, diff --git a/src/commands.rs b/src/commands.rs index 37d1d86..d115bc9 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -1,3 +1,5 @@ +//! Functions for Cli commands composed from library functions. + use crate::{ error::Error, file_stem_is_valid_txid, @@ -22,6 +24,7 @@ use url::Url; pub type CommandResult = Result<(), Error>; +/// Maps cli string argument to [`OutputFormat`]. pub fn get_output_format(output: &str) -> OutputFormat { match output { "quiet" => OutputFormat::DisplayQuiet, @@ -32,6 +35,7 @@ pub fn get_output_format(output: &str) -> OutputFormat { } } +/// Used by `estimate` command to return estimated cost of uploading a `glob` of files. pub async fn command_get_cost( arweave: &Arweave, glob_str: &str, @@ -98,6 +102,7 @@ pub async fn command_get_cost( Ok(()) } +/// Retrieves transaction from the network. pub async fn command_get_transaction(arweave: &Arweave, id: &str) -> CommandResult { let id = Base64::from_str(id)?; let transaction = arweave.get_transaction(&id).await?; @@ -105,6 +110,7 @@ pub async fn command_get_transaction(arweave: &Arweave, id: &str) -> CommandResu Ok(()) } +/// Gets status from the network for the provided [`crate::transaction::Transaction`] id. pub async fn command_get_status(arweave: &Arweave, id: &str, output_format: &str) -> CommandResult { let id = Base64::from_str(id)?; let output_format = get_output_format(output_format); @@ -122,6 +128,7 @@ pub async fn command_get_status(arweave: &Arweave, id: &str, output_format: &str Ok(()) } +/// Gets balance for provided wallet address. pub async fn command_wallet_balance( arweave: &Arweave, wallet_address: Option, @@ -154,6 +161,7 @@ pub async fn command_wallet_balance( Ok(()) } +/// Displays pending transaction count every second for one minute. pub async fn command_get_pending_count(arweave: &Arweave) -> CommandResult { println!(" {}\n{:-<84}", "pending tx", ""); @@ -174,6 +182,7 @@ pub async fn command_get_pending_count(arweave: &Arweave) -> CommandResult { Ok(()) } +/// Uploads files to Arweave. pub async fn command_upload( arweave: &Arweave, glob_str: &str, @@ -228,6 +237,7 @@ pub async fn command_upload( Ok(()) } +/// Uploads bundles created from provided glob to Arweave. pub async fn command_upload_bundles( arweave: &Arweave, glob_str: &str, @@ -285,6 +295,7 @@ pub async fn command_upload_bundles( Ok(()) } +/// Uploads files to Arweave, paying with SOL. pub async fn command_upload_with_sol( arweave: &Arweave, glob_str: &str, @@ -347,6 +358,7 @@ pub async fn command_upload_with_sol( Ok(()) } +/// Uploads bundles created from provided glob to Arweave, paying with SOL. pub async fn command_upload_bundles_with_sol( arweave: &Arweave, glob_str: &str, @@ -417,6 +429,7 @@ pub async fn command_upload_bundles_with_sol( Ok(()) } +/// Reads [`crate::status::Status`] for provided files in provided directory, filtered by statuses and max confirmations if provided. pub async fn command_list_statuses( arweave: &Arweave, glob_str: &str, @@ -450,6 +463,7 @@ pub async fn command_list_statuses( Ok(()) } +/// Updates [`crate::status::BundleStatus`]s for provided files in provided directory. pub async fn command_update_statuses( arweave: &Arweave, glob_str: &str, @@ -480,6 +494,7 @@ pub async fn command_update_statuses( Ok(()) } +/// Updates [`crate::status::BundleStatus`]s for provided files in provided directory. pub async fn command_update_bundle_statuses( arweave: &Arweave, log_dir: &str, @@ -597,7 +612,7 @@ pub async fn command_update_metadata( arweave: &Arweave, glob_str: &str, manifest_str: &str, - image_link_file: bool, + link_file: bool, ) -> CommandResult { let paths_iter = glob(glob_str)?.filter_map(Result::ok); let num_paths: usize = paths_iter.collect::>().len(); @@ -607,10 +622,37 @@ pub async fn command_update_metadata( .update_metadata( glob(glob_str)?.filter_map(Result::ok), manifest_path, - image_link_file, + link_file, ) .await?; println!("Successfully updated {} metadata files.", num_paths); Ok(()) } + +pub async fn command_write_metaplex_items( + arweave: &Arweave, + glob_str: &str, + manifest_str: &str, + log_dir: &str, + link_file: bool, +) -> CommandResult { + let paths_iter = glob(glob_str)?.filter_map(Result::ok); + let num_paths: usize = paths_iter.collect::>().len(); + let manifest_path = PathBuf::from(manifest_str); + + arweave + .write_metaplex_items( + glob(glob_str)?.filter_map(Result::ok), + manifest_path, + PathBuf::from(log_dir), + link_file, + ) + .await?; + + println!( + "Successfully wrote metaplex items for {} metadata files to {}", + num_paths, log_dir + ); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 1bd047a..2e1af8b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,10 @@ //! into larger transactions, making uploading much more efficient and reducing network congestion. //! The library supports both formats, with the recommended approach being to use the bundle format. //! +//! There are also two upload formats, whole transactions and in chunks. Whole transaction can up uploaded +//! to the `tx/` endpoint if they are less than 12 MB in total. Otherwise, you have to use the `chunk/`endpoint +//! and upload chunk sizes that are less than 256 KB. Arloader includes functions for both. +//! //! #### Transactions and DataItems //! Both formats start with chunking file data and creating merkle trees from the chunks. The merkle //! tree logic can be found in the [`merkle`] module. All of the hashing functions and other crypto @@ -29,6 +33,17 @@ //! [`transaction`] module, or a [`DataItem`] (if it is going to be included in a bundle format transaction), //! which can be found in the [`bundle`] module. //! +//! #### Tags +//! Tags are structs with `name` and `value` properties that can be included with either [`Transaction`]s or +//! [`DataItem`]s. One subtlety is that for [`Transaction`]s, Arweave expects the content at each key to be Base64 Url +//! encoded string, whereas for DataItems, Arweave expects utf8-encoded strings. Arloader includes two types of +//! tags to account for this, [`Tag`] and [`Tag`], used for [`Transaction`] and [`DataItem`], +//! respectively. +//! +//! The `Content-Type` tag is especially important as it is used by the Arweave gateways to communicate the content +//! type to browsers. Arloader includes a mime-type database that includes the appropriate content type +//! tag based on file extension for both [`Transaction`]s and [`DataItem`]s. +//! //! #### Bytes and Base64Url Data //! The library takes advantage of Rust's strong typing and trait model to store all data, signatures and //! addresses as a [`Base64`] struct with implementations for serialization and deserialization that automatically @@ -101,6 +116,8 @@ const VERSION: &'static str = env!("CARGO_PKG_VERSION"); /// Winstons are a sub unit of the native Arweave network token, AR. There are 1012 Winstons per AR. pub const WINSTONS_PER_AR: u64 = 1000000000000; + +/// Block size used for pricing calculations = 256 KB pub const BLOCK_SIZE: u64 = 1024 * 256; #[derive(Serialize, Deserialize, Debug)] @@ -113,9 +130,12 @@ struct OraclePrice { struct OraclePricePair { pub usd: f32, } + +/// Tuple struct includes two elements: chunk of paths and aggregatge data size of paths. #[derive(Clone, Debug)] pub struct PathsChunk(Vec, u64); +/// Uploads a stream of bundles from [`Vec`]s. pub fn upload_bundles_stream<'a>( arweave: &'a Arweave, paths_chunks: Vec, @@ -128,6 +148,7 @@ pub fn upload_bundles_stream<'a>( .buffer_unordered(buffer) } +/// Uploads a stream of bundles from [`Vec`]s, paying with SOL. pub fn upload_bundles_stream_with_sol<'a>( arweave: &'a Arweave, paths_chunks: Vec, @@ -217,7 +238,7 @@ where .buffer_unordered(buffer) } -/// Queries network and updates locally stored [`Status`] structs. +/// Queries network and updates locally stored [`BundleStatus`] structs. pub fn update_bundle_statuses_stream<'a, IP>( arweave: &'a Arweave, paths_iter: IP, @@ -231,6 +252,7 @@ where .buffer_unordered(buffer) } +/// Used when updating to determine wether files in a directory are [`BundleStatus`]s. pub fn file_stem_is_valid_txid(file_path: &PathBuf) -> bool { match Base64::from_str(file_path.file_stem().unwrap().to_str().unwrap()) { Ok(txid) => match txid.0.len() { @@ -1334,10 +1356,7 @@ impl Arweave { .unwrap() .to_str() .unwrap() - .split("_") - .collect::>() - .pop() - .unwrap(); + .replace("manifest_", ""); let data = fs::read_to_string(manifest_path.clone()).await?; let mut manifest: Value = serde_json::from_str(&data)?; let manifest = manifest.as_object_mut().unwrap(); @@ -1368,6 +1387,74 @@ impl Arweave { Err(Error::ManifestNotFound) } } + + pub async fn read_metadata_file(&self, file_path: PathBuf) -> Result { + let data = fs::read_to_string(file_path.clone()).await?; + let metadata: Value = serde_json::from_str(&data)?; + Ok(json!({"file_path": file_path.display().to_string(), "metadata": metadata})) + } + + pub async fn write_metaplex_items( + &self, + paths_iter: IP, + manifest_path: PathBuf, + log_dir: PathBuf, + link_file: bool, + ) -> Result<(), Error> + where + IP: Iterator + Send, + { + if manifest_path.exists() { + let manifest_id = manifest_path + .file_stem() + .unwrap() + .to_str() + .unwrap() + .replace("manifest_", ""); + let data = fs::read_to_string(manifest_path.clone()).await?; + let mut manifest: Value = serde_json::from_str(&data)?; + let manifest = manifest.as_object_mut().unwrap(); + + let metadata = try_join_all(paths_iter.map(|p| self.read_metadata_file(p))).await?; + + let items = + metadata + .iter() + .enumerate() + .fold(serde_json::Map::new(), |mut m, (i, meta)| { + let name = meta["metadata"]["name"].as_str().unwrap(); + let file_path = meta["file_path"].as_str().unwrap(); + let id = manifest + .get(file_path) + .unwrap() + .get("id") + .unwrap() + .as_str() + .unwrap(); + let link = if link_file { + format!("https://arweave.net/{}/{}", manifest_id, file_path) + } else { + format!("https://arweave.net/{}", id) + }; + m.insert( + i.to_string(), + json!({"name": name, "link": link, "onChain": false}), + ); + m + }); + + fs::write( + log_dir + .join(format!("metaplex_items_{}", manifest_id)) + .with_extension("json"), + serde_json::to_string(&json!(items))?, + ) + .await?; + Ok(()) + } else { + Err(Error::ManifestNotFound) + } + } } #[cfg(test)] diff --git a/src/main.rs b/src/main.rs index 06e5187..9cb9152 100644 --- a/src/main.rs +++ b/src/main.rs @@ -270,14 +270,14 @@ fn bundle_size_arg<'a, 'b>() -> Arg<'a, 'b> { .help("Sets the maximum file data bytes to include in a bundle.") } -fn image_link_file_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("image_link_file") - .long("image-link-file") - .value_name("IMAGE_LINK_FILE") +fn link_file_arg<'a, 'b>() -> Arg<'a, 'b> { + Arg::with_name("link_file") + .long("link-file") + .value_name("LINK_FILE") .required(false) .takes_value(false) .help( - "Specify whether to update `image` key in NFT metadata file with \ + "Specify whether to update key with \ file based link instead of id based link.", ) } @@ -427,7 +427,15 @@ fn get_app() -> App<'static, 'static> { .about("Update `image` and `files` keys in NFT metadata json files with links from provided manifest file.") .arg(glob_arg(true)) .arg(manifest_path_arg()) - .arg(image_link_file_arg()) + .arg(link_file_arg()) + ) + .subcommand( + SubCommand::with_name("write-metaplex-items") + .about("Write name and link for uploaded metadata files to `/metaplex_items_.json") + .arg(glob_arg(true)) + .arg(manifest_path_arg()) + .arg(log_dir_arg(true)) + .arg(link_file_arg()) ); app_matches } @@ -621,8 +629,15 @@ async fn main() -> CommandResult { ("update-metadata", Some(sub_arg_matches)) => { let glob_str = sub_arg_matches.value_of("glob").unwrap(); let manifest_str = sub_arg_matches.value_of("manifest_path").unwrap(); - let image_link_file = sub_arg_matches.is_present("image_link_file"); - command_update_metadata(&arweave, glob_str, manifest_str, image_link_file).await + let link_file = sub_arg_matches.is_present("link_file"); + command_update_metadata(&arweave, glob_str, manifest_str, link_file).await + } + ("write-metaplex-items", Some(sub_arg_matches)) => { + let glob_str = sub_arg_matches.value_of("glob").unwrap(); + let manifest_str = sub_arg_matches.value_of("manifest_path").unwrap(); + let log_dir = sub_arg_matches.value_of("log_dir").unwrap(); + let link_file = sub_arg_matches.is_present("link_file"); + command_write_metaplex_items(&arweave, glob_str, manifest_str, log_dir, link_file).await } _ => unreachable!(), } diff --git a/src/solana.rs b/src/solana.rs index 40fda6c..b3334d8 100644 --- a/src/solana.rs +++ b/src/solana.rs @@ -1,3 +1,5 @@ +//! Includes functionality for paying for transaction in SOL. + use crate::error::Error; use crate::transaction::{Base64, DeepHashItem}; use futures::future::try_join; @@ -8,11 +10,19 @@ use solana_sdk::{ }; use std::str::FromStr; +/// Solana address to which SOL payments are made. pub const SOL_AR_PUBKEY: &str = "6AaM5L2SeA7ciwDNaYLhKqQzsDVaQM9CRqXVDdWPeAQ9"; + +/// Uri of Solana payment api. pub const SOL_AR_BASE_URL: &str = "https://arloader.io/"; + +/// Winstons per lamports exchange rate for calculating SOL payment amounts. pub const RATE: u64 = 2500; + +/// Minimum SOL transaction amount. pub const FLOOR: u64 = 10000; +/// Returns recent blockhash neeed to create transaction. pub async fn get_recent_blockhash(base_url: url::Url) -> Result { let client = reqwest::Client::new(); @@ -40,6 +50,7 @@ pub async fn get_recent_blockhash(base_url: url::Url) -> Result { Ok(hash) } +/// Returns wallet balance. pub async fn get_sol_wallet_balance( base_url: url::Url, keypair: &keypair::Keypair, @@ -67,6 +78,7 @@ pub async fn get_sol_wallet_balance( Ok(balance) } +/// Airdrops tokens from devnet for testing purposes. pub async fn request_airdrop(base_url: url::Url, keypair: &keypair::Keypair) -> Result<(), Error> { let client = reqwest::Client::new(); @@ -86,6 +98,7 @@ pub async fn request_airdrop(base_url: url::Url, keypair: &keypair::Keypair) -> Ok(()) } +/// Creates Solana transaction. pub async fn create_sol_transaction( base_url: url::Url, from_keypair: &keypair::Keypair, @@ -112,6 +125,7 @@ pub async fn create_sol_transaction( Ok(bs58::encode(serialized).into_string()) } +/// Submits Solana transaction and required transaction elements and gets back signed AR transaction. pub async fn get_sol_ar_signature( base_url: url::Url, deep_hash_item: DeepHashItem, @@ -135,6 +149,7 @@ pub async fn get_sol_ar_signature( Ok(sig_response) } +/// Generic data structure for making json rpc requests. #[derive(Serialize, Deserialize, Debug)] pub struct PostObject { pub jsonrpc: String, @@ -154,6 +169,23 @@ impl Default for PostObject { } } +/// Struct for submitting required data to signature api. +#[derive(Debug, Deserialize, PartialEq, Serialize)] +pub struct TxData { + pub deep_hash_item: DeepHashItem, + pub sol_tx: String, +} + +/// Struct for receiving signature back from api. +#[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] +pub struct SigResponse { + pub ar_tx_sig: Base64, + pub ar_tx_id: Base64, + pub ar_tx_owner: Base64, + pub sol_tx_sig: String, + pub lamports: u64, +} + #[cfg(test)] mod tests { use super::{ @@ -192,18 +224,3 @@ mod tests { Ok(()) } } - -#[derive(Debug, Deserialize, PartialEq, Serialize)] -pub struct TxData { - pub deep_hash_item: DeepHashItem, - pub sol_tx: String, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, Clone)] -pub struct SigResponse { - pub ar_tx_sig: Base64, - pub ar_tx_id: Base64, - pub ar_tx_owner: Base64, - pub sol_tx_sig: String, - pub lamports: u64, -}