feat: workspace refactoring and downloader package (#18)

Cargo.toml

@@ -1,23 +1,14 @@
-[package]
-name = "era-compiler-common"
-version = "1.5.0"
+[workspace]
+
+members = [
+    "common",
+    "solc-downloader",
+]
+resolver = "2"
+
+[workspace.package]
 authors = [
     "Oleksandr Zarudnyi <[email protected]>",
 ]
 license = "MIT OR Apache-2.0"
 edition = "2021"
-description = "Shared constants of the ZKsync compilers"
-
-[lib]
-doctest = false
-
-[dependencies]
-anyhow = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0", features = [ "arbitrary_precision", "unbounded_depth" ] }
-serde_stacker = "0.1"
-
-sha3 = "0.10"
-hex = "0.4"
-ipfs-hasher = "0.13"
-base58 = "0.2"

common/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "era-compiler-common"
+version = "1.5.0"
+description = "Shared constants of the ZKsync compilers"
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[lib]
+doctest = false
+
+[dependencies]
+anyhow = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = [ "arbitrary_precision", "unbounded_depth" ] }
+serde_stacker = "0.1"
+
+sha3 = "0.10"
+hex = "0.4"
+ipfs-hasher = "0.13"
+base58 = "0.2"

solc-downloader/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "era-solc-downloader"
+version = "0.1.0"
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[dependencies]
+colored = "2.1.0"
+reqwest = { version = "0.11", features = ["blocking", "json"] }
+serde = { version = "1.0", "features" = [ "derive" ] }
+anyhow = "1.0"
+serde_json = { version = "1.0", features = [ "arbitrary_precision" ] }
+
+# Musl requires explicit openssl dependency
+[target.'cfg(target_env = "musl")'.dependencies]
+openssl = { version = "0.10", features = ["vendored"] }

solc-downloader/src/config/binary/mod.rs

@@ -0,0 +1,24 @@
+//!
+//! The compiler downloader binary config.
+//!
+
+pub mod protocol;
+
+use serde::Deserialize;
+
+use self::protocol::Protocol;
+
+///
+/// The compiler downloader binary config.
+///
+#[derive(Debug, Deserialize)]
+pub struct Binary {
+    /// Whether downloading the binary is enabled.
+    pub is_enabled: bool,
+    /// The downloading protocol.
+    pub protocol: Protocol,
+    /// The downloaded data source.
+    pub source: String,
+    /// The downloaded binary file destination.
+    pub destination: String,
+}

solc-downloader/src/config/binary/protocol.rs

@@ -0,0 +1,22 @@
+//!
+//! The compiler downloader binary download protocol.
+//!
+
+use serde::Deserialize;
+
+///
+/// The compiler downloader binary download protocol.
+///
+#[derive(Debug, Deserialize)]
+#[allow(clippy::upper_case_acronyms)]
+pub enum Protocol {
+    /// The local file copy.
+    #[serde(rename = "file")]
+    File,
+    /// Download via HTTPS.
+    #[serde(rename = "https")]
+    HTTPS,
+    /// Use the solc-bin JSON list.
+    #[serde(rename = "solc-bin-list")]
+    SolcBinList,
+}

solc-downloader/src/config/mod.rs

@@ -0,0 +1,63 @@
+//!
+//! The compiler downloader config.
+//!
+
+pub mod binary;
+pub(crate) mod solc_list;
+
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+
+use serde::Deserialize;
+
+use self::binary::Binary;
+
+///
+/// The compiler downloader config.
+///
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    /// The compiler binaries to download.
+    pub binaries: BTreeMap<String, Binary>,
+    /// The compiler platform directory names.
+    pub platforms: Option<HashMap<String, String>>,
+}
+
+impl Config {
+    ///
+    /// Returns the remote platform directory name for the specified platform.
+    ///
+    pub fn get_remote_platform_directory(&self) -> anyhow::Result<String> {
+        let platforms = match self.platforms {
+            Some(ref platform) => platform,
+            None => anyhow::bail!("Platforms are not defined"),
+        };
+
+        let platform = if cfg!(target_arch = "x86_64") {
+            if cfg!(target_os = "linux") {
+                "linux-amd64"
+            } else if cfg!(target_os = "macos") {
+                "macos-amd64"
+            } else if cfg!(target_os = "windows") {
+                "windows-amd64"
+            } else {
+                anyhow::bail!("This platform is not supported in `solc`!");
+            }
+        } else if cfg!(target_arch = "aarch64") {
+            if cfg!(target_os = "linux") {
+                "linux-arm64"
+            } else if cfg!(target_os = "macos") {
+                "macos-arm64"
+            } else {
+                anyhow::bail!("This platform is not supported in `solc`!");
+            }
+        } else {
+            anyhow::bail!("This platform is not supported in `solc`!");
+        };
+
+        platforms
+            .get(platform)
+            .cloned()
+            .ok_or_else(|| anyhow::anyhow!("Directory for platform `{}` is not defined", platform))
+    }
+}

solc-downloader/src/config/solc_list.rs

@@ -0,0 +1,35 @@
+//!
+//! The Solidity compiler JSON list metadata.
+//!
+
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::str::FromStr;
+
+use colored::Colorize;
+use serde::Deserialize;
+
+///
+/// The Solidity compiler JSON list metadata.
+///
+#[derive(Debug, Deserialize)]
+pub struct SolcList {
+    /// The collection of compiler releases.
+    pub releases: BTreeMap<String, String>,
+}
+
+impl TryFrom<&Path> for SolcList {
+    type Error = anyhow::Error;
+
+    fn try_from(path: &Path) -> Result<Self, Self::Error> {
+        let url =
+            reqwest::Url::from_str(path.to_str().expect("Always valid")).expect("Always valid");
+        println!(
+            " {} solc-bin JSON `{}`",
+            "Downloading".bright_green().bold(),
+            url
+        );
+        let list: SolcList = reqwest::blocking::get(url)?.json()?;
+        Ok(list)
+    }
+}

solc-downloader/src/lib.rs

@@ -0,0 +1,167 @@
+mod config;
+
+#[cfg(target_family = "unix")]
+use std::os::unix::fs::PermissionsExt;
+
+use std::path::Path;
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use colored::Colorize;
+
+use self::config::binary::protocol::Protocol;
+use self::config::solc_list::SolcList;
+use self::config::Config;
+
+///
+/// The compiler downloader.
+///
+#[derive(Debug)]
+pub struct Downloader {
+    /// The `reqwest` HTTP client.
+    http_client: reqwest::blocking::Client,
+    /// The solc-bin JSON list metadata.
+    solc_list: Option<SolcList>,
+}
+
+impl Downloader {
+    ///
+    /// A shortcut constructor.
+    ///
+    pub fn new(http_client: reqwest::blocking::Client) -> Self {
+        Self {
+            http_client,
+            solc_list: None,
+        }
+    }
+
+    ///
+    /// Downloads the compilers described in the config.
+    ///
+    pub fn download(mut self, config_path: &Path) -> anyhow::Result<Config> {
+        let config_file = std::fs::File::open(config_path).map_err(|error| {
+            anyhow::anyhow!(
+                "Binaries download config {:?} opening error: {}",
+                config_path,
+                error
+            )
+        })?;
+        let config_reader = std::io::BufReader::new(config_file);
+        let config: Config = serde_json::from_reader(config_reader).map_err(|error| {
+            anyhow::anyhow!(
+                "Binaries download config {:?} parsing error: {}",
+                config_path,
+                error
+            )
+        })?;
+
+        let platform_directory = config.get_remote_platform_directory()?;
+
+        for (version, binary) in config.binaries.iter() {
+            if !binary.is_enabled {
+                continue;
+            }
+
+            let source_path = binary
+                .source
+                .replace("${PLATFORM}", platform_directory.as_str())
+                .replace("${VERSION}", version.as_str())
+                + (std::env::consts::EXE_SUFFIX);
+
+            let destination_path = binary.destination.replace("${VERSION}", version.as_str());
+            let destination_path = PathBuf::from_str(
+                format!("{}{}", destination_path, std::env::consts::EXE_SUFFIX).as_str(),
+            )
+            .map_err(|_| anyhow::anyhow!("Binary `{}` destination is invalid", destination_path))?;
+
+            let data = match binary.protocol {
+                Protocol::File => {
+                    if source_path == destination_path.to_string_lossy() {
+                        continue;
+                    }
+
+                    println!(
+                        "     {} binary `{}` => {:?}",
+                        "Copying".bright_green().bold(),
+                        source_path,
+                        destination_path,
+                    );
+
+                    std::fs::copy(source_path.as_str(), binary.destination.as_str()).map_err(
+                        |error| {
+                            anyhow::anyhow!(
+                                "Binary {:?} copying error: {}",
+                                source_path.as_str(),
+                                error
+                            )
+                        },
+                    )?;
+                    continue;
+                }
+                Protocol::HTTPS => {
+                    if destination_path.exists() {
+                        continue;
+                    }
+
+                    let source_url =
+                        reqwest::Url::from_str(source_path.as_str()).expect("Always valid");
+                    println!(
+                        " {} binary `{}` => {:?}",
+                        "Downloading".bright_green().bold(),
+                        source_url,
+                        destination_path,
+                    );
+                    self.http_client.get(source_url).send()?.bytes()?
+                }
+                Protocol::SolcBinList => {
+                    if destination_path.exists() {
+                        continue;
+                    }
+
+                    let solc_list_path = PathBuf::from(source_path.as_str());
+                    let solc_list = self.solc_list.get_or_insert_with(|| {
+                        SolcList::try_from(solc_list_path.as_path())
+                            .expect("solc-bin JSON list downloading error")
+                    });
+                    if solc_list.releases.is_empty() {
+                        return Ok(config);
+                    }
+
+                    let source_binary_name =
+                        match solc_list.releases.get(version.to_string().as_str()) {
+                            Some(source_binary_name) => source_binary_name,
+                            None => anyhow::bail!(
+                                "Binary for version v{} not found in the solc JSON list",
+                                version
+                            ),
+                        };
+                    let mut source_path = solc_list_path;
+                    source_path.pop();
+                    source_path.push(source_binary_name);
+
+                    let source_url =
+                        reqwest::Url::from_str(source_path.to_str().expect("Always valid"))
+                            .expect("Always valid");
+                    println!(
+                        " {} binary `{}` => {:?}",
+                        "Downloading".bright_green().bold(),
+                        source_url,
+                        destination_path,
+                    );
+                    self.http_client.get(source_url).send()?.bytes()?
+                }
+            };
+
+            let mut destination_folder = destination_path.clone();
+            destination_folder.pop();
+            std::fs::create_dir_all(destination_folder)?;
+
+            std::fs::write(&destination_path, data)?;
+
+            #[cfg(target_family = "unix")]
+            std::fs::set_permissions(&destination_path, std::fs::Permissions::from_mode(0o755))?;
+        }
+
+        Ok(config)
+    }
+}