new: Add settings for controlling certs and proxies.

CHANGELOG.md

@@ -10,6 +10,15 @@
 - [Rust](https://github.com/moonrepo/rust-plugin/blob/master/CHANGELOG.md)
 - [Schema](https://github.com/moonrepo/schema-plugin/blob/master/CHANGELOG.md)
 
+## Unreleased
+
+#### 🚀 Updates
+
+- Added an `http` setting to `~/.proto/config.toml` to control proxies and certificates when making http/https requests, primarily for downloading tools.
+  - New `allow-invalid-certs` setting for allowing invalid certificates (be careful).
+  - New `proxies` setting for customizing internal proxy URLs.
+  - New `root-cert` setting for providing a root certificate (great for corporate environments).
+
 ## 0.17.1
 
 #### 🚀 Updates

Cargo.toml

@@ -5,7 +5,7 @@ default-members = ["crates/cli"]
 
 [workspace.dependencies]
 cached = "0.45.1"
-clap = "4.4.2"
+clap = "4.4.3"
 clap_complete = "4.4.1"
 convert_case = "0.6.0"
 extism = "0.5.0"
@@ -20,7 +20,7 @@ reqwest = { version = "0.11.20", default-features = false, features = [
 ] }
 semver = "1.0.18"
 serde = { version = "1.0.188", features = ["derive"] }
-serde_json = "1.0.105"
+serde_json = "1.0.107"
 sha2 = "0.10.7"
 starbase_archive = { version = "0.2.2", features = [
 	"tar-gz",

crates/core/src/error.rs

@@ -13,17 +13,6 @@ pub enum ProtoError {
     #[error("Tool inventory directory has been overridden but is not an absolute path. Only absolute paths are supported.")]
     AbsoluteInventoryDir,
 
-    #[diagnostic(
-        code(proto::download::missing),
-        help = "Please refer to the tool's official documentation."
-    )]
-    #[error("Tool download {} does not exist. This version may not be supported for your current operating system or architecture.", .url.style(Style::Url))]
-    DownloadNotFound { url: String },
-
-    #[diagnostic(code(proto::download::failed))]
-    #[error("Failed to download tool from {}: {status}", .url.style(Style::Url))]
-    DownloadFailed { url: String, status: String },
-
     #[diagnostic(code(proto::misc::offline))]
     #[error("Internet connection required, unable to download and install tools.")]
     InternetConnectionRequired,

crates/core/src/helpers.rs

@@ -195,57 +195,6 @@ pub fn hash_file_contents<P: AsRef<Path>>(path: P) -> miette::Result<String> {
     Ok(hash)
 }
 
-#[tracing::instrument(skip_all)]
-pub async fn download_from_url<U, F>(url: U, dest_file: F) -> miette::Result<()>
-where
-    U: AsRef<str>,
-    F: AsRef<Path>,
-{
-    if is_offline() {
-        return Err(ProtoError::InternetConnectionRequired.into());
-    }
-
-    let url = url.as_ref();
-    let dest_file = dest_file.as_ref();
-    let handle_http_error = |error: reqwest::Error| ProtoError::Http {
-        url: url.to_owned(),
-        error,
-    };
-
-    trace!(
-        dest_file = ?dest_file,
-        url,
-        "Downloading file from URL",
-    );
-
-    // Fetch the file from the HTTP source
-    let response = reqwest::get(url).await.map_err(handle_http_error)?;
-    let status = response.status();
-
-    if status.as_u16() == 404 {
-        return Err(ProtoError::DownloadNotFound {
-            url: url.to_owned(),
-        }
-        .into());
-    }
-
-    if !status.is_success() {
-        return Err(ProtoError::DownloadFailed {
-            url: url.to_owned(),
-            status: status.to_string(),
-        }
-        .into());
-    }
-
-    // Write the bytes to our local file
-    fs::write_file_with_lock(
-        dest_file,
-        response.bytes().await.map_err(handle_http_error)?,
-    )?;
-
-    Ok(())
-}
-
 pub fn read_json_file_with_lock<T: DeserializeOwned>(path: impl AsRef<Path>) -> miette::Result<T> {
     let path = path.as_ref();
     let content = fs::read_file_with_lock(path)?;

crates/core/src/proto.rs

@@ -1,8 +1,10 @@
 use crate::helpers::{get_home_dir, get_proto_home};
+use crate::user_config::UserConfig;
 use once_cell::sync::OnceCell;
 use std::env;
 use std::path::{Path, PathBuf};
-use warpgate::PluginLoader;
+use std::sync::Arc;
+use warpgate::{create_http_client_with_options, PluginLoader};
 
 #[derive(Clone, Debug)]
 pub struct ProtoEnvironment {
@@ -14,7 +16,8 @@ pub struct ProtoEnvironment {
     pub home: PathBuf, // ~
     pub root: PathBuf, // ~/.proto
 
-    loader: OnceCell<PluginLoader>,
+    client: Arc<OnceCell<reqwest::Client>>,
+    loader: Arc<OnceCell<PluginLoader>>,
 }
 
 impl ProtoEnvironment {
@@ -40,7 +43,17 @@ impl ProtoEnvironment {
             tools_dir: root.join("tools"),
             home: get_home_dir()?,
             root: root.to_owned(),
-            loader: OnceCell::new(),
+            client: Arc::new(OnceCell::new()),
+            loader: Arc::new(OnceCell::new()),
+        })
+    }
+
+    pub fn get_http_client(&self) -> miette::Result<&reqwest::Client> {
+        self.client.get_or_try_init(|| {
+            let user_config = UserConfig::load()?;
+            let client = create_http_client_with_options(user_config.http)?;
+
+            Ok(client)
         })
     }
 

crates/core/src/tool.rs

@@ -1,17 +1,16 @@
 use crate::error::ProtoError;
 use crate::events::*;
-use crate::helpers::{hash_file_contents, is_cache_enabled, is_offline};
+use crate::helpers::{
+    hash_file_contents, is_archive_file, is_cache_enabled, is_offline, read_json_file_with_lock,
+    write_json_file_with_lock, ENV_VAR,
+};
 use crate::proto::ProtoEnvironment;
 use crate::shimmer::{
     create_global_shim, create_local_shim, get_shim_file_name, ShimContext, SHIM_VERSION,
 };
 use crate::tool_manifest::ToolManifest;
 use crate::version::{UnresolvedVersionSpec, VersionSpec};
 use crate::version_resolver::VersionResolver;
-use crate::{
-    download_from_url, is_archive_file, read_json_file_with_lock, write_json_file_with_lock,
-    ENV_VAR,
-};
 use extism::{manifest::Wasm, Manifest as PluginManifest};
 use miette::IntoDiagnostic;
 use proto_pdk_api::*;
@@ -28,7 +27,7 @@ use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::time::{Duration, SystemTime};
 use tracing::{debug, trace};
-use warpgate::{Id, PluginContainer, VirtualPath};
+use warpgate::{download_from_url_to_file, Id, PluginContainer, VirtualPath};
 
 pub struct Tool {
     pub id: Id,
@@ -658,6 +657,7 @@ impl Tool {
             "Installing tool from a pre-built archive"
         );
 
+        let client = self.proto.get_http_client()?;
         let temp_dir = self
             .get_temp_dir()
             .join(self.get_resolved_version().to_string());
@@ -689,7 +689,7 @@ impl Tool {
         } else {
             debug!(tool = self.id.as_str(), "Tool not downloaded, downloading");
 
-            download_from_url(&download_url, &download_file).await?;
+            download_from_url_to_file(&download_url, &download_file, client).await?;
         }
 
         // Verify the checksum if applicable
@@ -703,7 +703,7 @@ impl Tool {
                     "Checksum does not exist, downloading"
                 );
 
-                download_from_url(&checksum_url, &checksum_file).await?;
+                download_from_url_to_file(&checksum_url, &checksum_file, client).await?;
             }
 
             self.verify_checksum(&checksum_file, &download_file).await?;
@@ -764,6 +764,10 @@ impl Tool {
             return Ok(false);
         }
 
+        if is_offline() {
+            return Err(ProtoError::InternetConnectionRequired.into());
+        }
+
         let install_dir = self.get_tool_dir();
         let _install_lock = fs::lock_directory(&install_dir).await?;
 

crates/core/src/tool_loader.rs

@@ -14,7 +14,7 @@ use std::{
     path::Path,
 };
 use tracing::debug;
-use warpgate::{to_virtual_path, Id, PluginLocator};
+use warpgate::{create_http_client_with_options, to_virtual_path, Id, PluginLocator};
 
 pub fn inject_default_manifest_config(
     id: &Id,
@@ -60,8 +60,11 @@ pub async fn load_tool_from_locator(
     let proto = proto.as_ref();
     let locator = locator.as_ref();
 
+    let http_client = create_http_client_with_options(user_config.http.clone())?;
     let plugin_loader = proto.get_plugin_loader();
-    let plugin_path = plugin_loader.load_plugin(&id, locator).await?;
+    let plugin_path = plugin_loader
+        .load_plugin(&id, locator, &http_client)
+        .await?;
 
     // If a TOML plugin, we need to load the WASM plugin for it,
     // wrap it, and modify the plugin manifest.
@@ -76,7 +79,7 @@ pub async fn load_tool_from_locator(
             proto,
             Wasm::file(
                 plugin_loader
-                    .load_plugin(id, ToolsConfig::schema_plugin())
+                    .load_plugin(id, ToolsConfig::schema_plugin(), &http_client)
                     .await?,
             ),
         )?;

crates/core/src/user_config.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::env;
 use std::path::{Path, PathBuf};
 use tracing::debug;
-use warpgate::{Id, PluginLocator};
+use warpgate::{HttpOptions, Id, PluginLocator};
 
 pub const USER_CONFIG_NAME: &str = "config.toml";
 
@@ -16,6 +16,7 @@ pub struct UserConfig {
     pub auto_clean: bool,
     pub auto_install: bool,
     pub node_intercept_globals: bool,
+    pub http: HttpOptions,
     pub plugins: BTreeMap<Id, PluginLocator>,
 
     #[serde(skip)]
@@ -39,17 +40,29 @@ impl UserConfig {
         let contents = fs::read_file_with_lock(&path)?;
         let mut config: UserConfig = toml::from_str(&contents).into_diagnostic()?;
 
+        let make_absolute = |file: &mut PathBuf| {
+            if file.is_absolute() {
+                file.to_owned()
+            } else {
+                dir.join(file)
+            }
+        };
+
         // Update plugin file paths to be absolute
         for locator in config.plugins.values_mut() {
             if let PluginLocator::SourceFile {
                 path: ref mut source_path,
                 ..
             } = locator
             {
-                *source_path = dir.join(&source_path);
+                *source_path = make_absolute(source_path);
             }
         }
 
+        if let Some(root_cert) = &mut config.http.root_cert {
+            *root_cert = make_absolute(root_cert);
+        }
+
         config.path = path;
 
         Ok(config)
@@ -73,6 +86,7 @@ impl Default for UserConfig {
             auto_clean: from_var("PROTO_AUTO_CLEAN", false),
             auto_install: from_var("PROTO_AUTO_INSTALL", false),
             node_intercept_globals: from_var("PROTO_NODE_INTERCEPT_GLOBALS", true),
+            http: HttpOptions::default(),
             plugins: BTreeMap::default(),
             path: PathBuf::new(),
         }

crates/core/tests/user_config_test.rs

@@ -2,7 +2,7 @@ use proto_core::{UserConfig, USER_CONFIG_NAME};
 use starbase_sandbox::create_empty_sandbox;
 use std::collections::BTreeMap;
 use std::env;
-use warpgate::{GitHubLocator, Id, PluginLocator};
+use warpgate::{GitHubLocator, HttpOptions, Id, PluginLocator};
 
 mod user_config {
     use super::*;
@@ -18,6 +18,7 @@ mod user_config {
                 auto_clean: false,
                 auto_install: false,
                 node_intercept_globals: true,
+                http: HttpOptions::default(),
                 plugins: BTreeMap::default(),
                 path: sandbox.path().join(USER_CONFIG_NAME),
             }
@@ -44,6 +45,7 @@ node-intercept-globals = false
                 auto_clean: true,
                 auto_install: true,
                 node_intercept_globals: false,
+                http: HttpOptions::default(),
                 plugins: BTreeMap::default(),
                 path: sandbox.path().join(USER_CONFIG_NAME),
             }
@@ -67,6 +69,7 @@ node-intercept-globals = false
                 auto_clean: true,
                 auto_install: true,
                 node_intercept_globals: false,
+                http: HttpOptions::default(),
                 plugins: BTreeMap::default(),
                 path: sandbox.path().join(USER_CONFIG_NAME),
             }