diff --git a/CHANGELOG.md b/CHANGELOG.md index d04ed017c..6d5195e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ #### 🚀 Updates - Added `--on-init` option to `proto activate`, which will trigger the activation hook immediately in the shell, instead of waiting for a directory/prompt change to occur. +- Added support for loading `.env` files through the special `env.file` and `tools.*.env.file` settings. + ```toml + [env] + file = ".env" + ``` #### 🐞 Fixes diff --git a/Cargo.lock b/Cargo.lock index 252956059..7ba850fa1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -978,6 +978,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dyn-clone" version = "1.0.17" @@ -2465,6 +2471,7 @@ version = "0.43.6" dependencies = [ "clap", "convert_case", + "dotenvy", "indexmap", "miette", "minisign-verify", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 7c24991f8..4c10cde85 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -9,17 +9,18 @@ repository = "https://github.com/moonrepo/proto" [dependencies] proto_pdk_api = { version = "0.24.3", path = "../pdk-api", features = [ - "schematic", + "schematic", ] } proto_shim = { version = "0.5.0", path = "../shim" } version_spec = { version = "0.7.0", path = "../version-spec", features = [ - "schematic", + "schematic", ] } warpgate = { version = "0.19.0", path = "../warpgate", features = [ - "schematic", + "schematic", ] } clap = { workspace = true, optional = true } convert_case = "0.6.0" +dotenvy = "0.15.7" indexmap = { workspace = true } miette = { workspace = true } minisign-verify = "0.2.2" @@ -28,12 +29,12 @@ regex = { workspace = true } reqwest = { workspace = true } rustc-hash = { workspace = true } schematic = { workspace = true, features = [ - "config", - "env", - "toml", - "type_indexmap", - "type_url", - "validate", + "config", + "env", + "toml", + "type_indexmap", + "type_url", + "validate", ] } semver = { workspace = true } serde = { workspace = true } diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index e1a4e2749..f1ea3d24a 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -181,4 +181,28 @@ pub enum ProtoError { #[source] error: Box, }, + + #[diagnostic(code(proto::env::missing_file))] + #[error( + "The .env file {} does not exist. This was configured as {} in the config {}.", + .path.style(Style::Path), + .config.style(Style::File), + .config_path.style(Style::Path), + )] + MissingEnvFile { + path: PathBuf, + config: String, + config_path: PathBuf, + }, + + #[diagnostic(code(proto::env::parse_failed))] + #[error( + "Failed to parse .env file {}.", + .path.style(Style::Path), + )] + EnvFileParseFailed { + path: PathBuf, + #[source] + error: Box, + }, } diff --git a/crates/core/src/proto_config.rs b/crates/core/src/proto_config.rs index 87d46762e..5dac34c42 100644 --- a/crates/core/src/proto_config.rs +++ b/crates/core/src/proto_config.rs @@ -1,3 +1,4 @@ +use crate::error::ProtoError; use crate::helpers::ENV_VAR_SUB; use indexmap::IndexMap; use once_cell::sync::OnceCell; @@ -9,10 +10,12 @@ use schematic::{ }; use serde::{Deserialize, Serialize}; use starbase_styles::color; +use starbase_utils::fs::FsError; use starbase_utils::json::JsonValue; use starbase_utils::toml::TomlValue; use starbase_utils::{fs, toml}; use std::collections::BTreeMap; +use std::ffi::OsStr; use std::fmt::Debug; use std::hash::Hash; use std::path::{Path, PathBuf}; @@ -24,6 +27,7 @@ use warpgate::{HttpOptions, Id, PluginLocator, UrlLocator}; pub const PROTO_CONFIG_NAME: &str = ".prototools"; pub const SCHEMA_PLUGIN_KEY: &str = "internal-schema"; pub const PROTO_PLUGIN_KEY: &str = "proto"; +pub const ENV_FILE_KEY: &str = "file"; fn merge_tools( mut prev: BTreeMap, @@ -131,6 +135,12 @@ derive_enum!( } ); +#[derive(Clone, Debug, PartialEq)] +pub struct EnvFile { + pub path: PathBuf, + pub weight: usize, +} + #[derive(Clone, Config, Debug, PartialEq, Serialize)] #[serde(untagged)] pub enum EnvVar { @@ -174,6 +184,10 @@ pub struct ProtoToolConfig { #[setting(merge = merge_fxhashmap)] #[serde(flatten, skip_serializing_if = "FxHashMap::is_empty")] pub config: FxHashMap, + + #[setting(exclude, merge = merge::append_vec)] + #[serde(skip)] + _env_files: Vec, } #[derive(Clone, Config, Debug, Serialize)] @@ -240,6 +254,10 @@ pub struct ProtoConfig { #[setting(merge = merge_fxhashmap)] #[serde(flatten, skip_serializing)] pub unknown: FxHashMap, + + #[setting(exclude, merge = merge::append_vec)] + #[serde(skip)] + _env_files: Vec, } impl ProtoConfig { @@ -402,7 +420,6 @@ impl ProtoConfig { debug!(file = ?path, "Loading {}", PROTO_CONFIG_NAME); - let config_path = path.to_string_lossy(); let config_content = if with_lock { fs::read_file_with_lock(path)? } else { @@ -415,7 +432,7 @@ impl ProtoConfig { config.validate(&(), true).map_err(|error| match error { ConfigError::Validator { error, .. } => ConfigError::Validator { - location: config_path.to_string(), + location: path.to_string_lossy().to_string(), error, help: Some(color::muted_light("https://moonrepo.dev/docs/proto/config")), }, @@ -454,7 +471,7 @@ impl ProtoConfig { if !error.errors.is_empty() { return Err(ConfigError::Validator { - location: config_path.to_string(), + location: path.to_string_lossy().to_string(), error: Box::new(error), help: Some(color::muted_light("https://moonrepo.dev/docs/proto/config")), } @@ -463,20 +480,22 @@ impl ProtoConfig { } // Update file paths to be absolute - let make_absolute = |file: &PathBuf| { + fn make_absolute>(file: T, current_path: &Path) -> PathBuf { + let file = PathBuf::from(file.as_ref()); + if file.is_absolute() { - file.to_owned() - } else if let Some(dir) = path.parent() { + file + } else if let Some(dir) = current_path.parent() { dir.join(file) } else { PathBuf::from("/").join(file) } - }; + } if let Some(plugins) = &mut config.plugins { for locator in plugins.values_mut() { if let PluginLocator::File(ref mut inner) = locator { - inner.path = Some(make_absolute(&inner.get_unresolved_path())); + inner.path = Some(make_absolute(inner.get_unresolved_path(), path)); } } } @@ -484,11 +503,49 @@ impl ProtoConfig { if let Some(settings) = &mut config.settings { if let Some(http) = &mut settings.http { if let Some(root_cert) = &mut http.root_cert { - *root_cert = make_absolute(root_cert); + *root_cert = make_absolute(&root_cert, path); + } + } + } + + let push_env_file = |env_map: Option<&mut IndexMap>, + file_list: &mut Option>, + extra_weight: usize| + -> miette::Result<()> { + if let Some(map) = env_map { + if let Some(PartialEnvVar::Value(env_file)) = map.get(ENV_FILE_KEY) { + let list = file_list.get_or_insert(vec![]); + let env_file_path = make_absolute(env_file, path); + + if !env_file_path.exists() { + return Err(ProtoError::MissingEnvFile { + path: env_file_path, + config: env_file.to_owned(), + config_path: path.to_path_buf(), + } + .into()); + } + + list.push(EnvFile { + path: env_file_path, + weight: (path.to_str().map_or(0, |p| p.len()) * 10) + extra_weight, + }); } + + map.shift_remove(ENV_FILE_KEY); + } + + Ok(()) + }; + + if let Some(tools) = &mut config.tools { + for tool in tools.values_mut() { + push_env_file(tool.env.as_mut(), &mut tool._env_files, 5)?; } } + push_env_file(config.env.as_mut(), &mut config._env_files, 0)?; + Ok(config) } @@ -521,25 +578,49 @@ impl ProtoConfig { Self::save_to(dir, config) } + pub fn get_env_files(&self, filter_id: Option<&Id>) -> Vec<&PathBuf> { + let mut paths: Vec<&EnvFile> = self._env_files.iter().collect(); + + if let Some(id) = filter_id { + if let Some(tool_config) = self.tools.get(id) { + paths.extend(&tool_config._env_files); + } + } + + // Sort by weight so that we persist the order of env files + // when layers across directories exist! + paths.sort_by(|a, d| a.weight.cmp(&d.weight)); + + // Then only return the paths + paths.into_iter().map(|file| &file.path).collect() + } + // We don't use a `BTreeMap` for env vars, so that variable interpolation // and order of declaration can work correctly! pub fn get_env_vars( &self, filter_id: Option<&Id>, ) -> miette::Result>> { + let env_files = self.get_env_files(filter_id); + let mut base_vars = IndexMap::new(); - base_vars.extend(self.env.iter()); + base_vars.extend(self.load_env_files(&env_files)?); + base_vars.extend(self.env.clone()); if let Some(id) = filter_id { if let Some(tool_config) = self.tools.get(id) { - base_vars.extend(tool_config.env.iter()) + base_vars.extend(tool_config.env.clone()) } } let mut vars = IndexMap::>::new(); for (key, value) in base_vars { - let key_exists = std::env::var(key).is_ok_and(|v| !v.is_empty()); + if key == ENV_FILE_KEY { + continue; + } + + let key_exists = std::env::var(&key).is_ok_and(|v| !v.is_empty()); let value = value.to_value(); // Don't override parent inherited vars @@ -564,7 +645,36 @@ impl ProtoConfig { .to_string() }); - vars.insert(key.to_owned(), value); + vars.insert(key, value); + } + + Ok(vars) + } + + pub fn load_env_files(&self, paths: &[&PathBuf]) -> miette::Result> { + let mut vars = IndexMap::default(); + + let map_error = |error: dotenvy::Error, path: &Path| -> miette::Report { + match error { + dotenvy::Error::Io(inner) => FsError::Read { + path: path.to_path_buf(), + error: Box::new(inner), + } + .into(), + other => ProtoError::EnvFileParseFailed { + path: path.to_path_buf(), + error: Box::new(other), + } + .into(), + } + }; + + for path in paths { + for item in dotenvy::from_path_iter(path).map_err(|error| map_error(error, path))? { + let (key, value) = item.map_err(|error| map_error(error, path))?; + + vars.insert(key, EnvVar::Value(value)); + } } Ok(vars) diff --git a/crates/core/tests/proto_config_test.rs b/crates/core/tests/proto_config_test.rs index 97bbcae28..93399465e 100644 --- a/crates/core/tests/proto_config_test.rs +++ b/crates/core/tests/proto_config_test.rs @@ -135,6 +135,37 @@ BAZ_QUX = "abc" }); } + #[test] + fn can_set_env_file() { + let sandbox = create_empty_sandbox(); + sandbox.create_file(".env", ""); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +KEY = "value" +"#, + ); + + let config = ProtoConfig::load_from(sandbox.path(), false).unwrap(); + + assert_eq!(config.env.unwrap(), { + let mut map = IndexMap::::default(); + map.insert("KEY".into(), PartialEnvVar::Value("value".into())); + map + }); + assert_eq!( + config + ._env_files + .unwrap() + .into_iter() + .map(|file| file.path) + .collect::>(), + vec![sandbox.path().join(".env")] + ); + } + #[test] fn can_set_plugins() { let sandbox = create_empty_sandbox(); @@ -307,6 +338,230 @@ foo = "file://./test.toml" ); } + mod envs { + use super::*; + + #[test] + #[should_panic(expected = "does not exist")] + fn errors_if_file_missing() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +"#, + ); + + ProtoConfigManager::load(sandbox.path(), None, None) + .unwrap() + .get_merged_config() + .unwrap(); + } + + #[test] + #[should_panic(expected = "Failed to parse .env file")] + fn errors_if_parse_fails() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".env", + r#" +.KEY={invalid} +"#, + ); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +"#, + ); + + ProtoConfigManager::load(sandbox.path(), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .get_env_vars(None) + .unwrap(); + } + + #[test] + fn merges_vars_and_files() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".env", + r#" +KEY1 = "file1" +KEY3 = "file3" +"#, + ); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +KEY1 = "value1" +KEY2 = "value2" +"#, + ); + + let config = ProtoConfigManager::load(sandbox.path(), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .to_owned(); + + assert_eq!(config.get_env_vars(None).unwrap(), { + let mut map = IndexMap::>::default(); + map.insert("KEY1".into(), Some("value1".into())); + map.insert("KEY2".into(), Some("value2".into())); + map.insert("KEY3".into(), Some("file3".into())); + map + }); + } + + #[test] + fn child_file_overwrites_parent() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".env", + r#" +KEY = "parent" +"#, + ); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +"#, + ); + sandbox.create_file( + "child/.env", + r#" +KEY = "child" +"#, + ); + sandbox.create_file( + "child/.prototools", + r#" +[env] +file = ".env" +"#, + ); + + let config = ProtoConfigManager::load(sandbox.path().join("child"), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .to_owned(); + + assert_eq!(config.get_env_vars(None).unwrap(), { + let mut map = IndexMap::>::default(); + map.insert("KEY".into(), Some("child".into())); + map + }); + } + + #[test] + fn files_can_substitute_from_self() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".env", + r#" +OTHER = "abc" +KEY = "other=${OTHER}" +"#, + ); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +"#, + ); + + let config = ProtoConfigManager::load(sandbox.path(), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .to_owned(); + + assert_eq!(config.get_env_vars(None).unwrap(), { + let mut map = IndexMap::>::default(); + map.insert("OTHER".into(), Some("abc".into())); + map.insert("KEY".into(), Some("other=abc".into())); + map + }); + } + + // #[test] + // fn files_can_substitute_from_process() { + // let sandbox = create_empty_sandbox(); + // sandbox.create_file( + // ".env", + // r#" + // KEY = "process=${PROCESS_KEY}" + // "#, + // ); + // sandbox.create_file( + // ".prototools", + // r#" + // [env] + // file = ".env" + // "#, + // ); + + // env::set_var("PROCESS_KEY", "abc"); + + // let config = ProtoConfigManager::load(sandbox.path(), None, None) + // .unwrap() + // .get_merged_config() + // .unwrap() + // .to_owned(); + + // env::remove_var("PROCESS_KEY"); + + // assert_eq!(config.get_env_vars(None).unwrap(), { + // let mut map = IndexMap::>::default(); + // map.insert("KEY".into(), Some("process=abc".into())); + // map + // }); + // } + + #[test] + fn vars_can_substitute_from_files() { + let sandbox = create_empty_sandbox(); + sandbox.create_file( + ".env", + r#" +FILE=file +"#, + ); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" +KEY = "from=${FILE}" +"#, + ); + + let config = ProtoConfigManager::load(sandbox.path(), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .to_owned(); + + assert_eq!(config.get_env_vars(None).unwrap(), { + let mut map = IndexMap::>::default(); + map.insert("FILE".into(), Some("file".into())); + map.insert("KEY".into(), Some("from=file".into())); + map + }); + } + } + mod builtins { use super::*; use proto_core::BuiltinPlugins; @@ -579,6 +834,70 @@ NODE_PATH = false map }); } + + #[test] + fn gathers_env_files() { + let sandbox = create_empty_sandbox(); + sandbox.create_file("a/b/.env.b", ""); + sandbox.create_file("a/b/.env.tool-b", ""); + sandbox.create_file( + "a/b/.prototools", + r#" +[env] +file = ".env.b" + +[tools.node.env] +file = ".env.tool-b" +"#, + ); + sandbox.create_file("a/.env.a", ""); + sandbox.create_file("a/.env.tool-a", ""); + sandbox.create_file( + "a/.prototools", + r#" +[env] +file = ".env.a" + +[tools.node.env] +file = ".env.tool-a" +"#, + ); + sandbox.create_file(".env", ""); + sandbox.create_file(".env.tool", ""); + sandbox.create_file( + ".prototools", + r#" +[env] +file = ".env" + +[tools.node.env] +file = ".env.tool" +"#, + ); + + let config = ProtoConfigManager::load(sandbox.path().join("a/b"), None, None) + .unwrap() + .get_merged_config() + .unwrap() + .to_owned(); + + assert_eq!(config.env, IndexMap::::default()); + assert_eq!( + config + .get_env_files(Some(&Id::raw("node"))) + .into_iter() + .cloned() + .collect::>(), + vec![ + sandbox.path().join(".env"), + sandbox.path().join(".env.tool"), + sandbox.path().join("a/.env.a"), + sandbox.path().join("a/.env.tool-a"), + sandbox.path().join("a/b/.env.b"), + sandbox.path().join("a/b/.env.tool-b"), + ] + ); + } } } diff --git a/package/src/api-types.ts b/package/src/api-types.ts index 7cd43d680..d782bc7db 100644 --- a/package/src/api-types.ts +++ b/package/src/api-types.ts @@ -107,8 +107,15 @@ export interface ToolMetadataOutput { configSchema?: unknown | null; /** Default alias or version to use as a fallback. */ defaultVersion?: UnresolvedVersionSpec | null; + /** + * List of deprecation messages that will be displayed to users + * of this plugin. + */ + deprecations?: string[]; /** Controls aspects of the tool inventory. */ inventory?: ToolInventoryMetadata; + /** Minimum version of proto required to execute this plugin. */ + minimumProtoVersion?: string | null; /** Human readable name of the tool. */ name: string; /** Version of the plugin. */ @@ -140,6 +147,8 @@ export interface ParseVersionFileInput { content: string; /** Name of file that's being parsed. */ file: string; + /** Virtual path to the file being parsed. */ + path: VirtualPath; } /** Output returned by the `parse_version_file` function. */ diff --git a/registry/data/third-party.json b/registry/data/third-party.json index c69aab36e..99074e16e 100644 --- a/registry/data/third-party.json +++ b/registry/data/third-party.json @@ -64,7 +64,7 @@ "name": "atlas", "description": "manage your database schema as code.", "author": "crashdump", - "homepageUrl": "https://atlasgo.io", + "homepageUrl": "https://atlasgo.io/", "repositoryUrl": "https://github.com/crashdump/proto-tools", "bins": [ "atlas"