Metadata & manifest-builder refactoring (#366)

* metadata refactoring, add cargo-targets tables that overrides main manifest' fields

* tests, improvements, minimal validation

* Fix possible recursion in env resolver, remove panic on missed var

* Add sources: unit-graph and full metadata by host cargo,

* use new manifest builder from `playdate-build` pre-0.4

* little `cargo-playdate` refactoring 🥵

* env-resolver: remove panic on missed var (missed part)

* Apply clippy suggestions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* add targets, improve validation for crate-level source

* update deps and rust toolchain

* simplification, get rid of `Package` dep in layout

* Apply clippy suggestions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix totally ignored assets

* improve deserialisation of legacy and error for `AssetsRules`

* fix missed feature-gate

* fix `AssetsRules` again

* improve deserialisation and errors

* upd deps

* parametrize strings in metadata format

* reduce usage of `Package` where only `PackageId` needed

* set correct version

* allow use other strings

* deps: downgraded clang-sys v1.8.2 -> v1.8.1

* improve `CrateInfoSource`

* add draft for meta-tree (unit-graph + metadata)

* apply clippy suggestions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Cargo.toml

@@ -28,7 +28,7 @@ system = { version = "0.3", path = "api/system", package = "playdate-system", de
 sys = { version = "0.4", path = "api/sys", package = "playdate-sys", default-features = false }
 
 tool = { version = "0.1", path = "support/tool", package = "playdate-tool" }
-build = { version = "0.3", path = "support/build", package = "playdate-build", default-features = false }
+build = { version = "=0.4.0-pre1", path = "support/build", package = "playdate-build", default-features = false }
 utils = { version = "0.3", path = "support/utils", package = "playdate-build-utils", default-features = false }
 device = { version = "0.2", path = "support/device", package = "playdate-device" }
 simulator = { version = "0.1", path = "support/sim-ctrl", package = "playdate-simulator-utils", default-features = false }

cargo/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cargo-playdate"
-version = "0.4.14"
+version = "0.5.0-beta.1"
 readme = "README.md"
 description = "Build tool for neat yellow console."
 keywords = ["playdate", "build", "cargo", "plugin", "cargo-subcommand"]
@@ -59,7 +59,7 @@ async-std = { version = "1.12", features = ["tokio1"] }
 [dependencies.build]
 workspace = true
 default-features = false
-features = ["assets-report", "toml"]
+features = ["assets-report", "toml", "json"]
 
 [dependencies.device]
 workspace = true
@@ -89,11 +89,11 @@ features = [
 
 
 [dev-dependencies]
-target = "2.0.0"
+target = "2.0.1"
 rand = "0.8"
 
 [target.'cfg(unix)'.dev-dependencies]
-nix = { version = "0.28", features = ["signal"] }
+nix = { version = "0.29", features = ["signal"] }
 
 
 [features]

cargo/src/assets/mod.rs

@@ -5,7 +5,9 @@ use std::path::{PathBuf, Path};
 use anstyle::AnsiColor as Color;
 use anyhow::bail;
 use cargo::CargoResult;
-use cargo::core::{Package, Verbosity};
+use cargo::core::{Package, PackageId, Verbosity};
+use playdate::manifest::ManifestSourceOpt as _;
+use playdate::metadata::source::MetadataSource as _;
 use playdate::metadata::METADATA_FIELD;
 use playdate::layout::Layout;
 
@@ -15,23 +17,23 @@ use crate::layout::{PlaydateAssets, LayoutLockable, Layout as _, CrossTargetLayo
 use crate::logger::LogErr;
 use crate::utils::LazyBuildContext;
 use crate::utils::path::AsRelativeTo;
-use self::plan::TomlMetadata;
+use self::plan::Metadata;
 
 
 mod plan;
 mod pdc;
 
 
 #[derive(Debug)]
-pub struct AssetsArtifact<'cfg> {
-	pub package: &'cfg Package,
+pub struct AssetsArtifact {
+	pub package_id: PackageId,
 	pub layout: PlaydateAssets<PathBuf>,
 	/// Cached metadata
-	pub metadata: Option<TomlMetadata>,
+	pub metadata: Option<Metadata>,
 }
 
 /// One artifact per package.
-pub type AssetsArtifacts<'cfg> = HashMap<&'cfg Package, AssetsArtifact<'cfg>>;
+pub type AssetsArtifacts<'cfg> = HashMap<&'cfg Package, AssetsArtifact>;
 
 
 pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
@@ -41,7 +43,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 	for (package, targets, ..) in config.possible_targets()? {
 		let env = plan::LazyEnvBuilder::new(config, package);
 		let mut plans: HashMap<&Package, _> = Default::default();
-		let global_layout = CrossTargetLayout::new(config, package, None)?;
+		let global_layout = CrossTargetLayout::new(config, package.package_id(), None)?;
 		let mut layout = global_layout.assets_layout(config);
 		let mut options = HashMap::new();
 
@@ -53,6 +55,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 			layout.clean()?;
 		}
 
+		// primary top-level package
 		let target_pid = package.package_id();
 		let has_dev = targets.iter()
 		                     .any(|t| t.is_example() || t.is_test() || t.is_bench());
@@ -111,7 +114,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 				                        .map(|plan| (plan, AssetKind::Package))
 				                        .chain(plan.dev.as_ref().into_iter().map(|plan| (plan, AssetKind::Dev)))
 				{
-					let message = plan.printable_serializable(package, kind);
+					let message = plan.printable_serializable(package.package_id(), kind);
 					config.workspace.config().shell().print_json(&message)?;
 				}
 			}
@@ -147,7 +150,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 			let mut has_errors = false;
 			let mut targets = HashMap::new();
 
-			let mut check_duplicates = |package: &Package, target_kind: AssetKind, plan| {
+			let mut check_duplicates = |package_id: PackageId, target_kind: AssetKind, plan| {
 				for target in plan {
 					if let Some((pid, kind)) = targets.get::<Cow<Path>>(&target) {
 						has_errors = true;
@@ -158,27 +161,28 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 							}
 						};
 						let a = err_msg(pid, *kind);
-						let b = err_msg(&package.package_id(), target_kind);
+						let b = err_msg(&package_id, target_kind);
 						let message = format!(
 						                      "Duplicate dev-asset destination: '{}':\n\t{a}\n\t{b}",
 						                      target.as_relative_to_root(config).display(),
 						);
 
 						config.log().error(message);
 					} else {
-						targets.insert(target, (package.package_id(), target_kind));
+						targets.insert(target, (package_id, target_kind));
 					}
 				}
 			};
 
 
 			for (package, plan) in plans.iter() {
+				let package_id = package.package_id();
 				if let Some(plan) = plan.main.as_ref() {
-					check_duplicates(package, AssetKind::Package, plan.as_inner().targets());
+					check_duplicates(package_id, AssetKind::Package, plan.as_inner().targets());
 				}
-				if package.package_id() == target_pid {
+				if package_id == target_pid {
 					if let Some(plan) = plan.dev.as_ref() {
-						check_duplicates(package, AssetKind::Dev, plan.as_inner().targets());
+						check_duplicates(package_id, AssetKind::Dev, plan.as_inner().targets());
 					}
 				}
 			}
@@ -232,6 +236,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 					            });
 
 
+					// FIXME: use primary (top-level) assets-options, but not options of dependency!
 					let metadata = options.get(dependency).expect("Metadata is gone, impossible!");
 					let report = plan.apply(&dest, &metadata.assets_options(), config)?;
 
@@ -348,7 +353,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 			let metadata = options.remove(package);
 			artifacts.insert(
 			                 package,
-			                 AssetsArtifact { package,
+			                 AssetsArtifact { package_id: package.package_id(),
 			                                  layout,
 			                                  metadata, },
 			);
@@ -369,7 +374,7 @@ pub fn build<'cfg>(config: &'cfg Config) -> CargoResult<AssetsArtifacts<'cfg>> {
 fn deps_tree_metadata<'cfg: 'r, 't: 'r, 'r>(package: &'cfg Package,
                                             bcx: &'t LazyBuildContext<'t, 'cfg>,
                                             config: &Config<'_>)
-                                            -> CargoResult<HashMap<&'r Package, TomlMetadata>> {
+                                            -> CargoResult<HashMap<&'r Package, Metadata>> {
 	let mut packages = HashMap::new();
 	if let Some(metadata) = playdate_metadata(package) {
 		// if explicitly allowed collect deps => scan deps-tree
@@ -470,11 +475,10 @@ fn deps_tree_metadata<'cfg: 'r, 't: 'r, 'r>(package: &'cfg Package,
 }
 
 
-pub fn playdate_metadata(package: &Package) -> Option<TomlMetadata> {
+pub fn playdate_metadata(package: &Package) -> Option<Metadata> {
 	package.manifest()
 	       .custom_metadata()
 	       .and_then(|m| m.as_table().map(|t| t.get(METADATA_FIELD)))
 	       .flatten()
-	       .and_then(|v| v.to_owned().try_into::<TomlMetadata>().log_err().ok())
-	       .and_then(|mut m| m.merge_opts().map(|_| m).log_err().ok())
+	       .and_then(|v| v.to_owned().try_into::<Metadata>().log_err().ok())
 }

cargo/src/assets/plan.rs

@@ -7,21 +7,19 @@ use playdate::assets::BuildReport;
 use playdate::assets::apply_build_plan;
 use playdate::config::Env;
 use playdate::metadata::format::AssetsOptions;
+use playdate::metadata::source::MetadataSource as _;
 use crate::config::Config;
 use crate::utils::path::AsRelativeTo;
 use playdate::consts::SDK_ENV_VAR;
 use cargo::util::CargoResult;
-use playdate::metadata::format::PlayDateMetadata;
+pub use playdate::metadata::format::Metadata;
 use playdate::assets::plan::BuildPlan as AssetsPlan;
 use playdate::assets::plan::build_plan as assets_build_plan;
 use try_lazy_init::Lazy;
 
 use crate::layout::{PlaydateAssets, LayoutLock};
 
 
-pub type TomlMetadata = PlayDateMetadata<toml::Value>;
-
-
 pub struct LazyEnvBuilder<'a, 'cfg> {
 	config: &'a Config<'cfg>,
 	package: &'cfg Package,
@@ -68,15 +66,15 @@ pub type LockedLayout<'t> = LayoutLock<&'t mut PlaydateAssets<PathBuf>>;
 /// Returns `None` if there is no `assets` metadata.
 pub fn plan_for<'cfg, 'env, 'l>(config: &'cfg Config,
                                 package: &'cfg Package,
-                                metadata: &TomlMetadata,
+                                metadata: &Metadata,
                                 env: &'cfg LazyEnvBuilder<'env, 'cfg>,
                                 layout: &'l LockedLayout<'l>,
                                 with_dev: bool)
                                 -> CargoResult<PackageAssetsPlan<'env, 'cfg>> {
 	let opts = metadata.assets_options();
 
-	let has_dev_assets = with_dev && metadata.dev_assets.iter().any(|t| !t.is_empty());
-	let is_empty = metadata.assets.is_empty() && !has_dev_assets;
+	let has_dev_assets = with_dev && !metadata.dev_assets().is_empty();
+	let is_empty = metadata.assets().is_empty() && !has_dev_assets;
 
 	if is_empty {
 		return Ok(PackageAssetsPlan { main: None,
@@ -88,11 +86,11 @@ pub fn plan_for<'cfg, 'env, 'l>(config: &'cfg Config,
 	                  .parent()
 	                  .ok_or(anyhow!("No parent of manifest-path"))?;
 
-	let main = if !metadata.assets.is_empty() {
-		let plan = assets_build_plan(env, &metadata.assets, opts.as_ref(), Some(root))?;
+	let main = if !metadata.assets().is_empty() {
+		let plan = assets_build_plan(env, metadata.assets(), opts.as_ref(), Some(root))?;
 
 		// main-assets plan:
-		let path = layout.as_inner().assets_plan_for(config, package);
+		let path = layout.as_inner().assets_plan_for(config, &package.package_id());
 		let mut cached = CachedPlan::new(path, plan)?;
 		if config.compile_options.build_config.force_rebuild {
 			cached.difference = Difference::Missing;
@@ -105,11 +103,12 @@ pub fn plan_for<'cfg, 'env, 'l>(config: &'cfg Config,
 
 
 	// dev-assets plan:
-	let dev = if has_dev_assets && metadata.dev_assets.is_some() {
-		let assets = metadata.dev_assets.as_ref().unwrap();
+	let dev = if has_dev_assets && !metadata.dev_assets().is_empty() {
+		let assets = metadata.dev_assets();
 		let dev_plan = assets_build_plan(env, assets, opts.as_ref(), Some(root))?;
 
-		let path = layout.as_inner().assets_plan_for_dev(config, package);
+		let path = layout.as_inner()
+		                 .assets_plan_for_dev(config, &package.package_id());
 		let mut dev_cached = CachedPlan::new(path, dev_plan)?;
 
 		// Inheritance, if main is stale or missing - this one is too:
@@ -224,8 +223,8 @@ impl<'t, 'cfg> CachedPlan<'t, 'cfg> {
 	}
 
 
-	pub fn printable_serializable(&self, source: &Package, kind: AssetKind) -> SerializablePlan<'_, 't, 'cfg> {
-		SerializablePlan { package: source.package_id(),
+	pub fn printable_serializable(&self, source: PackageId, kind: AssetKind) -> SerializablePlan<'_, 't, 'cfg> {
+		SerializablePlan { package: source,
 		                   plan: &self.plan,
 		                   difference: &self.difference,
 		                   path: &self.path,

cargo/src/build/mod.rs

@@ -39,7 +39,6 @@ use crate::utils::path::AsRelativeTo;
 use crate::utils::workspace::PossibleTargets;
 
 
-pub mod plan;
 pub mod rustflags;