new: Migrate to wax for better glob support. (#105)

* Move wax to utils. * Move utils. * Remove glob walk. * Remove globset. * Fix deps. * More globset work. * Add file group docs. * Add file pattern docs. * Bump versions. * Test windows. * Better git handling. * Fix windows prefix strip. * Use standardize. * Move wax to utils. * Move utils. * Remove glob walk. * Remove globset. * Fix deps. * More globset work. * Add file group docs. * Add file pattern docs. * Bump versions. * Test windows. * Better git handling. * Fix windows prefix strip. * Use standardize. * Bubble errors. * Debug hash. * Keep drive around. * Revert "Keep drive around." This reverts commit 55c3838. * Use double star. * Update test.
moonrepo · May 25, 2022 · 744761b · 744761b
1 parent cb744b7
commit 744761b
Show file tree

Hide file tree

Showing 27 changed files with 510 additions and 349 deletions.
diff --git a/.yarn/versions/dee9bb70.yml b/.yarn/versions/dee9bb70.yml
@@ -0,0 +1,2 @@
+releases:
+  website: patch
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -1,3 +1,43 @@
+# Backlog
+
+## Tasks
+
+- [ ] Add `@cache` token
+
+## Action runner
+
+- [ ] Add a debug layer so that the node processes can be inspected
+- [ ] Write output logs for every action
+
+## CLI
+
+- [ ] `run-many`
+- [ ] `graph`
+  - [ ] Spin up an interactive website with full project/task data
+
+## Node.js
+
+- [ ] Add chrome profiling support to spawned processes
+- [ ] Publish npm packages
+
+# 0.2.0
+
+## Cache
+
+- [ ] hashing
+  - [ ] ignore hashes for files that are gitignored
+- [ ] add docs on caching options
+
+## Tests
+
+- [ ] add code coverage reports in CI
+- [ ] increase code coverage and add more integration tests
+
+## Targets
+
+- [ ] macos arm/m1
+- [ ] linux x64 (musl)
+
 # 0.1.0
 
 - [x] website
@@ -92,44 +132,4 @@
   - [x] use `stdin` for commands that take long arguments
   - [x] dont load `package.json`/`tsconfig.json` so much
   - [x] delete old hashes when the hash changes
-  - [x] include local file changes in hash
-
-# 0.2.0
-
-## Cache
-
-- [ ] hashing
-  - [ ] ignore hashes for files that are gitignored
-- [ ] add docs on caching options
-
-## Tests
-
-- [ ] add code coverage reports in CI
-- [ ] increase code coverage and add more integration tests
-
-## Targets
-
-- [ ] macos arm/m1
-- [ ] linux x64 (musl)
-
-# Backlog
-
-## Tasks
-
-- [ ] Add `@cache` token
-
-## Action runner
-
-- [ ] Add a debug layer so that the node processes can be inspected
-- [ ] Write output logs for every action
-
-## CLI
-
-- [ ] `run-many`
-- [ ] `graph`
-  - [ ] Spin up an interactive website with full project/task data
-
-## Node.js
-
-- [ ] Add chrome profiling support to spawned processes
-- [ ] Publish npm packages
+  - [x] include local file changes in hash
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
@@ -31,7 +31,6 @@ strum = "0.24.0"
 strum_macros = "0.24.0"
 tera = "1.15.0"
 tokio = { version = "1.18.2", features = ["full"] }
-wax = "0.4.0"
 
 [dev-dependencies]
 moon_cache = { path = "../cache" }

diff --git a/crates/cli/src/commands/init.rs b/crates/cli/src/commands/init.rs
@@ -9,15 +9,13 @@ use moon_lang::is_using_package_manager;
 use moon_lang_node::{NODENV, NPM, NVMRC, PNPM, YARN};
 use moon_logger::color;
 use moon_terminal::create_theme;
-use moon_utils::fs;
-use moon_utils::path;
+use moon_utils::{fs, glob, path};
 use std::collections::BTreeMap;
 use std::env;
 use std::fs::{read_to_string, OpenOptions};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 use tera::{Context, Tera};
-use wax::Glob;
 
 type AnyError = Box<dyn std::error::Error>;
 
@@ -150,33 +148,11 @@ fn inherit_projects_from_workspaces(
     workspaces: Vec<String>,
     projects: &mut BTreeMap<String, String>,
 ) -> Result<(), AnyError> {
-    for pattern in workspaces {
-        if path::is_glob(&pattern) {
-            let glob = Glob::new(&pattern).unwrap();
-
-            for entry in glob.walk(dest_dir, usize::MAX) {
-                let entry = match entry {
-                    Ok(e) => e,
-                    // Will crash if the dir doesnt exist
-                    Err(_) => {
-                        continue;
-                    }
-                };
-
-                if entry.file_type().is_dir() {
-                    let (id, source) = infer_project_name_and_source(
-                        &entry
-                            .path()
-                            .strip_prefix(dest_dir)
-                            .unwrap()
-                            .to_string_lossy(),
-                    );
-
-                    projects.insert(id, source);
-                }
-            }
-        } else {
-            let (id, source) = infer_project_name_and_source(&pattern);
+    for path in glob::walk(dest_dir, &workspaces)? {
+        if path.is_dir() {
+            let (id, source) = infer_project_name_and_source(
+                &path.strip_prefix(dest_dir).unwrap().to_string_lossy(),
+            );
 
             projects.insert(id, source);
         }

diff --git a/crates/cli/tests/run_test.rs b/crates/cli/tests/run_test.rs
@@ -116,6 +116,13 @@ mod caching {
             .await
             .unwrap();
 
+        assert_snapshot!(read_to_string(
+            fixture
+                .path()
+                .join(format!(".moon/cache/hashes/{}.json", state.item.hash))
+        )
+        .unwrap());
+
         assert_eq!(state.item.exit_code, 0);
         assert_eq!(state.item.stdout, "stdout");
         assert_eq!(state.item.stderr, "stderr");

diff --git a/crates/cli/tests/snapshots/run_test__caching__creates_run_state_cache.snap b/crates/cli/tests/snapshots/run_test__caching__creates_run_state_cache.snap
@@ -0,0 +1,38 @@
+---
+source: crates/cli/tests/run_test.rs
+assertion_line: 119
+expression: "read_to_string(fixture.path().join(format!(\".moon/cache/hashes/{}.json\",\n                state.item.hash))).unwrap()"
+---
+{
+  "command": "node",
+  "args": [
+    "./standard.js"
+  ],
+  "deps": [],
+  "envVars": {},
+  "inputHashes": {
+    "node/cjsFile.cjs": "91382f667258361b9397214d0aec54d8d576ae19",
+    "node/cwd.js": "47d4aa44cd6363251821ab9c59f4c68df455445c",
+    "node/envVars.js": "44b0fda6bce364122223fbfc67b65ecbc52aec91",
+    "node/envVarsMoon.js": "69357e53fddeb9270d92a076e7edab3ebf004957",
+    "node/exitCodeNonZero.js": "1cea9d1c8c3776318c20d54a86c1b492df443397",
+    "node/exitCodeZero.js": "887c40bee2815c60cf4efde58a52716e4c393a24",
+    "node/mjsFile.mjs": "a5b6817de5cfa541e430072598de3f42d83bd6cf",
+    "node/passthroughArgs.js": "83cf03c37aa75b33592735611860e32c6f338693",
+    "node/processExitNonZero.js": "2fe41b280bb08f250ca7bf2263ec0b17fa7010f1",
+    "node/processExitZero.js": "eb885b8c763ab8b5d4e1da46494d965078c98e8b",
+    "node/project.yml": "7b3a3626af11cdc005b7042f3d28171f965e4545",
+    "node/standard.js": "8d4f87187c238d808f0505268febb619f2978c47",
+    "node/throwError.js": "9c5b8d2297fe4a6c23e8c18f1efb6871bec6faf4",
+    "node/topLevelAwait.mjs": "50945a9865eddbd52f7e648743d627749623da22",
+    "node/unhandledPromise.js": "909d23eeb442f4daff733fe3b601a1a1613c1282"
+  },
+  "nodeVersion": "16.0.0",
+  "packageDependencies": {},
+  "packageDevDependencies": {},
+  "packagePeerDependencies": {},
+  "projectDeps": [],
+  "target": "node:standard",
+  "tsconfigCompilerOptions": {},
+  "version": "1"
+}
diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml
@@ -9,8 +9,6 @@ moon_error = { path = "../error" }
 moon_logger = { path = "../logger" }
 moon_utils = { path = "../utils" }
 common-path = "1.0.0"
-globset = "0.4.8"
-globwalk = "0.8.1"
 itertools = "0.10.3"
 petgraph = "0.6.0"
 serde = { version = "1.0.137", features = ["derive"] }

diff --git a/crates/project/src/errors.rs b/crates/project/src/errors.rs
@@ -1,5 +1,6 @@
 use moon_config::{constants, ValidationErrors};
 use moon_error::MoonError;
+use moon_utils::glob::GlobError;
 use std::path::PathBuf;
 use thiserror::Error;
 
@@ -35,13 +36,10 @@ pub enum ProjectError {
     UnconfiguredTask(String, String),
 
     #[error(transparent)]
-    Moon(#[from] MoonError),
-
-    #[error(transparent)]
-    GlobWalk(#[from] globwalk::GlobError),
+    Glob(#[from] GlobError),
 
     #[error(transparent)]
-    GlobSet(#[from] globset::Error),
+    Moon(#[from] MoonError),
 
     #[error(transparent)]
     Target(#[from] TargetError),

diff --git a/crates/project/src/file_group.rs b/crates/project/src/file_group.rs
@@ -1,9 +1,8 @@
 use crate::errors::{ProjectError, TokenError};
 use common_path::common_path_all;
-use globwalk::GlobWalkerBuilder;
-use moon_utils::path::{expand_root_path, is_glob};
+use moon_utils::glob;
+use moon_utils::path::expand_root_path;
 use serde::{Deserialize, Serialize};
-use std::fs;
 use std::path::{Path, PathBuf};
 
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
@@ -56,7 +55,7 @@ impl FileGroup {
         let mut globs = vec![];
 
         for file in &self.files {
-            if is_glob(file) {
+            if glob::is_glob(file) {
                 globs.push(expand_root_path(file, workspace_root, project_root));
             }
         }
@@ -99,46 +98,35 @@ impl FileGroup {
         let mut list = vec![];
 
         for file in &self.files {
-            if is_glob(file) {
+            if glob::is_glob(file) {
                 let root = if file.starts_with('/') {
                     workspace_root
                 } else {
                     project_root
                 };
-                let walker = GlobWalkerBuilder::from_patterns(&root, &[file])
-                    .follow_links(false)
-                    .build()?;
-
-                for entry in walker {
-                    let entry_path = entry.unwrap(); // Handle error?
 
+                for path in glob::walk(root, &[file.clone()])? {
                     let allowed = if is_dir {
-                        entry_path.file_type().is_dir()
+                        path.is_dir()
                     } else {
-                        entry_path.file_type().is_file()
+                        path.is_file()
                     };
 
                     if allowed {
-                        list.push(entry_path.into_path());
+                        list.push(path);
                     }
                 }
             } else {
-                let file_path = expand_root_path(file, workspace_root, project_root);
-
-                let allowed = match fs::metadata(&file_path) {
-                    Ok(meta) => {
-                        if is_dir {
-                            meta.is_dir()
-                        } else {
-                            meta.is_file()
-                        }
-                    }
-                    // Branch exists for logging
-                    Err(_) => false,
+                let path = expand_root_path(file, workspace_root, project_root);
+
+                let allowed = if is_dir {
+                    path.is_dir()
+                } else {
+                    path.is_file()
                 };
 
                 if allowed {
-                    list.push(file_path.to_owned());
+                    list.push(path.to_owned());
                 }
             }
         }