Improved troubleshooting for go (CycloneDX#1096)

* Improved troubleshooting for go Signed-off-by: Prabhu Subramanian <[email protected]> * Fixes CycloneDX#1093. Adds a new property to represent indirect dependencies. Signed-off-by: Prabhu Subramanian <[email protected]> * Tweaks Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
timmyteo · May 19, 2024 · e5d2054 · e5d2054
1 parent 9bb16fe
commit e5d2054
Show file tree

Hide file tree

Showing 12 changed files with 48 additions and 22 deletions.
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -52,21 +52,21 @@ jobs:
         include:
           - os: windows
             build: |
-              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,osRelease --allow-write --allow-net --output cdxgenx.exe bin/cdxgen.js
+              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,osRelease,homedir --allow-write --allow-net --output cdxgenx.exe bin/cdxgen.js
               .\cdxgenx.exe --help
               (Get-FileHash .\cdxgenx.exe).hash | Out-File -FilePath .\cdxgenx.exe.sha256
             artifact: cdxgenx.exe
           - os: macos
             build: |
-              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net --target x86_64-apple-darwin --output cdxgenx-darwin-amd64 bin/cdxgen.js
+              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net --target x86_64-apple-darwin --output cdxgenx-darwin-amd64 bin/cdxgen.js
               ./cdxgenx-darwin-amd64 --help
               shasum -a 256 cdxgenx-darwin-amd64 > cdxgenx-darwin-amd64.sha256
-              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net --target aarch64-apple-darwin --output cdxgenx-darwin-arm64 bin/cdxgen.js
+              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net --target aarch64-apple-darwin --output cdxgenx-darwin-arm64 bin/cdxgen.js
               shasum -a 256 cdxgenx-darwin-arm64 > cdxgenx-darwin-arm64.sha256
             artifact: cdxgenx-darwin-amd64
           - os: ubuntu
             build: |
-              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net --output cdxgenx bin/cdxgen.js
+              deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net --output cdxgenx bin/cdxgen.js
               shasum -a 256 cdxgenx > cdxgenx.sha256
               chmod + cdxgenx
               ./cdxgenx --help

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -410,8 +410,8 @@ jobs:
         run: |
           deno info bin/cdxgen.js
           deno info bin/evinse.js
-          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-deno.json --deep
-          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net bin/cdxgen.js -p -t python repotests/django-DefectDojo -o bomresults/django-DefectDojo-deno.json --deep
+          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-deno.json --deep
+          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net bin/cdxgen.js -p -t python repotests/django-DefectDojo -o bomresults/django-DefectDojo-deno.json --deep
         env:
           FETCH_LICENSE: true
       - uses: actions/upload-artifact@v4

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -6,5 +6,5 @@
     "quickfix.biome": "explicit"
   },
   "editor.defaultFormatter": "biomejs.biome",
-  "deno.enable": true
+  "deno.enable": false
 }
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 [![JSR][badge-jsr]][jsr-cdxgen]
 [![NPM][badge-npm]][npmjs-cdxgen]
 [![GitHub Releases][badge-github-releases]][github-releases]
-[![NPM Downloads][badge-npm-downloads]][npmjs-cdxgen] 
-[![GitHub License][badge-github-license]][github-license] 
+[![NPM Downloads][badge-npm-downloads]][npmjs-cdxgen]
+[![GitHub License][badge-github-license]][github-license]
 [![GitHub Contributors][badge-github-contributors]][github-contributors]
 [![SWH][badge-swh]][swh-cdxgen]
 
@@ -65,7 +65,7 @@ $ brew install cdxgen
 Deno and bun runtime can be used with limited support.
 
 ```shell
-deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen"
+deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen"
 ```
 
 You can also use the cdxgen container image with node, deno, or bun runtime versions.

diff --git a/binary.js b/binary.js
@@ -754,7 +754,11 @@ export function executeOsQuery(query) {
       timeout: 60 * 1000,
     });
     if (result.status !== 0 || result.error) {
-      if (DEBUG_MODE && result.stderr) {
+      if (
+        DEBUG_MODE &&
+        result.stderr &&
+        !result.stderr.includes("no such table")
+      ) {
         console.error(result.stdout, result.stderr);
       }
     }

diff --git a/ci/Dockerfile-deno b/ci/Dockerfile-deno
@@ -69,7 +69,7 @@ RUN set -e; \
     && python${PYTHON_VERSION} -m pip install --upgrade pip virtualenv \
     && python${PYTHON_VERSION} -m pip install --user pipenv poetry blint \
     && curl -fsSL https://deno.land/x/install/install.sh | sh \
-    && deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen" \
+    && deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen" \
     && curl -s "https://get.sdkman.io" | bash \
     && source "$HOME/.sdkman/bin/sdkman-init.sh" \
     && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true" >> $HOME/.sdkman/etc/config \

diff --git a/contrib/deno/README.md b/contrib/deno/README.md
@@ -12,7 +12,7 @@ Install deno by following the [instructions](https://docs.deno.com/runtime/manua
 ## Develop and build cdxgen
 
 ```shell
-deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net main.ts <path to repo>
+deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net main.ts <path to repo>
 ```
 
 ### Produce native builds

diff --git a/deno.json b/deno.json
@@ -35,8 +35,8 @@
     "include": ["*.test.js"]
   },
   "tasks": {
-    "start": "deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net bin/cdxgen.js",
-    "exe": "deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net --output build/cdxgenx-devel bin/cdxgen.js",
+    "start": "deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net bin/cdxgen.js",
+    "exe": "deno compile --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net --output build/cdxgenx-devel bin/cdxgen.js",
     "lint:check": "biome check *",
     "lint": "biome check --apply-unsafe *",
     "gen-types": "npx -p typescript tsc"

diff --git a/docs/CLI.md b/docs/CLI.md
@@ -36,7 +36,7 @@ $ brew install cdxgen
 Deno install is also supported.
 
 ```shell
-deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen"
+deno install --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid,homedir --allow-write --allow-net -n cdxgen "npm:@cyclonedx/cdxgen/cdxgen"
 ```
 
 You can also use the cdxgen container image

diff --git a/index.js b/index.js
@@ -2994,10 +2994,8 @@ export async function createGoBom(path, options) {
             maxBuffer: MAX_BUFFER,
           },
         );
-        if (DEBUG_MODE) {
-          console.log("Executing go mod graph in", basePath);
-        }
         if (result.status !== 0 || result.error) {
+          console.log("go list -deps command has failed.");
           shouldManuallyParse = true;
           if (DEBUG_MODE && result.stdout) {
             console.log(result.stdout);
@@ -3022,6 +3020,9 @@ export async function createGoBom(path, options) {
             parentComponent = retMap.parentComponent;
             parentComponent.type = "application";
           }
+          if (DEBUG_MODE) {
+            console.log("Executing go mod graph in", basePath);
+          }
           // Next we use the go mod graph command to construct the dependency tree
           result = spawnSync("go", ["mod", "graph"], {
             cwd: basePath,
@@ -3031,8 +3032,14 @@ export async function createGoBom(path, options) {
           });
           // Check if got a mod graph successfully
           if (result.status !== 0 || result.error) {
+            console.log("go mod graph command has failed.");
             if (DEBUG_MODE && result.stdout) {
               console.log(result.stdout);
+              if (result?.stdout.includes("unrecognized import path")) {
+                console.log(
+                  "go couldn't download all the modules, including any private modules. Dependency tree would be missing.",
+                );
+              }
             }
             if (DEBUG_MODE && result.stderr) {
               console.log(result.stderr);
@@ -3062,8 +3069,14 @@ export async function createGoBom(path, options) {
           }
         } else {
           shouldManuallyParse = true;
-          console.error(
-            "go unexpectedly didn't return any output. Check if the correct version of golang is installed.",
+          console.log(
+            "1. Check if the correct version of golang is installed. Try building the application using go build or make command to troubleshoot.",
+          );
+          console.log(
+            "2. If the application uses private go modules, ensure the environment variable GOPRIVATE is set with the comma-separated repo names.\nEnsure $HOME/.netrc file contains a valid username and password for the private repos.",
+          );
+          console.log(
+            "3. Alternatively, consider generating a post-build SBOM from the built binary using blint. Use the official container image and invoke cdxgen with the arguments `-t binary --lifecycle post-build`.",
           );
           options.failOnError && process.exit(1);
         }

diff --git a/utils.js b/utils.js
@@ -3901,6 +3901,7 @@ export async function parseGoListDep(rawOutput, gosumMap) {
             version,
             gosumHash,
           );
+          // This is misusing the scope attribute to represent direct vs indirect
           if (verArr[2] === "false") {
             component.scope = "required";
           } else if (verArr[2] === "true") {
@@ -3915,6 +3916,10 @@ export async function parseGoListDep(rawOutput, gosumMap) {
               name: "ModuleGoVersion",
               value: verArr[4] || "",
             },
+            {
+              name: "cdx:go:indirect",
+              value: verArr[2],
+            },
           ];
           if (verArr.length > 5 && verArr[5] === "true") {
             parentComponent = component;
@@ -5155,7 +5160,7 @@ export function parseCargoDependencyData(cargoLockData) {
             // and continue.
             if (DEBUG_MODE) {
               console.warn(
-                `The package "${dependency.name}" appears as a dependency to "${pkg.name}" but is not itself listed in the Cargo.lock-file. The Cargo.lock-file is invalid! The produced SBOM will not list ${dependency.name} as a dependency.`,
+                `The package "${dependency.name}" appears as a dependency to "${pkg.name}" but is not itself listed in the Cargo.lock file. The Cargo.lock file is invalid! The produced SBOM will not list ${dependency.name} as a dependency.`,
               );
             }
             return undefined;

diff --git a/utils.test.js b/utils.test.js
@@ -803,6 +803,10 @@ test("parse go list dependencies", async () => {
           "/home/almalinux/go/pkg/mod/cache/download/github.com/gorilla/mux/@v/v1.7.4.mod",
       },
       { name: "ModuleGoVersion", value: "1.12" },
+      {
+        name: "cdx:go:indirect",
+        value: "false",
+      },
     ],
   });
 });