Handle multimodule project dependencies properly and only download so…

…urces that haven't been found

Signed-off-by: Juan Manuel Leflet Estrada <[email protected]>

provider/internal/java/dependency.go

@@ -195,7 +195,6 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI
 	args := []string{
 		"dependency:tree",
 		"-Djava.net.useSystemProxies=true",
-		fmt.Sprintf("-DoutputFile=%s", f.Name()),
 	}
 	if pom.Modules != nil {
 		args = append([]string{"compile"}, args...)
@@ -204,30 +203,34 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI
 	if p.mvnSettingsFile != "" {
 		args = append(args, "-s", p.mvnSettingsFile)
 	}
+
 	// get the graph output
 	cmd := exec.Command("mvn", args...)
 	cmd.Dir = moddir
-	err = cmd.Run()
+	mvnOutput, err := cmd.CombinedOutput()
 	if err != nil {
 		return nil, err
 	}
 
+	if _, err = f.Write(mvnOutput); err != nil {
+		return nil, err
+	}
+
 	b, err := os.ReadFile(f.Name())
 	if err != nil {
 		return nil, err
 	}
 
 	lines := strings.Split(string(b), "\n")
+	submoduleTrees := extractSubmoduleTrees(lines)
 
-	// strip first and last line of the output
-	// first line is the base package, last line empty
-	if len(lines) > 2 {
-		lines = lines[1 : len(lines)-1]
-	}
-
-	pomDeps, err := p.parseMavenDepLines(lines, localRepoPath)
-	if err != nil {
-		return nil, err
+	var pomDeps []provider.DepDAGItem
+	for _, tree := range submoduleTrees {
+		submoduleDeps, err := p.parseMavenDepLines(tree, localRepoPath)
+		if err != nil {
+			return nil, err
+		}
+		pomDeps = append(pomDeps, submoduleDeps...)
 	}
 
 	m := map[uri.URI][]provider.DepDAGItem{}
@@ -241,6 +244,37 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI
 	return m, nil
 }
 
+// extractSubmoduleTrees creates an array of lines for each submodule tree found in the mvn dependency:tree output
+func extractSubmoduleTrees(lines []string) [][]string {
+	submoduleTrees := [][]string{}
+
+	beginRegex := regexp.MustCompile(`maven-dependency-plugin:[\d\.]+:tree`)
+	endRegex := regexp.MustCompile(`\[INFO\] -*$`)
+
+	submod := 0
+	gather, skipmod := false, true
+	for _, line := range lines {
+		if beginRegex.Find([]byte(line)) != nil {
+			gather = true
+			submoduleTrees = append(submoduleTrees, []string{})
+		} else if gather {
+			if endRegex.Find([]byte(line)) != nil {
+				gather, skipmod = false, true
+				submod++
+			} else {
+				if skipmod { // we ignore the first module (base module)
+					skipmod = false
+				} else {
+					line = strings.TrimLeft(line, "[INFO] ")
+					submoduleTrees[submod] = append(submoduleTrees[submod], line)
+				}
+			}
+		}
+	}
+
+	return submoduleTrees
+}
+
 // discoverDepsFromJars walks given path to discover dependencies embedded as JARs
 func (p *javaServiceClient) discoverDepsFromJars(path string, ll map[uri.URI][]konveyor.DepDAGItem) {
 	// for binaries we only find JARs embedded in archive

provider/internal/java/provider.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/getkin/kin-openapi/openapi3"
@@ -326,6 +327,8 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet
 	}
 	defer mvnOutput.Close()
 
+	log.V(5).Info("resolving dependency sources")
+
 	pomPath := fmt.Sprintf("%s/pom.xml", location)
 	pom, err := gopom.Parse(pomPath)
 	if err != nil {
@@ -335,7 +338,6 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet
 	args := []string{
 		"dependency:sources",
 		"-Djava.net.useSystemProxies=true",
-		fmt.Sprintf("-DoutputFile=%s", mvnOutput.Name()),
 	}
 	if pom.Modules != nil {
 		args = append([]string{"compile"}, args...)
@@ -346,10 +348,18 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet
 	}
 	cmd := exec.CommandContext(ctx, "mvn", args...)
 	cmd.Dir = location
-	err = cmd.Run()
+	output, err := cmd.CombinedOutput()
 	if err != nil {
 		return err
 	}
+
+	if _, err = mvnOutput.Write(output); err != nil {
+		return err
+	}
+	if _, err = mvnOutput.Seek(0, 0); err != nil {
+		return err
+	}
+
 	artifacts, err := parseUnresolvedSources(mvnOutput)
 	if err != nil {
 		return err
@@ -359,6 +369,8 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet
 		return nil
 	}
 	for _, artifact := range artifacts {
+		log.V(5).WithValues("artifact", artifact).Info("sources for artifact not found, decompiling...")
+
 		groupDirs := filepath.Join(strings.Split(artifact.GroupId, ".")...)
 		artifactDirs := filepath.Join(strings.Split(artifact.ArtifactId, ".")...)
 		jarName := fmt.Sprintf("%s-%s.jar", artifact.ArtifactId, artifact.Version)
@@ -384,23 +396,76 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet
 	return nil
 }
 
+// filterExistingSubmodules takes a list of artifacts and takes out the ones existing in a pom's modules list
+func filterExistingSubmodules(artifacts []javaArtifact, pom *gopom.Project) []javaArtifact {
+	if pom.Modules == nil {
+		return artifacts
+	}
+
+	var filtered []javaArtifact
+	for _, artifact := range artifacts {
+		found := false
+		for _, module := range *pom.Modules {
+			if artifact.ArtifactId == module {
+				found = true
+				break
+			}
+		}
+		if !found {
+			filtered = append(filtered, artifact)
+		}
+	}
+
+	return filtered
+}
+
 func parseUnresolvedSources(output io.Reader) ([]javaArtifact, error) {
 	artifacts := []javaArtifact{}
 	scanner := bufio.NewScanner(output)
+
+	sourcesPluginSeparatorSeen := false
 	unresolvedSeparatorSeen := false
+	sourcesPluginRegex := regexp.MustCompile(`maven-dependency-plugin:[\d\.]+:sources`)
+	unresolvedRegex := regexp.MustCompile(`The following files have NOT been resolved`)
 	for scanner.Scan() {
 		line := scanner.Text()
-		line = strings.TrimLeft(line, " ")
-		if strings.HasPrefix(line, "The following files have NOT been resolved:") {
+		line = strings.TrimLeft(line, "[INFO] ")
+
+		if sourcesPluginRegex.Find([]byte(line)) != nil {
+			sourcesPluginSeparatorSeen = true
+			unresolvedSeparatorSeen = false
+		} else if unresolvedRegex.Find([]byte(line)) != nil {
 			unresolvedSeparatorSeen = true
-		} else if unresolvedSeparatorSeen {
+		} else if sourcesPluginSeparatorSeen && unresolvedSeparatorSeen {
+			line, _, _ = strings.Cut(line, "--") // ie: "org.apache.derby:derby:jar:10.14.2.0:test -- module derby (auto)"
+
 			parts := strings.Split(line, ":")
-			if len(parts) != 6 {
+			if len(parts) != 6 && len(parts) != 5 {
 				continue
 			}
+
+			// sometimes maven puts here artifacts that are not sources; don't count those as unresolved (ie: spring-petclinic project)
+			if len(parts) == 6 {
+				classifier := parts[3]
+				if classifier != "sources" {
+					continue
+				}
+			} else if len(parts) == 5 {
+				classifier := parts[2]
+				if classifier != "sources" {
+					continue
+				}
+			}
+
+			var version string
 			groupId := parts[0]
 			artifactId := parts[1]
-			version := parts[4]
+			if len(parts) == 6 {
+				version = parts[4]
+			} else {
+				version = parts[3]
+			}
+
 			artifacts = append(artifacts,
 				javaArtifact{
 					packaging:  JavaArchive,

provider/internal/java/provider_test.go

@@ -16,11 +16,18 @@ func Test_parseUnresolvedSources(t *testing.T) {
 		{
 			name: "valid sources output",
 			mvnOutput: `
-The following files have been resolved:
-   org.springframework.boot:spring-boot:jar:sources:2.5.0:compile
-
-The following files have NOT been resolved:
-   io.konveyor.demo:config-utils:jar:sources:1.0.0:compile
+[INFO] --- maven-dependency-plugin:3.5.0:sources (default-cli) @ spring-petclinic ---
+[INFO] The following files have been resolved:
+[INFO]    org.springframework.boot:spring-boot-starter-actuator:jar:sources:3.1.0 -- module spring.boot.starter.actuator [auto]
+[INFO]    org.springframework.boot:spring-boot-starter:jar:sources:3.1.0 -- module spring.boot.starter [auto]
+[INFO]    org.springframework.boot:spring-boot-starter-logging:jar:sources:3.1.0 -- module spring.boot.starter.logging [auto]
+[INFO] The following files have NOT been resolved:
+[INFO]    org.springframework.boot:spring-boot-starter-logging:jar:3.1.0:compile -- module spring.boot.starter.logging [auto]
+[INFO]    io.konveyor.demo:config-utils:jar:sources:1.0.0:compile
+[INFO] --- maven-dependency-plugin:3.5.0:sources (default-cli) @ spring-petclinic ---
+[INFO] -----------------------------------------------------------------------------
+[INFO] The following files have NOT been resolved:
+[INFO]    org.springframework.boot:spring-boot-actuator:jar:sources:3.1.0:compile
 `,
 			wantErr: false,
 			wantList: []javaArtifact{
@@ -30,6 +37,12 @@ The following files have NOT been resolved:
 					ArtifactId: "config-utils",
 					Version:    "1.0.0",
 				},
+				{
+					packaging:  JavaArchive,
+					GroupId:    "org.springframework.boot",
+					ArtifactId: "spring-boot-actuator",
+					Version:    "3.1.0",
+				},
 			},
 		},
 	}