✨ decompiel sources jar

Signed-off-by: Pranav Gaikwad <[email protected]>

provider/internal/java/dependency.go

@@ -78,12 +78,12 @@ func (p *javaServiceClient) GetDependencies(ctx context.Context) (map[uri.URI][]
 	return m, nil
 }
 
-func (p *javaServiceClient) getLocalRepoPath() string {
+func getMavenLocalRepoPath(mvnSettingsFile string) string {
 	args := []string{
 		"help:evaluate", "-Dexpression=settings.localRepository", "-q", "-DforceStdout",
 	}
-	if p.mvnSettingsFile != "" {
-		args = append(args, "-s", p.mvnSettingsFile)
+	if mvnSettingsFile != "" {
+		args = append(args, "-s", mvnSettingsFile)
 	}
 	cmd := exec.Command("mvn", args...)
 	var outb bytes.Buffer
@@ -147,7 +147,7 @@ func (p *javaServiceClient) GetDependencyFallback(ctx context.Context) (map[uri.
 }
 
 func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) {
-	localRepoPath := p.getLocalRepoPath()
+	localRepoPath := getMavenLocalRepoPath(p.mvnSettingsFile)
 
 	path := p.findPom()
 	file := uri.File(path)

provider/internal/java/provider.go

@@ -1,11 +1,14 @@
 package java
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/getkin/kin-openapi/openapi3"
@@ -187,26 +190,7 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide
 	}
 	log = log.WithValues("provider", "java")
 
-	isBinary := false
-	var returnErr error
-	// each service client should have their own context
-	ctx, cancelFunc := context.WithCancel(ctx)
-	extension := strings.ToLower(path.Ext(config.Location))
-	switch extension {
-	case JavaArchive, WebArchive, EnterpriseArchive:
-		depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location)
-		if err != nil {
-			cancelFunc()
-			return nil, err
-		}
-		config.Location = sourceLocation
-		// for binaries, we fallback to looking at .jar files only for deps
-		config.DependencyPath = depLocation
-		// for binaries, always run in source-only mode as we don't know how to correctly resolve deps
-		config.AnalysisMode = provider.SourceOnlyAnalysisMode
-		isBinary = true
-	}
-
+	// read provider settings
 	bundlesString, ok := config.ProviderSpecificConfig[BUNDLES_INIT_OPTION].(string)
 	if !ok {
 		bundlesString = ""
@@ -225,10 +209,37 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide
 
 	lspServerPath, ok := config.ProviderSpecificConfig[provider.LspServerPathConfigKey].(string)
 	if !ok || lspServerPath == "" {
-		cancelFunc()
 		return nil, fmt.Errorf("invalid lspServerPath provided, unable to init java provider")
 	}
 
+	isBinary := false
+	var returnErr error
+	// each service client should have their own context
+	ctx, cancelFunc := context.WithCancel(ctx)
+	extension := strings.ToLower(path.Ext(config.Location))
+	switch extension {
+	case JavaArchive, WebArchive, EnterpriseArchive:
+		depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location)
+		if err != nil {
+			cancelFunc()
+			return nil, err
+		}
+		config.Location = sourceLocation
+		// for binaries, we fallback to looking at .jar files only for deps
+		config.DependencyPath = depLocation
+		// for binaries, always run in source-only mode as we don't know how to correctly resolve deps
+		config.AnalysisMode = provider.SourceOnlyAnalysisMode
+		isBinary = true
+	default:
+		// when location points to source code, we attempt to decompile
+		// JARs of dependencies that don't have a sources JAR attached
+		err := resolveSourcesJars(ctx, log, config.Location, mavenSettingsFile)
+		if err != nil {
+			// TODO (pgaikwad): should we ignore this failure?
+			log.Error(err, "failed to resolve sources jar for location", "location", config.Location)
+		}
+	}
+
 	// handle proxy settings
 	for k, v := range config.Proxy.ToEnvVars() {
 		os.Setenv(k, v)
@@ -305,3 +316,88 @@ func (p *javaProvider) GetDependencies(ctx context.Context) (map[uri.URI][]*prov
 func (p *javaProvider) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) {
 	return provider.FullDepDAGResponse(ctx, p.clients)
 }
+
+// resolveSourcesJars for a given source code location, runs maven to find
+// deps that don't have sources attached and decompiles them
+func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSettings string) error {
+	decompileJobs := []decompileJob{}
+	mvnOutput, err := os.CreateTemp("", "mvn-sources-")
+	if err != nil {
+		return err
+	}
+	args := []string{
+		"dependency:sources",
+		"-Djava.net.useSystemProxies=true",
+		fmt.Sprintf("-DoutputFile=%s", mvnOutput.Name()),
+	}
+	if mavenSettings != "" {
+		args = append(args, "-s", mavenSettings)
+	}
+	cmd := exec.CommandContext(ctx, "mvn", args...)
+	cmd.Dir = location
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	artifacts, err := parseUnresolvedSources(mvnOutput)
+	if err != nil {
+		return err
+	}
+	m2Repo := getMavenLocalRepoPath(mavenSettings)
+	if m2Repo == "" {
+		return nil
+	}
+	for _, artifact := range artifacts {
+		groupDirs := filepath.Join(strings.Split(artifact.groupId, ".")...)
+		artifactDirs := filepath.Join(strings.Split(artifact.artifactId, ".")...)
+		jarName := fmt.Sprintf("%s-%s.jar", artifact.artifactId, artifact.version)
+		decompileJobs = append(decompileJobs, decompileJob{
+			artifact: artifact,
+			inputPath: filepath.Join(
+				m2Repo, groupDirs, artifactDirs, artifact.version, jarName),
+			outputPath: filepath.Join(
+				m2Repo, groupDirs, artifactDirs, artifact.version, "decompiled", jarName),
+		})
+	}
+	err = decompile(ctx, log, alwaysDecompileFilter(true), 10, decompileJobs, "")
+	if err != nil {
+		return err
+	}
+	// move decompiled files to base location of the jar
+	for _, decompileJob := range decompileJobs {
+		jarName := strings.TrimSuffix(filepath.Base(decompileJob.inputPath), ".jar")
+		moveFile(decompileJob.outputPath,
+			filepath.Join(filepath.Dir(decompileJob.inputPath),
+				fmt.Sprintf("%s-sources.jar", jarName)))
+	}
+	return nil
+}
+
+func parseUnresolvedSources(output io.Reader) ([]javaArtifact, error) {
+	artifacts := []javaArtifact{}
+	scanner := bufio.NewScanner(output)
+	unresolvedSeparatorSeen := false
+	for scanner.Scan() {
+		line := scanner.Text()
+		line = strings.TrimLeft(line, " ")
+		if strings.HasPrefix(line, "The following files have NOT been resolved:") {
+			unresolvedSeparatorSeen = true
+		} else if unresolvedSeparatorSeen {
+			parts := strings.Split(line, ":")
+			if len(parts) != 6 {
+				continue
+			}
+			groupId := parts[0]
+			artifactId := parts[1]
+			version := parts[4]
+			artifacts = append(artifacts,
+				javaArtifact{
+					packaging:  JavaArchive,
+					artifactId: artifactId,
+					groupId:    groupId,
+					version:    version,
+				})
+		}
+	}
+	return artifacts, scanner.Err()
+}

provider/internal/java/provider_test.go

@@ -0,0 +1,43 @@
+package java
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func Test_parseUnresolvedSources(t *testing.T) {
+	tests := []struct {
+		name      string
+		mvnOutput string
+		wantErr   bool
+		wantList  []string
+	}{
+		{
+			name: "valid sources output",
+			mvnOutput: `
+The following files have been resolved:
+   org.springframework.boot:spring-boot:jar:sources:2.5.0:compile
+
+The following files have NOT been resolved:
+   io.konveyor.demo:config-utils:jar:sources:1.0.0:compile
+`,
+			wantErr: false,
+			wantList: []string{
+				"io/konveyor/demo/config-utils/1.0.0/config-utils-1.0.0.jar",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			outputReader := strings.NewReader(tt.mvnOutput)
+			gotList, gotErr := parseUnresolvedSources(outputReader)
+			if (gotErr != nil) != tt.wantErr {
+				t.Errorf("parseUnresolvedSources() gotErr = %v, wantErr %v", gotErr, tt.wantErr)
+			}
+			if !reflect.DeepEqual(gotList, tt.wantList) {
+				t.Errorf("parseUnresolvedSources() gotList = %v, wantList %v", gotList, tt.wantList)
+			}
+		})
+	}
+}

provider/internal/java/service_client.go

@@ -19,7 +19,6 @@ import (
 
 type javaServiceClient struct {
 	rpc              *jsonrpc2.Conn
-	ctx              context.Context
 	cancelFunc       context.CancelFunc
 	config           provider.InitConfig
 	log              logr.Logger

provider/internal/java/util.go

@@ -7,6 +7,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"os/exec"
 	"path"
@@ -85,6 +86,7 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 	wg := &sync.WaitGroup{}
 	jobChan := make(chan decompileJob)
 
+	workerCount = int(math.Min(float64(len(jobs)), float64(workerCount)))
 	// init workers
 	for i := 0; i < workerCount; i++ {
 		wg.Add(1)
@@ -113,7 +115,7 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 				}
 				// if we just decompiled a java archive, we need to
 				// explode it further and copy files to project
-				if job.artifact.packaging == JavaArchive {
+				if job.artifact.packaging == JavaArchive && projectPath != "" {
 					_, _, err = explode(ctx, log, job.outputPath, projectPath)
 					if err != nil {
 						log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath)
@@ -251,7 +253,7 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
 					packaging: ClassFile,
 				},
 			})
-		// when it's a java file, it's already decompiled, copy it to project path
+		// when it's a java file, it's already decompiled, move it to project path
 		case strings.HasSuffix(f.Name, JavaFile):
 			destPath := filepath.Join(
 				projectPath, "src", "main", "java",
@@ -262,31 +264,11 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
 				log.V(8).Error(err, "error creating directory for java file", "path", destPath)
 				continue
 			}
-			log.V(6).Info("copying file", "src", filePath, "dest", destPath)
-			inputFile, err := os.Open(filePath)
-			if err != nil {
-				log.V(8).Error(err, "failed to open input file", "path", filePath)
-				continue
-			}
-			outputFile, err := os.Create(destPath)
-			if err != nil {
-				inputFile.Close()
-				log.V(8).Error(err, "failed to open output file", "path", destPath)
-				continue
-			}
-			_, err = io.Copy(outputFile, inputFile)
-			inputFile.Close()
-			if err != nil {
-				log.V(8).Error(err, "failed to move java file to project", "src", filePath, "dest", destPath)
+			if err := moveFile(filePath, destPath); err != nil {
+				log.V(8).Error(err, "error moving decompiled file to project path",
+					"src", filePath, "dest", destPath)
 				continue
 			}
-			// The copy was successful, so now delete the original file
-			err = os.Remove(filePath)
-			if err != nil {
-				log.V(8).Error(err, "failed to remove source file", "src", filePath)
-				continue
-			}
-			defer outputFile.Close()
 		// decompile web archives
 		case strings.HasSuffix(f.Name, WebArchive):
 			_, nestedJobs, err := explode(ctx, log, filePath, projectPath)
@@ -332,6 +314,29 @@ func createJavaProject(ctx context.Context, dir string) error {
 	return nil
 }
 
+func moveFile(srcPath string, destPath string) error {
+	inputFile, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return err
+	}
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return err
+	}
+	err = os.Remove(srcPath)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+	return nil
+}
+
 type project struct {
 	XMLName    xml.Name     `xml:"project"`
 	Dependency dependencies `xml:"dependencies"`