From f737724314faa603c1d45f31d716941a7dbeaf6a Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 9 Oct 2023 18:38:12 +0200 Subject: [PATCH 1/6] Must compile for dependency:sources to work properly Signed-off-by: Juan Manuel Leflet Estrada --- provider/internal/java/provider.go | 1 + 1 file changed, 1 insertion(+) diff --git a/provider/internal/java/provider.go b/provider/internal/java/provider.go index 7fa37e96..60344128 100644 --- a/provider/internal/java/provider.go +++ b/provider/internal/java/provider.go @@ -325,6 +325,7 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet } defer mvnOutput.Close() args := []string{ + "compile", "dependency:sources", "-Djava.net.useSystemProxies=true", fmt.Sprintf("-DoutputFile=%s", mvnOutput.Name()), From b62ee1f588dd4feb1dfb0aee1bc14181377d19b6 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Tue, 10 Oct 2023 14:07:49 +0200 Subject: [PATCH 2/6] Compile only if the project has submodules Signed-off-by: Juan Manuel Leflet Estrada --- go.mod | 1 + go.sum | 3 +++ provider/internal/java/dependency.go | 7 +++++++ provider/internal/java/provider.go | 9 ++++++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 930de582..43a43a51 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 + github.com/vifraa/gopom v1.0.0 go.lsp.dev/uri v0.3.0 go.opentelemetry.io/otel/trace v1.11.2 google.golang.org/grpc v1.54.0 diff --git a/go.sum b/go.sum index 5d70a32b..7cfcccd5 100644 --- a/go.sum +++ b/go.sum @@ -71,11 +71,14 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/vifraa/gopom v1.0.0 h1:L9XlKbyvid8PAIK8nr0lihMApJQg/12OBvMA28BcWh0= +github.com/vifraa/gopom v1.0.0/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o= go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo= go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I= go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index 5d75492f..631126dd 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -18,6 +18,7 @@ import ( "github.com/konveyor/analyzer-lsp/engine/labels" "github.com/konveyor/analyzer-lsp/output/v1/konveyor" "github.com/konveyor/analyzer-lsp/provider" + "github.com/vifraa/gopom" "go.lsp.dev/uri" ) @@ -160,11 +161,17 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI defer os.Remove(f.Name()) moddir := filepath.Dir(path) + + pom, err := gopom.Parse(path) args := []string{ "dependency:tree", "-Djava.net.useSystemProxies=true", fmt.Sprintf("-DoutputFile=%s", f.Name()), } + if pom.Modules != nil { + args = append([]string{"compile"}, args...) + } + if p.mvnSettingsFile != "" { args = append(args, "-s", p.mvnSettingsFile) } diff --git a/provider/internal/java/provider.go b/provider/internal/java/provider.go index 60344128..b4e3ae4b 100644 --- a/provider/internal/java/provider.go +++ b/provider/internal/java/provider.go @@ -16,6 +16,7 @@ import ( "github.com/konveyor/analyzer-lsp/jsonrpc2" "github.com/konveyor/analyzer-lsp/lsp/protocol" "github.com/konveyor/analyzer-lsp/provider" + "github.com/vifraa/gopom" "go.lsp.dev/uri" ) @@ -324,12 +325,18 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet return err } defer mvnOutput.Close() + + pomPath := fmt.Sprintf("%s/pom.xml", location) + pom, err := gopom.Parse(pomPath) args := []string{ - "compile", "dependency:sources", "-Djava.net.useSystemProxies=true", fmt.Sprintf("-DoutputFile=%s", mvnOutput.Name()), } + if pom.Modules != nil { + args = append([]string{"compile"}, args...) + } + if mavenSettings != "" { args = append(args, "-s", mavenSettings) } From c0affda57c8cf06fe99fd62c8597b629ec621cfd Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Tue, 10 Oct 2023 14:26:28 +0200 Subject: [PATCH 3/6] Handle errors Signed-off-by: Juan Manuel Leflet Estrada --- provider/internal/java/dependency.go | 6 +++++- provider/internal/java/provider.go | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index 631126dd..9f2a3851 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -122,7 +122,7 @@ func (p *javaServiceClient) GetDependencyFallback(ctx context.Context) (map[uri. } deps := []*provider.Dep{} dep := &provider.Dep{} - // TODO this is comedically janky + // TODO this is comically janky for _, node := range list { if node.Data == "groupId" { if dep.Name != "" { @@ -163,6 +163,10 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI moddir := filepath.Dir(path) pom, err := gopom.Parse(path) + if err != nil { + return nil, err + } + args := []string{ "dependency:tree", "-Djava.net.useSystemProxies=true", diff --git a/provider/internal/java/provider.go b/provider/internal/java/provider.go index b4e3ae4b..6c0ba487 100644 --- a/provider/internal/java/provider.go +++ b/provider/internal/java/provider.go @@ -328,6 +328,10 @@ func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSet pomPath := fmt.Sprintf("%s/pom.xml", location) pom, err := gopom.Parse(pomPath) + if err != nil { + return err + } + args := []string{ "dependency:sources", "-Djava.net.useSystemProxies=true", From 277d4d9d4e5dcae4d48d645284ea81bc4d2cdbe4 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Wed, 11 Oct 2023 12:37:51 +0200 Subject: [PATCH 4/6] Refactor dependency fallback function Signed-off-by: Juan Manuel Leflet Estrada --- provider/internal/java/dependency.go | 95 ++++++++++++++++------------ 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index 9f2a3851..ee6d98ab 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -10,11 +10,9 @@ import ( "os" "os/exec" "path/filepath" - "reflect" "regexp" "strings" - "github.com/antchfx/xmlquery" "github.com/konveyor/analyzer-lsp/engine/labels" "github.com/konveyor/analyzer-lsp/output/v1/konveyor" "github.com/konveyor/analyzer-lsp/provider" @@ -58,12 +56,12 @@ func (p *javaServiceClient) GetDependencies(ctx context.Context) (map[uri.URI][] } else { ll, err = p.GetDependenciesDAG(ctx) if err != nil { - p.log.Info("unable to get dependencies using fallback", "error", err) - return p.GetDependencyFallback(ctx) + p.log.Info("unable to get dependencies, using fallback", "error", err) + return p.GetDependenciesFallback(ctx, "") } if len(ll) == 0 { - p.log.Info("unable to get dependencies non found using fallback") - return p.GetDependencyFallback(ctx) + p.log.Info("unable to get dependencies (none found), using fallback") + return p.GetDependenciesFallback(ctx, "") } } for f, ds := range ll { @@ -98,52 +96,65 @@ func getMavenLocalRepoPath(mvnSettingsFile string) string { return string(outb.String()) } -func (p *javaServiceClient) GetDependencyFallback(ctx context.Context) (map[uri.URI][]*provider.Dep, error) { - pomDependencyQuery := "//dependencies/dependency/*" - path := p.findPom() - file := uri.File(path) - - absPath, err := filepath.Abs(path) - if err != nil { - return nil, err - } +func (p *javaServiceClient) GetDependenciesFallback(ctx context.Context, location string) (map[uri.URI][]*provider.Dep, error) { + deps := []*provider.Dep{} - f, err := os.Open(absPath) - if err != nil { - return nil, err - } - doc, err := xmlquery.Parse(f) - if err != nil { - return nil, err + path := p.findPom() + if location != "" { + path = location } - list, err := xmlquery.QueryAll(doc, pomDependencyQuery) + pom, err := gopom.Parse(path) if err != nil { return nil, err } - deps := []*provider.Dep{} - dep := &provider.Dep{} - // TODO this is comically janky - for _, node := range list { - if node.Data == "groupId" { - if dep.Name != "" { - deps = append(deps, dep) - dep = &provider.Dep{} + + // have to get both and dependencies (if present) + var pomDeps []gopom.Dependency + if pom.Dependencies != nil { + pomDeps = append(pomDeps, *pom.Dependencies...) + } + if pom.DependencyManagement != nil { + pomDeps = append(pomDeps, *pom.DependencyManagement.Dependencies...) + } + + // add each dependency found + for _, d := range pomDeps { + dep := provider.Dep{} + dep.Name = fmt.Sprintf("%s.%s", *d.GroupID, *d.ArtifactID) + if *d.Version != "" { + if strings.Contains(*d.Version, "$") { + version := strings.TrimSuffix(strings.TrimPrefix(*d.Version, "${"), "}") + version = pom.Properties.Entries[version] + if version != "" { + dep.Version = version + } + } else { + dep.Version = *d.Version } - dep.Name = node.InnerText() - } else if node.Data == "artifactId" { - dep.Name += "." + node.InnerText() - } else if node.Data == "version" { - dep.Version = node.InnerText() } - // Ignore the others - } - if !reflect.DeepEqual(dep, provider.Dep{}) { - dep.Labels = []string{fmt.Sprintf("%v=%v", provider.DepSourceLabel, javaDepSourceInternal)} - deps = append(deps, dep) + deps = append(deps, &dep) } + m := map[uri.URI][]*provider.Dep{} - m[file] = deps + m[uri.File(path)] = deps p.depsCache = m + + // recursively find deps in submodules + if pom.Modules != nil { + for _, mod := range *pom.Modules { + mPath := fmt.Sprintf("%s/%s/pom.xml", filepath.Dir(path), mod) + moreDeps, err := p.GetDependenciesFallback(ctx, mPath) + if err != nil { + return nil, err + } + + // add found dependencies to map + for depPath := range moreDeps { + m[depPath] = moreDeps[depPath] + } + } + } + return m, nil } From 430f4897e309819b0214ab236993155c69dd828d Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Wed, 11 Oct 2023 17:40:56 +0200 Subject: [PATCH 5/6] DependencyManagement filed can have no children Signed-off-by: Juan Manuel Leflet Estrada --- provider/internal/java/dependency.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index ee6d98ab..b00a4a17 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -114,7 +114,9 @@ func (p *javaServiceClient) GetDependenciesFallback(ctx context.Context, locatio pomDeps = append(pomDeps, *pom.Dependencies...) } if pom.DependencyManagement != nil { - pomDeps = append(pomDeps, *pom.DependencyManagement.Dependencies...) + if pom.DependencyManagement.Dependencies != nil { + pomDeps = append(pomDeps, *pom.DependencyManagement.Dependencies...) + } } // add each dependency found From 4d149f90aad403229083f33fab1434be53213ffa Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Wed, 11 Oct 2023 20:03:49 +0200 Subject: [PATCH 6/6] Use absolute path Signed-off-by: Juan Manuel Leflet Estrada --- provider/internal/java/dependency.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index b00a4a17..53b8f071 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -99,7 +99,11 @@ func getMavenLocalRepoPath(mvnSettingsFile string) string { func (p *javaServiceClient) GetDependenciesFallback(ctx context.Context, location string) (map[uri.URI][]*provider.Dep, error) { deps := []*provider.Dep{} - path := p.findPom() + path, err := filepath.Abs(p.findPom()) + if err != nil { + return nil, err + } + if location != "" { path = location }