Skip to content

Commit

Permalink
🐛 improve decompile performance (#530)
Browse files Browse the repository at this point in the history 
This improves decompilation performance significantly. Also adds more
tracing spans so we get more visibility into decomp perf in future.

Before:

* Full analysis via kantra in source + deps mode
    _InProgress_

* Decompile performance (41 sec per file, 22451 sec total)


![Screenshot_20240308_133015](https://github.com/konveyor/analyzer-lsp/assets/9839757/f4e282c0-bce2-4930-a828-204eb3c98a4d)


After:

* Full analysis via kantra in source + deps mode (improved by )
     ```
     real	10m21.651s
     user	0m1.978s
     sys	0m2.554s
     ```
     
* Decompile performance (3.1 sec on per file, 1683 sec total, improved
by ~93%)


![Screenshot_20240308_133054](https://github.com/konveyor/analyzer-lsp/assets/9839757/22d9b883-baef-4aef-8f07-36f4e18d693b)

    
There's one more issue we need to fix. We are decompiling for dependency
CLI too. That's because we use the same _Init()_ function for analyzer
and dep cli. We either need to combine these two commands, or have an
alternate _InitDep()_ function interface on providers that does a light
weight init of providers. Captured in
#529

---------

Signed-off-by: Pranav Gaikwad <[email protected]>
  • Loading branch information
@pranavgaikwad
pranavgaikwad authored Mar 11, 2024
1 parent 6ec6477 commit 073e142
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 14 deletions.
15 changes: 10 additions & 5 deletions cmd/analyzer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/swaggest/openapi-go/openapi3"
"go.opentelemetry.io/otel/attribute"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -112,8 +113,8 @@ func AnalysisCmd() *cobra.Command {

defer tracing.Shutdown(ctx, log, tp)

ctx, span := tracing.StartNewSpan(ctx, "main")
defer span.End()
ctx, mainSpan := tracing.StartNewSpan(ctx, "main")
defer mainSpan.End()

// Get the configs
configs, err := provider.GetConfig(settingsFile)
Expand All @@ -122,16 +123,16 @@ func AnalysisCmd() *cobra.Command {
os.Exit(1)
}

engineCtx, engineSpan := tracing.StartNewSpan(ctx, "rule-engine")
//start up the rule eng
eng := engine.CreateRuleEngine(ctx,
eng := engine.CreateRuleEngine(engineCtx,
10,
log,
engine.WithIncidentLimit(limitIncidents),
engine.WithCodeSnipLimit(limitCodeSnips),
engine.WithContextLines(contextLines),
engine.WithIncidentSelector(incidentSelector),
)

providers := map[string]provider.InternalProviderClient{}

for _, config := range configs {
Expand Down Expand Up @@ -194,14 +195,18 @@ func AnalysisCmd() *cobra.Command {
}
// Now that we have all the providers, we need to start them.
for name, provider := range needProviders {
err := provider.ProviderInit(ctx)
initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
attribute.Key("provider").String(name))
err := provider.ProviderInit(initCtx)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", name)
os.Exit(1)
}
initSpan.End()
}

rulesets := eng.RunRules(ctx, ruleSets, selectors...)
engineSpan.End()
eng.Stop()

for _, provider := range needProviders {
Expand Down
8 changes: 7 additions & 1 deletion provider/internal/java/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/konveyor/analyzer-lsp/lsp/protocol"
"github.com/konveyor/analyzer-lsp/output/v1/konveyor"
"github.com/konveyor/analyzer-lsp/provider"
"github.com/konveyor/analyzer-lsp/tracing"
"github.com/swaggest/openapi-go/openapi3"
"go.lsp.dev/uri"
)
Expand Down Expand Up @@ -234,7 +235,8 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide
extension := strings.ToLower(path.Ext(config.Location))
switch extension {
case JavaArchive, WebArchive, EnterpriseArchive:
depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location)
depLocation, sourceLocation, err := decompileJava(ctx, log,
config.Location, getMavenLocalRepoPath(mavenSettingsFile))
if err != nil {
cancelFunc()
return nil, err
Expand Down Expand Up @@ -404,6 +406,10 @@ func (j *javaProvider) GetLocation(ctx context.Context, dep konveyor.Dep) (engin
// resolveSourcesJars for a given source code location, runs maven to find
// deps that don't have sources attached and decompiles them
func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSettings string) error {
// TODO (pgaikwad): when we move to external provider, inherit context from parent
ctx, span := tracing.StartNewSpan(ctx, "resolve-sources")
defer span.End()

decompileJobs := []decompileJob{}

log.V(5).Info("resolving dependency sources")
Expand Down
37 changes: 29 additions & 8 deletions provider/internal/java/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/konveyor/analyzer-lsp/engine/labels"
"github.com/konveyor/analyzer-lsp/provider"
"github.com/konveyor/analyzer-lsp/tracing"
"go.opentelemetry.io/otel/attribute"
)

const javaProjectPom = `<?xml version="1.0" encoding="UTF-8"?>
Expand Down Expand Up @@ -93,6 +94,7 @@ type decompileJob struct {
inputPath string
outputPath string
artifact javaArtifact
m2RepoPath string
}

// decompile decompiles files submitted via a list of decompileJob concurrently
Expand All @@ -107,11 +109,14 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
for i := 0; i < workerCount; i++ {
logger := log.WithName(fmt.Sprintf("decompileWorker-%d", i))
wg.Add(1)
go func(log logr.Logger) {
go func(log logr.Logger, workerId int) {
defer log.V(6).Info("shutting down decompile worker")
defer wg.Done()
log.V(6).Info("init decompile worker")
for job := range jobChan {
// TODO (pgaikwad): when we move to external provider, inherit context from parent
jobCtx, span := tracing.StartNewSpan(ctx, "decomp-job",
attribute.Key("worker").Int(workerId))
// apply decompile filter
if !filter.shouldDecompile(job.artifact) {
continue
Expand All @@ -126,8 +131,9 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
"failed to create directories for decompiled file", "path", outputPathDir)
continue
}
// -mpm (max processing method) is required to keep decomp time low
cmd := exec.CommandContext(
ctx, "java", "-jar", "/bin/fernflower.jar", job.inputPath, outputPathDir)
jobCtx, "java", "-jar", "/bin/fernflower.jar", "-mpm=30", job.inputPath, outputPathDir)
err := cmd.Run()
if err != nil {
log.V(5).Error(err, "failed to decompile file", "file", job.inputPath, job.outputPath)
Expand All @@ -137,13 +143,15 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
// if we just decompiled a java archive, we need to
// explode it further and copy files to project
if job.artifact.packaging == JavaArchive && projectPath != "" {
_, _, _, err = explode(ctx, log, job.outputPath, projectPath)
_, _, _, err = explode(jobCtx, log, job.outputPath, projectPath, job.m2RepoPath)
if err != nil {
log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath)
}
}
span.End()
jobCtx.Done()
}
}(logger)
}(logger, i)
}

seenJobs := map[string]bool{}
Expand All @@ -165,15 +173,15 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
// decompileJava unpacks archive at archivePath, decompiles all .class files in it
// creates new java project and puts the java files in the tree of the project
// returns path to exploded archive, path to java project, and an error when encountered
func decompileJava(ctx context.Context, log logr.Logger, archivePath string) (explodedPath, projectPath string, err error) {
func decompileJava(ctx context.Context, log logr.Logger, archivePath string, m2RepoPath string) (explodedPath, projectPath string, err error) {
ctx, span := tracing.StartNewSpan(ctx, "decompile")
defer span.End()

projectPath = filepath.Join(filepath.Dir(archivePath), "java-project")

decompFilter := alwaysDecompileFilter(true)

explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath)
explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath, m2RepoPath)
if err != nil {
log.Error(err, "failed to decompile archive", "path", archivePath)
return "", "", err
Expand Down Expand Up @@ -212,7 +220,7 @@ func deduplicateJavaArtifacts(artifacts []javaArtifact) []javaArtifact {
// explode explodes the given JAR, WAR or EAR archive, generates javaArtifact struct for given archive
// and identifies all .class found recursively. returns output path, a list of decompileJob for .class files
// it also returns a list of any javaArtifact we could interpret from jars
func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string) (string, []decompileJob, []javaArtifact, error) {
func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string, m2Repo string) (string, []decompileJob, []javaArtifact, error) {
var dependencies []javaArtifact
fileInfo, err := os.Stat(archivePath)
if err != nil {
Expand Down Expand Up @@ -313,7 +321,7 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
// decompile web archives
case strings.HasSuffix(f.Name, WebArchive):
// TODO(djzager): Should we add these deps to the pom?
_, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath)
_, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath, m2Repo)
if err != nil {
log.Error(err, "failed to decompile file", "file", filePath)
}
Expand Down Expand Up @@ -344,6 +352,16 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
if (dep != javaArtifact{}) {
if dep.foundOnline {
dependencies = append(dependencies, dep)
// copy this into m2 repo to avoid downloading again
groupPath := filepath.Join(strings.Split(dep.GroupId, ".")...)
artifactPath := filepath.Join(strings.Split(dep.ArtifactId, ".")...)
destPath := filepath.Join(m2Repo, groupPath, artifactPath,
dep.Version, filepath.Base(filePath))
if err := moveFile(filePath, destPath); err != nil {
log.V(8).Error(err, "failed moving jar to m2 local repo")
} else {
log.V(8).Info("moved jar file", "src", filePath, "dest", destPath)
}
} else {
// when it isn't found online, decompile it
outputPath := filepath.Join(
Expand Down Expand Up @@ -387,6 +405,9 @@ func createJavaProject(ctx context.Context, dir string, dependencies []javaArtif
}

func moveFile(srcPath string, destPath string) error {
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
inputFile, err := os.Open(srcPath)
if err != nil {
return err
Expand Down

0 comments on commit 073e142

Please sign in to comment.