Skip to content

Commit

Permalink
adding scopes that allow for the condition context to be updated by t…
Browse files Browse the repository at this point in the history
…he running engine and the caller of run rules

add scoped run rules to interface

Signed-off-by: Shawn Hurley <[email protected]>
  • Loading branch information
shawn-hurley committed Nov 20, 2024
1 parent 5811cce commit 49c5fe4
Show file tree
Hide file tree
Showing 9 changed files with 262 additions and 33 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/pr-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.21'

- name: Test
run: go test -v ./...
Expand Down
12 changes: 12 additions & 0 deletions engine/conditions.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package engine
import (
"context"
"fmt"
"maps"
"regexp"
"strings"

Expand All @@ -26,6 +27,17 @@ type ConditionResponse struct {
type ConditionContext struct {
Tags map[string]interface{} `yaml:"tags"`
Template map[string]ChainTemplate `yaml:"template"`
RuleID string `yaml:ruleID`
}

// This will copy the condition, but this will not copy the ruleID
func (c *ConditionContext) Copy() ConditionContext {
newTags := maps.Clone(c.Tags)
newTemplate := maps.Clone(c.Template)
return ConditionContext{
Tags: newTags,
Template: newTemplate,
}
}

type ConditionEntry struct {
Expand Down
39 changes: 32 additions & 7 deletions engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,15 @@ import (

type RuleEngine interface {
RunRules(context context.Context, rules []RuleSet, selectors ...RuleSelector) []konveyor.RuleSet
RunRulesScoped(ctx context.Context, ruleSets []RuleSet, scopes Scope, selectors ...RuleSelector) []konveyor.RuleSet
Stop()
}

type ruleMessage struct {
rule Rule
ruleSetName string
ctx ConditionContext
scope Scope
returnChan chan response
}

Expand Down Expand Up @@ -128,7 +130,12 @@ func processRuleWorker(ctx context.Context, ruleMessages chan ruleMessage, logge
case m := <-ruleMessages:
logger.V(5).Info("taking rule", "ruleset", m.ruleSetName, "rule", m.rule.RuleID)
newLogger := logger.WithValues("ruleID", m.rule.RuleID)
//We createa new rule context for a every rule run, here we need to apply the scope
m.ctx.Template = make(map[string]ChainTemplate)
if m.scope != nil {
m.scope.AddToContext(&m.ctx)
}

bo, err := processRule(ctx, m.rule, m.ctx, newLogger)
logger.V(5).Info("finished rule", "found", len(bo.Incidents), "error", err, "rule", m.rule.RuleID)
m.returnChan <- response{
Expand Down Expand Up @@ -162,13 +169,32 @@ func (r *ruleEngine) createRuleSet(ruleSet RuleSet) *konveyor.RuleSet {
// This will run tagging rules first, synchronously, generating tags to pass on further as context to other rules
// then runs remaining rules async, fanning them out, fanning them in, finally generating the results. will block until completed.
func (r *ruleEngine) RunRules(ctx context.Context, ruleSets []RuleSet, selectors ...RuleSelector) []konveyor.RuleSet {
return r.RunRulesScoped(ctx, ruleSets, nil, selectors...)
}

func (r *ruleEngine) RunRulesScoped(ctx context.Context, ruleSets []RuleSet, scopes Scope, selectors ...RuleSelector) []konveyor.RuleSet {
// determine if we should run

conditionContext := ConditionContext{
Tags: make(map[string]interface{}),
Template: make(map[string]ChainTemplate),
}
if scopes != nil {
r.logger.Info("using scopes", "scope", scopes.Name())
err := scopes.AddToContext(&conditionContext)
if err != nil {
r.logger.Error(err, "unable to apply scopes to ruleContext")
// Call this, even though it is not used, to make sure that
// we don't leak anything.
return []konveyor.RuleSet{}
}
r.logger.Info("added scopes to condition context", "scopes", scopes, "conditionContext", conditionContext)
}
ctx, cancelFunc := context.WithCancel(ctx)

taggingRules, otherRules, mapRuleSets := r.filterRules(ruleSets, selectors...)

ruleContext := r.runTaggingRules(ctx, taggingRules, mapRuleSets)
ruleContext := r.runTaggingRules(ctx, taggingRules, mapRuleSets, conditionContext)

// Need a better name for this thing
ret := make(chan response)
Expand Down Expand Up @@ -241,6 +267,7 @@ func (r *ruleEngine) RunRules(ctx context.Context, ruleSets []RuleSet, selectors
wg.Add(1)
rule.returnChan = ret
rule.ctx = ruleContext
rule.scope = scopes
r.ruleProcessing <- rule
}
r.logger.V(5).Info("All rules added buffer, waiting for engine to complete", "size", len(otherRules))
Expand Down Expand Up @@ -318,11 +345,7 @@ func (r *ruleEngine) filterRules(ruleSets []RuleSet, selectors ...RuleSelector)

// runTaggingRules filters and runs info rules synchronously
// returns list of non-info rules, a context to pass to them
func (r *ruleEngine) runTaggingRules(ctx context.Context, infoRules []ruleMessage, mapRuleSets map[string]*konveyor.RuleSet) ConditionContext {
context := ConditionContext{
Tags: make(map[string]interface{}),
Template: make(map[string]ChainTemplate),
}
func (r *ruleEngine) runTaggingRules(ctx context.Context, infoRules []ruleMessage, mapRuleSets map[string]*konveyor.RuleSet, context ConditionContext) ConditionContext {
// track unique tags per ruleset
rulesetTagsCache := map[string]map[string]bool{}
for _, ruleMessage := range infoRules {
Expand Down Expand Up @@ -427,9 +450,11 @@ func processRule(ctx context.Context, rule Rule, ruleCtx ConditionContext, log l
ctx, span := tracing.StartNewSpan(
ctx, "process-rule", attribute.Key("rule").String(rule.RuleID))
defer span.End()
newContext := ruleCtx.Copy()
newContext.RuleID = rule.RuleID
// Here is what a worker should run when getting a rule.
// For now, lets not fan out the running of conditions.
return rule.When.Evaluate(ctx, log, ruleCtx)
return rule.When.Evaluate(ctx, log, newContext)

}

Expand Down
88 changes: 88 additions & 0 deletions engine/scopes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package engine

import (
"fmt"

"github.com/go-logr/logr"
)

const TemplateContextPathScopeKey = "konveyor.io/path-scope"

// Scopes apply to individual calls to the providers and will add inforamtion to the ConditionContext
// To apply the scope. It is the responsiblity of the provider to use these correctly.
type Scope interface {
Name() string
// For now this is the only place that we are considering adding a scope
// in the future, we could scope other things
AddToContext(*ConditionContext) error
}

type scopeWrapper struct {
scopes []Scope
}

func (s *scopeWrapper) Name() string {
name := ""
for i, s := range s.scopes {
if i == 0 {
name = s.Name()

} else {
name = fmt.Sprintf("%s -- %s", name, s.Name())
}
}
return name
}

func (s *scopeWrapper) AddToContext(conditionCTX *ConditionContext) error {
for _, s := range s.scopes {
err := s.AddToContext(conditionCTX)
if err != nil {
return err
}
}
return nil
}

var _ Scope = &scopeWrapper{}

func NewScope(scopes ...Scope) Scope {
return &scopeWrapper{scopes: scopes}
}

type includedPathScope struct {
log logr.Logger
paths []string
}

var _ Scope = &includedPathScope{}

func (i *includedPathScope) Name() string {
return "IncludedPathScope"
}

// This will only update conditionCTX if filepaths is not set.
func (i *includedPathScope) AddToContext(conditionCTX *ConditionContext) error {
// If any chain template has the filepaths set, only use those.
for k, chainTemplate := range conditionCTX.Template {
if chainTemplate.Filepaths != nil && len(chainTemplate.Filepaths) > 0 {
i.log.V(5).Info("includedPathScope not used because filepath set", "filepaths", chainTemplate.Filepaths, "key", k)
return nil
}
}

// if no As clauses have filepaths, then assume we need to add the special cased filepath for scopes here
conditionCTX.Template[TemplateContextPathScopeKey] = ChainTemplate{
Filepaths: i.paths,
Extras: nil,
}
return nil

}

func IncludedPathsScope(paths []string, log logr.Logger) Scope {
return &includedPathScope{
paths: paths,
log: log,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"time"

"github.com/go-logr/logr"
"github.com/konveyor/analyzer-lsp/engine"
"github.com/konveyor/analyzer-lsp/jsonrpc2"
"github.com/konveyor/analyzer-lsp/lsp/protocol"
"github.com/konveyor/analyzer-lsp/provider"
Expand Down Expand Up @@ -59,7 +58,7 @@ func (p *javaServiceClient) Evaluate(ctx context.Context, cap string, conditionI
cond.Referenced.Filepaths = strings.Split(cond.Referenced.Filepaths[0], " ")
}

condCtx := &engine.ConditionContext{}
condCtx := &provider.ProviderContext{}
err = yaml.Unmarshal(conditionInfo, condCtx)
if err != nil {
return provider.ProviderEvaluateResponse{}, fmt.Errorf("unable to get condition context info: %v", err)
Expand All @@ -68,7 +67,7 @@ func (p *javaServiceClient) Evaluate(ctx context.Context, cap string, conditionI
if cond.Referenced.Pattern == "" {
return provider.ProviderEvaluateResponse{}, fmt.Errorf("provided query pattern empty")
}
symbols, err := p.GetAllSymbols(ctx, *cond)
symbols, err := p.GetAllSymbols(ctx, *cond, condCtx)
if err != nil {
p.log.Error(err, "unable to get symbols", "symbols", symbols, "cap", cap, "conditionInfo", cond)
return provider.ProviderEvaluateResponse{}, err
Expand Down Expand Up @@ -121,7 +120,7 @@ func (p *javaServiceClient) Evaluate(ctx context.Context, cap string, conditionI
}, nil
}

func (p *javaServiceClient) GetAllSymbols(ctx context.Context, c javaCondition) ([]protocol.WorkspaceSymbol, error) {
func (p *javaServiceClient) GetAllSymbols(ctx context.Context, c javaCondition, condCTX *provider.ProviderContext) ([]protocol.WorkspaceSymbol, error) {
// This command will run the added bundle to the language server. The command over the wire needs too look like this.
// in this case the project is hardcoded in the init of the Langauge Server above
// workspace/executeCommand '{"command": "io.konveyor.tackle.ruleEntry", "arguments": {"query":"*customresourcedefinition","project": "java"}}'
Expand All @@ -136,9 +135,14 @@ func (p *javaServiceClient) GetAllSymbols(ctx context.Context, c javaCondition)
argumentsMap["annotationQuery"] = c.Referenced.Annotated
}

if p.includedPaths != nil && len(p.includedPaths) > 0 {
log := p.log.WithValues("ruleID", condCTX.RuleID)

if len(p.includedPaths) > 0 {
argumentsMap[provider.IncludedPathsConfigKey] = p.includedPaths
p.log.V(5).Info("setting search scope by filepaths", "paths", p.includedPaths)
log.V(8).Info("setting search scope by filepaths", "paths", p.includedPaths)
} else if ok, paths := condCTX.GetScopedFilepaths(); ok {
argumentsMap[provider.IncludedPathsConfigKey] = paths
log.V(8).Info("setting search scope by filepaths", "paths", p.includedPaths, "argumentMap", argumentsMap)
}

argumentsBytes, _ := json.Marshal(argumentsMap)
Expand All @@ -155,10 +159,10 @@ func (p *javaServiceClient) GetAllSymbols(ctx context.Context, c javaCondition)
err := p.rpc.Call(timeOutCtx, "workspace/executeCommand", wsp, &refs)
if err != nil {
if jsonrpc2.IsRPCClosed(err) {
p.log.Error(err, "connection to the language server is closed, language server is not running")
log.Error(err, "connection to the language server is closed, language server is not running")
return refs, fmt.Errorf("connection to the language server is closed, language server is not running")
} else {
p.log.Error(err, "unable to ask for Konveyor rule entry")
log.Error(err, "unable to ask for Konveyor rule entry")
return refs, fmt.Errorf("unable to ask for Konveyor rule entry")
}
}
Expand Down
2 changes: 1 addition & 1 deletion external-providers/yq-external-provider/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.20 as go-builder
FROM golang:1.21 as go-builder

copy / /analyzer-lsp

Expand Down
Loading

0 comments on commit 49c5fe4

Please sign in to comment.