Skip to content

Commit

Permalink
allow .index.json to embed item content (#3145)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmetc
mmetc authored Jul 22, 2024
1 parent 35f97d4 commit 5b4f924
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 23 deletions.
1 change: 1 addition & 0 deletions pkg/cwhub/item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ type Item struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name"
FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Content string `json:"content,omitempty" yaml:"-"`
Author string `json:"author,omitempty" yaml:"author,omitempty"`
References []string `json:"references,omitempty" yaml:"references,omitempty"`

Expand Down
50 changes: 44 additions & 6 deletions pkg/cwhub/itemupgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@ package cwhub

import (
"context"
"crypto"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"

"github.com/sirupsen/logrus"

Expand Down Expand Up @@ -110,16 +114,50 @@ func (i *Item) downloadLatest(ctx context.Context, overwrite bool, updateOnly bo

// FetchContentTo downloads the last version of the item's YAML file to the specified path.
func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) {
wantHash := i.latestHash()
if wantHash == "" {
return false, "", errors.New("latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again")
}

// Use the embedded content if available
if i.Content != "" {
// the content was historically base64 encoded
content, err := base64.StdEncoding.DecodeString(i.Content)
if err != nil {
content = []byte(i.Content)
}

dir := filepath.Dir(destPath)

if err := os.MkdirAll(dir, 0o755); err != nil {
return false, "", fmt.Errorf("while creating %s: %w", dir, err)
}

// check sha256
hash := crypto.SHA256.New()
if _, err := hash.Write(content); err != nil {
return false, "", fmt.Errorf("while hashing %s: %w", i.Name, err)
}

gotHash := hex.EncodeToString(hash.Sum(nil))
if gotHash != wantHash {
return false, "", fmt.Errorf("hash mismatch: expected %s, got %s. The index file is invalid, please run 'cscli hub update' and try again", wantHash, gotHash)
}

if err := os.WriteFile(destPath, content, 0o600); err != nil {
return false, "", fmt.Errorf("while writing %s: %w", destPath, err)
}

i.hub.logger.Debugf("Wrote %s content from .index.json to %s", i.Name, destPath)

return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil
}

url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return false, "", fmt.Errorf("failed to build request: %w", err)
}

wantHash := i.latestHash()
if wantHash == "" {
return false, "", errors.New("latest hash missing from index")
}

d := downloader.
New().
WithHTTPClient(hubClient).
Expand Down Expand Up @@ -167,7 +205,7 @@ func (i *Item) download(ctx context.Context, overwrite bool) (bool, error) {

downloaded, _, err := i.FetchContentTo(ctx, finalPath)
if err != nil {
return false, fmt.Errorf("while downloading %s: %w", i.Name, err)
return false, err
}

if downloaded {
Expand Down
8 changes: 1 addition & 7 deletions pkg/cwhub/testdata/index1.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "foobar collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand All @@ -34,7 +33,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "test_collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand All @@ -52,7 +50,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "foobar collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand All @@ -73,7 +70,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "A foobar parser",
"author": "crowdsecurity",
"labels": null
Expand All @@ -89,7 +85,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "A foobar parser",
"author": "crowdsecurity",
"labels": null
Expand All @@ -107,7 +102,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "a foobar scenario",
"author": "crowdsecurity",
"labels": {
Expand All @@ -118,4 +112,4 @@
}
}
}
}
}
9 changes: 1 addition & 8 deletions pkg/cwhub/testdata/index2.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "foobar collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand Down Expand Up @@ -38,7 +37,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "test_collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand All @@ -57,7 +55,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "foobar collection : foobar",
"author": "crowdsecurity",
"labels": null,
Expand All @@ -78,7 +75,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "A foobar parser",
"author": "crowdsecurity",
"labels": null
Expand All @@ -94,7 +90,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "A foobar parser",
"author": "crowdsecurity",
"labels": null
Expand All @@ -112,7 +107,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "a foobar scenario",
"author": "crowdsecurity",
"labels": {
Expand All @@ -132,7 +126,6 @@
}
},
"long_description": "bG9uZyBkZXNjcmlwdGlvbgo=",
"content": "bG9uZyBkZXNjcmlwdGlvbgo=",
"description": "a foobar scenario",
"author": "crowdsecurity",
"labels": {
Expand All @@ -143,4 +136,4 @@
}
}
}
}
}
2 changes: 1 addition & 1 deletion test/bats/20_hub.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ teardown() {
assert_stderr --partial "invalid hub item appsec-rules:crowdsecurity/vpatch-laravel-debug-mode: latest version missing from index"

rune -1 cscli appsec-rules install crowdsecurity/vpatch-laravel-debug-mode --force
assert_stderr --partial "error while installing 'crowdsecurity/vpatch-laravel-debug-mode': while downloading crowdsecurity/vpatch-laravel-debug-mode: latest hash missing from index"
assert_stderr --partial "error while installing 'crowdsecurity/vpatch-laravel-debug-mode': latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again"
}

@test "missing reference in hub index" {
Expand Down
2 changes: 1 addition & 1 deletion test/lib/setup_file.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ hub_strip_index() {
local INDEX
INDEX=$(config_get .config_paths.index_path)
local hub_min
hub_min=$(jq <"$INDEX" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
hub_min=$(jq <"$INDEX" 'del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$INDEX"
}
export -f hub_strip_index
Expand Down

0 comments on commit 5b4f924

Please sign in to comment.