Merge branch 'main' into em/dry-run-support-bundle

CONTRIBUTING.md

@@ -94,6 +94,12 @@ go tool pprof -http=":8000" cpu.prof
 go tool pprof -http=":8001" mem.prof
 ```
 
+**Additional flags for memory profiling**
+- `inuse_space`: Amount of memory allocated and not released yet (default).
+- `inuse_objects`: Amount of objects allocated and not released yet.
+- `alloc_space`: Total amount of memory allocated (regardless of released).
+- `alloc_objects`: Total amount of objects allocated (regardless of released).
+
 More on profiling please visit https://go.dev/doc/diagnostics#profiling
 
 ## Contribution workflow

examples/preflight/host/cpu.yaml

@@ -19,4 +19,3 @@ spec:
               message: At least 16 CPU cores preferred
           - pass:
               message: This server has sufficient CPU cores
-

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/manifoldco/promptui v0.9.0
 	github.com/mattn/go-isatty v0.0.19
 	github.com/mholt/archiver/v3 v3.5.1
-	github.com/microsoft/go-mssqldb v1.5.0
+	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/pkg/errors v0.9.1
 	github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851
@@ -35,20 +35,20 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tj/go-spin v1.1.0
-	go.opentelemetry.io/otel v1.17.0
+	go.opentelemetry.io/otel v1.18.0
 	go.opentelemetry.io/otel/sdk v1.17.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/sync v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.28.1
+	k8s.io/api v0.28.2
 	k8s.io/apiextensions-apiserver v0.28.1
-	k8s.io/apimachinery v0.28.1
+	k8s.io/apimachinery v0.28.2
 	k8s.io/apiserver v0.28.1
-	k8s.io/cli-runtime v0.28.1
-	k8s.io/client-go v0.28.1
+	k8s.io/cli-runtime v0.28.2
+	k8s.io/client-go v0.28.2
 	k8s.io/klog/v2 v2.100.1
 	oras.land/oras-go v1.2.3
-	sigs.k8s.io/controller-runtime v0.15.1
+	sigs.k8s.io/controller-runtime v0.16.1
 	sigs.k8s.io/e2e-framework v0.3.0
 )
 
@@ -98,10 +98,10 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	go.opentelemetry.io/otel/metric v1.17.0 // indirect
-	go.opentelemetry.io/otel/trace v1.17.0 // indirect
+	go.opentelemetry.io/otel/metric v1.18.0 // indirect
+	go.opentelemetry.io/otel/trace v1.18.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 	k8s.io/component-base v0.28.1 // indirect
@@ -220,11 +220,11 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/net v0.13.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.13.0
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -238,7 +238,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.12.3
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/metrics v0.28.1
+	k8s.io/metrics v0.28.2
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 	periph.io/x/host/v3 v3.8.2
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

pkg/analyze/common_status.go

@@ -2,10 +2,11 @@ package analyzer
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
-	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"strconv"
 	"strings"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 )
 
 func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey string, iconURI string, readyReplicas int, exists bool, resourceType string) (*AnalyzeResult, error) {
@@ -64,7 +65,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 			if exists == false && outcome.Warn.When != "absent" {
 				result.IsFail = true
 				result.Message = fmt.Sprintf("The %s %q was not found", resourceType, name)
-				result.URI = outcome.Fail.URI
+				result.URI = outcome.Warn.URI
 				return result, nil
 			}
 
@@ -104,7 +105,7 @@ func commonStatus(outcomes []*troubleshootv1beta2.Outcome, name string, iconKey
 			if exists == false && outcome.Pass.When != "absent" {
 				result.IsFail = true
 				result.Message = fmt.Sprintf("The %s %q was not found", resourceType, name)
-				result.URI = outcome.Fail.URI
+				result.URI = outcome.Pass.URI
 				return result, nil
 			}
 

pkg/collect/redact.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -17,18 +16,29 @@ import (
 	"k8s.io/klog/v2"
 )
 
+// Max number of concurrent redactors to run
+// Ensure the number is low enough since each of the redactors
+// also spawns goroutines to redact files in tar archives and
+// other goroutines for each redactor spec.
+const MAX_CONCURRENT_REDACTORS = 10
+
 func RedactResult(bundlePath string, input CollectorResult, additionalRedactors []*troubleshootv1beta2.Redact) error {
 	wg := &sync.WaitGroup{}
 
 	// Error channel to capture errors from goroutines
 	errorCh := make(chan error, len(input))
+	limitCh := make(chan struct{}, MAX_CONCURRENT_REDACTORS)
+	defer close(limitCh)
 
 	for k, v := range input {
+		limitCh <- struct{}{}
 
 		wg.Add(1)
 
 		go func(file string, data []byte) {
 			defer wg.Done()
+			defer func() { <-limitCh }() // free up after the function execution has run
+
 			var reader io.Reader
 			if data == nil {
 
@@ -84,7 +94,7 @@ func RedactResult(bundlePath string, input CollectorResult, additionalRedactors
 			// If the file is .tar, .tgz or .tar.gz, it must not be redacted. Instead it is
 			// decompressed and each file inside the tar redacted and compressed back into the archive.
 			if filepath.Ext(file) == ".tar" || filepath.Ext(file) == ".tgz" || strings.HasSuffix(file, ".tar.gz") {
-				tmpDir, err := ioutil.TempDir("", "troubleshoot-subresult-")
+				tmpDir, err := os.MkdirTemp("", "troubleshoot-subresult-")
 				if err != nil {
 					errorCh <- errors.Wrap(err, "failed to create temp dir")
 					return

pkg/collect/result.go

@@ -132,7 +132,7 @@ func (r CollectorResult) SaveResult(bundlePath string, relativePath string, read
 		return errors.Wrap(err, "failed to stat file")
 	}
 
-	klog.V(2).Infof("Added %q (%d MB) to bundle output", relativePath, fileInfo.Size()/(1024*1024))
+	klog.V(2).Infof("Added %q (%d KB) to bundle output", relativePath, fileInfo.Size()/(1024))
 	return nil
 }
 

pkg/constants/constants.go

@@ -83,6 +83,11 @@ const (
 	MESSAGE_TEXT_PADDING                = 4
 	MESSAGE_TEXT_LINES_MARGIN_TO_BOTTOM = 4
 
-	// Bufio Reader Constants
-	MAX_BUFFER_CAPACITY = 1024 * 1024
+	// This is the initial size of the buffer allocated.
+	// Under the hood, an array of size N is allocated in memory
+	BUF_INIT_SIZE = 4096 // 4KB
+
+	// This is the muximum size the buffer can grow to
+	// Its not what the buffer will be allocated to initially
+	SCANNER_MAX_SIZE = 10 * 1024 * 1024 // 10MB
 )

pkg/redact/literal.go

@@ -2,23 +2,26 @@ package redact
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
-	"strings"
+
+	"github.com/replicatedhq/troubleshoot/pkg/constants"
+	"k8s.io/klog/v2"
 )
 
 type literalRedactor struct {
-	matchString string
-	filePath    string
-	redactName  string
-	isDefault   bool
+	match      []byte
+	filePath   string
+	redactName string
+	isDefault  bool
 }
 
-func literalString(matchString, path, name string) Redactor {
+func literalString(match []byte, path, name string) Redactor {
 	return literalRedactor{
-		matchString: matchString,
-		filePath:    path,
-		redactName:  name,
+		match:      match,
+		filePath:   path,
+		redactName: name,
 	}
 }
 
@@ -28,32 +31,37 @@ func (r literalRedactor) Redact(input io.Reader, path string) io.Reader {
 	go func() {
 		var err error
 		defer func() {
-			if err == io.EOF {
+			if err == nil || err == io.EOF {
 				writer.Close()
 			} else {
+				if err == bufio.ErrTooLong {
+					s := fmt.Sprintf("Error redacting %q. A line in the file exceeded %d MB max length", path, constants.SCANNER_MAX_SIZE/1024/1024)
+					klog.V(2).Info(s)
+				} else {
+					klog.V(2).Info(fmt.Sprintf("Error redacting %q: %v", path, err))
+				}
 				writer.CloseWithError(err)
 			}
 		}()
 
-		reader := bufio.NewReader(input)
+		buf := make([]byte, constants.BUF_INIT_SIZE)
+		scanner := bufio.NewScanner(input)
+		scanner.Buffer(buf, constants.SCANNER_MAX_SIZE)
+
 		lineNum := 0
-		for {
+		for scanner.Scan() {
 			lineNum++
-			var line string
-			line, err = readLine(reader)
-			if err != nil {
-				return
-			}
+			line := scanner.Bytes()
 
-			clean := strings.ReplaceAll(line, r.matchString, MASK_TEXT)
+			clean := bytes.ReplaceAll(line, r.match, maskTextBytes)
 
-			// io.WriteString would be nicer, but scanner strips new lines
-			fmt.Fprintf(writer, "%s\n", clean)
+			// Append newline since scanner strips it
+			err = writeBytes(writer, clean, NEW_LINE)
 			if err != nil {
 				return
 			}
 
-			if clean != line {
+			if !bytes.Equal(clean, line) {
 				addRedaction(Redaction{
 					RedactorName:      r.redactName,
 					CharactersRemoved: len(line) - len(clean),
@@ -63,6 +71,9 @@ func (r literalRedactor) Redact(input io.Reader, path string) io.Reader {
 				})
 			}
 		}
+		if scanErr := scanner.Err(); scanErr != nil {
+			err = scanErr
+		}
 	}()
 	return out
 }