Add heuristic search option for GoPCLnTab upload

cli_flags.go

@@ -36,36 +36,37 @@ const (
 )
 
 type arguments struct {
-	bpfVerifierLogLevel     uint64
-	agentURL                string
-	copyright               bool
-	mapScaleFactor          uint64
-	monitorInterval         time.Duration
-	clockSyncInterval       time.Duration
-	noKernelVersionCheck    bool
-	node                    string
-	probabilisticInterval   time.Duration
-	probabilisticThreshold  uint64
-	reporterInterval        time.Duration
-	samplesPerSecond        uint64
-	pprofPrefix             string
-	sendErrorFrames         bool
-	serviceName             string
-	environment             string
-	uploadSymbols           bool
-	uploadDynamicSymbols    bool
-	uploadGoPCLnTab         bool
-	uploadSymbolsDryRun     bool
-	tags                    string
-	timeline                bool
-	tracers                 string
-	verboseMode             bool
-	apiKey                  string
-	appKey                  string
-	site                    string
-	agentless               bool
-	enableGoRuntimeProfiler bool
-	cmd                     *cli.Command
+	bpfVerifierLogLevel         uint64
+	agentURL                    string
+	copyright                   bool
+	mapScaleFactor              uint64
+	monitorInterval             time.Duration
+	clockSyncInterval           time.Duration
+	noKernelVersionCheck        bool
+	node                        string
+	probabilisticInterval       time.Duration
+	probabilisticThreshold      uint64
+	reporterInterval            time.Duration
+	samplesPerSecond            uint64
+	pprofPrefix                 string
+	sendErrorFrames             bool
+	serviceName                 string
+	environment                 string
+	uploadSymbols               bool
+	uploadDynamicSymbols        bool
+	uploadGoPCLnTab             bool
+	useGoPCLnTabHeuristicSearch bool
+	uploadSymbolsDryRun         bool
+	tags                        string
+	timeline                    bool
+	tracers                     string
+	verboseMode                 bool
+	apiKey                      string
+	appKey                      string
+	site                        string
+	agentless                   bool
+	enableGoRuntimeProfiler     bool
+	cmd                         *cli.Command
 }
 
 func parseArgs() (*arguments, error) {
@@ -257,6 +258,14 @@ func parseArgs() (*arguments, error) {
 				Sources:     cli.EnvVars("DD_HOST_PROFILING_EXPERIMENTAL_UPLOAD_GOPCLNTAB"),
 				Destination: &args.uploadGoPCLnTab,
 			},
+			&cli.BoolFlag{
+				Name:        "upload-gopclntab-use-heuristic-search",
+				Usage:       "Use heurisitc search when uploading gopclntab. This can lead to uploading data beyond the actual gopclntab section.",
+				Value:       false,
+				Hidden:      true,
+				Sources:     cli.EnvVars("DD_HOST_PROFILING_EXPERIMENTAL_UPLOAD_GOPCLNTAB_USE_HEURISTIC_SEARCH"),
+				Destination: &args.useGoPCLnTabHeuristicSearch,
+			},
 			&cli.BoolFlag{
 				Name:        "upload-symbols-dry-run",
 				Value:       false,

main.go

@@ -208,14 +208,15 @@ func mainWithExitCode() exitCode {
 		Timeline:               args.timeline,
 		APIKey:                 apiKey,
 		SymbolUploaderConfig: reporter.SymbolUploaderConfig{
-			Enabled:              args.uploadSymbols,
-			UploadDynamicSymbols: args.uploadDynamicSymbols,
-			UploadGoPCLnTab:      args.uploadGoPCLnTab,
-			DryRun:               args.uploadSymbolsDryRun,
-			APIKey:               args.apiKey,
-			APPKey:               args.appKey,
-			Site:                 args.site,
-			Version:              versionInfo.Version,
+			Enabled:                     args.uploadSymbols,
+			UploadDynamicSymbols:        args.uploadDynamicSymbols,
+			UploadGoPCLnTab:             args.uploadGoPCLnTab,
+			UseGoPCLnTabHeuristicSearch: args.useGoPCLnTabHeuristicSearch,
+			DryRun:                      args.uploadSymbolsDryRun,
+			APIKey:                      args.apiKey,
+			APPKey:                      args.appKey,
+			Site:                        args.site,
+			Version:                     versionInfo.Version,
 		},
 	}, containerMetadataProvider)
 	if err != nil {

reporter/config.go

@@ -57,6 +57,9 @@ type SymbolUploaderConfig struct {
 	UploadDynamicSymbols bool
 	// UploadGoPCLnTab defines whether the agent should upload GoPCLnTab section for Go binaries to the backend.
 	UploadGoPCLnTab bool
+	// UseGoPCLnTabHeuristicSearch defines whether the agent should use heuristic search to find GoPCLnTab section for Go binaries when sections/symbols are not present.
+	// This is useful for stripped Go binaries but currently leads to uploading data that is beyond the actual GoPCLnTab section.
+	UseGoPCLnTabHeuristicSearch bool
 	// DryRun defines whether the agent should upload debug symbols to the backend in dry-run mode.
 	DryRun bool
 	// DataDog API key

reporter/symbol_uploader.go

@@ -51,6 +51,14 @@ const (
 	maxBytesGoPclntab = 128 * 1024 * 1024
 )
 
+type GoPCLnTabSearchMode int64
+
+const (
+	NoPCLnTabSearch GoPCLnTabSearchMode = iota
+	PCLnTabSearchWithSectionAndSymbols
+	FullPCLnTabSearch
+)
+
 var debugStrSectionNames = []string{".debug_str", ".zdebug_str", ".debug_str.dwo"}
 var debugInfoSectionNames = []string{".debug_info", ".zdebug_info"}
 var globalDebugDirectories = []string{"/usr/lib/debug"}
@@ -65,14 +73,15 @@ type uploadData struct {
 type goPCLnTabData []byte
 
 type DatadogSymbolUploader struct {
-	ddAPIKey             string
-	ddAPPKey             string
-	intakeURL            string
-	version              string
-	dryRun               bool
-	uploadDynamicSymbols bool
-	uploadGoPCLnTab      bool
-	workerCount          int
+	ddAPIKey                    string
+	ddAPPKey                    string
+	intakeURL                   string
+	version                     string
+	dryRun                      bool
+	uploadDynamicSymbols        bool
+	uploadGoPCLnTab             bool
+	useGoPCLnTabHeuristicSearch bool
+	workerCount                 int
 
 	uploadCache   *lru.SyncedLRU[libpf.FileID, struct{}]
 	client        *http.Client
@@ -114,18 +123,19 @@ func NewDatadogSymbolUploader(cfg SymbolUploaderConfig) (*DatadogSymbolUploader,
 	}
 
 	return &DatadogSymbolUploader{
-		ddAPIKey:             cfg.APIKey,
-		ddAPPKey:             cfg.APPKey,
-		intakeURL:            intakeURL,
-		version:              cfg.Version,
-		dryRun:               cfg.DryRun,
-		uploadDynamicSymbols: cfg.UploadDynamicSymbols,
-		uploadGoPCLnTab:      cfg.UploadGoPCLnTab,
-		workerCount:          uploadWorkerCount,
-		client:               &http.Client{Timeout: uploadTimeout},
-		uploadCache:          uploadCache,
-		uploadQueue:          make(chan uploadData, uploadQueueSize),
-		symbolQuerier:        symbolQuerier,
+		ddAPIKey:                    cfg.APIKey,
+		ddAPPKey:                    cfg.APPKey,
+		intakeURL:                   intakeURL,
+		version:                     cfg.Version,
+		dryRun:                      cfg.DryRun,
+		uploadDynamicSymbols:        cfg.UploadDynamicSymbols,
+		uploadGoPCLnTab:             cfg.UploadGoPCLnTab,
+		useGoPCLnTabHeuristicSearch: cfg.UseGoPCLnTabHeuristicSearch,
+		workerCount:                 uploadWorkerCount,
+		client:                      &http.Client{Timeout: uploadTimeout},
+		uploadCache:                 uploadCache,
+		uploadQueue:                 make(chan uploadData, uploadQueueSize),
+		symbolQuerier:               symbolQuerier,
 	}, nil
 }
 
@@ -184,6 +194,16 @@ func (d *DatadogSymbolUploader) GetExistingSymbolsOnBackend(ctx context.Context,
 	return symbolSource, nil
 }
 
+func (d *DatadogSymbolUploader) getPCLnTabSearchMode() GoPCLnTabSearchMode {
+	if !d.uploadGoPCLnTab {
+		return NoPCLnTabSearch
+	}
+	if d.useGoPCLnTabHeuristicSearch {
+		return FullPCLnTabSearch
+	}
+	return PCLnTabSearchWithSectionAndSymbols
+}
+
 // Returns true if the upload was successful, false otherwise
 func (d *DatadogSymbolUploader) upload(ctx context.Context, uploadData uploadData) bool {
 	filePath := uploadData.filePath
@@ -200,7 +220,7 @@ func (d *DatadogSymbolUploader) upload(ctx context.Context, uploadData uploadDat
 	}
 	defer elfWrapper.Close()
 
-	debugElf, symbolSource, goPCLNTabData := elfWrapper.findSymbols(d.uploadGoPCLnTab)
+	debugElf, symbolSource, goPCLNTabData := elfWrapper.findSymbols(d.getPCLnTabSearchMode())
 	if debugElf == nil {
 		log.Debugf("Skipping symbol upload for executable %s: no debug symbols found", filePath)
 		return false
@@ -541,7 +561,7 @@ func HasDWARFData(f *pfelf.File) bool {
 	return len(f.Progs) == 0 && hasBuildID && hasDebugStr
 }
 
-func findGoPCLnTab(ef *pfelf.File) (goPCLnTabData, error) {
+func findGoPCLnTab(ef *pfelf.File, useHeuristicSearchAsFallback bool) (goPCLnTabData, error) {
 	var err error
 	var data []byte
 
@@ -556,6 +576,9 @@ func findGoPCLnTab(ef *pfelf.File) (goPCLnTabData, error) {
 	} else if s := ef.Section(".go.buildinfo"); s != nil {
 		symtab, err := ef.ReadSymbols()
 		if err != nil {
+			if !useHeuristicSearchAsFallback {
+				return nil, fmt.Errorf("failed to read symbols: %w", err)
+			}
 			// It seems the Go binary was stripped, use the heuristic approach to get find gopclntab.
 			// Note that `SearchGoPclntab` returns a slice starting from gopcltab header to the end of segment
 			// containing gopclntab. Therefore this slice might contain additional data after gopclntab.
@@ -630,10 +653,10 @@ func openELF(filePath string, opener process.FileOpener) (*elfWrapper, error) {
 
 // findSymbols attempts to find a symbol source for the elf file, it returns an elfWrapper around the elf file
 // with symbols if found, or nil if no symbols were found.
-func (e *elfWrapper) findSymbols(checkGoPCLnTab bool) (*elfWrapper, SymbolSource, goPCLnTabData) {
+func (e *elfWrapper) findSymbols(pcLnTabSearchMode GoPCLnTabSearchMode) (*elfWrapper, SymbolSource, goPCLnTabData) {
 	// Check if the elf file has a GoPCLnTab
-	if checkGoPCLnTab {
-		goPCLnTabData, err := findGoPCLnTab(e.elfFile)
+	if pcLnTabSearchMode != NoPCLnTabSearch {
+		goPCLnTabData, err := findGoPCLnTab(e.elfFile, pcLnTabSearchMode == FullPCLnTabSearch)
 		if err == nil {
 			if goPCLnTabData != nil {
 				return e, GoPCLnTab, goPCLnTabData

reporter/symbol_uploader_test.go

@@ -42,6 +42,19 @@ func checkGoPCLnTab(t *testing.T, filename string) {
 	assert.Equal(t, expectedHeader, data[:8])
 }
 
+func checkGoPCLnTabExtraction(t *testing.T, filename, tmpDir string, useGoPCLnTabHeuristicSearch bool) {
+	f, err := pfelf.Open(filename)
+	require.NoError(t, err)
+	goPCLnTabInfo, err := findGoPCLnTab(f, useGoPCLnTabHeuristicSearch)
+	require.NoError(t, err)
+	assert.NotNil(t, goPCLnTabInfo)
+
+	outputFile := filepath.Join(tmpDir, "output.dbg")
+	err = copySymbolsAndGoPCLnTab(context.Background(), filename, outputFile, goPCLnTabInfo)
+	require.NoError(t, err)
+	checkGoPCLnTab(t, outputFile)
+}
+
 func TestGoPCLnTabExtraction(t *testing.T) {
 	t.Parallel()
 	srcFile := "./testdata/helloworld.go"
@@ -62,21 +75,17 @@ func TestGoPCLnTabExtraction(t *testing.T) {
 			t.Parallel()
 			tmpDir := t.TempDir()
 			exe := filepath.Join(tmpDir, strings.TrimRight(srcFile, ".go")+"."+name)
+			exeStripped := exe + ".stripped"
 			cmd := exec.Command("go", append([]string{"build", "-o", exe}, test.buildArgs...)...) // #nosec G204
 			cmd.Args = append(cmd.Args, srcFile)
 			out, err := cmd.CombinedOutput()
 			require.NoError(t, err, "failed to build test binary with `%v`: %s\n%s", cmd.Args, err, out)
 
-			f, err := pfelf.Open(exe)
-			require.NoError(t, err)
-			goPCLnTabInfo, err := findGoPCLnTab(f)
-			require.NoError(t, err)
-			assert.NotNil(t, goPCLnTabInfo)
+			checkGoPCLnTabExtraction(t, exe, tmpDir, false)
 
-			outputFile := filepath.Join(tmpDir, "output.dbg")
-			err = copySymbolsAndGoPCLnTab(context.Background(), exe, outputFile, goPCLnTabInfo)
-			require.NoError(t, err)
-			checkGoPCLnTab(t, outputFile)
+			out, err = exec.Command("objcopy", "-S", "--rename-section", ".data.rel.ro.gopclntab=.foo1", "--rename-section", ".gopclntab=.foo2", exe, exeStripped).CombinedOutput() // #nosec G204
+			require.NoError(t, err, "failed to rename section: %s\n%s", err, out)
+			checkGoPCLnTabExtraction(t, exeStripped, tmpDir, true)
 		})
 	}
 }