From 6cd14791f00b945492661c355c07743bb053e1d0 Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:28:58 -0400 Subject: [PATCH 1/6] add new methods for extracting .tar.gz and .zip; --- go.mod | 10 -- go.sum | 20 --- .../observiq_downloadable_file_manager.go | 131 +++++++++++++++++- 3 files changed, 129 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index d6d0754c8..e9e16a580 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.22.6 require ( github.com/google/uuid v1.6.0 - github.com/mholt/archiver/v3 v3.5.1 github.com/observiq/bindplane-agent/exporter/azureblobexporter v1.59.0 github.com/observiq/bindplane-agent/exporter/chronicleexporter v1.59.0 github.com/observiq/bindplane-agent/exporter/chronicleforwarderexporter v1.59.0 @@ -518,7 +517,6 @@ require ( github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect @@ -603,7 +601,6 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.17.0 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect github.com/knadh/koanf v1.5.0 // indirect github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect @@ -632,7 +629,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/nginxinc/nginx-prometheus-exporter v0.11.0 // indirect - github.com/nwaples/rardecode v1.1.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudexporter v0.107.0 // indirect; indi72.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.107.0 // indirect; indi72.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.107.0 // indirect; indi72.0 @@ -706,7 +702,6 @@ require ( github.com/tinylib/msgp v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.9 // indirect github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/vmware/govmomi v0.39.0 // indirect @@ -714,7 +709,6 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -836,10 +830,6 @@ replace github.com/observiq/bindplane-agent/internal/measurements => ./internal/ // Relevant issue https://github.com/mattn/go-ieproxy/issues/45 replace github.com/mattn/go-ieproxy => github.com/mattn/go-ieproxy v0.0.1 -// TODO(dakota): This could be removed after https://github.com/mholt/archiver/pull/396 is merged. -// This replace includes a resolution for https://github.com/advisories/GHSA-rhh4-rh7c-7r5v. -replace github.com/mholt/archiver/v3 => github.com/anchore/archiver/v3 v3.5.2 - // Replaces below this are required by datadog exporter in v0.83.0 https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.83.0/exporter/datadogexporter/go.mod#L266-L275 // openshift removed all tags from their repo, use the pseudoversion from the release-3.9 branch HEAD diff --git a/go.sum b/go.sum index 18e1cc583..6a2d18109 100644 --- a/go.sum +++ b/go.sum @@ -950,9 +950,6 @@ github.com/aliyun/aliyun-log-go-sdk v0.1.81 h1:OXQ78LSL0yZw60VLd+YjeevdK0XofKaHb github.com/aliyun/aliyun-log-go-sdk v0.1.81/go.mod h1:aCY/2vkzJxMpoAVuyuMMTYFHur5otoWXSo9ofKJ67Ek= github.com/aliyun/credentials-go v1.1.2 h1:qU1vwGIBb3UJ8BwunHDRFtAhS6jnQLnde/yk0+Ih2GY= github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= -github.com/anchore/archiver/v3 v3.5.2 h1:Bjemm2NzuRhmHy3m0lRe5tNoClB9A4zYyDV58PaB6aA= -github.com/anchore/archiver/v3 v3.5.2/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -1170,9 +1167,6 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02 h1:tR3jsKPiO/mb6ntzk/dJlHZtm37CPfVp1C9KIo534+4= github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02/go.mod h1:7NQ3kWOx2cZOSjtcveTa5nqupVr2s6/83sG+rTlI7uA= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -1404,7 +1398,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -1708,18 +1701,13 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs= github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= @@ -1877,8 +1865,6 @@ github.com/nginxinc/nginx-prometheus-exporter v0.11.0 h1:21xjnqNgxtni2jDgAQ90bl1 github.com/nginxinc/nginx-prometheus-exporter v0.11.0/go.mod h1:GdyHnWAb8q8OW1Pssrrqbcqra0SH0Vn6UXICMmyWkw8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -2301,7 +2287,6 @@ github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -2518,9 +2503,6 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= -github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA= @@ -2544,8 +2526,6 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 4b84d3c6c..4b90e69af 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -15,6 +15,9 @@ package observiq import ( + "archive/tar" + "archive/zip" + "compress/gzip" "crypto/sha256" "crypto/subtle" "errors" @@ -24,8 +27,8 @@ import ( "net/url" "os" "path/filepath" + "strings" - archiver "github.com/mholt/archiver/v3" "github.com/observiq/bindplane-agent/opamp" "github.com/open-telemetry/opamp-go/protobufs" "go.uber.org/zap" @@ -75,7 +78,7 @@ func (m DownloadableFileManager) FetchAndExtractArchive(file *protobufs.Download return fmt.Errorf("error cleaning archive extraction target path: %w", err) } - if err := archiver.Unarchive(archiveFilePath, extractPath); err != nil { + if err := unarchive(archiveFilePath, extractPath); err != nil { return fmt.Errorf("failed to extract file: %w", err) } @@ -171,3 +174,127 @@ func (m DownloadableFileManager) CleanupArtifacts() { m.logger.Error("Failed to remove temporary directory", zap.Error(err)) } } + +// unarchive will unpack the package at archivePath(.tar.gz or .zip) into the directory found at extractPath +func unarchive(archivePath, extractPath string) error { + if strings.HasSuffix(archivePath, ".tar.gz") { + // Handle tar.gz files + if err := extractTarGz(archivePath, extractPath); err != nil { + return fmt.Errorf("extract tar.gz: %w", err) + } + } else if strings.HasSuffix(archivePath, ".zip") { + // Handle zip files + if err := extractZip(archivePath, extractPath); err != nil { + return fmt.Errorf("extract .zip: %w", err) + } + } else { + return fmt.Errorf("unsupported file type: %s", archivePath) + } + + return nil +} + +// extractTarGz will extract the .tar package at archivePath into the dir at extractPath +func extractTarGz(archivePath, extractPath string) error { + if err := os.MkdirAll(extractPath, 0750); err != nil { + return fmt.Errorf("mkdir extract path: %w", err) + } + + file, err := os.Open(archivePath) + if err != nil { + return fmt.Errorf("open archive package: %w", err) + } + defer file.Close() + + gzipReader, err := gzip.NewReader(file) + if err != nil { + return fmt.Errorf("new gzip reader: %w", err) + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("read next tarball header: %w", err) + } + + outputPath := filepath.Join(extractPath, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(outputPath, os.FileMode(header.Mode)); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + + case tar.TypeReg: + outFile, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("create file: %w", err) + } + defer outFile.Close() + + if _, err := io.Copy(outFile, tarReader); err != nil { + return fmt.Errorf("write to file: %w", err) + } + + if err := os.Chmod(outputPath, os.FileMode(header.Mode)); err != nil { + return fmt.Errorf("chmod on file: %w", err) + } + + default: + fmt.Printf("Unsupported type: %v in %s\n", header.Typeflag, header.Name) + } + } + return nil +} + +// extractZip will extract the .zip package at archivePath into the dir at extractPath +func extractZip(archivePath, extractPath string) error { + // Ensure the output directory exists + if err := os.MkdirAll(extractPath, 0755); err != nil { + return fmt.Errorf("mkdir extract path: %w", err) + } + + r, err := zip.OpenReader(archivePath) + if err != nil { + return fmt.Errorf("new zip reader: %w", err) + } + defer r.Close() + + for _, f := range r.File { + outputPath := filepath.Join(extractPath, f.Name) + + if f.FileInfo().IsDir() { + if err := os.MkdirAll(outputPath, f.Mode()); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + continue + } + + if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { + return fmt.Errorf("create file: %w", err) + } + + outFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return fmt.Errorf("open output file: %w", err) + } + defer outFile.Close() + + rc, err := f.Open() + if err != nil { + return fmt.Errorf("open source file: %w", err) + } + defer rc.Close() + + if _, err := io.Copy(outFile, rc); err != nil { + return fmt.Errorf("write source file to output file: %w", err) + } + } + return nil +} From 2af0007e8bef58d2a9ea693d08612bc84123fb1a Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Thu, 29 Aug 2024 08:57:17 -0400 Subject: [PATCH 2/6] gosec fixes --- .../observiq_downloadable_file_manager.go | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 4b90e69af..7cee4d3df 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -35,6 +35,7 @@ import ( ) const extractFolder = "latest" +const maxArchiveObjectByteSize = 1024 // Ensure interface is satisfied var _ opamp.DownloadableFileManager = (*DownloadableFileManager)(nil) @@ -200,7 +201,8 @@ func extractTarGz(archivePath, extractPath string) error { return fmt.Errorf("mkdir extract path: %w", err) } - file, err := os.Open(archivePath) + archivePathClean := filepath.Clean(archivePath) + file, err := os.Open(archivePathClean) if err != nil { return fmt.Errorf("open archive package: %w", err) } @@ -223,7 +225,10 @@ func extractTarGz(archivePath, extractPath string) error { return fmt.Errorf("read next tarball header: %w", err) } - outputPath := filepath.Join(extractPath, header.Name) + outputPath, err := sanitizeArchivePath(extractPath, header.Name) + if err != nil { + return fmt.Errorf("sanitize archive path: %w", err) + } switch header.Typeflag { case tar.TypeDir: @@ -232,13 +237,14 @@ func extractTarGz(archivePath, extractPath string) error { } case tar.TypeReg: - outFile, err := os.Create(outputPath) + outputPathClean := filepath.Clean(outputPath) + outFile, err := os.Create(outputPathClean) if err != nil { return fmt.Errorf("create file: %w", err) } defer outFile.Close() - if _, err := io.Copy(outFile, tarReader); err != nil { + if _, err := io.CopyN(outFile, tarReader, maxArchiveObjectByteSize); err != nil { return fmt.Errorf("write to file: %w", err) } @@ -256,7 +262,7 @@ func extractTarGz(archivePath, extractPath string) error { // extractZip will extract the .zip package at archivePath into the dir at extractPath func extractZip(archivePath, extractPath string) error { // Ensure the output directory exists - if err := os.MkdirAll(extractPath, 0755); err != nil { + if err := os.MkdirAll(extractPath, 0750); err != nil { return fmt.Errorf("mkdir extract path: %w", err) } @@ -267,7 +273,10 @@ func extractZip(archivePath, extractPath string) error { defer r.Close() for _, f := range r.File { - outputPath := filepath.Join(extractPath, f.Name) + outputPath, err := sanitizeArchivePath(extractPath, f.Name) + if err != nil { + return fmt.Errorf("sanitize archive path: %w", err) + } if f.FileInfo().IsDir() { if err := os.MkdirAll(outputPath, f.Mode()); err != nil { @@ -276,11 +285,12 @@ func extractZip(archivePath, extractPath string) error { continue } - if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(outputPath), 0750); err != nil { return fmt.Errorf("create file: %w", err) } - outFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + outputPathClean := filepath.Clean(outputPath) + outFile, err := os.OpenFile(outputPathClean, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return fmt.Errorf("open output file: %w", err) } @@ -292,9 +302,17 @@ func extractZip(archivePath, extractPath string) error { } defer rc.Close() - if _, err := io.Copy(outFile, rc); err != nil { + if _, err := io.CopyN(outFile, rc, maxArchiveObjectByteSize); err != nil { return fmt.Errorf("write source file to output file: %w", err) } } return nil } + +func sanitizeArchivePath(dir, file string) (string, error) { + s := filepath.Join(dir, file) + if strings.HasPrefix(s, filepath.Clean(dir)) { + return s, nil + } + return "", fmt.Errorf("content filepath is tainted: %q", file) +} From dd91304f6e374c12df9e17cd9fb069ce5f3125ff Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:23:11 -0400 Subject: [PATCH 3/6] ignore EOF err when writing to dst file --- opamp/observiq/observiq_downloadable_file_manager.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 7cee4d3df..25bdb6eea 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -35,7 +35,7 @@ import ( ) const extractFolder = "latest" -const maxArchiveObjectByteSize = 1024 +const maxArchiveObjectByteSize = 1000000000 // Ensure interface is satisfied var _ opamp.DownloadableFileManager = (*DownloadableFileManager)(nil) @@ -244,7 +244,8 @@ func extractTarGz(archivePath, extractPath string) error { } defer outFile.Close() - if _, err := io.CopyN(outFile, tarReader, maxArchiveObjectByteSize); err != nil { + _, err = io.CopyN(outFile, tarReader, maxArchiveObjectByteSize) + if err != nil && err != io.EOF { return fmt.Errorf("write to file: %w", err) } @@ -302,7 +303,8 @@ func extractZip(archivePath, extractPath string) error { } defer rc.Close() - if _, err := io.CopyN(outFile, rc, maxArchiveObjectByteSize); err != nil { + _, err = io.CopyN(outFile, rc, maxArchiveObjectByteSize) + if err != nil && err != io.EOF { return fmt.Errorf("write source file to output file: %w", err) } } From 831f10f1f4a82f13880520ce85e11d0c9c170c53 Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:15:32 -0400 Subject: [PATCH 4/6] use OpenFile for tar --- opamp/observiq/observiq_downloadable_file_manager.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 25bdb6eea..99f263b51 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -238,9 +238,9 @@ func extractTarGz(archivePath, extractPath string) error { case tar.TypeReg: outputPathClean := filepath.Clean(outputPath) - outFile, err := os.Create(outputPathClean) + outFile, err := os.OpenFile(outputPathClean, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) if err != nil { - return fmt.Errorf("create file: %w", err) + return fmt.Errorf("open output file: %w", err) } defer outFile.Close() @@ -249,10 +249,6 @@ func extractTarGz(archivePath, extractPath string) error { return fmt.Errorf("write to file: %w", err) } - if err := os.Chmod(outputPath, os.FileMode(header.Mode)); err != nil { - return fmt.Errorf("chmod on file: %w", err) - } - default: fmt.Printf("Unsupported type: %v in %s\n", header.Typeflag, header.Name) } From ef511f61ac97d6d9ed57a9bb7ef99bca7e3bb83a Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:36:09 -0400 Subject: [PATCH 5/6] ensure archive file parent dir exists --- opamp/observiq/observiq_downloadable_file_manager.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 99f263b51..69bb5ba66 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -237,6 +237,10 @@ func extractTarGz(archivePath, extractPath string) error { } case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(outputPath), 0750); err != nil { + return fmt.Errorf("create file's dir: %w", err) + } + outputPathClean := filepath.Clean(outputPath) outFile, err := os.OpenFile(outputPathClean, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) if err != nil { @@ -283,7 +287,7 @@ func extractZip(archivePath, extractPath string) error { } if err := os.MkdirAll(filepath.Dir(outputPath), 0750); err != nil { - return fmt.Errorf("create file: %w", err) + return fmt.Errorf("create file's dir: %w", err) } outputPathClean := filepath.Clean(outputPath) From 7567069bba4887957fd1579651945086eb235479 Mon Sep 17 00:00:00 2001 From: Dakota Paasman <122491662+dpaasman00@users.noreply.github.com> Date: Fri, 30 Aug 2024 07:18:34 -0400 Subject: [PATCH 6/6] 0700 permission on files parent --- opamp/observiq/observiq_downloadable_file_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opamp/observiq/observiq_downloadable_file_manager.go b/opamp/observiq/observiq_downloadable_file_manager.go index 69bb5ba66..34e7215a9 100644 --- a/opamp/observiq/observiq_downloadable_file_manager.go +++ b/opamp/observiq/observiq_downloadable_file_manager.go @@ -237,7 +237,7 @@ func extractTarGz(archivePath, extractPath string) error { } case tar.TypeReg: - if err := os.MkdirAll(filepath.Dir(outputPath), 0750); err != nil { + if err := os.MkdirAll(filepath.Dir(outputPath), 0700); err != nil { return fmt.Errorf("create file's dir: %w", err) } @@ -286,7 +286,7 @@ func extractZip(archivePath, extractPath string) error { continue } - if err := os.MkdirAll(filepath.Dir(outputPath), 0750); err != nil { + if err := os.MkdirAll(filepath.Dir(outputPath), 0700); err != nil { return fmt.Errorf("create file's dir: %w", err) }