Allow downloading private objects from S3 (#8229)

release/cli/pkg/assets/archives/archives.go

@@ -235,7 +235,7 @@ func GetArchiveAssets(rc *releasetypes.ReleaseConfig, archive *assettypes.Archiv
 		ProjectPath:       projectPath,
 		SourcedFromBranch: sourcedFromBranch,
 		ImageFormat:       archive.Format,
-		PrivateUpload:     archive.Private,
+		Private:           archive.Private,
 	}
 
 	return archiveArtifact, nil

release/cli/pkg/assets/config/bundle_release.go

@@ -74,7 +74,7 @@ var bundleReleaseAssetsConfigMap = []assettypes.AssetConfig{
 		Archives: []*assettypes.Archive{
 			{
 				Name:                "rtos",
-				Format:              "ami",
+				Format:              "raw",
 				OSName:              "ubuntu",
 				OSVersion:           "22.04",
 				ArchiveS3PathGetter: archives.RTOSArtifactPathGetter,

release/cli/pkg/assets/manifests/manifests.go

@@ -68,7 +68,7 @@ func GetManifestAssets(rc *releasetypes.ReleaseConfig, manifestComponent *assett
 		ProjectPath:       projectPath,
 		SourcedFromBranch: sourcedFromBranch,
 		Component:         componentName,
-		PrivateUpload:     manifestComponent.Private,
+		Private:           manifestComponent.Private,
 	}
 
 	return manifestArtifact, nil

release/cli/pkg/aws/s3/s3.go

@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
@@ -43,7 +44,7 @@ func Read(bucket, key string) (io.ReadCloser, error) {
 	return resp.Body, nil
 }
 
-func DownloadFile(filePath, bucket, key string) error {
+func DownloadFile(filePath, bucket, key string, s3Downloader *s3manager.Downloader, private bool) error {
 	if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
 		return errors.Cause(err)
 	}
@@ -54,15 +55,25 @@ func DownloadFile(filePath, bucket, key string) error {
 	}
 	defer fd.Close()
 
-	body, err := Read(bucket, key)
-	if err != nil {
-		return err
-	}
-
-	defer body.Close()
-
-	if _, err = io.Copy(fd, body); err != nil {
-		return err
+	if private {
+		_, err = s3Downloader.Download(fd, &s3.GetObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		body, err := Read(bucket, key)
+		if err != nil {
+			return err
+		}
+
+		defer body.Close()
+
+		if _, err = io.Copy(fd, body); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -93,13 +104,30 @@ func UploadFile(filePath string, bucket, key *string, s3Uploader *s3manager.Uplo
 	return nil
 }
 
-func KeyExists(bucket, key string) bool {
-	objectUrl := fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucket, key)
-
-	resp, err := http.Head(objectUrl)
-	if err != nil || resp.StatusCode != http.StatusOK {
-		return false
+func KeyExists(s3Client *s3.S3, bucket, key string, private bool) (bool, error) {
+	if private {
+		_, err := s3Client.HeadObject(&s3.HeadObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		})
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+				return false, nil
+			}
+			return false, fmt.Errorf("calling S3 HeadObject API to check if object is present: %v", err)
+		}
+	} else {
+		objectUrl := fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucket, key)
+
+		resp, err := http.Head(objectUrl)
+		if err != nil {
+			return false, fmt.Errorf("making HTTP HEAD request to check if object is present: %v", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return false, nil
+		}
 	}
 
-	return true
+	return true, nil
 }

release/cli/pkg/clients/clients.go

@@ -41,7 +41,8 @@ type ReleaseClients struct {
 }
 
 type SourceS3Clients struct {
-	Client *s3.S3
+	Client     *s3.S3
+	Downloader *s3manager.Downloader
 }
 
 type ReleaseS3Clients struct {
@@ -88,6 +89,7 @@ func CreateDevReleaseClients(dryRun bool) (*SourceClients, *ReleaseClients, erro
 
 	// S3 client and uploader
 	s3Client := s3.New(pdxSession)
+	downloader := s3manager.NewDownloader(pdxSession)
 	uploader := s3manager.NewUploader(pdxSession)
 
 	// Get source ECR auth config
@@ -107,7 +109,8 @@ func CreateDevReleaseClients(dryRun bool) (*SourceClients, *ReleaseClients, erro
 	// Constructing source clients
 	sourceClients := &SourceClients{
 		S3: &SourceS3Clients{
-			Client: s3Client,
+			Client:     s3Client,
+			Downloader: downloader,
 		},
 		ECR: &SourceECRClient{
 			EcrClient:  ecrClient,
@@ -162,6 +165,7 @@ func CreateStagingReleaseClients() (*SourceClients, *ReleaseClients, error) {
 
 	// Release S3 client and uploader
 	releaseS3Client := s3.New(releaseSession)
+	downloader := s3manager.NewDownloader(releaseSession)
 	uploader := s3manager.NewUploader(releaseSession)
 
 	// Get source ECR auth config
@@ -181,7 +185,8 @@ func CreateStagingReleaseClients() (*SourceClients, *ReleaseClients, error) {
 	// Constructing source clients
 	sourceClients := &SourceClients{
 		S3: &SourceS3Clients{
-			Client: sourceS3Client,
+			Client:     sourceS3Client,
+			Downloader: downloader,
 		},
 		ECR: &SourceECRClient{
 			EcrClient:  ecrClient,
@@ -237,6 +242,7 @@ func CreateProdReleaseClients() (*SourceClients, *ReleaseClients, error) {
 
 	// Release S3 client and uploader
 	releaseS3Client := s3.New(releaseSession)
+	downloader := s3manager.NewDownloader(releaseSession)
 	uploader := s3manager.NewUploader(releaseSession)
 
 	// Get source ECR Public auth config
@@ -256,7 +262,8 @@ func CreateProdReleaseClients() (*SourceClients, *ReleaseClients, error) {
 	// Constructing release clients
 	sourceClients := &SourceClients{
 		S3: &SourceS3Clients{
-			Client: sourceS3Client,
+			Client:     sourceS3Client,
+			Downloader: downloader,
 		},
 		ECR: &SourceECRClient{
 			EcrPublicClient: sourceEcrPublicClient,

release/cli/pkg/filereader/file_reader.go

@@ -180,6 +180,9 @@ func GetEksDReleaseManifestUrl(releaseChannel, releaseNumber string, dev bool) s
 
 // GetNextEksADevBuildNumber computes next eksa dev build number for the current eks-a dev build
 func GetNextEksADevBuildNumber(releaseVersion string, r *releasetypes.ReleaseConfig) (int, error) {
+	if r.DryRun {
+		return 0, nil
+	}
 	tempFileName := "latest-dev-release-version"
 
 	var latestReleaseKey, latestBuildVersion string
@@ -189,8 +192,14 @@ func GetNextEksADevBuildNumber(releaseVersion string, r *releasetypes.ReleaseCon
 	} else {
 		latestReleaseKey = fmt.Sprintf("%s/LATEST_RELEASE_VERSION", r.BuildRepoBranchName)
 	}
-	if s3.KeyExists(r.ReleaseBucket, latestReleaseKey) {
-		err := s3.DownloadFile(tempFileName, r.ReleaseBucket, latestReleaseKey)
+
+	keyExists, err := s3.KeyExists(r.ReleaseClients.S3.Client, r.ReleaseBucket, latestReleaseKey, false)
+	if err != nil {
+		return -1, errors.Cause(err)
+	}
+
+	if keyExists {
+		err := s3.DownloadFile(tempFileName, r.ReleaseBucket, latestReleaseKey, r.SourceClients.S3.Downloader, false)
 		if err != nil {
 			return -1, errors.Cause(err)
 		}

release/cli/pkg/images/images.go

@@ -83,7 +83,7 @@ func PollForExistence(devRelease bool, authConfig *docker.AuthConfiguration, ima
 
 		bodyStr := string(body)
 		if strings.Contains(bodyStr, "MANIFEST_UNKNOWN") {
-			return fmt.Errorf("requested image not found")
+			return fmt.Errorf("requested image not found: %v", imageUri)
 		}
 
 		return nil
@@ -321,7 +321,12 @@ func GetPreviousReleaseImageSemver(r *releasetypes.ReleaseConfig, releaseImageUr
 		bundles := &anywherev1alpha1.Bundles{}
 		bundleReleaseManifestKey := r.BundlesManifestFilepath()
 		bundleManifestUrl := fmt.Sprintf("https://%s.s3.amazonaws.com/%s", r.ReleaseBucket, bundleReleaseManifestKey)
-		if s3.KeyExists(r.ReleaseBucket, bundleReleaseManifestKey) {
+
+		keyExists, err := s3.KeyExists(r.ReleaseClients.S3.Client, r.ReleaseBucket, bundleReleaseManifestKey, false)
+		if err != nil {
+			return "", fmt.Errorf("checking if object [%s] is present in S3 bucket: %v", bundleReleaseManifestKey, err)
+		}
+		if keyExists {
 			contents, err := filereader.ReadHttpFile(bundleManifestUrl)
 			if err != nil {
 				return "", fmt.Errorf("Error reading bundle manifest from S3: %v", err)

release/cli/pkg/operations/download.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	s3sdk "github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
@@ -42,6 +44,12 @@ func DownloadArtifacts(ctx context.Context, r *releasetypes.ReleaseConfig, eksaA
 		}
 		return false, 0
 	}))
+	var s3Client *s3sdk.S3
+	var s3Downloader *s3manager.Downloader
+	if !r.DryRun {
+		s3Client = r.SourceClients.S3.Client
+		s3Downloader = r.SourceClients.S3.Downloader
+	}
 	fmt.Println("==========================================================")
 	fmt.Println("                  Artifacts Download")
 	fmt.Println("==========================================================")
@@ -55,12 +63,12 @@ func DownloadArtifacts(ctx context.Context, r *releasetypes.ReleaseConfig, eksaA
 			errGroup.Go(func() error {
 				// Check if there is an archive to be downloaded
 				if artifact.Archive != nil {
-					return handleArchiveDownload(ctx, r, artifact, s3Retrier)
+					return handleArchiveDownload(ctx, r, artifact, s3Retrier, s3Client, s3Downloader)
 				}
 
 				// Check if there is a manifest to be downloaded
 				if artifact.Manifest != nil {
-					return handleManifestDownload(ctx, r, artifact, s3Retrier)
+					return handleManifestDownload(ctx, r, artifact, s3Retrier, s3Client, s3Downloader)
 				}
 
 				return nil
@@ -76,7 +84,7 @@ func DownloadArtifacts(ctx context.Context, r *releasetypes.ReleaseConfig, eksaA
 	return nil
 }
 
-func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, artifact releasetypes.Artifact, s3Retrier *retrier.Retrier) error {
+func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, artifact releasetypes.Artifact, s3Retrier *retrier.Retrier, s3Client *s3sdk.S3, s3Downloader *s3manager.Downloader) error {
 	sourceS3Prefix := artifact.Archive.SourceS3Prefix
 	sourceS3Key := artifact.Archive.SourceS3Key
 	artifactPath := artifact.Archive.ArtifactPath
@@ -87,8 +95,13 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 		fmt.Println("Skipping OS image downloads in dry-run mode")
 	} else {
 		err := s3Retrier.Retry(func() error {
-			if !s3.KeyExists(r.SourceBucket, objectKey) {
-				return fmt.Errorf("requested object not found")
+			keyExists, err := s3.KeyExists(s3Client, r.SourceBucket, objectKey, artifact.Archive.Private)
+			if err != nil {
+				return fmt.Errorf("checking if object [%s] is present in S3 bucket: %v", objectKey, err)
+			}
+
+			if !keyExists {
+				return fmt.Errorf("requested object not found: %v", objectKey)
 			}
 			return nil
 		})
@@ -107,11 +120,11 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 				}
 				objectKey = filepath.Join(latestSourceS3PrefixFromMain, sourceS3Key)
 			} else {
-				return fmt.Errorf("retries exhausted waiting for archive [%s] to be uploaded to source location: %v", objectKey, err)
+				return fmt.Errorf("retries exhausted waiting for source archive [%s] to be available for download: %v", objectKey, err)
 			}
 		}
 
-		err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey)
+		err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey, s3Downloader, artifact.Archive.Private)
 		if err != nil {
 			return errors.Cause(err)
 		}
@@ -136,8 +149,13 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 			fmt.Printf("Checksum file - %s\n", objectShasumFileKey)
 
 			err := s3Retrier.Retry(func() error {
-				if !s3.KeyExists(r.SourceBucket, objectShasumFileKey) {
-					return fmt.Errorf("requested object not found")
+				keyExists, err := s3.KeyExists(s3Client, r.SourceBucket, objectShasumFileKey, artifact.Archive.Private)
+				if err != nil {
+					return fmt.Errorf("checking if object [%s] is present in S3 bucket: %v", objectShasumFileKey, err)
+				}
+
+				if !keyExists {
+					return fmt.Errorf("requested object not found: %v", objectShasumFileKey)
 				}
 				return nil
 			})
@@ -156,11 +174,11 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 					}
 					objectShasumFileKey = filepath.Join(latestSourceS3PrefixFromMain, objectShasumFileName)
 				} else {
-					return fmt.Errorf("retries exhausted waiting for checksum file [%s] to be uploaded to source location: %v", objectShasumFileKey, err)
+					return fmt.Errorf("retries exhausted waiting for source checksum file [%s] to be available for download: %v", objectShasumFileKey, err)
 				}
 			}
 
-			err = s3.DownloadFile(objectShasumFileLocalFilePath, r.SourceBucket, objectShasumFileKey)
+			err = s3.DownloadFile(objectShasumFileLocalFilePath, r.SourceBucket, objectShasumFileKey, s3Downloader, artifact.Archive.Private)
 			if err != nil {
 				return errors.Cause(err)
 			}
@@ -170,7 +188,7 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 	return nil
 }
 
-func handleManifestDownload(_ context.Context, r *releasetypes.ReleaseConfig, artifact releasetypes.Artifact, s3Retrier *retrier.Retrier) error {
+func handleManifestDownload(_ context.Context, r *releasetypes.ReleaseConfig, artifact releasetypes.Artifact, s3Retrier *retrier.Retrier, s3Client *s3sdk.S3, s3Downloader *s3manager.Downloader) error {
 	sourceS3Prefix := artifact.Manifest.SourceS3Prefix
 	sourceS3Key := artifact.Manifest.SourceS3Key
 	artifactPath := artifact.Manifest.ArtifactPath
@@ -179,8 +197,13 @@ func handleManifestDownload(_ context.Context, r *releasetypes.ReleaseConfig, ar
 	fmt.Printf("Manifest - %s\n", objectKey)
 
 	err := s3Retrier.Retry(func() error {
-		if !s3.KeyExists(r.SourceBucket, objectKey) {
-			return fmt.Errorf("requested object not found")
+		keyExists, err := s3.KeyExists(s3Client, r.SourceBucket, objectKey, artifact.Manifest.Private)
+		if err != nil {
+			return fmt.Errorf("checking if object [%s] is present in S3 bucket: %v", objectKey, err)
+		}
+
+		if !keyExists {
+			return fmt.Errorf("requested object not found: %v", objectKey)
 		}
 		return nil
 	})
@@ -194,11 +217,11 @@ func handleManifestDownload(_ context.Context, r *releasetypes.ReleaseConfig, ar
 			latestSourceS3PrefixFromMain := strings.NewReplacer(r.BuildRepoBranchName, "latest", artifact.Manifest.GitTag, gitTagFromMain).Replace(sourceS3Prefix)
 			objectKey = filepath.Join(latestSourceS3PrefixFromMain, sourceS3Key)
 		} else {
-			return fmt.Errorf("retries exhausted waiting for manifest [%s] to be uploaded to source location: %v", objectKey, err)
+			return fmt.Errorf("retries exhausted waiting for source manifest [%s] to be available for download: %v", objectKey, err)
 		}
 	}
 
-	err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey)
+	err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey, s3Downloader, artifact.Manifest.Private)
 	if err != nil {
 		return errors.Cause(err)
 	}