diff --git a/cmd/openshift-install/upi.go b/cmd/openshift-install/upi.go index 3cce1e04dd8..96d2d40c36b 100644 --- a/cmd/openshift-install/upi.go +++ b/cmd/openshift-install/upi.go @@ -2,12 +2,18 @@ package main import ( "context" + "fmt" "path/filepath" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/client-go/tools/clientcmd" + + "github.com/openshift/installer/pkg/rhcos" + "github.com/openshift/installer/pkg/types/aws" + "github.com/openshift/installer/pkg/types/libvirt" + "github.com/openshift/installer/pkg/types/validation" ) var ( @@ -22,7 +28,8 @@ provides entry points to support the following workflow: 1. Call 'create ignition-configs' to create the bootstrap Ignition config and admin kubeconfig. 2. Creates all required cluster resources, after which the cluster - will being bootstrapping. + will being bootstrapping. 'user-provided-infrastructure bootimage' + may help with this. 3. Call 'user-provided-infrastructure bootstrap-complete' to wait until the bootstrap phase has completed. 4. Destroy the bootstrap resources. @@ -42,11 +49,51 @@ func newUPICmd() *cobra.Command { return cmd.Help() }, } + cmd.AddCommand(newUPIBootimageCmd()) cmd.AddCommand(newUPIBootstrapCompleteCmd()) cmd.AddCommand(newUPIFinishCmd()) return cmd } +func newUPIBootimageCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "bootimage", + Short: "Show the suggested RHCOS bootimage for a given platform", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + + platformName := args[0] + switch platformName { + case aws.Name: + region, err := cmd.Flags().GetString("region") + if err != nil { + logrus.Fatal(err) + } + + ami, err := rhcos.AMI(ctx, region) + if err != nil { + logrus.Fatal(err) + } + fmt.Println(ami) + case libvirt.Name: + qemu, err := rhcos.QEMU(ctx) + if err != nil { + logrus.Fatal(err) + } + fmt.Println(qemu) + default: + err := validation.PlatformName(platformName) + if err != nil { + logrus.Fatal(errors.Wrapf(err, "unrecognized %q", platformName)) + } + } + }, + } + cmd.Flags().StringP("region", "r", "", "AMI region (required for AWS)") + return cmd +} + func newUPIBootstrapCompleteCmd() *cobra.Command { return &cobra.Command{ Use: "bootstrap-complete", diff --git a/data/data/rhcos.json b/data/data/rhcos.json new file mode 100644 index 00000000000..60878c249ca --- /dev/null +++ b/data/data/rhcos.json @@ -0,0 +1,94 @@ +{ + "amis": { + "ap-northeast-1": { + "hvm": "ami-0fbc0018a8310e53e" + }, + "ap-northeast-2": { + "hvm": "ami-0ba6edf20991dee91" + }, + "ap-south-1": { + "hvm": "ami-0acf1668760f8f8e9" + }, + "ap-southeast-1": { + "hvm": "ami-0bcecfdf9ff5fd5ca" + }, + "ap-southeast-2": { + "hvm": "ami-0480abd0220d56ae2" + }, + "ca-central-1": { + "hvm": "ami-0de4e822461cb671b" + }, + "eu-central-1": { + "hvm": "ami-056c9291dce6d5023" + }, + "eu-west-1": { + "hvm": "ami-0f0159b00648b0bf4" + }, + "eu-west-2": { + "hvm": "ami-044f299a3abcb4d96" + }, + "eu-west-3": { + "hvm": "ami-095776c2e71c62b2f" + }, + "sa-east-1": { + "hvm": "ami-00aefabf6d653ff5d" + }, + "us-east-1": { + "hvm": "ami-07f604c5f18a1e7a9" + }, + "us-east-2": { + "hvm": "ami-0eef624367320ec26" + }, + "us-west-1": { + "hvm": "ami-0f89dba68d747846b" + }, + "us-west-2": { + "hvm": "ami-0eac581fbaa9fa9c6" + } + }, + "baseURI": "https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190325.0/", + "buildid": "410.8.20190325.0", + "images": { + "metal-bios": { + "path": "rhcos-410.8.20190325.0-metal-bios.raw", + "sha256": "3caaf8d714ac8bb820af6b31d595b8afae5ef951c68f8cee3701e236e9600f30", + "size": "735841103", + "uncompressed-sha256": "32d19b94cfb2aa45799caed2dda70f4e7ac1d0fca2b834a1cbd4459a6b0d056a", + "uncompressed-size": "17179869184" + }, + "metal-uefi": { + "path": "rhcos-410.8.20190325.0-metal-uefi.raw", + "sha256": "1686bb2e3b01873804325b50e286d858435afe1fbc3e88479d54b7a21e58d0f1", + "size": "732781221", + "uncompressed-sha256": "4c9a34f8b30ff7c5b107c87faac0f34a6b591d07a43ae729c45d98d15f3a8fa1", + "uncompressed-size": "17179869184" + }, + "openstack": { + "path": "rhcos-410.8.20190325.0-openstack.qcow2", + "sha256": "1b16214e59d38b5c1e6d291aea35f2539845232ca3300462104d022f9877fede", + "size": "721092654", + "uncompressed-sha256": "f2f1362c155d8d331387ce759a46b16c83c8a454f59d2ab5087d9781cdc29c97", + "uncompressed-size": "2014380032" + }, + "qemu": { + "path": "rhcos-410.8.20190325.0-qemu.qcow2", + "sha256": "12b2db0cae8ea4019f24183dfc905609ab27e8a0fa01bf3631dc6ad7d2afe664", + "size": "721096885", + "uncompressed-sha256": "3b288cf2e02b63f8852c731836d5923a75230a836c63814a3988a96c26268257", + "uncompressed-size": "2014314496" + }, + "vmware": { + "path": "rhcos-410.8.20190325.0-vmware.ova", + "sha256": "d8c78e90f3a6fccd21c848a53b669c1c51080972377ad3f6bc81cdcc2cb48300", + "size": "710257080", + "uncompressed-sha256": "f73f5ef77095b0c79a648e2957a965f60544da08c78176028393948500da0dfc", + "uncompressed-size": "743802880" + } + }, + "oscontainer": { + "digest": "sha256:007512169406dfa78f2eefdcd1553d94c399340bd1e871b4c884baf609a967ba", + "image": "docker-registry-default.cloud.registry.upshift.redhat.com/redhat-coreos/ootpa" + }, + "ostree-commit": "ed3e5f10b22db9db37e48b72e907ee60113ed259eeef5c019faa0109feeeccf8", + "ostree-version": "410.8.20190325.0" +} \ No newline at end of file diff --git a/hack/build.sh b/hack/build.sh index 5a01524fc0b..991e8803e2e 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -2,8 +2,6 @@ set -ex -RHCOS_BUILD_NAME="${RHCOS_BUILD_NAME:-410.8.20190325.0}" - # shellcheck disable=SC2068 version() { IFS="."; printf "%03d%03d%03d\\n" $@; unset IFS;} @@ -46,10 +44,6 @@ release) then LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/asset/ignition/bootstrap.defaultReleaseImage=${RELEASE_IMAGE}" fi - if test -n "${RHCOS_BUILD_NAME}" - then - LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/rhcos.buildName=${RHCOS_BUILD_NAME}" - fi if test "${SKIP_GENERATION}" != y then go generate ./data diff --git a/hack/update-rhcos-bootimage.py b/hack/update-rhcos-bootimage.py new file mode 100755 index 00000000000..ac2f64944fb --- /dev/null +++ b/hack/update-rhcos-bootimage.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 +# Usage: ./hack/update-rhcos-bootimage.py https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190401.0/meta.json +import codecs,os,sys,json,argparse +import urllib.parse +import urllib.request + +dn = os.path.abspath(os.path.dirname(sys.argv[0])) + +parser = argparse.ArgumentParser() +parser.add_argument("meta", action='store') +args = parser.parse_args() + +with urllib.request.urlopen(args.meta) as f: + string_f = codecs.getreader('utf-8')(f) # support for Python < 3.6 + meta = json.load(string_f) +newmeta = {} +for k in ['images', 'buildid', 'oscontainer', + 'ostree-commit', 'ostree-version']: + newmeta[k] = meta[k] +newmeta['amis'] = { + entry['name']: { + 'hvm': entry['hvm'], + } + for entry in meta['amis'] +} +newmeta['baseURI'] = urllib.parse.urljoin(args.meta, '.') +with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f: + json.dump(newmeta, f, sort_keys=True, indent=4) diff --git a/pkg/asset/installconfig/aws/aws.go b/pkg/asset/installconfig/aws/aws.go index 21191627577..64ea7f3feb0 100644 --- a/pkg/asset/installconfig/aws/aws.go +++ b/pkg/asset/installconfig/aws/aws.go @@ -2,6 +2,7 @@ package aws import ( + "context" "fmt" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" + "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/aws/validation" "github.com/openshift/installer/pkg/version" @@ -25,7 +27,14 @@ import ( func Platform() (*aws.Platform, error) { longRegions := make([]string, 0, len(validation.Regions)) shortRegions := make([]string, 0, len(validation.Regions)) + rhcosRegions, err := rhcos.AMIRegions(context.TODO()) + if err != nil { + return nil, err + } for id, location := range validation.Regions { + if _, ok := rhcosRegions[id]; !ok { + continue + } longRegions = append(longRegions, fmt.Sprintf("%s (%s)", id, location)) shortRegions = append(shortRegions, id) } diff --git a/pkg/asset/rhcos/image.go b/pkg/asset/rhcos/image.go index 956b85bc320..ab082524e77 100644 --- a/pkg/asset/rhcos/image.go +++ b/pkg/asset/rhcos/image.go @@ -55,9 +55,9 @@ func (i *Image) Generate(p asset.Parents) error { defer cancel() switch config.Platform.Name() { case aws.Name: - osimage, err = rhcos.AMI(ctx, rhcos.DefaultChannel, config.Platform.AWS.Region) + osimage, err = rhcos.AMI(ctx, config.Platform.AWS.Region) case libvirt.Name: - osimage, err = rhcos.QEMU(ctx, rhcos.DefaultChannel) + osimage, err = rhcos.QEMU(ctx) case openstack.Name: osimage = "rhcos" case none.Name: diff --git a/pkg/rhcos/ami.go b/pkg/rhcos/ami.go index ee43dba5f89..44e1f399155 100644 --- a/pkg/rhcos/ami.go +++ b/pkg/rhcos/ami.go @@ -2,22 +2,46 @@ package rhcos import ( "context" + "sort" + "strings" "github.com/pkg/errors" ) -// AMI fetches the HVM AMI ID of the latest Red Hat Enterprise Linux CoreOS release. -func AMI(ctx context.Context, channel, region string) (string, error) { - meta, err := fetchLatestMetadata(ctx, channel) +// AMI fetches the HVM AMI ID of the Red Hat Enterprise Linux CoreOS release. +func AMI(ctx context.Context, region string) (string, error) { + meta, err := fetchRHCOSBuild(ctx) if err != nil { return "", errors.Wrap(err, "failed to fetch RHCOS metadata") } - for _, ami := range meta.AMIs { - if ami.Name == region { - return ami.HVM, nil + ami, ok := meta.AMIs[region] + if !ok { + regions := make([]string, 0, len(meta.AMIs)) + for rgn := range meta.AMIs { + regions = append(regions, rgn) } + sort.Strings(regions) + + return "", errors.Errorf("no RHCOS AMIs found in %q (%s)", region, strings.Join(regions, ", ")) + } + + return ami.HVM, nil +} + +// AMIRegions returns a set of AWS regions with HVM AMIs of the Red +// Hat Enterprise Linux CoreOS release. +func AMIRegions(ctx context.Context) (map[string]struct{}, error) { + meta, err := fetchRHCOSBuild(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch RHCOS metadata") + } + + exists := struct{}{} + regions := make(map[string]struct{}, len(meta.AMIs)) + for region := range meta.AMIs { + regions[region] = exists } - return "", errors.Errorf("no RHCOS AMIs found in %s", region) + return regions, nil } diff --git a/pkg/rhcos/builds.go b/pkg/rhcos/builds.go index b758460d5b4..c0a75165e36 100644 --- a/pkg/rhcos/builds.go +++ b/pkg/rhcos/builds.go @@ -3,31 +3,18 @@ package rhcos import ( "context" "encoding/json" - "fmt" "io/ioutil" - "net/http" + "github.com/openshift/installer/data" "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -var ( - // DefaultChannel is the default RHCOS channel for the cluster. - DefaultChannel = "ootpa" - - // buildName is the name of the build in the channel that will be picked up - // empty string means the first one in the build list (latest) will be used - buildName = "" - - baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases" ) type metadata struct { - AMIs []struct { - HVM string `json:"hvm"` - Name string `json:"name"` + AMIs map[string]struct { + HVM string `json:"hvm"` } `json:"amis"` - Images struct { + BaseURI string `json:"baseURI"` + Images struct { QEMU struct { Path string `json:"path"` SHA256 string `json:"sha256"` @@ -36,81 +23,22 @@ type metadata struct { OSTreeVersion string `json:"ostree-version"` } -func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) { - build := buildName - var err error - if build == "" { - build, err = fetchLatestBuild(ctx, channel) - if err != nil { - return metadata{}, errors.Wrap(err, "failed to fetch latest build") - } - } - - url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build) - logrus.Debugf("Fetching RHCOS metadata from %q", url) - req, err := http.NewRequest("GET", url, nil) +func fetchRHCOSBuild(ctx context.Context) (*metadata, error) { + file, err := data.Assets.Open("rhcos.json") if err != nil { - return metadata{}, errors.Wrap(err, "failed to build request") + return nil, err } + defer file.Close() - client := &http.Client{} - resp, err := client.Do(req.WithContext(ctx)) + body, err := ioutil.ReadAll(file) if err != nil { - return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status) + return nil, err } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return metadata{}, errors.Wrap(err, "failed to read HTTP response") - } - - var meta metadata + var meta *metadata if err := json.Unmarshal(body, &meta); err != nil { - return meta, errors.Wrap(err, "failed to parse HTTP response") + return meta, errors.Wrap(err, "failed to parse RHCOS build metadata") } return meta, nil } - -func fetchLatestBuild(ctx context.Context, channel string) (string, error) { - url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel) - logrus.Debugf("Fetching RHCOS builds from %q", url) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", errors.Wrap(err, "failed to build request") - } - - client := &http.Client{} - resp, err := client.Do(req.WithContext(ctx)) - if err != nil { - return "", errors.Wrap(err, "failed to fetch builds") - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", errors.Wrap(err, "failed to read HTTP response") - } - - var builds struct { - Builds []string `json:"builds"` - } - if err := json.Unmarshal(body, &builds); err != nil { - return "", errors.Wrap(err, "failed to parse HTTP response") - } - - if len(builds.Builds) == 0 { - return "", errors.Errorf("no builds found") - } - - return builds.Builds[0], nil -} diff --git a/pkg/rhcos/qemu.go b/pkg/rhcos/qemu.go index 98b0d33e88b..e4408447cfa 100644 --- a/pkg/rhcos/qemu.go +++ b/pkg/rhcos/qemu.go @@ -2,17 +2,27 @@ package rhcos import ( "context" - "fmt" + "net/url" "github.com/pkg/errors" ) -// QEMU fetches the URL of the latest Red Hat Enterprise Linux CoreOS release. -func QEMU(ctx context.Context, channel string) (string, error) { - meta, err := fetchLatestMetadata(ctx, channel) +// QEMU fetches the URL of the Red Hat Enterprise Linux CoreOS release. +func QEMU(ctx context.Context) (string, error) { + meta, err := fetchRHCOSBuild(ctx) if err != nil { return "", errors.Wrap(err, "failed to fetch RHCOS metadata") } - return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil + base, err := url.Parse(meta.BaseURI) + if err != nil { + return "", err + } + + relQEMU, err := url.Parse(meta.Images.QEMU.Path) + if err != nil { + return "", err + } + + return base.ResolveReference(relQEMU).String(), nil } diff --git a/pkg/types/aws/validation/platform.go b/pkg/types/aws/validation/platform.go index 6ca7eca6873..f2f2c058867 100644 --- a/pkg/types/aws/validation/platform.go +++ b/pkg/types/aws/validation/platform.go @@ -15,25 +15,25 @@ var ( Regions = map[string]string{ "ap-northeast-1": "Tokyo", "ap-northeast-2": "Seoul", - //"ap-northeast-3": "Osaka-Local", + "ap-northeast-3": "Osaka-Local", "ap-south-1": "Mumbai", "ap-southeast-1": "Singapore", "ap-southeast-2": "Sydney", "ca-central-1": "Central", - //"cn-north-1": "Beijing", - //"cn-northwest-1": "Ningxia", - "eu-central-1": "Frankfurt", - //"eu-north-1": "Stockholm", - "eu-west-1": "Ireland", - "eu-west-2": "London", - "eu-west-3": "Paris", - "sa-east-1": "São Paulo", - "us-east-1": "N. Virginia", - "us-east-2": "Ohio", - //"us-gov-east-1": "AWS GovCloud (US-East)", - //"us-gov-west-1": "AWS GovCloud (US-West)", - "us-west-1": "N. California", - "us-west-2": "Oregon", + "cn-north-1": "Beijing", + "cn-northwest-1": "Ningxia", + "eu-central-1": "Frankfurt", + "eu-north-1": "Stockholm", + "eu-west-1": "Ireland", + "eu-west-2": "London", + "eu-west-3": "Paris", + "sa-east-1": "São Paulo", + "us-east-1": "N. Virginia", + "us-east-2": "Ohio", + "us-gov-east-1": "AWS GovCloud (US-East)", + "us-gov-west-1": "AWS GovCloud (US-West)", + "us-west-1": "N. California", + "us-west-2": "Oregon", } validRegionValues = func() []string { diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index 1684205c157..6d226b9b6c6 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/validation/field" @@ -189,16 +190,26 @@ func validateCompute(pools []types.MachinePool, fldPath *field.Path, platform st return allErrs } -func validatePlatform(platform *types.Platform, fldPath *field.Path, openStackValidValuesFetcher openstackvalidation.ValidValuesFetcher) field.ErrorList { - allErrs := field.ErrorList{} - activePlatform := platform.Name() +// PlatformName name validates a platform name against the set of names +// understood by the installer. +func PlatformName(platform string) error { platforms := make([]string, len(types.PlatformNames)) copy(platforms, types.PlatformNames) platforms = append(platforms, types.HiddenPlatformNames...) sort.Strings(platforms) - i := sort.SearchStrings(platforms, activePlatform) - if i == len(platforms) || platforms[i] != activePlatform { - allErrs = append(allErrs, field.Invalid(fldPath, platform, fmt.Sprintf("must specify one of the platforms (%s)", strings.Join(platforms, ", ")))) + i := sort.SearchStrings(platforms, platform) + if i == len(platforms) || platforms[i] != platform { + return errors.Errorf("must specify one of the platforms (%s)", strings.Join(platforms, ", ")) + } + return nil +} + +func validatePlatform(platform *types.Platform, fldPath *field.Path, openStackValidValuesFetcher openstackvalidation.ValidValuesFetcher) field.ErrorList { + allErrs := field.ErrorList{} + activePlatform := platform.Name() + err := PlatformName(activePlatform) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, platform, err.Error())) } validate := func(n string, value interface{}, validation func(*field.Path) field.ErrorList) { if n != activePlatform { diff --git a/pkg/types/validation/installconfig_test.go b/pkg/types/validation/installconfig_test.go index a02681d7489..613184cfcc6 100644 --- a/pkg/types/validation/installconfig_test.go +++ b/pkg/types/validation/installconfig_test.go @@ -370,7 +370,7 @@ func TestValidateInstallConfig(t *testing.T) { } return c }(), - expectedError: `^platform\.aws\.region: Unsupported value: "": supported values: "ap-northeast-1", "ap-northeast-2", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"$`, + expectedError: `^platform\.aws\.region: Unsupported value: "": supported values: "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2"$`, }, { name: "valid libvirt platform",