cosalib, ore: add ability to cross upload aarch64 AMIs

This adds the ability to specify an architecture to buildextend-aws
so that we can upload an aarch64 AMI even if we're on x86_64.

Also removes --board option from `ore aws upload` as that wasn't used
for anything useful any longer.

Author: dustymabe

mantle/cmd/ore/aws/upload.go

@@ -50,7 +50,7 @@ After a successful run, the final line of output will be a line of JSON describi
 	uploadSourceObject       string
 	uploadBucket             string
 	uploadImageName          string
-	uploadBoard              string
+	uploadImageArchitecture  string
 	uploadFile               string
 	uploadDiskSizeGiB        uint
 	uploadDiskSizeInspect    bool
@@ -70,7 +70,7 @@ func init() {
 	cmdUpload.Flags().StringVar(&uploadSourceObject, "source-object", "", "'s3://' URI pointing to image data (default: same as upload)")
 	cmdUpload.Flags().StringVar(&uploadBucket, "bucket", "", "s3://bucket/prefix/ (defaults to a regional bucket and prefix defaults to $USER/board/name)")
 	cmdUpload.Flags().StringVar(&uploadImageName, "name", "", "name of uploaded image")
-	cmdUpload.Flags().StringVar(&uploadBoard, "board", "amd64-usr", "board used for naming with default prefix only")
+	cmdUpload.Flags().StringVar(&uploadImageArchitecture, "arch", "", "The target architecture for the AMI")
 	cmdUpload.Flags().StringVar(&uploadFile, "file", "", "path to CoreOS image")
 	cmdUpload.Flags().UintVarP(&uploadDiskSizeGiB, "disk-size-gib", "", aws.ContainerLinuxDiskSizeGiB, "AMI disk size in GiB")
 	cmdUpload.Flags().BoolVar(&uploadDiskSizeInspect, "disk-size-inspect", false, "set AMI disk size to size of local file")
@@ -92,7 +92,7 @@ func defaultBucketNameForRegion(region string) string {
 // defaultBucketURL determines the location the tool should upload to.
 // The 'urlPrefix' parameter, if it contains a path, will override all other
 // arguments
-func defaultBucketURL(urlPrefix, imageName, board, file, region string) (*url.URL, error) {
+func defaultBucketURL(urlPrefix, file, region string) (*url.URL, error) {
 	if urlPrefix == "" {
 		urlPrefix = fmt.Sprintf("s3://%s", defaultBucketNameForRegion(region))
 	}
@@ -108,9 +108,9 @@ func defaultBucketURL(urlPrefix, imageName, board, file, region string) (*url.UR
 		return nil, fmt.Errorf("URL missing bucket name %v\n", urlPrefix)
 	}
 
-	// if prefix not specified, default to /$USER/$BOARD/$VERSION
+	// if prefix not specified, default to 'ami-import'
 	if s3URL.Path == "" {
-		s3URL.Path = fmt.Sprintf("/%s/%s/%s", os.Getenv("USER"), board, imageName)
+		s3URL.Path = "/ami-import"
 	}
 
 	if s3URL.Path[len(s3URL.Path)-1] != '/' {
@@ -171,7 +171,7 @@ func runUpload(cmd *cobra.Command, args []string) error {
 			os.Exit(1)
 		}
 	} else {
-		s3URL, err = defaultBucketURL(uploadBucket, uploadImageName, uploadBoard, uploadFile, region)
+		s3URL, err = defaultBucketURL(uploadBucket, uploadFile, region)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%v\n", err)
 			os.Exit(1)
@@ -241,7 +241,7 @@ func runUpload(cmd *cobra.Command, args []string) error {
 	}
 
 	// create AMIs and grant permissions
-	amiID, err := API.CreateHVMImage(sourceSnapshot, uploadDiskSizeGiB, uploadAMIName, uploadAMIDescription)
+	amiID, err := API.CreateHVMImage(sourceSnapshot, uploadDiskSizeGiB, uploadAMIName, uploadAMIDescription, uploadImageArchitecture)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create HVM image: %v\n", err)
 		os.Exit(1)

mantle/cmd/plume/prerelease.go

@@ -462,7 +462,7 @@ func awsUploadToPartition(spec *channelSpec, part *awsPartitionSpec, imagePath s
 
 	plog.Printf("Creating AMIs from %v...", snapshot.SnapshotID)
 
-	imageID, err := api.CreateHVMImage(snapshot.SnapshotID, aws.ContainerLinuxDiskSizeGiB, imageName, imageDescription)
+	imageID, err := api.CreateHVMImage(snapshot.SnapshotID, aws.ContainerLinuxDiskSizeGiB, imageName, imageDescription, "x86_64")
 	if err != nil {
 		return nil, fmt.Errorf("unable to create image: %v", err)
 	}

mantle/platform/api/aws/images.go

@@ -329,15 +329,18 @@ func (a *API) CreateImportRole(bucket string) error {
 	return nil
 }
 
-func (a *API) CreateHVMImage(snapshotID string, diskSizeGiB uint, name string, description string) (string, error) {
+func (a *API) CreateHVMImage(snapshotID string, diskSizeGiB uint, name string, description string, architecture string) (string, error) {
 	var awsArch string
-	switch runtime.GOARCH {
-	case "amd64":
+	if architecture == "" {
+		architecture = runtime.GOARCH
+	}
+	switch architecture {
+	case "amd64", "x86_64":
 		awsArch = ec2.ArchitectureTypeX8664
-	case "arm64":
+	case "arm64", "aarch64":
 		awsArch = ec2.ArchitectureTypeArm64
 	default:
-		return "", fmt.Errorf("unsupported ec2 architecture %q", runtime.GOARCH)
+		return "", fmt.Errorf("unsupported ec2 architecture %q", architecture)
 	}
 
 	return a.createImage(&ec2.RegisterImageInput{
@@ -431,6 +434,7 @@ func (a *API) createImage(params *ec2.RegisterImageInput) (string, error) {
 	var imageID string
 	if err == nil {
 		imageID = *res.ImageId
+		plog.Infof("created image %v", imageID)
 	} else if awserr, ok := err.(awserr.Error); ok && awserr.Code() == "InvalidAMIName.Duplicate" {
 		// The AMI already exists. Get its ID. Due to races, this
 		// may take several attempts.

src/cmd-ore-wrapper

@@ -88,6 +88,7 @@ Each target has its own sub options. To access them us:
                         help="Upload/replicate to specific regions",
                         nargs='+')
     parser.add_argument("--source-region", help="Region to copy AMI from")
+    parser.add_argument("--arch", dest="arch", help="Architecture to target")
 
     parser.description = (
         f"'ore' interface for running ore commands for {target.upper()}"

src/cosalib/aws.py

@@ -125,6 +125,8 @@ def aws_run_ore(build, args):
         '--disk-size-inspect',
         '--delete-object'
     ])
+    if args.arch:
+        ore_args.extend(['--arch', f"{args.arch}"])
     for user in args.grant_user:
         ore_args.extend(['--grant-user', user])
     for user in args.grant_user_snapshot:

src/cosalib/build.py

@@ -88,10 +88,12 @@ def __init__(self, **kwargs):
         else:
             build = builds.get_latest()
 
+        self._basearch = kwargs.get("arch", None) or BASEARCH
+        log.info("Targeting architecture: %s", self.basearch)
         log.info("Targeting build: %s", build)
         self._build_dir = builds.get_build_dir(
             build,
-            basearch=kwargs.get("arch", BASEARCH)
+            basearch=self.basearch
         )
 
         # This is a bit subtle; but essentially, we want to verify that the
@@ -115,7 +117,8 @@ def __init__(self, **kwargs):
             "commit": None,
             "config": None,
             "image": None,
-            "meta": Meta(self.workdir, build, schema=schema)
+            "meta": Meta(self.workdir, build,
+                         basearch=self.basearch, schema=schema)
         }
 
         os.environ['workdir'] = self._workdir
@@ -252,7 +255,8 @@ def refresh_meta(self):
 
     @property
     def basearch(self):
-        return self.meta.get("coreos-assembler.basearch", BASEARCH)
+        """ get the target architecture """
+        return self._basearch
 
     def ensure_built(self):
         if not self.have_artifact:

src/cosalib/meta.py

@@ -260,15 +260,17 @@ class GenericBuildMeta(GenericMeta):
     """
 
     def __init__(self, workdir=None, build='latest',
-                 schema=SCHEMA_PATH):
+                 basearch=None, schema=SCHEMA_PATH):
         builds = Builds(workdir)
         if build != "latest":
             if not builds.has(build):
                 raise Exception('Build was not found in builds.json')
         else:
             build = builds.get_latest()
 
-        self._build_dir = builds.get_build_dir(build)
+        self._build_dir = \
+            builds.get_build_dir(build,
+                                 basearch=basearch)
         path = os.path.join(self._build_dir, 'meta.json')
         super().__init__(schema=schema, path=path)
 

src/cosalib/qemuvariants.py

@@ -132,6 +132,7 @@ def get_qemu_variant(variant, parser, kwargs={}):
         schema=parser.schema,
         variant=variant,
         force=parser.force,
+        arch=parser.arch,
         **kwargs)