Replace s3:// urls with signed urls for metal.server (#51)

When creating the bootscript bss will now convert s3:// urls to signed
urls. This conversion is done for the params field for the metal.server field.

For example, it will convert the following to a signed url:
'metal.server=s3://bucket/path'

It will leave other s3 uris alone. For example this will remain untouched:
'root=craycps-s3:s3://boot-images/2b63caf8'

Jira: CASMHMS-5656

.version

@@ -1 +1 @@
-1.19.0
+1.20.0

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.20.0] - 2022-08-08
+
+### Changed
+
+- Added conversion of the metal.server s3 URI in params to a signed URL when creating the boot script.
+
 ## [1.19.0] - 2022-07-19
 
 ### Changed

cmd/boot-script-service/default_api.go

@@ -47,6 +47,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -74,6 +75,15 @@ var gwURI = getEnvVal("BSS_GW_URI", "/apis/bss")
 // Store ptr to S3 client
 var s3Client *hms_s3.S3Client
 
+// regex for matching s3 URIs in the params field
+var s3ParamsRegex = "(^|[ ])((metal.server=)(s3://[^ ]*))"
+
+type (
+	// function interface for checkURL()
+	// this enables writing unit tests for replaceS3Params()
+	signedS3UrlGetter func(string) (string, error)
+)
+
 type scriptParams struct {
 	xname         string
 	nid           string
@@ -88,6 +98,40 @@ func getEnvVal(envVar, defVal string) string {
 	return defVal
 }
 
+func replaceS3Params(params string, getSignedS3Url signedS3UrlGetter) (newParams string, err error) {
+	newParams = params // always return the params even when there is an error
+
+	// regex groups created when this matches:
+	// 0: full match                   example: ' metal.server=s3://bucket/path'
+	// 1: params beginning or a space  example: '' or ' '
+	// 2: key and value                example: 'metal.server=s3://bucket/path'
+	// 3: key                          example: 'metal.server='
+	// 4: value (s3 uri)               example: 's3://bucket/path'
+	r, err := regexp.Compile(s3ParamsRegex)
+	if err != nil {
+		err = fmt.Errorf("Failed to replace s3 URIs in the params because the regex failed to compile: %s, error: %v", s3ParamsRegex, err)
+		return params, err
+	}
+
+	matches := r.FindAllStringSubmatch(params, -1)
+	for _, m := range matches {
+		if len(m) >= 5 {
+			httpS3SignedUrl, err := getSignedS3Url(m[4])
+			if err != nil {
+				return newParams, err
+			}
+
+			oldParam := m[1] + m[2]
+			newParam := m[1] + m[3] + httpS3SignedUrl
+			newParams = strings.Replace(newParams, oldParam, newParam, 1)
+		} else {
+			err = fmt.Errorf("The matched pattern contained fewer groups than expected. has: %d, expected: %d, matches: %v", len(m), 5, m)
+			return params, err
+		}
+	}
+	return newParams, nil
+}
+
 func checkURL(u string) (string, error) {
 	p, err := url.Parse(u)
 	if err != nil || !strings.EqualFold(p.Scheme, "s3") {
@@ -564,6 +608,12 @@ func buildBootScript(bd BootData, sp scriptParams, chain, descr string) (string,
 		return "", err
 	}
 
+	params, err = replaceS3Params(params, checkURL)
+	if err != nil {
+		log.Printf("Error replacing s3 URIs. error: %v, params:\n%s", err, params)
+		err = nil
+	}
+
 	script := "#!ipxe\n"
 	if bd.Initrd.Path != "" {
 		start := strings.Index(params, "initrd")

cmd/boot-script-service/default_api_test.go

@@ -0,0 +1,157 @@
+// MIT License
+//
+// (C) Copyright [2022] Hewlett Packard Enterprise Development LP
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+func mockGetSignedS3Url(s3Url string) (string, error) {
+	return s3Url + "_signed", nil
+}
+
+func mockGetSignedS3UrlError(s3Url string) (string, error) {
+	return s3Url, fmt.Errorf("error")
+}
+
+func TestReplaceS3Params_regex(t *testing.T) {
+	r, err := regexp.Compile(s3ParamsRegex)
+	if err != nil {
+		t.Errorf("Failed to compile the regex: %s, error: %v\n", s3ParamsRegex, err)
+		return
+	}
+	params := fmt.Sprintf("%s%s",
+		"metal.server=s3://b1/p1/p2",
+		" metal.server=s3://b2/p1/p2")
+
+	expected := [][]string{
+		[]string{
+			"metal.server=s3://b1/p1/p2",
+			"",
+			"metal.server=s3://b1/p1/p2",
+			"metal.server=",
+			"s3://b1/p1/p2",
+		},
+		[]string{
+			" metal.server=s3://b2/p1/p2",
+			" ",
+			"metal.server=s3://b2/p1/p2",
+			"metal.server=",
+			"s3://b2/p1/p2",
+		},
+	}
+
+	matches := r.FindAllStringSubmatch(params, -1)
+	if len(matches) != 2 {
+		t.Errorf("Failed expected two matches for: %s, using: %s\n", params, s3ParamsRegex)
+		return
+	}
+
+	for i, match := range matches {
+		if len(match) != 5 {
+			t.Errorf("Failed. Expected %d match to have 5. groups: %v, params: %s\n", i, match, params)
+			return
+		}
+		for j, group := range match {
+			if group != expected[i][j] {
+				t.Errorf("Failed wrong string for match %d group %d. expected: '%s', actual: '%s'\n",
+					i, j, expected[i][j], group)
+			}
+		}
+	}
+}
+
+func TestReplaceS3Params_replace(t *testing.T) {
+	params := fmt.Sprintf("%s %s  %s %s %s",
+		"metal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs",
+		"bond=bond0",
+		"metal.server=s3://bucket/path",
+		"root=craycps-s3:s3://boot-images",
+		"m=s3://b/p")
+
+	expected_params := fmt.Sprintf("%s %s  %s %s %s",
+		"metal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs_signed",
+		"bond=bond0",
+		"metal.server=s3://bucket/path_signed",
+		"root=craycps-s3:s3://boot-images",
+		"m=s3://b/p")
+
+	newParams, err := replaceS3Params(params, mockGetSignedS3Url)
+	if err != nil {
+		t.Errorf("replaceS3Params returned an error for params: %s, error: %v\n", params, err)
+	}
+	if newParams != expected_params {
+		t.Errorf("replaceS3Params failed.\n  expected: %s\n  actual: %s\n", expected_params, newParams)
+	}
+}
+
+func TestReplaceS3Params_replace2(t *testing.T) {
+	params := fmt.Sprintf("%s %s",
+		"xmetal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs",
+		"metal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs")
+	expected_params := fmt.Sprintf("%s %s",
+		"xmetal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs",
+		"metal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs_signed")
+
+	newParams, err := replaceS3Params(params, mockGetSignedS3Url)
+	if err != nil {
+		t.Errorf("replaceS3Params returned an error for params: %s, error: %v\n", params, err)
+	}
+	if newParams != expected_params {
+		t.Errorf("replaceS3Params failed.\n  expected: %s\n  actual: %s\n", expected_params, newParams)
+	}
+}
+
+func TestReplaceS3Params_no_replace(t *testing.T) {
+	// This test expects the string to remain unchanged
+	params := fmt.Sprintf(
+		"%s %s %s %s %s",
+		"made_up_key=s3://ncn-images/path",
+		"xmetal.server=s3://ncn-images/k8s/0.2.78/filesystem.squashfs",
+		"nmd_data=url=s3://boot-images/bb-86/rootfs,etag=c8-204",
+		"bos_update_frequency=4h",
+		"root=craycps-s3:s3://boot-images/bb-78/rootfs:c8-204:dvs:api-gw-service-nmn.local:300:hsn0,nmn0:0")
+	expected_params := params
+
+	newParams, err := replaceS3Params(params, mockGetSignedS3Url)
+	if err != nil {
+		t.Errorf("replaceS3Params returned an error for params: %s, error: %v\n", params, err)
+	}
+	if newParams != expected_params {
+		t.Errorf("replaceS3Params failed.\n  expected: %s\n  actual: %s\n", expected_params, newParams)
+	}
+}
+
+func TestReplaceS3Params_error(t *testing.T) {
+	params := "bond=bond0 metal.server=s3://bucket/path"
+	expected_params := params
+	newParams, err := replaceS3Params(params, mockGetSignedS3UrlError)
+	if err == nil {
+		t.Errorf("replaceS3Params failed to return an error when using mock that injects an error. params: %s\n", params)
+	}
+	if newParams != expected_params {
+		t.Errorf("replaceS3Params failed.\n  expected: %s\n  actual: %s\n", expected_params, newParams)
+	}
+}