Enabled tracking of last access time for bootscript and cloud-init (u…

…ser-data) resources (#36)

* Enabled tracking of last access time for bootscript and cloud-init (user-data) resources.

* General refactor to change "access" to "endpoint" to clarify what this references.

* Always make sure to give back at least an empty array instead of `null`.

* Added enum to endpoint-history swagger to identify types.

.version

@@ -1 +1 @@
-1.12.0
+1.13.0

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.13.0] - 2021-10-26
+
+### Added
+
+- Enabled tracking of last access time for bootscript and cloud-init (user-data) resources.
+
+## [1.12.0] - 2021-11-30
+
+## Changed
+
+- Enable multi IP support by targeting v2 of HSM.
+
 ## [1.11.0] - 2021-10-19
 
 ## Changed

api/swagger.yaml

@@ -556,6 +556,34 @@ paths:
           description: Internal Server Error
           schema:
             $ref: '#/definitions/Error'
+  /endpoint-history:
+    get:
+      summary: Retrieve access information for xname and endpoint
+      tags:
+        - endpoint-history
+      description: >-
+                   Retrieve access information for xname and endpoint. Every time a node requests special
+                   types of endpoint (its bootscript or cloud-init data) that is recorded in the database. This is
+                   useful for determining a number of things most notably as a way to monitor boot progress.
+      parameters:
+        - name: name
+          in: query
+          type: string
+          description: Xname of the node.
+        - name: endpoint
+          in: query
+          type: string
+          enum:
+            - bootscript
+            - user-data
+          description: The endpoint to get the last access informaion for.
+      responses:
+        '200':
+          description: Endpoint access information
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/EndpointAccess'
 definitions:
   BootParams:
     description: >-
@@ -672,6 +700,24 @@ definitions:
       properties:
         schema:
           $ref: '#/definitions/Component'
+  EndpointAccess:
+    description: >-
+                 This data structure is used to return the endpoint access information for a given resource.
+    type: object
+    properties:
+      name:
+        type: string
+        description: Xname of the node
+        example: x3000c0s1b0n0
+      endpoint:
+        type: string
+        enum:
+          - bootscript
+          - user-data
+      last_epoch:
+        type: integer
+        description: Unix epoch time of last request.
+        example: 1635284155
   Error:
     description: Return an RFC7808 error response.
     type: object

cmd/boot-script-service/boot_data.go

@@ -40,16 +40,19 @@ import (
 	"log"
 	"net/http"
 	"reflect"
+	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
 const (
-	kernelImageType = "kernel"
-	initrdImageType = "initrd"
-	keyMin          = " "
-	keyMax          = "~"
-	paramsPfx       = "/params/"
+	kernelImageType   = "kernel"
+	initrdImageType   = "initrd"
+	keyMin            = " "
+	keyMax            = "~"
+	paramsPfx         = "/params/"
+	endpointAccessPfx = "/endpoint-access"
 )
 
 type BootDataStore struct {
@@ -613,6 +616,106 @@ func updateCloudInit(d *bssTypes.CloudInit, p bssTypes.CloudInit) bool {
 	return changed
 }
 
+func updateEndpointAccessed(name string, accessType bssTypes.EndpointType) {
+	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+	key := fmt.Sprintf("%s/%s/%s", endpointAccessPfx, name, accessType)
+	if err := kvstore.Store(key, timestamp); err != nil {
+		log.Printf("Failed to store last access timestamp %s to key %s: %s",
+			timestamp, key, err)
+	}
+}
+
+func searchKeyspace(prefix string) ([]hmetcd.Kvi_KV, error) {
+	// No kidding, the way you search in etcd is to search for a range where the first part of the range is the actual
+	// prefix and the second part of the range is that same prefix with the last character 1 unicode greater.
+	// > If range_end is key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"), then the range request gets all keys
+	// > prefixed with key.
+	// https://github.com/etcd-io/etcd/pull/7206/commits/7e31ddd32a4511c436b14e30ef43756ac782d080
+	rangePrefix := prefix[:len(prefix)-1]
+	rangeLastNextChar := prefix[len(prefix)-1:][0] + 1
+	rangeEnd := fmt.Sprintf("%s%c", rangePrefix, rangeLastNextChar)
+
+	return kvstore.GetRange(rangePrefix, rangeEnd)
+}
+
+func getAccessesForPrefix(prefix string) (accesses []bssTypes.EndpointAccess, err error) {
+	kvs, searchErr := searchKeyspace(prefix)
+	if searchErr != nil {
+		err = fmt.Errorf("failed to search keyspace: %w", searchErr)
+		return
+	}
+
+	for _, kv := range kvs {
+		endpointParts := strings.Split(kv.Key, "/")
+		endpoint := endpointParts[len(endpointParts)-1]
+		name := endpointParts[len(endpointParts)-2]
+
+		lastEpoch, err := strconv.ParseInt(kv.Value, 0, 64)
+		if err != nil {
+			err = fmt.Errorf("failed to convert timestamp to int: %w", err)
+		}
+
+		newAccess := bssTypes.EndpointAccess{
+			Name: name,
+			Endpoint:  bssTypes.EndpointType(endpoint),
+			LastEpoch: lastEpoch,
+		}
+
+		accesses = append(accesses, newAccess)
+	}
+
+	return
+}
+
+func SearchEndpointAccessed(name string, endpointType bssTypes.EndpointType) (accesses []bssTypes.EndpointAccess,
+	err error) {
+	if name == "" && endpointType == "" {
+		return getAccessesForPrefix(endpointAccessPfx)
+	} else if name != "" && endpointType == "" {
+		return getAccessesForPrefix(fmt.Sprintf("%s/%s", endpointAccessPfx, name))
+	} else if name != "" && endpointType != "" {
+		var epoch int64
+		epoch, err = getEndpointAccessed(name, endpointType)
+		if err != nil {
+			return
+		}
+
+		access := bssTypes.EndpointAccess{
+			Name:      name,
+			Endpoint:  endpointType,
+			LastEpoch: epoch,
+		}
+		accesses = append(accesses, access)
+
+		return
+	} else {
+		err = fmt.Errorf("invalid search combination of name (%s) and endpoint (%s)", name, endpointType)
+	}
+
+	return
+}
+
+func getEndpointAccessed(name string, endpointType bssTypes.EndpointType) (int64, error) {
+	key := fmt.Sprintf("%s/%s/%s", endpointAccessPfx, name, endpointType)
+	timestampString, exists, err := kvstore.Get(key)
+
+	if err != nil {
+		return -1, fmt.Errorf("failed to retreive last access timestamp at key %s: %w", key, err)
+	}
+
+	if !exists {
+		// Magic number, 0 meaning never accessed.
+		return 0, nil
+	}
+
+	ts, err := strconv.ParseInt(timestampString, 0, 64)
+	if err != nil {
+		return -1, fmt.Errorf("failed to convert timestamp to int: %w", err)
+	}
+
+	return ts, nil
+}
+
 func getTags() ([]hmetcd.Kvi_KV, error) {
 	return kvstore.GetRange(paramsPfx+keyMin, paramsPfx+keyMax)
 }

cmd/boot-script-service/cloudInitAPI.go

@@ -272,9 +272,47 @@ func userDataGetAPI(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/yaml")
 	w.WriteHeader(httpStatus)
 	_, _ = fmt.Fprintf(w, "#cloud-config\n%s", string(databytes))
+
+	// Record the fact this was asked for.
+	updateEndpointAccessed(xname, bssTypes.EndpointTypeUserData)
+
 	return
 }
 
+func endpointHistoryGetAPI(w http.ResponseWriter, r *http.Request) {
+	debugf("endpointHistoryGetAPI(): Received request %v\n", r.URL)
+
+	r.ParseForm() // r.Form is empty until after parsing
+	name := strings.Join(r.Form["name"], "")
+	endpoint := strings.Join(r.Form["endpoint"], "")
+
+	var lastAccessTypeStruct bssTypes.EndpointType
+
+	if endpoint != "" {
+		lastAccessTypeStruct = bssTypes.EndpointType(endpoint)
+	}
+
+	accesses, err := SearchEndpointAccessed(name, lastAccessTypeStruct)
+	if err != nil {
+		errMsg := fmt.Sprintf("Failed to search for name: %s, endpoint: %s", name, endpoint)
+		base.SendProblemDetailsGeneric(w, http.StatusInternalServerError, errMsg)
+		log.Printf("BSS request failed: %s", errMsg)
+		return
+	}
+
+	if len(accesses) == 0 {
+		// Always make sure to give back at least an empty array instead of `null`.
+		accesses = []bssTypes.EndpointAccess{}
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+	w.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(w).Encode(accesses)
+	if err != nil {
+		log.Printf("Yikes, I couldn't encode a JSON status response: %s\n", err)
+	}
+}
+
 func phoneHomePostAPI(w http.ResponseWriter, r *http.Request) {
 	var bp bssTypes.BootParams
 	var hosts []string

cmd/boot-script-service/default_api.go

@@ -751,6 +751,9 @@ func BootscriptGet(w http.ResponseWriter, r *http.Request) {
 				log.Printf("BSS request delayed for %s while updating state", descr)
 			} else {
 				log.Printf("BSS request succeeded for %s", descr)
+
+				// Record the fact this was asked for.
+				updateEndpointAccessed(comp.ID, bssTypes.EndpointTypeBootscript)
 			}
 		} else {
 			log.Printf("BSS request failed writing response for %s: %s", descr, err.Error())

cmd/boot-script-service/routers.go

@@ -45,9 +45,9 @@ const (
 	baseEndpoint     = "/boot/v1"
 	notifierEndpoint = baseEndpoint + "/scn"
 	// We don't use the baseEndpoint here because cloud-init doesn't like them
-	metaDataRoute  = "/meta-data"
-	userDataRoute  = "/user-data"
-	phoneHomeRoute = "/phone-home"
+	metaDataRoute   = "/meta-data"
+	userDataRoute   = "/user-data"
+	phoneHomeRoute  = "/phone-home"
 )
 
 func initHandlers() {
@@ -65,6 +65,8 @@ func initHandlers() {
 	http.HandleFunc(phoneHomeRoute, phoneHomePost)
 	// notifications
 	http.HandleFunc(notifierEndpoint, scn)
+	// endpoint-access
+	http.HandleFunc(baseEndpoint+"/endpoint-history", endpointHistoryGet)
 }
 
 func Index(w http.ResponseWriter, r *http.Request) {
@@ -166,3 +168,12 @@ func phoneHomePost(w http.ResponseWriter, r *http.Request) {
 		sendAllowable(w, "POST")
 	}
 }
+
+func endpointHistoryGet(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		endpointHistoryGetAPI(w, r)
+	default:
+		sendAllowable(w, "GET")
+	}
+}

kubernetes/cray-hms-bss/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+description: "Kubernetes resources for cray-hms-bss"
+name: "cray-hms-bss"
+home: "HMS/hms-bss"
+version: 1.12.1

kubernetes/cray-hms-bss/requirements.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: cray-service
+  repository: https://artifactory.algol60.net/artifactory/csm-helm-charts
+  version: 6.2.0
+digest: sha256:b37b5af4dc631d05ee5635dab90ab43582e7ef99919852d219edeb33db723b46
+generated: "2021-11-09T18:08:38.486284-06:00"

pkg/bssTypes/types.go

@@ -59,3 +59,18 @@ type BootParams struct {
 	Initrd    string    `json:"initrd,omitempty"`
 	CloudInit CloudInit `json:"cloud-init,omitempty"`
 }
+
+// The following structures and types all related to the last access information for bootscripts and cloud-init data.
+
+type EndpointType string
+
+const (
+	EndpointTypeBootscript EndpointType = "bootscript"
+	EndpointTypeUserData                = "user-data"
+)
+
+type EndpointAccess struct {
+	Name      string       `json:"name"`
+	Endpoint  EndpointType `json:"endpoint"`
+	LastEpoch int64        `json:"last_epoch"`
+}