From 990abad080cae6de6aca6282715f168cb77f583c Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 4 Aug 2023 11:08:37 -0500 Subject: [PATCH 01/99] Implement mechanism for leader designation. --- attestation_test.go | 2 +- enclave.go | 103 ++++++++++++++++++++++++++++++-------------- handlers.go | 8 ++++ handlers_test.go | 8 ++-- main.go | 22 +++++++--- metrics_test.go | 2 +- 6 files changed, 99 insertions(+), 46 deletions(-) diff --git a/attestation_test.go b/attestation_test.go index a79e223..a13e29b 100644 --- a/attestation_test.go +++ b/attestation_test.go @@ -49,7 +49,7 @@ func TestAttestationHashes(t *testing.T) { rec := httptest.NewRecorder() buf := bytes.NewBufferString(base64.StdEncoding.EncodeToString(appKeyHash[:])) req := httptest.NewRequest(http.MethodPost, pathHash, buf) - e.privSrv.Handler.ServeHTTP(rec, req) + e.intSrv.Handler.ServeHTTP(rec, req) s := e.hashes.Serialize() expectedLen := sha256.Size*2 + len(hashPrefix)*2 + len(hashSeparator) diff --git a/enclave.go b/enclave.go index de8622d..1a1d195 100644 --- a/enclave.go +++ b/enclave.go @@ -50,6 +50,7 @@ const ( pathReady = "/enclave/ready" pathProfiling = "/enclave/debug" pathConfig = "/enclave/config" + pathLeader = "/enclave/leader" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" @@ -65,8 +66,9 @@ var ( type Enclave struct { sync.RWMutex cfg *Config - pubSrv *http.Server - privSrv *http.Server + extPubSrv *http.Server + extPrivSrv *http.Server + intSrv *http.Server promSrv *http.Server revProxy *httputil.ReverseProxy hashes *AttestationHashes @@ -84,26 +86,42 @@ type Config struct { // is required. FQDN string - // ExtPort contains the TCP port that the Web server should + // FQDNLeader contains the fully qualified domain name of the leader + // enclave, which coordinates enclave synchronization. Only set this field + // if horizontal scaling is required. + FQDNLeader string + + // IsLeader will be set to true if the enclave is designated as leader for + // enclave synchronization. Leader designation happens at runtime, by + // calling an HTTP handler. + IsLeader bool + + // ExtPubPort contains the TCP port that the public Web server should // listen on, e.g. 443. This port is not *directly* reachable by the // Internet but the EC2 host's proxy *does* forward Internet traffic to // this port. This field is required. - ExtPort uint16 + ExtPubPort uint16 + + // ExtPrivPort contains the TCP port that the non-public Web server should + // listen on. The Web server behind this port exposes confidential + // endpoints and is therefore only meant to be reachable by the enclave + // administrator but *not* the public Internet. + ExtPrivPort uint16 + + // IntPort contains the enclave-internal TCP port of the Web server that + // provides an HTTP API to the enclave application. This field is + // required. + IntPort uint16 // UseVsockForExtPort must be set to true if direct communication // between the host and Web server via VSOCK is desired. The daemon will listen - // on the enclave's VSOCK address and the port defined in ExtPort. + // on the enclave's VSOCK address and the port defined in ExtPubPort. UseVsockForExtPort bool // DisableKeepAlives must be set to true if keep-alive connections // should be disabled for the HTTPS service. DisableKeepAlives bool - // IntPort contains the enclave-internal TCP port of the Web server that - // provides an HTTP API to the enclave application. This field is - // required. - IntPort uint16 - // HostProxyPort indicates the TCP port of the proxy application running on // the EC2 host. Note that VSOCK ports are 32 bits large. This field is // required. @@ -174,7 +192,7 @@ type Config struct { // Validate returns an error if required fields in the config are not set. func (c *Config) Validate() error { - if c.ExtPort == 0 || c.IntPort == 0 || c.HostProxyPort == 0 { + if c.ExtPubPort == 0 || c.IntPort == 0 || c.HostProxyPort == 0 { return errCfgMissingPort } if c.FQDN == "" { @@ -201,10 +219,14 @@ func NewEnclave(cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ cfg: cfg, - pubSrv: &http.Server{ + extPubSrv: &http.Server{ + Handler: chi.NewRouter(), + }, + extPrivSrv: &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.ExtPrivPort), Handler: chi.NewRouter(), }, - privSrv: &http.Server{ + intSrv: &http.Server{ Addr: fmt.Sprintf("127.0.0.1:%d", cfg.IntPort), Handler: chi.NewRouter(), }, @@ -228,30 +250,36 @@ func NewEnclave(cfg *Config) (*Enclave, error) { http.DefaultTransport.(*http.Transport).MaxIdleConns = 500 if cfg.Debug { - e.pubSrv.Handler.(*chi.Mux).Use(middleware.Logger) - e.privSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.extPubSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.extPrivSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.intSrv.Handler.(*chi.Mux).Use(middleware.Logger) } if cfg.PrometheusPort > 0 { - e.pubSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) - e.privSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.extPubSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.extPrivSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.intSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) } if cfg.UseProfiling { - e.pubSrv.Handler.(*chi.Mux).Mount(pathProfiling, middleware.Profiler()) + e.extPubSrv.Handler.(*chi.Mux).Mount(pathProfiling, middleware.Profiler()) } if cfg.DisableKeepAlives { - e.pubSrv.SetKeepAlivesEnabled(false) + e.extPubSrv.SetKeepAlivesEnabled(false) } - // Register public HTTP API. - m := e.pubSrv.Handler.(*chi.Mux) + // Register external public HTTP API. + m := e.extPubSrv.Handler.(*chi.Mux) m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes)) m.Get(pathNonce, nonceHandler(e)) m.Get(pathRoot, rootHandler(e.cfg)) m.Post(pathSync, respSyncHandler(e)) m.Get(pathConfig, configHandler(e.cfg)) + // Register external but private HTTP API. + m = e.extPrivSrv.Handler.(*chi.Mux) + m.Get(pathLeader, leaderHandler(e.cfg)) + // Register enclave-internal HTTP API. - m = e.privSrv.Handler.(*chi.Mux) + m = e.intSrv.Handler.(*chi.Mux) m.Get(pathSync, reqSyncHandler(e)) m.Get(pathReady, readyHandler(e)) m.Get(pathState, getStateHandler(e)) @@ -263,7 +291,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { if cfg.AppWebSrv != nil { e.revProxy = httputil.NewSingleHostReverseProxy(cfg.AppWebSrv) e.revProxy.BufferPool = newBufPool() - e.pubSrv.Handler.(*chi.Mux).Handle(pathProxy, e.revProxy) + e.extPubSrv.Handler.(*chi.Mux).Handle(pathProxy, e.revProxy) // If we expose Prometheus metrics, we keep track of the HTTP backend's // responses. if cfg.PrometheusPort > 0 { @@ -315,10 +343,10 @@ func (e *Enclave) Start() error { // Stop stops the enclave. func (e *Enclave) Stop() error { close(e.stop) - if err := e.privSrv.Shutdown(context.Background()); err != nil { + if err := e.intSrv.Shutdown(context.Background()); err != nil { return err } - if err := e.pubSrv.Shutdown(context.Background()); err != nil { + if err := e.extPubSrv.Shutdown(context.Background()); err != nil { return err } if err := e.promSrv.Shutdown(context.Background()); err != nil { @@ -331,9 +359,9 @@ func (e *Enclave) Stop() error { // via AF_INET or AF_VSOCK. func (e *Enclave) getExtListener() (net.Listener, error) { if e.cfg.UseVsockForExtPort { - return vsock.Listen(uint32(e.cfg.ExtPort), nil) + return vsock.Listen(uint32(e.cfg.ExtPubPort), nil) } else { - return net.Listen("tcp", fmt.Sprintf(":%d", e.cfg.ExtPort)) + return net.Listen("tcp", fmt.Sprintf(":%d", e.cfg.ExtPubPort)) } } @@ -350,13 +378,20 @@ func (e *Enclave) startWebServers() error { }() } - elog.Printf("Starting public (%s) and private (%s) Web servers.", e.pubSrv.Addr, e.privSrv.Addr) go func() { - err := e.privSrv.ListenAndServe() + elog.Printf("Starting internal Web server at %s.", e.intSrv.Addr) + err := e.intSrv.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { elog.Fatalf("Private Web server error: %v", err) } }() + go func() { + elog.Printf("Starting external private Web server at %s.", e.extPrivSrv.Addr) + err := e.extPrivSrv.ListenAndServeTLS("", "") + if err != nil && !errors.Is(err, http.ErrServerClosed) { + elog.Fatalf("External private Web server error: %v", err) + } + }() go func() { // If desired, don't launch our Internet-facing Web server until the // application signalled that it's ready. @@ -370,9 +405,10 @@ func (e *Enclave) startWebServers() error { elog.Fatalf("Failed to listen on external port: %v", err) } - err = e.pubSrv.ServeTLS(listener, "", "") + elog.Printf("Starting external public Web server at :%d.", e.cfg.ExtPubPort) + err = e.extPubSrv.ServeTLS(listener, "", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { - elog.Fatalf("Public Web server error: %v", err) + elog.Fatalf("External public Web server error: %v", err) } }() @@ -439,9 +475,10 @@ func (e *Enclave) genSelfSignedCert() error { return err } - e.pubSrv.TLSConfig = &tls.Config{ + e.extPubSrv.TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, } + e.extPrivSrv.TLSConfig = e.extPubSrv.TLSConfig // Both servers share a TLS config. return nil } @@ -470,7 +507,7 @@ func (e *Enclave) setupAcme() error { Prompt: autocert.AcceptTOS, HostPolicy: autocert.HostWhitelist([]string{e.cfg.FQDN}...), } - e.pubSrv.TLSConfig = certManager.TLSConfig() + e.extPubSrv.TLSConfig = certManager.TLSConfig() go func() { var rawData []byte diff --git a/handlers.go b/handlers.go index d054438..db6b6bb 100644 --- a/handlers.go +++ b/handlers.go @@ -225,3 +225,11 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl fmt.Fprintln(w, b64Doc) } } + +func leaderHandler(cfg *Config) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + cfg.IsLeader = true + elog.Println("Designated enclave as leader.") + w.WriteHeader(http.StatusOK) + } +} diff --git a/handlers_test.go b/handlers_test.go index 2f8bfb2..48453a3 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -79,7 +79,7 @@ func TestRootHandler(t *testing.T) { // instructing it to spin up its Internet-facing Web server. func signalReady(t *testing.T, e *Enclave) { t.Helper() - makeReq := makeRequestFor(e.privSrv) + makeReq := makeRequestFor(e.intSrv) assertResponse(t, makeReq(http.MethodGet, pathReady, nil), @@ -94,7 +94,7 @@ func signalReady(t *testing.T, e *Enclave) { } func TestSyncHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).privSrv) + makeReq := makeRequestFor(createEnclave(&defaultCfg).intSrv) assertResponse(t, makeReq(http.MethodGet, pathSync, nil), @@ -113,7 +113,7 @@ func TestSyncHandler(t *testing.T) { } func TestStateHandlers(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).privSrv) + makeReq := makeRequestFor(createEnclave(&defaultCfg).intSrv) tooLargeKey := make([]byte, 1024*1024+1) assertResponse(t, @@ -194,7 +194,7 @@ func TestHashHandler(t *testing.T) { validHash := [sha256.Size]byte{} validHashB64 := base64.StdEncoding.EncodeToString(validHash[:]) e := createEnclave(&defaultCfg) - makeReq := makeRequestFor(e.privSrv) + makeReq := makeRequestFor(e.intSrv) // Send invalid Base64. assertResponse(t, diff --git a/main.go b/main.go index 5168d90..a701786 100644 --- a/main.go +++ b/main.go @@ -33,13 +33,15 @@ func init() { } func main() { - var fqdn, appURL, appWebSrv, appCmd, prometheusNamespace, mockCertFp string - var extPort, intPort, hostProxyPort, prometheusPort uint + var fqdn, fqdnLeader, appURL, appWebSrv, appCmd, prometheusNamespace, mockCertFp string + var extPubPort, extPrivPort, intPort, hostProxyPort, prometheusPort uint var useACME, waitForApp, useProfiling, useVsockForExtPort, disableKeepAlives, debug bool var err error flag.StringVar(&fqdn, "fqdn", "", "FQDN of the enclave application (e.g., \"example.com\").") + flag.StringVar(&fqdnLeader, "fqdn-leader", "", + "FQDN of the leader enclave (e.g., \"leader.example.com\").") flag.StringVar(&appURL, "appurl", "", "Code repository of the enclave application (e.g., \"github.com/foo/bar\").") flag.StringVar(&appWebSrv, "appwebsrv", "", @@ -48,8 +50,10 @@ func main() { "Launch enclave application via the given command.") flag.StringVar(&prometheusNamespace, "prometheus-namespace", "", "Prometheus namespace for exported metrics.") - flag.UintVar(&extPort, "extport", 443, - "Nitriding's HTTPS port. Must match port forwarding rules on EC2 host.") + flag.UintVar(&extPubPort, "ext-pub-port", 443, + "Nitriding's external, public HTTPS port. Must match port forwarding rules on EC2 host.") + flag.UintVar(&extPrivPort, "ext-priv-port", 444, + "Nitriding's external, non-public HTTPS port. Must match port forwarding rules on the EC2 host.") flag.BoolVar(&disableKeepAlives, "disable-keep-alives", false, "Disables keep-alive connections for the HTTPS service.") flag.BoolVar(&useVsockForExtPort, "vsock-ext", false, @@ -75,9 +79,12 @@ func main() { if fqdn == "" { elog.Fatalf("-fqdn must be set.") } - if extPort < 1 || extPort > math.MaxUint16 { + if extPubPort < 1 || extPubPort > math.MaxUint16 { elog.Fatalf("-extport must be in interval [1, %d]", math.MaxUint16) } + if extPrivPort < 1 || extPrivPort > math.MaxUint16 { + elog.Fatalf("-extPrivPort must be in interval [1, %d]", math.MaxUint16) + } if intPort < 1 || intPort > math.MaxUint16 { elog.Fatalf("-intport must be in interval [1, %d]", math.MaxUint16) } @@ -93,10 +100,11 @@ func main() { c := &Config{ FQDN: fqdn, - ExtPort: uint16(extPort), + ExtPubPort: uint16(extPubPort), + ExtPrivPort: uint16(extPrivPort), + IntPort: uint16(intPort), UseVsockForExtPort: useVsockForExtPort, DisableKeepAlives: disableKeepAlives, - IntPort: uint16(intPort), PrometheusPort: uint16(prometheusPort), PrometheusNamespace: prometheusNamespace, HostProxyPort: uint32(hostProxyPort), diff --git a/metrics_test.go b/metrics_test.go index 13e4351..299468c 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -50,7 +50,7 @@ func TestHandlerMetrics(t *testing.T) { ), float64(1)) // POST /enclave/hash - makeReq = makeRequestFor(enclave.privSrv) + makeReq = makeRequestFor(enclave.intSrv) assertResponse(t, makeReq(http.MethodPost, pathHash, bytes.NewBufferString("foo")), newResp(http.StatusBadRequest, errNoBase64.Error()), From 718214e4c39275dcb12536ef11170d49d0456a49 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 4 Aug 2023 15:33:36 -0500 Subject: [PATCH 02/99] Implement mechanism to register worker enclaves. --- enclave.go | 5 ++++- handlers.go | 46 ++++++++++++++++++++++++++++++++++++++++++++-- workers.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 workers.go diff --git a/enclave.go b/enclave.go index 1a1d195..259fa6b 100644 --- a/enclave.go +++ b/enclave.go @@ -51,6 +51,7 @@ const ( pathProfiling = "/enclave/debug" pathConfig = "/enclave/config" pathLeader = "/enclave/leader" + pathRegistation = "/enclave/registration" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" @@ -74,6 +75,7 @@ type Enclave struct { hashes *AttestationHashes promRegistry *prometheus.Registry metrics *metrics + workers workers nonceCache *cache keyMaterial any ready, stop chan bool @@ -238,6 +240,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { metrics: newMetrics(reg, cfg.PrometheusNamespace), nonceCache: newCache(defaultItemExpiry), hashes: new(AttestationHashes), + workers: workers{}, stop: make(chan bool), ready: make(chan bool), } @@ -276,7 +279,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) - m.Get(pathLeader, leaderHandler(e.cfg)) + m.Get(pathLeader, leaderHandler(e)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) diff --git a/handlers.go b/handlers.go index db6b6bb..4fe4216 100644 --- a/handlers.go +++ b/handlers.go @@ -4,15 +4,21 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" + + "github.com/go-chi/chi/v5" ) const ( + // The maximum length of a worker registration, which is essentially just a + // URL. + maxRegistrationLen = 1024 // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 @@ -121,6 +127,8 @@ func putStateHandler(e *Enclave) http.HandlerFunc { } e.SetKeyMaterial(body) w.WriteHeader(http.StatusOK) + + // TODO: The leader must push new keys to the workers. } } @@ -226,10 +234,44 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl } } -func leaderHandler(cfg *Config) http.HandlerFunc { +func leaderHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - cfg.IsLeader = true + e.cfg.IsLeader = true elog.Println("Designated enclave as leader.") + + e.extPrivSrv.Handler.(*chi.Mux).Put(pathRegistation, workerRegistrationHandler(e.workers, e.keyMaterial)) + elog.Println("Set up worker registration endpoint.") + w.WriteHeader(http.StatusOK) } } + +func workerRegistrationHandler(ws workers, keyMaterial any) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Extract the worker's endpoint for the pushing of key material. + body, err := io.ReadAll(newLimitReader(r.Body, maxRegistrationLen)) + if err != nil { + http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + return + } + // We expect enclaves to PUT a JSON body that contains a "url". + registration := struct { + URL string `json:"url"` + }{} + if err = json.Unmarshal(body, ®istration); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + u, err := url.Parse(registration.URL) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + + go func() { + ws.registerAndPush(&worker{URL: u}, keyMaterial) + elog.Printf("New worker registered; now managing:\n%s", ws.String()) + }() + } +} diff --git a/workers.go b/workers.go new file mode 100644 index 0000000..3e890d9 --- /dev/null +++ b/workers.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "net/url" +) + +// worker represents a worker enclave. The leader enclave is responsible for +// managing key material (for nitriding itself and the enclave application) and +// pushes this key material to worker enclaves. +type worker struct { + URL *url.URL +} + +func (w *worker) String() string { + return w.URL.String() +} + +// workers represents a set of worker enclaves. +type workers []*worker + +func (ws workers) push(keyMaterial interface{}) error { + // TODO: Contact worker enclave and sync key material. + return nil +} + +func (ws workers) pushTo(w *worker, keyMaterial any) error { + // TODO: Contact worker enclave and sync key material. + return nil +} + +func (ws *workers) register(w *worker) { + *ws = append(*ws, w) +} + +func (ws *workers) registerAndPush(w *worker, keyMaterial any) { + ws.register(w) + elog.Printf("Registered new worker enclave %s.", w) + ws.pushTo(w, keyMaterial) + elog.Printf("Pushed key material to new worker enclave %s.", w) +} + +func (ws *workers) unregister(w *worker) { + // TODO: Unregister worker enclave. +} + +func (ws workers) String() string { + var s string + for i, w := range ws { + s += fmt.Sprintf("%2d: %s\n", i, w) + } + return s +} From b19e1f7f87ce78a8df1d28c0906900781986c2a5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 8 Aug 2023 11:47:41 -0500 Subject: [PATCH 03/99] Add handlers for key (re-)synchronization. --- attester.go | 173 ++++++++++++++++++++++++++++++++++++ certcache.go | 28 ++++++ enclave.go | 203 +++++++++++++++++++++++++++++++++++-------- handlers.go | 164 ++++++++++++++++++++++++++++++---- keysync_responder.go | 2 +- main.go | 9 ++ workers.go | 56 ++++-------- 7 files changed, 544 insertions(+), 91 deletions(-) create mode 100644 attester.go diff --git a/attester.go b/attester.go new file mode 100644 index 0000000..19bf117 --- /dev/null +++ b/attester.go @@ -0,0 +1,173 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "github.com/hf/nitrite" + "github.com/hf/nsm" + "github.com/hf/nsm/request" +) + +// attester defines functions for the creation and verification of attestation +// documents. Making this an interface helps with testing: It allows us to +// implement a dummy attester that works without the AWS Nitro hypervisor. +type attester interface { + createAttstn(auxInfo) ([]byte, error) + verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) +} + +type auxInfo interface{} + +// workerAuxInfo holds the auxiliary information of the worker's attestation +// document. +type workerAuxInfo struct { + WorkersNonce nonce `json:"workers_nonce"` + LeadersNonce nonce `json:"leaders_nonce"` + PublicKey []byte `json:"public_key"` +} + +func (w workerAuxInfo) String() string { + return fmt.Sprintf("Worker's auxiliary info:\n"+ + "Worker's nonce: %x\nLeader's nonce: %x\nPublic key: %x", + w.WorkersNonce, w.LeadersNonce, w.PublicKey) +} + +// leaderAuxInfo holds the auxiliary information of the leader's attestation +// document. +type leaderAuxInfo struct { + WorkersNonce nonce `json:"workers_nonce"` + EnclaveKeys []byte `json:"enclave_keys"` +} + +func (l leaderAuxInfo) String() string { + return fmt.Sprintf("Leader's auxiliary info:\n"+ + "Worker's nonce: %x\nEnclave keys: %x", + l.WorkersNonce, l.EnclaveKeys) +} + +// dummyAttester helps with local testing. The interface simply turns +// auxiliary information into JSON, and does not do any cryptography. +type dummyAttester struct{} + +func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) { + return json.Marshal(aux) +} + +func (*dummyAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) { + var w workerAuxInfo + var l leaderAuxInfo + + // First, assume we're dealing with a worker's auxiliary information. + if err := json.Unmarshal(doc, &w); err != nil { + return nil, err + } + if len(w.WorkersNonce) == nonceLen && len(w.LeadersNonce) == nonceLen && w.PublicKey != nil { + if !isOurNonce(w.LeadersNonce.B64()) { + return nil, errors.New("leader nonce not in cache") + } + elog.Println(w) + return &w, nil + } + + // Next, let's assume it's a leader. + if err := json.Unmarshal(doc, &l); err != nil { + return nil, err + } + if len(l.WorkersNonce) == nonceLen && l.EnclaveKeys != nil { + if !isOurNonce(l.WorkersNonce.B64()) { + return nil, errors.New("worker nonce not in cache") + } + elog.Println(l) + return &l, nil + } + + return nil, errors.New("invalid auxiliary information") +} + +// nitroAttester implements production functions for the creation and +// verification of attestation documents. +type nitroAttester struct{} + +func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { + var nonce, userData, publicKey []byte + + // Prepare our auxiliary information. + switch v := aux.(type) { + case workerAuxInfo: + nonce = v.WorkersNonce[:] + userData = v.LeadersNonce[:] + publicKey = v.PublicKey + case leaderAuxInfo: + nonce = v.WorkersNonce[:] + userData = v.EnclaveKeys + } + + s, err := nsm.OpenDefaultSession() + if err != nil { + return nil, err + } + defer func() { + if err = s.Close(); err != nil { + elog.Printf("Attestation: Failed to close default NSM session: %s", err) + } + }() + + res, err := s.Send(&request.Attestation{ + Nonce: nonce, + UserData: userData, + PublicKey: publicKey, + }) + if err != nil { + return nil, err + } + if res.Attestation == nil || res.Attestation.Document == nil { + return nil, errors.New("NSM device did not return an attestation") + } + + return res.Attestation.Document, nil +} + +func (*nitroAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) { + errStr := "error verifying attestation document" + // Verify the remote enclave's attestation document before doing anything + // with it. + opts := nitrite.VerifyOptions{CurrentTime: currentTime()} + their, err := nitrite.Verify(doc, opts) + if err != nil { + return nil, fmt.Errorf("%s: %w", errStr, err) + } + + // Verify that the remote enclave's PCR values (e.g., the image ID) are + // identical to ours. + ourPCRs, err := getPCRValues() + if err != nil { + return nil, fmt.Errorf("%s: %w", errStr, err) + } + if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { + return nil, fmt.Errorf("%s: PCR values of remote enclave not identical to ours", errStr) + } + + // Verify that the remote enclave's attestation document contains the nonce + // that we asked it to embed. + b64Nonce := base64.StdEncoding.EncodeToString(their.Document.Nonce) + if !isOurNonce(b64Nonce) { + return nil, fmt.Errorf("%s: nonce %s not in cache", errStr, b64Nonce) + } + + // If the "public key" field is unset, we know that we're dealing with a + // worker's auxiliary information. + if their.Document.PublicKey != nil { + return &workerAuxInfo{ + WorkersNonce: nonce(their.Document.Nonce), + LeadersNonce: nonce(their.Document.UserData), + PublicKey: their.Document.PublicKey, + }, nil + } + return &leaderAuxInfo{ + WorkersNonce: nonce(their.Document.Nonce), + EnclaveKeys: their.Document.UserData, + }, nil +} diff --git a/certcache.go b/certcache.go index 3de03db..a871935 100644 --- a/certcache.go +++ b/certcache.go @@ -2,11 +2,39 @@ package main import ( "context" + "crypto/tls" + "errors" "sync" "golang.org/x/crypto/acme/autocert" ) +// certRetriever stores an HTTPS certificate and implements the GetCertificate +// function signature, which allows our Web servers to retrieve the +// certificate when clients connect: +// https://pkg.go.dev/crypto/tls#Config +type certRetriever struct { + sync.RWMutex + cert *tls.Certificate +} + +func (c *certRetriever) set(cert *tls.Certificate) { + c.Lock() + defer c.Unlock() + + c.cert = cert +} + +func (c *certRetriever) get(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + c.RLock() + defer c.RUnlock() + + if c.cert == nil { + return nil, errors.New("certificate not yet initialized") + } + return c.cert, nil +} + // certCache implements the autocert.Cache interface. type certCache struct { sync.RWMutex diff --git a/enclave.go b/enclave.go index 259fa6b..cf0510c 100644 --- a/enclave.go +++ b/enclave.go @@ -5,21 +5,25 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + cryptoRand "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" + "io" "math/big" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "net/url" + "strings" "sync" "time" @@ -30,8 +34,14 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/crypto/acme/autocert" + "golang.org/x/crypto/nacl/box" ) +// TODO: Support Let's Encrypt (if we choose to). +// TODO: Remove defunct workers from worker set. +// TODO: Handle key sync error edge cases. +// TODO: Handle the case of the leader restarting. Workers must not break. + const ( acmeCertCacheDir = "cert-cache" certificateOrg = "AWS Nitro enclave application" @@ -41,17 +51,17 @@ const ( // https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-concepts.html parentCID = 3 // The following paths are handled by nitriding. - pathRoot = "/enclave" - pathNonce = "/enclave/nonce" - pathAttestation = "/enclave/attestation" - pathState = "/enclave/state" - pathSync = "/enclave/sync" - pathHash = "/enclave/hash" - pathReady = "/enclave/ready" - pathProfiling = "/enclave/debug" - pathConfig = "/enclave/config" - pathLeader = "/enclave/leader" - pathRegistation = "/enclave/registration" + pathRoot = "/enclave" + pathNonce = "/enclave/nonce" + pathAttestation = "/enclave/attestation" + pathState = "/enclave/state" + pathSync = "/enclave/sync" + pathHash = "/enclave/hash" + pathReady = "/enclave/ready" + pathProfiling = "/enclave/debug" + pathConfig = "/enclave/config" + pathLeader = "/enclave/leader" + pathRegistration = "/enclave/registration" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" @@ -63,22 +73,35 @@ var ( errCfgMissingPort = errors.New("given config is missing port") ) +// enclaveKeys holds key material for nitriding itself (the HTTPS certificate) +// and for the enclave application (whatever the application wants to "store" +// in nitriding). All these keys are meant to be managed by a leader enclave +// and -- if horizontal scaling is required -- synced to worker enclaves. +type enclaveKeys struct { + NitridingKey []byte `json:"nitriding_key"` + NitridingCert []byte `json:"nitriding_cert"` + AppKeys []byte `json:"app_keys"` +} + // Enclave represents a service running inside an AWS Nitro Enclave. type Enclave struct { sync.RWMutex - cfg *Config - extPubSrv *http.Server - extPrivSrv *http.Server - intSrv *http.Server - promSrv *http.Server - revProxy *httputil.ReverseProxy - hashes *AttestationHashes - promRegistry *prometheus.Registry - metrics *metrics - workers workers - nonceCache *cache - keyMaterial any - ready, stop chan bool + attester + cfg *Config + extPubSrv *http.Server + extPrivSrv *http.Server + intSrv *http.Server + promSrv *http.Server + revProxy *httputil.ReverseProxy + hashes *AttestationHashes + promRegistry *prometheus.Registry + metrics *metrics + workers *workers + nonceCache *cache + keys *enclaveKeys + ephemeralSyncKeys *boxKey // TODO + ready, stop chan bool + httpsCert *certRetriever } // Config represents the configuration of our enclave service. @@ -220,7 +243,8 @@ func NewEnclave(cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ - cfg: cfg, + attester: &dummyAttester{}, + cfg: cfg, extPubSrv: &http.Server{ Handler: chi.NewRouter(), }, @@ -236,11 +260,13 @@ func NewEnclave(cfg *Config) (*Enclave, error) { Addr: fmt.Sprintf(":%d", cfg.PrometheusPort), Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), }, + httpsCert: &certRetriever{}, + keys: &enclaveKeys{}, promRegistry: reg, metrics: newMetrics(reg, cfg.PrometheusNamespace), nonceCache: newCache(defaultItemExpiry), hashes: new(AttestationHashes), - workers: workers{}, + workers: newWorkers(), stop: make(chan bool), ready: make(chan bool), } @@ -274,16 +300,16 @@ func NewEnclave(cfg *Config) (*Enclave, error) { m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes)) m.Get(pathNonce, nonceHandler(e)) m.Get(pathRoot, rootHandler(e.cfg)) - m.Post(pathSync, respSyncHandler(e)) m.Get(pathConfig, configHandler(e.cfg)) // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(e)) + m.Get(pathSync, initSyncHandler(e)) + m.Post(pathSync, finishSyncHandler(e)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) - m.Get(pathSync, reqSyncHandler(e)) m.Get(pathReady, readyHandler(e)) m.Get(pathState, getStateHandler(e)) m.Put(pathState, putStateHandler(e)) @@ -340,6 +366,24 @@ func (e *Enclave) Start() error { return fmt.Errorf("%s: %w", errPrefix, err) } + if e.cfg.FQDNLeader == "" { + return errors.New("leader enclave's FQDN is unset") + } + // Prepare the leader's registration URL and register the worker. At the + // time of making this request, the leader is still untrusted but that's + // fine. The sensitive part (i.e., key synchronization) happens in a + // subsequent step, and is secured by mutual attestation document + // verification. + // TODO: Is this where/how we want to initiate the process? + // TODO: Ideally, the leader enclave would not call this function. + if err := e.syncWithLeader(&url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, 8444), // TODO: Use e.cfg.ExtPrivPort. + Path: pathRegistration, + }); err != nil { + elog.Printf("Error syncing with leader: %v", err) + } + return nil } @@ -472,14 +516,16 @@ func (e *Enclave) genSelfSignedCert() error { if pemKey == nil { elog.Fatal("Failed to encode key to PEM.") } + e.keys.NitridingKey = pemKey + e.keys.NitridingCert = pemCert cert, err := tls.X509KeyPair(pemCert, pemKey) if err != nil { return err } - + e.httpsCert.set(&cert) e.extPubSrv.TLSConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, + GetCertificate: e.httpsCert.get, } e.extPrivSrv.TLSConfig = e.extPubSrv.TLSConfig // Both servers share a TLS config. @@ -576,11 +622,11 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { // // This is only necessary if you intend to scale enclaves horizontally. If you // will only ever run a single enclave, ignore this function. -func (e *Enclave) SetKeyMaterial(keyMaterial any) { +func (e *Enclave) SetKeyMaterial(appKeys []byte) { e.Lock() defer e.Unlock() - e.keyMaterial = keyMaterial + e.keys.AppKeys = appKeys } // KeyMaterial returns the key material or, if none was registered, an error. @@ -588,8 +634,97 @@ func (e *Enclave) KeyMaterial() (any, error) { e.RLock() defer e.RUnlock() - if e.keyMaterial == nil { + if e.keys.AppKeys == nil { return nil, errNoKeyMaterial } - return e.keyMaterial, nil + return e.keys.AppKeys, nil +} + +// syncWithLeader is called by worker enclaves to initiate the process of key +// synchronization. +func (e *Enclave) syncWithLeader(leader *url.URL) error { + resp, err := http.Post(leader.String(), "text/plain", nil) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + return nil +} + +// syncWithWorker is called by the leader enclave right after a worker +// registers itself with the leader enclave. +func (e *Enclave) syncWithWorker(worker *url.URL) error { + elog.Println("Initiating key synchronization with worker.") + + // Step 1: Create a nonce that the worker must embed in its attestation + // document to prevent replay attacks. + nonce, err := newNonce() + if err != nil { + return err + } + e.nonceCache.Add(nonce.B64()) + + // Step 2: Request the worker's attestation document, and provide the + // previously-generated nonce. + reqURL := *worker + reqURL.RawQuery = fmt.Sprintf("nonce=%x", nonce) + resp, err := http.Get(reqURL.String()) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + // Step 3: Verify the worker's attestation document and extract its + // auxiliary information. + b64Attstn, err := io.ReadAll(newLimitReader(resp.Body, maxAttDocLen)) + resp.Body.Close() + attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + if err != nil { + return err + } + workerAux, err := e.verifyAttstn(attstn, e.nonceCache.Exists) + if err != nil { + return err + } + + // Step 4: Encrypt the leader's enclave keys with the ephemeral public key + // that the worker put into its auxiliary information. + pubKey := &[boxKeyLen]byte{} + copy(pubKey[:], workerAux.(*workerAuxInfo).PublicKey[:]) + jsonKeys, err := json.Marshal(e.keys) + if err != nil { + return err + } + var encrypted []byte + encrypted, err = box.SealAnonymous(nil, jsonKeys, pubKey, cryptoRand.Reader) + if err != nil { + return err + } + + // Step 5: Create the leader's auxiliary information, consisting of the + // worker's nonce and the encrypted enclave keys. + leaderAux := &leaderAuxInfo{ + WorkersNonce: workerAux.(*workerAuxInfo).WorkersNonce, + EnclaveKeys: encrypted, + } + attstn, err = e.createAttstn(leaderAux) + if err != nil { + return err + } + strAttstn := base64.StdEncoding.EncodeToString(attstn) + + // Step 6: Send the leader's attestation document to the worker. + resp, err = http.Post(worker.String(), "text/plain", strings.NewReader(strAttstn)) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + return nil } diff --git a/handlers.go b/handlers.go index 4fe4216..5390564 100644 --- a/handlers.go +++ b/handlers.go @@ -2,17 +2,20 @@ package main import ( "crypto/sha256" + "crypto/tls" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "io" + "net" "net/http" "net/url" "strings" "github.com/go-chi/chi/v5" + "golang.org/x/crypto/nacl/box" ) const ( @@ -34,6 +37,10 @@ var ( errHashWrongSize = errors.New("given hash is of invalid size") ) +func errNo200(code int) error { + return fmt.Errorf("peer responded with HTTP code %d", code) +} + func formatIndexPage(appURL *url.URL) string { page := indexPage if appURL != nil { @@ -128,7 +135,18 @@ func putStateHandler(e *Enclave) http.HandlerFunc { e.SetKeyMaterial(body) w.WriteHeader(http.StatusOK) - // TODO: The leader must push new keys to the workers. + // The leader's keys have changed. Re-synchronize the key material + // with all registered workers. + go func() { + for w := range e.workers.set { + if err := e.syncWithWorker(&w); err != nil { + // TODO: Failure to sync means that the enclave set is in + // an inconsistent state. We should log this in + // Prometheus. + elog.Printf("Error syncing with worker: %v", err) + } + } + }() } } @@ -239,39 +257,149 @@ func leaderHandler(e *Enclave) http.HandlerFunc { e.cfg.IsLeader = true elog.Println("Designated enclave as leader.") - e.extPrivSrv.Handler.(*chi.Mux).Put(pathRegistation, workerRegistrationHandler(e.workers, e.keyMaterial)) + e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) elog.Println("Set up worker registration endpoint.") w.WriteHeader(http.StatusOK) } } -func workerRegistrationHandler(ws workers, keyMaterial any) http.HandlerFunc { +func workerRegistrationHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Extract the worker's endpoint for the pushing of key material. - body, err := io.ReadAll(newLimitReader(r.Body, maxRegistrationLen)) + // Go's HTTP server sets RemoteAddr to IP:port: + // https://pkg.go.dev/net/http#Request + strIP, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + http.Error(w, "error extracting IP address", http.StatusInternalServerError) return } - // We expect enclaves to PUT a JSON body that contains a "url". - registration := struct { - URL string `json:"url"` - }{} - if err = json.Unmarshal(body, ®istration); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + workerURL := &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", strIP, 9444), // TODO: Use e.cfg.ExtPrivPort. + Path: pathSync, + } + w.WriteHeader(http.StatusOK) + + go func() { + if err := e.syncWithWorker(workerURL); err != nil { + elog.Printf("Error syncing with worker %s: %v", workerURL, err) + return + } + e.workers.register(workerURL) + elog.Printf("Successfully registered and synced with worker %s.", workerURL) + }() + } +} + +const errNonceRequired = "nonce is required" + +func initSyncHandler(e *Enclave) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + elog.Println("New request to init key sync.") + + // Extract the leader's nonce from the URL. + hexNonce := r.URL.Query().Get("nonce") + if hexNonce == "" { + http.Error(w, errNonceRequired, http.StatusBadRequest) return } - u, err := url.Parse(registration.URL) + + nonceSlice, err := hex.DecodeString(hexNonce) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - w.WriteHeader(http.StatusOK) + if len(nonceSlice) != nonceLen { + http.Error(w, "invalid nonce length", http.StatusBadRequest) + return + } + var leadersNonce nonce + copy(leadersNonce[:], nonceSlice) - go func() { - ws.registerAndPush(&worker{URL: u}, keyMaterial) - elog.Printf("New worker registered; now managing:\n%s", ws.String()) - }() + // Create the worker's nonce and add it to our nonce cache, so it can + // later be verified. + workersNonce, err := newNonce() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + e.nonceCache.Add(workersNonce.B64()) + + // Create an ephemeral key that the leader is going to use to encrypt + // its enclave keys. + boxKey, err := newBoxKey() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + e.ephemeralSyncKeys = boxKey // TODO: Could be more elegant. + + // Create and return the worker's Base64-encoded attestation document. + aux := &workerAuxInfo{ + WorkersNonce: workersNonce, + LeadersNonce: leadersNonce, + PublicKey: boxKey.pubKey[:], + } + attstn, err := e.createAttstn(aux) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + fmt.Fprintln(w, base64.StdEncoding.EncodeToString(attstn)) + } +} + +// finishSyncHandler is called by the leader to finish key synchronization. +func finishSyncHandler(e *Enclave) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + elog.Println("New request to finish key sync.") + + // Read the leader's Base64-encoded attestation document. + maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) + b64Attstn, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) + if err != nil { + elog.Printf("Failed to read http body: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + elog.Printf("Leader's attstn doc: %v", string(b64Attstn)) + + // Decode Base64 to byte slice. + attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + aux, err := e.verifyAttstn(attstn, e.nonceCache.Exists) + + // Decrypt the leader's enclave keys, which are encrypted with the + // public key that we provided earlier. + decrypted, ok := box.OpenAnonymous( + nil, + aux.(*leaderAuxInfo).EnclaveKeys, + e.ephemeralSyncKeys.pubKey, + e.ephemeralSyncKeys.privKey) + if !ok { + http.Error(w, "error decrypting enclave keys", http.StatusBadRequest) + return + } + e.ephemeralSyncKeys = nil // Clear the ephemeral key material. + + // Set the leader's enclave keys. + var keys enclaveKeys + if err := json.Unmarshal(decrypted, &keys); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + e.httpsCert.set(&cert) + + elog.Printf("Leader's enclave keys: %s (%s)", string(decrypted), decrypted) } } diff --git a/keysync_responder.go b/keysync_responder.go index 186ee53..c199119 100644 --- a/keysync_responder.go +++ b/keysync_responder.go @@ -98,7 +98,7 @@ func respSyncHandler(e *Enclave) http.HandlerFunc { copy(theirBoxPubKey[:], theirAttDoc.PublicKey[:]) // Encrypt our key material with the provided key. - jsonKeyMaterial, err := json.Marshal(e.keyMaterial) + jsonKeyMaterial, err := json.Marshal(e.keys) if err != nil { http.Error(w, "failed to marshal key material", http.StatusInternalServerError) return diff --git a/main.go b/main.go index a701786..6e84e66 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,13 @@ package main import ( "bufio" + "crypto/tls" "errors" "flag" "io" "log" "math" + "net/http" "net/url" "os" "os/exec" @@ -98,8 +100,15 @@ func main() { elog.Fatalf("-prometheus-namespace must be set when Prometheus is used.") } + // TODO: If we choose to abandon Let's Encrypt, we need to tell Go to + // forego certificate validation. We shouldn't do this globally though. + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + c := &Config{ FQDN: fqdn, + FQDNLeader: fqdnLeader, ExtPubPort: uint16(extPubPort), ExtPrivPort: uint16(extPrivPort), IntPort: uint16(intPort), diff --git a/workers.go b/workers.go index 3e890d9..289e977 100644 --- a/workers.go +++ b/workers.go @@ -1,53 +1,33 @@ package main import ( - "fmt" "net/url" + "sync" ) -// worker represents a worker enclave. The leader enclave is responsible for -// managing key material (for nitriding itself and the enclave application) and -// pushes this key material to worker enclaves. -type worker struct { - URL *url.URL +// workers represents a set of worker enclaves. The leader enclave keeps track +// of workers. +type workers struct { + sync.RWMutex + set map[url.URL]struct{} } -func (w *worker) String() string { - return w.URL.String() -} - -// workers represents a set of worker enclaves. -type workers []*worker - -func (ws workers) push(keyMaterial interface{}) error { - // TODO: Contact worker enclave and sync key material. - return nil -} - -func (ws workers) pushTo(w *worker, keyMaterial any) error { - // TODO: Contact worker enclave and sync key material. - return nil +func newWorkers() *workers { + return &workers{ + set: make(map[url.URL]struct{}), + } } -func (ws *workers) register(w *worker) { - *ws = append(*ws, w) -} +func (w *workers) register(u *url.URL) { + w.Lock() + defer w.Unlock() -func (ws *workers) registerAndPush(w *worker, keyMaterial any) { - ws.register(w) - elog.Printf("Registered new worker enclave %s.", w) - ws.pushTo(w, keyMaterial) - elog.Printf("Pushed key material to new worker enclave %s.", w) + w.set[*u] = struct{}{} } -func (ws *workers) unregister(w *worker) { - // TODO: Unregister worker enclave. -} +func (w *workers) unregister(u *url.URL) { + w.Lock() + defer w.Unlock() -func (ws workers) String() string { - var s string - for i, w := range ws { - s += fmt.Sprintf("%2d: %s\n", i, w) - } - return s + delete(w.set, *u) } From e724427a92a3fa7a6b088e07c4c528717b574e6e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 11 Aug 2023 10:50:26 -0500 Subject: [PATCH 04/99] Improve key sync protocol. --- enclave.go | 136 +++++++++++++++++++++++++++++++--------------------- handlers.go | 65 +++++++++++++++---------- main.go | 2 +- workers.go | 49 ++++++++++++++++--- 4 files changed, 163 insertions(+), 89 deletions(-) diff --git a/enclave.go b/enclave.go index cf0510c..a8110bc 100644 --- a/enclave.go +++ b/enclave.go @@ -38,7 +38,6 @@ import ( ) // TODO: Support Let's Encrypt (if we choose to). -// TODO: Remove defunct workers from worker set. // TODO: Handle key sync error edge cases. // TODO: Handle the case of the leader restarting. Workers must not break. @@ -78,30 +77,39 @@ var ( // in nitriding). All these keys are meant to be managed by a leader enclave // and -- if horizontal scaling is required -- synced to worker enclaves. type enclaveKeys struct { + // TODO: Add another field that contains high-entropy bytes. NitridingKey []byte `json:"nitriding_key"` NitridingCert []byte `json:"nitriding_cert"` AppKeys []byte `json:"app_keys"` } +// hashAndB64 returns the Base64-encoded hash over our key material. The +// resulting string is not confidential as it's impractical to reverse the key +// material. +func (e *enclaveKeys) hashAndB64() string { + keys := append(append(e.NitridingCert, e.NitridingKey...), e.AppKeys...) + hash := sha256.Sum256(keys) + return base64.StdEncoding.EncodeToString(hash[:]) +} + // Enclave represents a service running inside an AWS Nitro Enclave. type Enclave struct { sync.RWMutex attester - cfg *Config - extPubSrv *http.Server - extPrivSrv *http.Server - intSrv *http.Server - promSrv *http.Server - revProxy *httputil.ReverseProxy - hashes *AttestationHashes - promRegistry *prometheus.Registry - metrics *metrics - workers *workers - nonceCache *cache - keys *enclaveKeys - ephemeralSyncKeys *boxKey // TODO - ready, stop chan bool - httpsCert *certRetriever + cfg *Config + extPubSrv, extPrivSrv *http.Server + intSrv *http.Server + promSrv *http.Server + revProxy *httputil.ReverseProxy + hashes *AttestationHashes + promRegistry *prometheus.Registry + metrics *metrics + workers *workers + nonceCache *cache + keys *enclaveKeys + ephemeralSyncKeys *boxKey // TODO + ready, stop, isLeader chan bool + httpsCert *certRetriever } // Config represents the configuration of our enclave service. @@ -116,11 +124,6 @@ type Config struct { // if horizontal scaling is required. FQDNLeader string - // IsLeader will be set to true if the enclave is designated as leader for - // enclave synchronization. Leader designation happens at runtime, by - // calling an HTTP handler. - IsLeader bool - // ExtPubPort contains the TCP port that the public Web server should // listen on, e.g. 443. This port is not *directly* reachable by the // Internet but the EC2 host's proxy *does* forward Internet traffic to @@ -226,6 +229,12 @@ func (c *Config) Validate() error { return nil } +// isScalingEnabled returns true if horizontal enclave scaling is enabled in our +// enclave configuration. +func (c *Config) isScalingEnabled() bool { + return c.FQDNLeader != "" +} + // String returns a string representation of the enclave's configuration. func (c *Config) String() string { s, err := json.MarshalIndent(c, "", " ") @@ -266,9 +275,10 @@ func NewEnclave(cfg *Config) (*Enclave, error) { metrics: newMetrics(reg, cfg.PrometheusNamespace), nonceCache: newCache(defaultItemExpiry), hashes: new(AttestationHashes), - workers: newWorkers(), + workers: newWorkers(time.Minute), stop: make(chan bool), ready: make(chan bool), + isLeader: make(chan bool), } // Increase the maximum number of idle connections per host. This is @@ -366,22 +376,16 @@ func (e *Enclave) Start() error { return fmt.Errorf("%s: %w", errPrefix, err) } - if e.cfg.FQDNLeader == "" { - return errors.New("leader enclave's FQDN is unset") - } - // Prepare the leader's registration URL and register the worker. At the - // time of making this request, the leader is still untrusted but that's - // fine. The sensitive part (i.e., key synchronization) happens in a - // subsequent step, and is secured by mutual attestation document - // verification. - // TODO: Is this where/how we want to initiate the process? // TODO: Ideally, the leader enclave would not call this function. - if err := e.syncWithLeader(&url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, 8444), // TODO: Use e.cfg.ExtPrivPort. - Path: pathRegistration, - }); err != nil { - elog.Printf("Error syncing with leader: %v", err) + if e.cfg.isScalingEnabled() { + if err := e.syncWithLeader(&url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), + Path: pathRegistration, + }); err != nil { + elog.Fatalf("Error syncing with leader: %v", err) + } + elog.Println("Successfully synced with leader.") } return nil @@ -396,6 +400,9 @@ func (e *Enclave) Stop() error { if err := e.extPubSrv.Shutdown(context.Background()); err != nil { return err } + if err := e.extPrivSrv.Shutdown(context.Background()); err != nil { + return err + } if err := e.promSrv.Shutdown(context.Background()); err != nil { return err } @@ -615,22 +622,20 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { return nil } -// SetKeyMaterial registers the enclave's key material (e.g., secret encryption -// keys) as being ready to be synchronized to other, identical enclaves. Note -// that the key material's underlying data structure must be marshallable to -// JSON. -// -// This is only necessary if you intend to scale enclaves horizontally. If you -// will only ever run a single enclave, ignore this function. -func (e *Enclave) SetKeyMaterial(appKeys []byte) { +// SetAppKeys registers the enclave application's key material (e.g., secret +// encryption keys, or what have you), so it can be synchronized with worker +// enclaves. This is only necessary if horizontal enclave scaling is required. +func (e *Enclave) SetAppKeys(appKeys []byte) { e.Lock() defer e.Unlock() e.keys.AppKeys = appKeys } -// KeyMaterial returns the key material or, if none was registered, an error. -func (e *Enclave) KeyMaterial() (any, error) { +// AppKeys returns the enclave application's key material. This allows a worker +// enclave to retrieve the key material that it synchronized from the leader +// enclave. +func (e *Enclave) AppKeys() ([]byte, error) { e.RLock() defer e.RUnlock() @@ -641,16 +646,34 @@ func (e *Enclave) KeyMaterial() (any, error) { } // syncWithLeader is called by worker enclaves to initiate the process of key -// synchronization. +// synchronization. The worker tries to register with the leader for a total of +// one minute, after which it gives up and terminates. func (e *Enclave) syncWithLeader(leader *url.URL) error { - resp, err := http.Post(leader.String(), "text/plain", nil) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errNo200(resp.StatusCode) + elog.Println("Attempting to sync with leader.") + + // Keep on trying every five seconds for a minute. + ctx, _ := context.WithTimeout(context.Background(), time.Minute) + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + return errors.New("timed out syncing with leader") + case <-e.isLeader: + elog.Println("This enclave became leader. Aborting key sync.") + return nil + case <-ticker.C: + resp, err := http.Post(leader.String(), "text/plain", nil) + if err != nil { + elog.Printf("POST request to leader failed: %v", err) + break + } + if resp.StatusCode != http.StatusOK { + elog.Printf("Leader returned HTTP code %d.", resp.StatusCode) + break + } + return nil + } } - return nil } // syncWithWorker is called by the leader enclave right after a worker @@ -681,6 +704,9 @@ func (e *Enclave) syncWithWorker(worker *url.URL) error { // Step 3: Verify the worker's attestation document and extract its // auxiliary information. b64Attstn, err := io.ReadAll(newLimitReader(resp.Body, maxAttDocLen)) + if err != nil { + return err + } resp.Body.Close() attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) if err != nil { diff --git a/handlers.go b/handlers.go index 5390564..947030d 100644 --- a/handlers.go +++ b/handlers.go @@ -19,9 +19,6 @@ import ( ) const ( - // The maximum length of a worker registration, which is essentially just a - // URL. - maxRegistrationLen = 1024 // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 @@ -85,7 +82,7 @@ func reqSyncHandler(e *Enclave) http.HandlerFunc { return } - if err := RequestKeys(addr, e.KeyMaterial); err != nil { + if err := RequestKeys(addr, e.AppKeys); err != nil { http.Error(w, fmt.Sprintf("failed to synchronize state: %v", err), http.StatusInternalServerError) return } @@ -101,17 +98,17 @@ func reqSyncHandler(e *Enclave) http.HandlerFunc { func getStateHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") - s, err := e.KeyMaterial() + keys, err := e.AppKeys() if err != nil { http.Error(w, errFailedGetState.Error(), http.StatusInternalServerError) return } - n, err := w.Write(s.([]byte)) + n, err := w.Write(keys) if err != nil { elog.Printf("Error writing state to client: %v", err) return } - expected := len(s.([]byte)) + expected := len(keys) if n != expected { elog.Printf("Only wrote %d out of %d-byte state to client.", n, expected) return @@ -127,26 +124,29 @@ func getStateHandler(e *Enclave) http.HandlerFunc { // trusted enclave application. func putStateHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) + keys, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) if err != nil { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - e.SetKeyMaterial(body) + e.SetAppKeys(keys) w.WriteHeader(http.StatusOK) - // The leader's keys have changed. Re-synchronize the key material - // with all registered workers. - go func() { - for w := range e.workers.set { - if err := e.syncWithWorker(&w); err != nil { - // TODO: Failure to sync means that the enclave set is in - // an inconsistent state. We should log this in - // Prometheus. - elog.Printf("Error syncing with worker: %v", err) + // The leader's application keys have changed. Re-synchronize the key + // material with all registered workers. If synchronization fails for a + // given worker, unregister it. + for worker := range e.workers.set { + go func(worker *url.URL) { + if err := e.syncWithWorker(worker); err != nil { + // TODO: Log in Prometheus. + // TODO: We may want to re-attempt synchronization. + elog.Printf("Error syncing with worker %s: %v", worker.String(), err) + e.workers.unregister(worker) + } else { + elog.Printf("Successfully synced with worker %s.", worker.String()) } - } - }() + }(&worker) + } } } @@ -254,8 +254,8 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl func leaderHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - e.cfg.IsLeader = true elog.Println("Designated enclave as leader.") + close(e.isLeader) // Signal to other parts of the code. e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) elog.Println("Set up worker registration endpoint.") @@ -273,7 +273,7 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { http.Error(w, "error extracting IP address", http.StatusInternalServerError) return } - workerURL := &url.URL{ + worker := &url.URL{ Scheme: "https", Host: fmt.Sprintf("%s:%d", strIP, 9444), // TODO: Use e.cfg.ExtPrivPort. Path: pathSync, @@ -281,18 +281,26 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { w.WriteHeader(http.StatusOK) go func() { - if err := e.syncWithWorker(workerURL); err != nil { - elog.Printf("Error syncing with worker %s: %v", workerURL, err) + if err := e.syncWithWorker(worker); err != nil { + elog.Printf("Error syncing with worker %s: %v", worker.String(), err) return } - e.workers.register(workerURL) - elog.Printf("Successfully registered and synced with worker %s.", workerURL) + e.workers.register(worker) + elog.Printf("Successfully registered and synced with worker %s.", worker.String()) }() } } const errNonceRequired = "nonce is required" +func heartbeatHandler(keys *enclaveKeys) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, keys.hashAndB64()) + // e.workers.updateAndPrune() // TODO + } +} + +// TODO: Terminate worker if sync fails. func initSyncHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { elog.Println("New request to init key sync.") @@ -350,6 +358,7 @@ func initSyncHandler(e *Enclave) http.HandlerFunc { } } +// TODO: Terminate worker if sync fails. // finishSyncHandler is called by the leader to finish key synchronization. func finishSyncHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -372,6 +381,10 @@ func finishSyncHandler(e *Enclave) http.HandlerFunc { return } aux, err := e.verifyAttstn(attstn, e.nonceCache.Exists) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Decrypt the leader's enclave keys, which are encrypted with the // public key that we provided earlier. diff --git a/main.go b/main.go index 6e84e66..caad6ba 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ func main() { flag.StringVar(&fqdn, "fqdn", "", "FQDN of the enclave application (e.g., \"example.com\").") flag.StringVar(&fqdnLeader, "fqdn-leader", "", - "FQDN of the leader enclave (e.g., \"leader.example.com\").") + "FQDN of the leader enclave (e.g., \"leader.example.com\"). Setting this enables key synchronization.") flag.StringVar(&appURL, "appurl", "", "Code repository of the enclave application (e.g., \"github.com/foo/bar\").") flag.StringVar(&appWebSrv, "appwebsrv", "", diff --git a/workers.go b/workers.go index 289e977..42eaf5c 100644 --- a/workers.go +++ b/workers.go @@ -3,31 +3,66 @@ package main import ( "net/url" "sync" + "time" ) // workers represents a set of worker enclaves. The leader enclave keeps track // of workers. type workers struct { sync.RWMutex - set map[url.URL]struct{} + timeout time.Duration + set map[url.URL]time.Time } -func newWorkers() *workers { +func newWorkers(timeout time.Duration) *workers { return &workers{ - set: make(map[url.URL]struct{}), + set: make(map[url.URL]time.Time), + timeout: timeout, } } -func (w *workers) register(u *url.URL) { +func (w *workers) register(worker *url.URL) { w.Lock() defer w.Unlock() - w.set[*u] = struct{}{} + w.set[*worker] = time.Now() + elog.Printf("Registered worker %s. %d workers now registered.", + worker.String(), len(w.set)) } -func (w *workers) unregister(u *url.URL) { +func (w *workers) unregister(worker *url.URL) { w.Lock() defer w.Unlock() - delete(w.set, *u) + delete(w.set, *worker) + elog.Printf("Unregistered worker %s. %d workers left.", + worker.String(), len(w.set)) +} + +func (w *workers) updateAndPrune(worker *url.URL) { + w.updateHeartbeat(worker) + w.pruneDefunctWorkers() +} + +func (w *workers) updateHeartbeat(worker *url.URL) { + w.Lock() + defer w.Unlock() + + _, exists := w.set[*worker] + if !exists { + elog.Printf("Updating heartbeat for previously-unregistered worker %s.", worker) + } + w.set[*worker] = time.Now() +} + +func (w *workers) pruneDefunctWorkers() { + w.RLock() + defer w.RUnlock() + + now := time.Now() + for worker, lastSeen := range w.set { + if now.Sub(lastSeen) > w.timeout { + w.unregister(&worker) + } + } } From 904468fa978f6a3773568d29c8556860a98f40f2 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 11 Aug 2023 11:15:39 -0500 Subject: [PATCH 05/99] Move log message to the correct place. --- enclave.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index a8110bc..f502595 100644 --- a/enclave.go +++ b/enclave.go @@ -385,7 +385,6 @@ func (e *Enclave) Start() error { }); err != nil { elog.Fatalf("Error syncing with leader: %v", err) } - elog.Println("Successfully synced with leader.") } return nil @@ -671,6 +670,7 @@ func (e *Enclave) syncWithLeader(leader *url.URL) error { elog.Printf("Leader returned HTTP code %d.", resp.StatusCode) break } + elog.Println("Successfully synced with leader.") return nil } } From f8f6e89d154621085fb13643dad3bb3a2f0e3ce7 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 11 Aug 2023 11:15:55 -0500 Subject: [PATCH 06/99] Remove debug code. --- handlers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers.go b/handlers.go index 947030d..c3c8100 100644 --- a/handlers.go +++ b/handlers.go @@ -275,14 +275,14 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { } worker := &url.URL{ Scheme: "https", - Host: fmt.Sprintf("%s:%d", strIP, 9444), // TODO: Use e.cfg.ExtPrivPort. + Host: fmt.Sprintf("%s:%d", strIP, e.cfg.ExtPrivPort), Path: pathSync, } w.WriteHeader(http.StatusOK) go func() { if err := e.syncWithWorker(worker); err != nil { - elog.Printf("Error syncing with worker %s: %v", worker.String(), err) + elog.Printf("Error syncing with worker: %v", err) return } e.workers.register(worker) From efeeb8bf7585d744a821e577f9b20656e0d56ee2 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 11 Aug 2023 11:22:22 -0500 Subject: [PATCH 07/99] Improve debug messages. --- enclave.go | 2 +- handlers.go | 7 +++---- workers.go | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/enclave.go b/enclave.go index f502595..89c7a8f 100644 --- a/enclave.go +++ b/enclave.go @@ -670,7 +670,7 @@ func (e *Enclave) syncWithLeader(leader *url.URL) error { elog.Printf("Leader returned HTTP code %d.", resp.StatusCode) break } - elog.Println("Successfully synced with leader.") + elog.Println("Successfully registered with leader.") return nil } } diff --git a/handlers.go b/handlers.go index c3c8100..a6b5758 100644 --- a/handlers.go +++ b/handlers.go @@ -303,7 +303,7 @@ func heartbeatHandler(keys *enclaveKeys) http.HandlerFunc { // TODO: Terminate worker if sync fails. func initSyncHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - elog.Println("New request to init key sync.") + elog.Println("Received leader's request to initiate key sync.") // Extract the leader's nonce from the URL. hexNonce := r.URL.Query().Get("nonce") @@ -362,7 +362,7 @@ func initSyncHandler(e *Enclave) http.HandlerFunc { // finishSyncHandler is called by the leader to finish key synchronization. func finishSyncHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - elog.Println("New request to finish key sync.") + elog.Println("Received leader's request to complete key sync.") // Read the leader's Base64-encoded attestation document. maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) @@ -412,7 +412,6 @@ func finishSyncHandler(e *Enclave) http.HandlerFunc { return } e.httpsCert.set(&cert) - - elog.Printf("Leader's enclave keys: %s (%s)", string(decrypted), decrypted) + elog.Println("Successfully synced with leader.") } } diff --git a/workers.go b/workers.go index 42eaf5c..74ff0e2 100644 --- a/workers.go +++ b/workers.go @@ -26,7 +26,7 @@ func (w *workers) register(worker *url.URL) { defer w.Unlock() w.set[*worker] = time.Now() - elog.Printf("Registered worker %s. %d workers now registered.", + elog.Printf("Registered worker %s; %d worker(s) now registered.", worker.String(), len(w.set)) } @@ -35,7 +35,7 @@ func (w *workers) unregister(worker *url.URL) { defer w.Unlock() delete(w.set, *worker) - elog.Printf("Unregistered worker %s. %d workers left.", + elog.Printf("Unregistered worker %s; %d worker(s) left.", worker.String(), len(w.set)) } From 0b357523917ce73f374f3a7dc786530f5a150389 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 11 Aug 2023 11:35:37 -0500 Subject: [PATCH 08/99] Improve registration process. --- enclave.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/enclave.go b/enclave.go index 89c7a8f..1357b5a 100644 --- a/enclave.go +++ b/enclave.go @@ -650,28 +650,37 @@ func (e *Enclave) AppKeys() ([]byte, error) { func (e *Enclave) syncWithLeader(leader *url.URL) error { elog.Println("Attempting to sync with leader.") + errChan := make(chan error) + register := func(e chan error) { + resp, err := http.Post(leader.String(), "text/plain", nil) + if err != nil { + e <- err + } + if resp.StatusCode != http.StatusOK { + e <- fmt.Errorf("leader returned HTTP code %d", resp.StatusCode) + } + e <- nil + } + go register(errChan) + // Keep on trying every five seconds for a minute. ctx, _ := context.WithTimeout(context.Background(), time.Minute) ticker := time.NewTicker(5 * time.Second) for { select { + case err := <-errChan: + if err == nil { + elog.Println("Successfully registered with leader.") + return nil + } + elog.Printf("Error registering with leader: %v", err) case <-ctx.Done(): return errors.New("timed out syncing with leader") case <-e.isLeader: - elog.Println("This enclave became leader. Aborting key sync.") + elog.Println("We became leader. Aborting key sync.") return nil case <-ticker.C: - resp, err := http.Post(leader.String(), "text/plain", nil) - if err != nil { - elog.Printf("POST request to leader failed: %v", err) - break - } - if resp.StatusCode != http.StatusOK { - elog.Printf("Leader returned HTTP code %d.", resp.StatusCode) - break - } - elog.Println("Successfully registered with leader.") - return nil + go register(errChan) } } } From 75c059ac97cf7cb4de6c86fe5afc3947fd06fecf Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 15 Aug 2023 09:24:15 -0500 Subject: [PATCH 09/99] Refactor and test key synchronization. --- attester.go | 13 +- enclave.go | 223 +++++++-------------------------- enclave_keys.go | 72 +++++++++++ enclave_keys_test.go | 72 +++++++++++ enclave_test.go | 26 +--- handlers.go | 203 +++--------------------------- handlers_test.go | 33 ++--- keysync_initiator.go | 212 -------------------------------- keysync_initiator_test.go | 238 ------------------------------------ keysync_responder.go | 128 ------------------- keysync_responder_test.go | 251 -------------------------------------- keysync_shared_test.go | 24 ---- main.go | 8 -- metrics_test.go | 2 +- sync_leader.go | 107 ++++++++++++++++ sync_worker.go | 210 +++++++++++++++++++++++++++++++ sync_worker_test.go | 83 +++++++++++++ util.go | 17 +++ workers.go | 7 ++ workers_test.go | 33 +++++ 20 files changed, 683 insertions(+), 1279 deletions(-) create mode 100644 enclave_keys.go create mode 100644 enclave_keys_test.go delete mode 100644 keysync_initiator.go delete mode 100644 keysync_initiator_test.go delete mode 100644 keysync_responder.go delete mode 100644 keysync_responder_test.go create mode 100644 sync_leader.go create mode 100644 sync_worker.go create mode 100644 sync_worker_test.go create mode 100644 util.go create mode 100644 workers_test.go diff --git a/attester.go b/attester.go index 19bf117..0a9dc37 100644 --- a/attester.go +++ b/attester.go @@ -16,7 +16,7 @@ import ( // implement a dummy attester that works without the AWS Nitro hypervisor. type attester interface { createAttstn(auxInfo) ([]byte, error) - verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) + verifyAttstn([]byte, nonce) (auxInfo, error) } type auxInfo interface{} @@ -56,7 +56,7 @@ func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) { return json.Marshal(aux) } -func (*dummyAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) { +func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { var w workerAuxInfo var l leaderAuxInfo @@ -65,7 +65,8 @@ func (*dummyAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (au return nil, err } if len(w.WorkersNonce) == nonceLen && len(w.LeadersNonce) == nonceLen && w.PublicKey != nil { - if !isOurNonce(w.LeadersNonce.B64()) { + if n.B64() != w.LeadersNonce.B64() { + fmt.Printf("'%s' / '%s'", n.B64(), w.LeadersNonce.B64()) return nil, errors.New("leader nonce not in cache") } elog.Println(w) @@ -77,7 +78,7 @@ func (*dummyAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (au return nil, err } if len(l.WorkersNonce) == nonceLen && l.EnclaveKeys != nil { - if !isOurNonce(l.WorkersNonce.B64()) { + if n.B64() != l.WorkersNonce.B64() { return nil, errors.New("worker nonce not in cache") } elog.Println(l) @@ -130,7 +131,7 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { return res.Attestation.Document, nil } -func (*nitroAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (auxInfo, error) { +func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { errStr := "error verifying attestation document" // Verify the remote enclave's attestation document before doing anything // with it. @@ -153,7 +154,7 @@ func (*nitroAttester) verifyAttstn(doc []byte, isOurNonce func(string) bool) (au // Verify that the remote enclave's attestation document contains the nonce // that we asked it to embed. b64Nonce := base64.StdEncoding.EncodeToString(their.Document.Nonce) - if !isOurNonce(b64Nonce) { + if n.B64() == b64Nonce { return nil, fmt.Errorf("%s: nonce %s not in cache", errStr, b64Nonce) } diff --git a/enclave.go b/enclave.go index 1357b5a..519dcf2 100644 --- a/enclave.go +++ b/enclave.go @@ -5,7 +5,6 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" - cryptoRand "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" @@ -16,14 +15,12 @@ import ( "encoding/pem" "errors" "fmt" - "io" "math/big" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "net/url" - "strings" "sync" "time" @@ -34,11 +31,9 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/crypto/acme/autocert" - "golang.org/x/crypto/nacl/box" ) // TODO: Support Let's Encrypt (if we choose to). -// TODO: Handle key sync error edge cases. // TODO: Handle the case of the leader restarting. Workers must not break. const ( @@ -61,28 +56,17 @@ const ( pathConfig = "/enclave/config" pathLeader = "/enclave/leader" pathRegistration = "/enclave/registration" + pathHeartbeat = "/enclave/heartbeat" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" ) var ( - errNoKeyMaterial = errors.New("no key material registered") errCfgMissingFQDN = errors.New("given config is missing FQDN") errCfgMissingPort = errors.New("given config is missing port") ) -// enclaveKeys holds key material for nitriding itself (the HTTPS certificate) -// and for the enclave application (whatever the application wants to "store" -// in nitriding). All these keys are meant to be managed by a leader enclave -// and -- if horizontal scaling is required -- synced to worker enclaves. -type enclaveKeys struct { - // TODO: Add another field that contains high-entropy bytes. - NitridingKey []byte `json:"nitriding_key"` - NitridingCert []byte `json:"nitriding_cert"` - AppKeys []byte `json:"app_keys"` -} - // hashAndB64 returns the Base64-encoded hash over our key material. The // resulting string is not confidential as it's impractical to reverse the key // material. @@ -96,20 +80,18 @@ func (e *enclaveKeys) hashAndB64() string { type Enclave struct { sync.RWMutex attester - cfg *Config - extPubSrv, extPrivSrv *http.Server - intSrv *http.Server - promSrv *http.Server - revProxy *httputil.ReverseProxy - hashes *AttestationHashes - promRegistry *prometheus.Registry - metrics *metrics - workers *workers - nonceCache *cache - keys *enclaveKeys - ephemeralSyncKeys *boxKey // TODO - ready, stop, isLeader chan bool - httpsCert *certRetriever + cfg *Config + extPubSrv, extPrivSrv *http.Server + intSrv *http.Server + promSrv *http.Server + revProxy *httputil.ReverseProxy + hashes *AttestationHashes + promRegistry *prometheus.Registry + metrics *metrics + workers *workers + keys *enclaveKeys + ready, stop, becameLeader chan struct{} + httpsCert *certRetriever } // Config represents the configuration of our enclave service. @@ -273,12 +255,11 @@ func NewEnclave(cfg *Config) (*Enclave, error) { keys: &enclaveKeys{}, promRegistry: reg, metrics: newMetrics(reg, cfg.PrometheusNamespace), - nonceCache: newCache(defaultItemExpiry), hashes: new(AttestationHashes), workers: newWorkers(time.Minute), - stop: make(chan bool), - ready: make(chan bool), - isLeader: make(chan bool), + stop: make(chan struct{}), + ready: make(chan struct{}), + becameLeader: make(chan struct{}), } // Increase the maximum number of idle connections per host. This is @@ -308,15 +289,14 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Register external public HTTP API. m := e.extPubSrv.Handler.(*chi.Mux) m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes)) - m.Get(pathNonce, nonceHandler(e)) m.Get(pathRoot, rootHandler(e.cfg)) m.Get(pathConfig, configHandler(e.cfg)) // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(e)) - m.Get(pathSync, initSyncHandler(e)) - m.Post(pathSync, finishSyncHandler(e)) + m.Get(pathHeartbeat, heartbeatHandler(e)) + m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) @@ -342,6 +322,18 @@ func NewEnclave(cfg *Config) (*Enclave, error) { return e, nil } +// installKeys installs the given enclave keys. Worker enclaves do this after +// key synchronization. +func (e *Enclave) installKeys(keys *enclaveKeys) error { + e.keys.set(keys) + cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) + if err != nil { + return err + } + e.httpsCert.set(&cert) + return nil +} + // Start starts the Nitro Enclave. If something goes wrong, the function // returns an error. func (e *Enclave) Start() error { @@ -376,9 +368,8 @@ func (e *Enclave) Start() error { return fmt.Errorf("%s: %w", errPrefix, err) } - // TODO: Ideally, the leader enclave would not call this function. if e.cfg.isScalingEnabled() { - if err := e.syncWithLeader(&url.URL{ + if err := asWorker(e.installKeys, e.becameLeader).registerWith(&url.URL{ Scheme: "https", Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), Path: pathRegistration, @@ -522,8 +513,7 @@ func (e *Enclave) genSelfSignedCert() error { if pemKey == nil { elog.Fatal("Failed to encode key to PEM.") } - e.keys.NitridingKey = pemKey - e.keys.NitridingCert = pemCert + e.keys.setNitridingKeys(pemKey, pemCert) cert, err := tls.X509KeyPair(pemCert, pemKey) if err != nil { @@ -621,145 +611,16 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { return nil } -// SetAppKeys registers the enclave application's key material (e.g., secret -// encryption keys, or what have you), so it can be synchronized with worker -// enclaves. This is only necessary if horizontal enclave scaling is required. -func (e *Enclave) SetAppKeys(appKeys []byte) { - e.Lock() - defer e.Unlock() - - e.keys.AppKeys = appKeys -} - -// AppKeys returns the enclave application's key material. This allows a worker -// enclave to retrieve the key material that it synchronized from the leader -// enclave. -func (e *Enclave) AppKeys() ([]byte, error) { - e.RLock() - defer e.RUnlock() - - if e.keys.AppKeys == nil { - return nil, errNoKeyMaterial - } - return e.keys.AppKeys, nil -} - -// syncWithLeader is called by worker enclaves to initiate the process of key -// synchronization. The worker tries to register with the leader for a total of -// one minute, after which it gives up and terminates. -func (e *Enclave) syncWithLeader(leader *url.URL) error { - elog.Println("Attempting to sync with leader.") - - errChan := make(chan error) - register := func(e chan error) { - resp, err := http.Post(leader.String(), "text/plain", nil) - if err != nil { - e <- err - } - if resp.StatusCode != http.StatusOK { - e <- fmt.Errorf("leader returned HTTP code %d", resp.StatusCode) - } - e <- nil - } - go register(errChan) - - // Keep on trying every five seconds for a minute. - ctx, _ := context.WithTimeout(context.Background(), time.Minute) - ticker := time.NewTicker(5 * time.Second) - for { - select { - case err := <-errChan: - if err == nil { - elog.Println("Successfully registered with leader.") - return nil - } - elog.Printf("Error registering with leader: %v", err) - case <-ctx.Done(): - return errors.New("timed out syncing with leader") - case <-e.isLeader: - elog.Println("We became leader. Aborting key sync.") - return nil - case <-ticker.C: - go register(errChan) - } - } -} - -// syncWithWorker is called by the leader enclave right after a worker -// registers itself with the leader enclave. -func (e *Enclave) syncWithWorker(worker *url.URL) error { - elog.Println("Initiating key synchronization with worker.") - - // Step 1: Create a nonce that the worker must embed in its attestation - // document to prevent replay attacks. - nonce, err := newNonce() - if err != nil { - return err - } - e.nonceCache.Add(nonce.B64()) - - // Step 2: Request the worker's attestation document, and provide the - // previously-generated nonce. - reqURL := *worker - reqURL.RawQuery = fmt.Sprintf("nonce=%x", nonce) - resp, err := http.Get(reqURL.String()) +func (e *Enclave) httpClientToSyncURL(r *http.Request) (*url.URL, error) { + // Go's HTTP server sets RemoteAddr to IP:port: + // https://pkg.go.dev/net/http#Request + strIP, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errNo200(resp.StatusCode) - } - - // Step 3: Verify the worker's attestation document and extract its - // auxiliary information. - b64Attstn, err := io.ReadAll(newLimitReader(resp.Body, maxAttDocLen)) - if err != nil { - return err - } - resp.Body.Close() - attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) - if err != nil { - return err - } - workerAux, err := e.verifyAttstn(attstn, e.nonceCache.Exists) - if err != nil { - return err - } - - // Step 4: Encrypt the leader's enclave keys with the ephemeral public key - // that the worker put into its auxiliary information. - pubKey := &[boxKeyLen]byte{} - copy(pubKey[:], workerAux.(*workerAuxInfo).PublicKey[:]) - jsonKeys, err := json.Marshal(e.keys) - if err != nil { - return err + return nil, err } - var encrypted []byte - encrypted, err = box.SealAnonymous(nil, jsonKeys, pubKey, cryptoRand.Reader) - if err != nil { - return err - } - - // Step 5: Create the leader's auxiliary information, consisting of the - // worker's nonce and the encrypted enclave keys. - leaderAux := &leaderAuxInfo{ - WorkersNonce: workerAux.(*workerAuxInfo).WorkersNonce, - EnclaveKeys: encrypted, - } - attstn, err = e.createAttstn(leaderAux) - if err != nil { - return err - } - strAttstn := base64.StdEncoding.EncodeToString(attstn) - - // Step 6: Send the leader's attestation document to the worker. - resp, err = http.Post(worker.String(), "text/plain", strings.NewReader(strAttstn)) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errNo200(resp.StatusCode) - } - - return nil + return &url.URL{ + Scheme: "https", // Leader and workers use HTTPS to communicate. + Host: fmt.Sprintf("%s:%d", strIP, e.cfg.ExtPrivPort), + Path: pathSync, + }, nil } diff --git a/enclave_keys.go b/enclave_keys.go new file mode 100644 index 0000000..75d5f19 --- /dev/null +++ b/enclave_keys.go @@ -0,0 +1,72 @@ +package main + +import ( + "bytes" + "sync" +) + +// enclaveKeys holds key material for nitriding itself (the HTTPS certificate) +// and for the enclave application (whatever the application wants to "store" +// in nitriding). These keys are meant to be managed by a leader enclave and -- +// if horizontal scaling is required -- synced to worker enclaves. The struct +// implements getters and setters that allow for thread-safe setting and getting +// of members. +type enclaveKeys struct { + sync.RWMutex + NitridingKey []byte `json:"nitriding_key"` + NitridingCert []byte `json:"nitriding_cert"` + AppKeys []byte `json:"app_keys"` +} + +func (e1 *enclaveKeys) equal(e2 *enclaveKeys) bool { + e1.RLock() + e2.RLock() + defer e1.RUnlock() + defer e2.RUnlock() + + return bytes.Equal(e1.NitridingCert, e2.NitridingCert) && + bytes.Equal(e1.NitridingKey, e2.NitridingKey) && + bytes.Equal(e1.AppKeys, e2.AppKeys) +} + +func (e *enclaveKeys) setAppKeys(appKeys []byte) { + e.Lock() + defer e.Unlock() + + e.AppKeys = appKeys +} + +func (e *enclaveKeys) setNitridingKeys(key, cert []byte) { + e.Lock() + defer e.Unlock() + + e.NitridingKey = key + e.NitridingCert = cert +} + +func (e *enclaveKeys) set(newKeys *enclaveKeys) { + e.Lock() + defer e.Unlock() + + e.NitridingKey = newKeys.NitridingKey + e.NitridingCert = newKeys.NitridingCert + e.AppKeys = newKeys.AppKeys +} + +func (e *enclaveKeys) get() *enclaveKeys { + e.RLock() + defer e.RUnlock() + + return &enclaveKeys{ + NitridingKey: e.NitridingKey, + NitridingCert: e.NitridingCert, + AppKeys: e.AppKeys, + } +} + +func (e *enclaveKeys) getAppKeys() []byte { + e.RLock() + defer e.RUnlock() + + return e.AppKeys +} diff --git a/enclave_keys_test.go b/enclave_keys_test.go new file mode 100644 index 0000000..c445534 --- /dev/null +++ b/enclave_keys_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "bytes" + "testing" +) + +// testKeys holds arbitrary keys that we use for testing. +var testKeys = &enclaveKeys{ + NitridingKey: []byte("NitridingTestKey"), + NitridingCert: []byte("NitridingTestCert"), + AppKeys: []byte("AppTestKeys"), +} + +func TestSetKeys(t *testing.T) { + var ( + keys enclaveKeys + appKeys = []byte("AppKeys") + ) + + // Ensure that the application keys are set correctly. + keys = enclaveKeys{} + keys.setAppKeys(appKeys) + if !bytes.Equal(keys.AppKeys, appKeys) { + t.Fatal("Application keys not set correctly.") + } + + // Ensure that the nitriding keys are set correctly. + keys = enclaveKeys{} + keys.setNitridingKeys(testKeys.NitridingKey, testKeys.NitridingCert) + if !bytes.Equal(keys.NitridingKey, testKeys.NitridingKey) { + t.Fatal("Nitriding key not set correctly.") + } + if !bytes.Equal(keys.NitridingCert, testKeys.NitridingCert) { + t.Fatal("Nitriding cert not set correctly.") + } + + // Ensure that a new set of keys is set correctly. + keys = enclaveKeys{} + keys.set(testKeys) + if !keys.equal(testKeys) { + t.Fatal("Enclave keys not set correctly.") + } +} + +func TestGetKeys(t *testing.T) { + var ( + appKeys = testKeys.getAppKeys() + keys = testKeys.get() + ) + + // Ensure that the application key is retrieved correctly. + if !bytes.Equal(appKeys, testKeys.AppKeys) { + t.Fatal("Application keys not retrieved correctly.") + } + + // Ensure that a new set of keys is retrieved correctly. + if !keys.equal(testKeys) { + t.Fatal("Enclave keys not retrieved correctly.") + } +} + +func TestModifyCloneObject(t *testing.T) { + newKeys := testKeys.get() + newKeys.setAppKeys([]byte("foobar")) + + // Make sure that setting the clone's application keys does not affect the + // original object. + if bytes.Equal(newKeys.getAppKeys(), testKeys.getAppKeys()) { + t.Fatal("Cloned object must not affect original object.") + } +} diff --git a/enclave_test.go b/enclave_test.go index c33a096..224e0bc 100644 --- a/enclave_test.go +++ b/enclave_test.go @@ -6,8 +6,9 @@ import ( var defaultCfg = Config{ FQDN: "example.com", - ExtPort: 50000, - IntPort: 50001, + ExtPubPort: 50000, + ExtPrivPort: 50001, + IntPort: 50002, HostProxyPort: 1024, UseACME: false, Debug: false, @@ -46,7 +47,8 @@ func TestValidateConfig(t *testing.T) { } // Set the remaining required fields. - c.ExtPort = 1 + c.ExtPubPort = 1 + c.ExtPrivPort = 1 c.IntPort = 1 c.HostProxyPort = 1 if err = c.Validate(); err != nil { @@ -60,21 +62,3 @@ func TestGenSelfSignedCert(t *testing.T) { t.Fatalf("Failed to create self-signed certificate: %s", err) } } - -func TestKeyMaterial(t *testing.T) { - e := createEnclave(&defaultCfg) - k := struct{ Foo string }{"foobar"} - - if _, err := e.KeyMaterial(); err != errNoKeyMaterial { - t.Fatal("Expected error because we're trying to retrieve non-existing key material.") - } - - e.SetKeyMaterial(k) - r, err := e.KeyMaterial() - if err != nil { - t.Fatalf("Failed to retrieve key material: %s", err) - } - if r != k { - t.Fatal("Retrieved key material is unexpected.") - } -} diff --git a/handlers.go b/handlers.go index a6b5758..8212a52 100644 --- a/handlers.go +++ b/handlers.go @@ -2,20 +2,16 @@ package main import ( "crypto/sha256" - "crypto/tls" "encoding/base64" "encoding/hex" - "encoding/json" "errors" "fmt" "io" - "net" "net/http" "net/url" "strings" "github.com/go-chi/chi/v5" - "golang.org/x/crypto/nacl/box" ) const ( @@ -27,11 +23,9 @@ const ( ) var ( - errFailedReqBody = errors.New("failed to read request body") - errFailedGetState = errors.New("failed to retrieve saved state") - errNoAddr = errors.New("parameter 'addr' not found") - errBadSyncAddr = errors.New("invalid 'addr' parameter for sync") - errHashWrongSize = errors.New("given hash is of invalid size") + errFailedReqBody = errors.New("failed to read request body") + errHashWrongSize = errors.New("given hash is of invalid size") + errNoBase64 = errors.New("no Base64 given") ) func errNo200(code int) error { @@ -56,40 +50,6 @@ func rootHandler(cfg *Config) http.HandlerFunc { } } -// reqSyncHandler returns a handler that lets the enclave application request -// state synchronization, which copies the given remote enclave's state into -// our state. -// -// This is an enclave-internal endpoint that can only be accessed by the -// trusted enclave application. -// -// FIXME: https://github.com/brave/nitriding-daemon/issues/10 -func reqSyncHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - // The 'addr' parameter must have the following form: - // https://example.com:443 - addrs, ok := q["addr"] - if !ok { - http.Error(w, errNoAddr.Error(), http.StatusBadRequest) - return - } - addr := addrs[0] - - // Are we dealing with a well-formed URL? - if _, err := url.Parse(addr); err != nil { - http.Error(w, errBadSyncAddr.Error(), http.StatusBadRequest) - return - } - - if err := RequestKeys(addr, e.AppKeys); err != nil { - http.Error(w, fmt.Sprintf("failed to synchronize state: %v", err), http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusOK) - } -} - // getStateHandler returns a handler that lets the enclave application retrieve // previously-set state. // @@ -98,17 +58,13 @@ func reqSyncHandler(e *Enclave) http.HandlerFunc { func getStateHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") - keys, err := e.AppKeys() - if err != nil { - http.Error(w, errFailedGetState.Error(), http.StatusInternalServerError) - return - } - n, err := w.Write(keys) + appKeys := e.keys.getAppKeys() + n, err := w.Write(appKeys) if err != nil { elog.Printf("Error writing state to client: %v", err) return } - expected := len(keys) + expected := len(appKeys) if n != expected { elog.Printf("Only wrote %d out of %d-byte state to client.", n, expected) return @@ -129,7 +85,7 @@ func putStateHandler(e *Enclave) http.HandlerFunc { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - e.SetAppKeys(keys) + e.keys.setAppKeys(keys) w.WriteHeader(http.StatusOK) // The leader's application keys have changed. Re-synchronize the key @@ -137,13 +93,12 @@ func putStateHandler(e *Enclave) http.HandlerFunc { // given worker, unregister it. for worker := range e.workers.set { go func(worker *url.URL) { - if err := e.syncWithWorker(worker); err != nil { + if err := asLeader(e.keys.get()).syncWith(worker); err != nil { // TODO: Log in Prometheus. - // TODO: We may want to re-attempt synchronization. - elog.Printf("Error syncing with worker %s: %v", worker.String(), err) + elog.Printf("Error re-syncing with worker %s: %v", worker.String(), err) e.workers.unregister(worker) } else { - elog.Printf("Successfully synced with worker %s.", worker.String()) + elog.Printf("Successfully re-synced with worker %s.", worker.String()) } }(&worker) } @@ -255,7 +210,7 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl func leaderHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { elog.Println("Designated enclave as leader.") - close(e.isLeader) // Signal to other parts of the code. + close(e.becameLeader) // Signal to other parts of the code. e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) elog.Println("Set up worker registration endpoint.") @@ -266,22 +221,14 @@ func leaderHandler(e *Enclave) http.HandlerFunc { func workerRegistrationHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Go's HTTP server sets RemoteAddr to IP:port: - // https://pkg.go.dev/net/http#Request - strIP, _, err := net.SplitHostPort(r.RemoteAddr) + worker, err := e.httpClientToSyncURL(r) if err != nil { - http.Error(w, "error extracting IP address", http.StatusInternalServerError) - return - } - worker := &url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s:%d", strIP, e.cfg.ExtPrivPort), - Path: pathSync, + http.Error(w, err.Error(), http.StatusInternalServerError) } w.WriteHeader(http.StatusOK) go func() { - if err := e.syncWithWorker(worker); err != nil { + if err := asLeader(e.keys.get()).syncWith(worker); err != nil { elog.Printf("Error syncing with worker: %v", err) return } @@ -291,127 +238,17 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { } } -const errNonceRequired = "nonce is required" - -func heartbeatHandler(keys *enclaveKeys) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, keys.hashAndB64()) - // e.workers.updateAndPrune() // TODO - } -} - -// TODO: Terminate worker if sync fails. -func initSyncHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - elog.Println("Received leader's request to initiate key sync.") - - // Extract the leader's nonce from the URL. - hexNonce := r.URL.Query().Get("nonce") - if hexNonce == "" { - http.Error(w, errNonceRequired, http.StatusBadRequest) - return - } - - nonceSlice, err := hex.DecodeString(hexNonce) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if len(nonceSlice) != nonceLen { - http.Error(w, "invalid nonce length", http.StatusBadRequest) - return - } - var leadersNonce nonce - copy(leadersNonce[:], nonceSlice) - - // Create the worker's nonce and add it to our nonce cache, so it can - // later be verified. - workersNonce, err := newNonce() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - e.nonceCache.Add(workersNonce.B64()) - - // Create an ephemeral key that the leader is going to use to encrypt - // its enclave keys. - boxKey, err := newBoxKey() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - e.ephemeralSyncKeys = boxKey // TODO: Could be more elegant. - - // Create and return the worker's Base64-encoded attestation document. - aux := &workerAuxInfo{ - WorkersNonce: workersNonce, - LeadersNonce: leadersNonce, - PublicKey: boxKey.pubKey[:], - } - attstn, err := e.createAttstn(aux) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - fmt.Fprintln(w, base64.StdEncoding.EncodeToString(attstn)) - } -} - -// TODO: Terminate worker if sync fails. -// finishSyncHandler is called by the leader to finish key synchronization. -func finishSyncHandler(e *Enclave) http.HandlerFunc { +func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - elog.Println("Received leader's request to complete key sync.") - - // Read the leader's Base64-encoded attestation document. - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - b64Attstn, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) - if err != nil { - elog.Printf("Failed to read http body: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - elog.Printf("Leader's attstn doc: %v", string(b64Attstn)) - - // Decode Base64 to byte slice. - attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - aux, err := e.verifyAttstn(attstn, e.nonceCache.Exists) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Decrypt the leader's enclave keys, which are encrypted with the - // public key that we provided earlier. - decrypted, ok := box.OpenAnonymous( - nil, - aux.(*leaderAuxInfo).EnclaveKeys, - e.ephemeralSyncKeys.pubKey, - e.ephemeralSyncKeys.privKey) - if !ok { - http.Error(w, "error decrypting enclave keys", http.StatusBadRequest) - return - } - e.ephemeralSyncKeys = nil // Clear the ephemeral key material. - - // Set the leader's enclave keys. - var keys enclaveKeys - if err := json.Unmarshal(decrypted, &keys); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + // TODO: Use AttestationHashes instead. + fmt.Fprintln(w, e.keys.hashAndB64()) - cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) + // Take note of the worker still being alive. + worker, err := e.httpClientToSyncURL(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - e.httpsCert.set(&cert) - elog.Println("Successfully synced with leader.") + e.workers.updateAndPrune(worker) } } diff --git a/handlers_test.go b/handlers_test.go index 48453a3..7336e0d 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -67,7 +67,7 @@ func assertResponse(t *testing.T, actual, expected *http.Response) { } func TestRootHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathRoot, nil), @@ -93,25 +93,6 @@ func signalReady(t *testing.T, e *Enclave) { time.Sleep(100 * time.Millisecond) } -func TestSyncHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).intSrv) - - assertResponse(t, - makeReq(http.MethodGet, pathSync, nil), - newResp(http.StatusBadRequest, errNoAddr.Error()), - ) - - assertResponse(t, - makeReq(http.MethodGet, pathSync+"?addr=:foo", nil), - newResp(http.StatusBadRequest, errBadSyncAddr.Error()), - ) - - assertResponse(t, - makeReq(http.MethodGet, pathSync+"?addr=foobar", nil), - newResp(http.StatusInternalServerError, ""), // The exact error is convoluted, so we skip comparison. - ) -} - func TestStateHandlers(t *testing.T) { makeReq := makeRequestFor(createEnclave(&defaultCfg).intSrv) @@ -172,7 +153,7 @@ func TestProxyHandler(t *testing.T) { // Skip certificate validation because we are using a self-signed // certificate in this test. http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - nitridingSrv := "https://127.0.0.1" + e.pubSrv.Addr + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) // Request the enclave's index page. Nitriding is going to return it. resp, err := http.Get(nitridingSrv + pathRoot) @@ -242,7 +223,7 @@ func TestReadiness(t *testing.T) { } defer e.Stop() //nolint:errcheck - nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPort) + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) u := nitridingSrv + pathRoot // Make sure that the Internet-facing Web server is already running because // we didn't ask nitriding to wait for the application. The Web server may @@ -279,7 +260,7 @@ func TestReadyHandler(t *testing.T) { defer e.Stop() //nolint:errcheck // Check if the Internet-facing Web server is running. - nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPort) + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) _, err := http.Get(nitridingSrv + pathRoot) if !errors.Is(err, syscall.ECONNREFUSED) { t.Fatal("Expected 'connection refused'.") @@ -299,7 +280,7 @@ func TestReadyHandler(t *testing.T) { func TestAttestationHandlerWhileProfiling(t *testing.T) { cfg := defaultCfg cfg.UseProfiling = true - makeReq := makeRequestFor(createEnclave(&cfg).pubSrv) + makeReq := makeRequestFor(createEnclave(&cfg).extPubSrv) // Ensure that the attestation handler aborts if profiling is enabled. assertResponse(t, @@ -309,7 +290,7 @@ func TestAttestationHandlerWhileProfiling(t *testing.T) { } func TestAttestationHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodPost, pathAttestation, nil), @@ -337,7 +318,7 @@ func TestAttestationHandler(t *testing.T) { } func TestConfigHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathConfig, nil), diff --git a/keysync_initiator.go b/keysync_initiator.go deleted file mode 100644 index 79bd9b2..0000000 --- a/keysync_initiator.go +++ /dev/null @@ -1,212 +0,0 @@ -package main - -// AWS Nitro Enclave attestation documents contain three fields (called -// "nonce", "user data", and "public key") that can be set by the requester. -// We are using those fields as follows: -// -// When the requesting enclave sends a request to the remote enclave, it sets -// the following fields in the attestation document: -// -// Attestation document( -// Nonce: Remote enclave's nonce -// User data: Requesting enclave's nonce -// Public key: Requesting enclave's NaCl box public key -// ) -// -// The remote enclave then generates its own attestation document containing -// the following fields: -// -// Attestation document( -// Nonce: The nonce the requester provided in its attestation document -// User data: Encrypted key material -// Public key: -// ) - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/hf/nitrite" - "golang.org/x/crypto/nacl/box" -) - -// RequestKeys asks a remote enclave to share its key material with us, which -// is then written to the provided variable. -// -// This is only necessary if you intend to scale enclaves horizontally. If -// you will only ever run a single enclave, ignore this function. -func RequestKeys(addr string, keyMaterial any) error { - errStr := "failed to request key material" - - // First, request a nonce from the remote enclave. - theirNonce, err := requestNonce(addr) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Now, create our own nonce. - ourNonce, err := newNonce() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Next, create a key that the remote enclave is going to use to encrypt - // its key material. - boxKey, err := newBoxKey() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Now create an attestation document containing our nonce, the remote - // enclave's nonce, and the key material that they remote enclave is - // supposed to use. - ourAttDoc, err := attest(theirNonce[:], ourNonce[:], boxKey.pubKey[:]) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Send our attestation document to the remote enclave, and get theirs in - // return. - theirAttDoc, err := requestAttDoc(addr, ourAttDoc) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Finally, verify the attestation document and extract the key material. - if err := processAttDoc(theirAttDoc, &ourNonce, boxKey, keyMaterial); err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - return nil -} - -// requestNonce requests a nonce from the remote enclave specified by 'addr'. -func requestNonce(addr string) (nonce, error) { - errStr := "failed to fetch nonce from remote enclave" - - endpoint := fmt.Sprintf("%s%s", addr, pathNonce) - resp, err := http.Get(endpoint) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - defer resp.Body.Close() - - // Add an extra byte to account for the "\n". - maxReadLen := base64.StdEncoding.EncodedLen(nonceLen) + 1 - body, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - - // Decode the Base64-encoded nonce. - raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(body))) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - - if len(raw) != nonceLen { - return nonce{}, errors.New("remote enclave's nonce has incorrect length") - } - var n nonce - copy(n[:], raw) - return n, nil -} - -// requestAttDoc takes as input the remote enclave's address (e.g., -// ) and our attestation document. The function then -// submits our attestation document to the remote enclave, and returns the -// remote enclave's attestation document. -func requestAttDoc(addr string, ourAttDoc []byte) ([]byte, error) { - errStr := "failed to fetch attestation doc from remote enclave" - - endpoint := fmt.Sprintf("%s%s", addr, pathSync) - - // Finally, send our attestation document to the remote enclave. If - // everything works out, the remote enclave is going to respond with its - // attestation document. - b64AttDoc := base64.StdEncoding.EncodeToString(ourAttDoc) - resp, err := http.Post( - endpoint, - "text/plain", - bytes.NewBufferString(b64AttDoc), - ) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%s: expected status code %d but got %d", errStr, http.StatusOK, resp.StatusCode) - } - - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - body, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - - theirAttDoc, err := base64.StdEncoding.DecodeString(string(body)) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - - return theirAttDoc, nil -} - -// processAttDoc first verifies that the remote enclave's attestation document -// is authentic, and then attempts to decrypt and extract the key material that -// the remote enclave provided in its attestation document. -func processAttDoc( - theirAttDoc []byte, - ourNonce *nonce, - boxKey *boxKey, - keyMaterial any, -) error { - errStr := "failed to process attestation doc from remote enclave" - // Verify the remote enclave's attestation document before doing anything - // with it. - opts := nitrite.VerifyOptions{CurrentTime: currentTime()} - their, err := nitrite.Verify(theirAttDoc, opts) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Are the PCR values (i.e. image IDs) identical? - ourPCRs, err := getPCRValues() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { - return fmt.Errorf("%s: PCR values of remote enclave not identical to ours", errStr) - } - - // Now verify that the remote enclave's attestation document contains the - // nonce that we provided earlier. - if !bytes.Equal(their.Document.Nonce, ourNonce[:]) { - return fmt.Errorf("%s: expected nonce %x but got %x", - errStr, ourNonce[:], their.Document.Nonce) - } - - // Attempt to decrypt the key material. - decrypted, ok := box.OpenAnonymous( - nil, - their.Document.UserData, - boxKey.pubKey, - boxKey.privKey) - if !ok { - return fmt.Errorf("%s: failed to decrypt key material", errStr) - } - - // Finally, write the JSON-encoded key material to the provided interface. - if err := json.Unmarshal(decrypted, keyMaterial); err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - return nil -} diff --git a/keysync_initiator_test.go b/keysync_initiator_test.go deleted file mode 100644 index ebf3146..0000000 --- a/keysync_initiator_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "encoding/base64" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -func TestRequestNonce(t *testing.T) { - expNonce := nonce{ - 0x14, 0x56, 0x82, 0x13, 0x1f, 0xff, 0x9c, 0xf7, 0xeb, 0xb6, - 0x9e, 0x7b, 0xea, 0x29, 0x16, 0x49, 0xeb, 0x03, 0xa2, 0x47, - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, expNonce.B64()) - })) - defer srv.Close() - - retNonce, err := requestNonce(srv.URL) - if err != nil { - t.Fatalf("Failed to request nonce: %s", err) - } - if expNonce != retNonce { - t.Fatal("Returned nonce not as expected.") - } -} - -func TestRequestNonceDoS(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - nonce1 := nonce{} - nonce2 := nonce{} - fmt.Fprintf(w, "%s%s", nonce1.B64(), nonce2.B64()) - })) - defer srv.Close() - - _, err := requestNonce(srv.URL) - if !errors.Is(err, errTooMuchToRead) { - t.Fatalf("Expected error %q but got %q.", errTooMuchToRead, err) - } -} - -func TestRequestAttDoc(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "foobar") - })) - defer srv.Close() - - _, err := requestAttDoc(srv.URL, []byte{}) - if err == nil { - t.Fatal("Client code should have rejected non-Base64 data but didn't.") - } -} - -func TestRequestAttDocDoS(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - // Send one byte more than the client is willing to read. - buf := make([]byte, maxReadLen+1) - fmt.Fprint(w, string(buf)) - })) - defer srv.Close() - - _, err := requestAttDoc(srv.URL, []byte{}) - if !errors.Is(err, errTooMuchToRead) { - t.Fatalf("Expected error %q but got %q.", errTooMuchToRead, err) - } -} - -func TestProcessAttDoc(t *testing.T) { - // Mock functions for our tests to pass. - getPCRValues = func() (map[uint][]byte, error) { - return respAttInfo.pcr, nil - } - currentTime = func() time.Time { return respAttInfo.attDocTime } - - rawAttDoc, err := base64.StdEncoding.DecodeString(respAttInfo.attDoc) - if err != nil { - t.Fatalf("Failed to Base64-decode attestation document: %s", err) - } - keyMaterial := struct { - SecretKey string `json:"secret_key"` - }{} - - if err := processAttDoc( - rawAttDoc, - &respAttInfo.nonce, - &boxKey{ - pubKey: &respAttInfo.pubKey, - privKey: &respAttInfo.privKey, - }, - &keyMaterial, - ); err != nil { - t.Fatalf("Failed to verify valid attestation document: %s", err) - } - - // Make sure that processAttDoc successfully decrypted and recovered the - // secret key material, "foobar". - if keyMaterial.SecretKey != "foobar" { - t.Fatalf("Expected secret key 'foobar' but got %q.", keyMaterial.SecretKey) - } -} - -var respAttInfo = &remoteAttInfo{ - pubKey: [boxKeyLen]byte{ - 213, 156, 108, 34, 179, 183, 69, 26, 209, 218, 58, 186, 9, 32, 237, - 253, 46, 80, 36, 200, 169, 239, 97, 200, 17, 188, 203, 99, 151, 40, - 10, 113, - }, - privKey: [boxKeyLen]byte{ - 74, 137, 121, 11, 209, 38, 48, 48, 167, 157, 184, 58, 2, 110, 9, 204, - 174, 148, 243, 154, 191, 74, 118, 90, 11, 240, 246, 131, 187, 157, - 157, 25, - }, - nonce: nonce{}, - pcr: map[uint][]byte{ - 0: { - 0xb0, 0x61, 0xbc, 0xe3, 0x1a, 0x85, 0x50, 0xc2, 0x4c, 0xb8, - 0xc1, 0xdc, 0x0e, 0x53, 0x98, 0xe5, 0xc8, 0x0f, 0xab, 0xa6, - 0x7f, 0x75, 0xfd, 0x3b, 0x06, 0x21, 0xc0, 0xb8, 0x66, 0x36, - 0xfc, 0xe0, 0xd6, 0x4c, 0x4d, 0x7d, 0x37, 0x47, 0x89, 0x08, - 0xe1, 0xf8, 0xfc, 0xe9, 0xdf, 0x66, 0xe1, 0xb9}, - 1: { - 0xbc, 0xdf, 0x05, 0xfe, 0xfc, 0xca, 0xa8, 0xe5, 0x5b, 0xf2, - 0xc8, 0xd6, 0xde, 0xe9, 0xe7, 0x9b, 0xbf, 0xf3, 0x1e, 0x34, - 0xbf, 0x28, 0xa9, 0x9a, 0xa1, 0x9e, 0x6b, 0x29, 0xc3, 0x7e, - 0xe8, 0x0b, 0x21, 0x4a, 0x41, 0x4b, 0x76, 0x07, 0x23, 0x6e, - 0xdf, 0x26, 0xfc, 0xb7, 0x86, 0x54, 0xe6, 0x3f}, - 2: { - 0x6a, 0xe6, 0x79, 0x76, 0xd7, 0x40, 0x38, 0x0d, 0x50, 0x64, - 0x36, 0x91, 0xac, 0x3a, 0xae, 0xbb, 0xa6, 0x0f, 0x27, 0xd7, - 0xb8, 0xa0, 0xe1, 0xa9, 0xea, 0xf2, 0x38, 0x6d, 0x25, 0xee, - 0xab, 0x88, 0x1c, 0x09, 0xac, 0xc5, 0xc8, 0x09, 0xeb, 0xec, - 0xf9, 0x9b, 0x49, 0x71, 0x05, 0xf6, 0xcb, 0x5b}, - 3: null, - 4: { - 0xd8, 0xa8, 0xe8, 0xee, 0xe9, 0x6d, 0x81, 0xb7, 0x7a, 0x25, - 0x14, 0x10, 0xb7, 0xa9, 0xb1, 0x80, 0x78, 0x76, 0x53, 0xf1, - 0x25, 0xd1, 0xdb, 0xca, 0x79, 0x68, 0x5c, 0x93, 0xfb, 0x88, - 0x5b, 0x33, 0x5e, 0x0b, 0x8d, 0x17, 0x2c, 0x98, 0x21, 0xa8, - 0x62, 0x51, 0x5a, 0x60, 0x3c, 0xc3, 0x3a, 0xb2}, - 5: null, - 6: null, - 7: null, - 8: null, - 9: null, - 10: null, - 11: null, - 12: null, - 13: null, - 14: null, - 15: null, - }, - attDocTime: mustParse("2022-07-27T05:00:00Z"), - // The following attestation document was generated on 2022-07-27 and - // contains a nonce (set to all 0 bytes) and user data (contains encrypted - // key information). - attDoc: ` -hEShATgioFkRG6lpbW9kdWxlX2lkeCdpLTA4MDk4NDk3MTBiZjFiNjFiLWVuYzAxODIzZDY0M2U2Mzl -hYTBmZGlnZXN0ZlNIQTM4NGl0aW1lc3RhbXAbAAABgj1kWVpkcGNyc7AAWDCwYbzjGoVQwky4wdwOU5 -jlyA+rpn91/TsGIcC4Zjb84NZMTX03R4kI4fj86d9m4bkBWDC83wX+/Mqo5VvyyNbe6eebv/MeNL8oq -Zqhnmspw37oCyFKQUt2ByNu3yb8t4ZU5j8CWDBq5nl210A4DVBkNpGsOq67pg8n17ig4anq8jhtJe6r -iBwJrMXICevs+ZtJcQX2y1sDWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAEWDDYqOju6W2Bt3olFBC3qbGAeHZT8SXR28p5aFyT+4hbM14LjRcsmCGoYlFaYDzDOr -IFWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGWDAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHWDAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAKWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAALWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAMWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWDAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOWDAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPWDAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABrY2VydGlmaWNhdGVZAn8wggJ7MIICAaADAgECAhAB -gj1kPmOaoAAAAABi4JzCMAoGCCqGSM49BAMDMIGOMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGl -uZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxOTA3Bg -NVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsYXZlczAeFw0yM -jA3MjcwMjAyMzlaFw0yMjA3MjcwNTAyNDJaMIGTMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGlu -Z3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxPjA8BgN -VBAMMNWktMDgwOTg0OTcxMGJmMWI2MWItZW5jMDE4MjNkNjQzZTYzOWFhMC51cy1lYXN0LTIuYXdzMH -YwEAYHKoZIzj0CAQYFK4EEACIDYgAE7il+oEijv6hrLpdsG4T/TbjSNxla6LnM2/2IGyCFNblCghVv1 -VNv7JF1zu+pP4jT7VbeVEj2z5T0lQMc/bLLxXUcbVlaA8qzAIX5yTkwAA53zU6m7frzvWVwdSuSNvXw -ox0wGzAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIGwDAKBggqhkjOPQQDAwNoADBlAjEAiKzrNPjQug4 -lt4wfSuIxvyr4BoiS0en2pLM7NtI9QnQKwXKT7V1Rk4oKr7zVBeiJAjAMnKjSMZn3cID2nL55qgoeCF -0PXntyuGXwkh8J5bsN5BUKP38CiqmONjvyxPOiQWpoY2FidW5kbGWEWQIVMIICETCCAZagAwIBAgIRA -PkxdWgbkK/hHUbMtOTn+FYwCgYIKoZIzj0EAwMwSTELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpv -bjEMMAoGA1UECwwDQVdTMRswGQYDVQQDDBJhd3Mubml0cm8tZW5jbGF2ZXMwHhcNMTkxMDI4MTMyODA -1WhcNNDkxMDI4MTQyODA1WjBJMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDA -NBV1MxGzAZBgNVBAMMEmF3cy5uaXRyby1lbmNsYXZlczB2MBAGByqGSM49AgEGBSuBBAAiA2IABPwCV -OumCMHzaHDimtqQvkY4MpJzbolL//Zy2YlES1BR5TSksfbb48C8WBoyt7F2Bw7eEtaaP+ohG2bnUs99 -0d0JX28TcPQXCEPZ3BABIeTPYwEoCWZEh8l5YoQwTcU/9KNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgN -VHQ4EFgQUkCW1DdkFR+eWw5b6cp3PmanfS5YwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2kAMG -YCMQCjfy+Rocm9Xue4YnwWmNJVA44fA0P5W2OpYow9OYCVRaEevL8uO1XYru5xtMPWrfMCMQCi85sWB -bJwKKXdS6BptQFuZbT73o/gBh1qUxl/nNr12UO8Yfwr6wPLb+6NIwLz3/ZZAsIwggK+MIICRKADAgEC -AhA990CB9kGNMZfvZChwdlgnMAoGCCqGSM49BAMDMEkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF -6b24xDDAKBgNVBAsMA0FXUzEbMBkGA1UEAwwSYXdzLm5pdHJvLWVuY2xhdmVzMB4XDTIyMDcyNTA2ND -gwOFoXDTIyMDgxNDA3NDgwOFowZDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpvbjEMMAoGA1UEC -wwDQVdTMTYwNAYDVQQDDC05ZjJmYTNhYWRlZTBhMzZhLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jbGF2 -ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASJrZH+NsENK8xQ+r4qIT56spgyQ0rqLuQOUv7CmfHg19Z -giX4k1tUbTIGc1hFVphxMaahoM6N3e1mBRMkX9Y/gxYmPnSrom/cq6BnWW8yYWpocaFuXqq/VjOJ9Ba -TcO4qjgdUwgdIwEgYDVR0TAQH/BAgwBgEB/wIBAjAfBgNVHSMEGDAWgBSQJbUN2QVH55bDlvpync+Zq -d9LljAdBgNVHQ4EFgQUPZaPBc+bF0kz5B3MQi4KoWP+hRIwDgYDVR0PAQH/BAQDAgGGMGwGA1UdHwRl -MGMwYaBfoF2GW2h0dHA6Ly9hd3Mtbml0cm8tZW5jbGF2ZXMtY3JsLnMzLmFtYXpvbmF3cy5jb20vY3J -sL2FiNDk2MGNjLTdkNjMtNDJiZC05ZTlmLTU5MzM4Y2I2N2Y4NC5jcmwwCgYIKoZIzj0EAwMDaAAwZQ -IxANMhikJw9gtb6vBdlgVKT1gOgX8g8HhmL764kGCqNUcQEx87vPMhiVamVtUsCIB/awIwbV4Neqsy1 -H4Cq3JWZG9lR2+D8s+nMCVDpUlThEK2K8p0EJP5lPF8N5e0V8oZuA0JWQMYMIIDFDCCApqgAwIBAgIQ -NF6Fd19DpZgwKsWQtzHX8TAKBggqhkjOPQQDAzBkMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1hem9 -uMQwwCgYDVQQLDANBV1MxNjA0BgNVBAMMLTlmMmZhM2FhZGVlMGEzNmEudXMtZWFzdC0yLmF3cy5uaX -Ryby1lbmNsYXZlczAeFw0yMjA3MjYxMTQ1MDBaFw0yMjA4MDEwNDQ0NTlaMIGJMTwwOgYDVQQDDDM0M -TE4MGUyNmU3ZWNjNWY0LnpvbmFsLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jbGF2ZXMxDDAKBgNVBAsM -A0FXUzEPMA0GA1UECgwGQW1hem9uMQswCQYDVQQGEwJVUzELMAkGA1UECAwCV0ExEDAOBgNVBAcMB1N -lYXR0bGUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARay8thWSHgPinGev1LSpKiNMhpY3uGlzdNtGrl4D -vRC3tYm3e4y1WC8zjR96rPEPqPaImtmJ4GuXUC8oP5u1g4Cr4jFqqL4KwvwvZFeOhY5FdIEidNByaFQ -1PmdLWM7OGjgeowgecwEgYDVR0TAQH/BAgwBgEB/wIBATAfBgNVHSMEGDAWgBQ9lo8Fz5sXSTPkHcxC -LgqhY/6FEjAdBgNVHQ4EFgQUQjKkC8oNyrWpWkdVSGw8wOOYa9IwDgYDVR0PAQH/BAQDAgGGMIGABgN -VHR8EeTB3MHWgc6Bxhm9odHRwOi8vY3JsLXVzLWVhc3QtMi1hd3Mtbml0cm8tZW5jbGF2ZXMuczMudX -MtZWFzdC0yLmFtYXpvbmF3cy5jb20vY3JsLzI1NDY2N2JmLWY2ZDMtNDNlZS1iMGNiLTYyZWNmZWNiZ -TZmMS5jcmwwCgYIKoZIzj0EAwMDaAAwZQIxAKuYyI19bA8mHLo88O1epcirSbOfK348e6SbhdyJazZb -cIkko5zyvgKmskjACB2IpwIwVo0cIeP+2C4L+5CW8iVr5DrRVhtESi+qta4DzYNJlUXl2X3HiV23fqz -2/3XY9uyqWQKCMIICfjCCAgSgAwIBAgIUfFo+I5v6VGh7k5qouGsLv7Mv57owCgYIKoZIzj0EAwMwgY -kxPDA6BgNVBAMMMzQxMTgwZTI2ZTdlY2M1ZjQuem9uYWwudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsY -XZlczEMMAoGA1UECwwDQVdTMQ8wDQYDVQQKDAZBbWF6b24xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJX -QTEQMA4GA1UEBwwHU2VhdHRsZTAeFw0yMjA3MjYxNzIyMThaFw0yMjA3MjcxNzIyMThaMIGOMQswCQY -DVQQGEwJVUzETMBEGA1UECAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW -1hem9uMQwwCgYDVQQLDANBV1MxOTA3BgNVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yL -mF3cy5uaXRyby1lbmNsYXZlczB2MBAGByqGSM49AgEGBSuBBAAiA2IABHauNrI7BTIweN+zwPt+cchE -nzuRwHLILTAHh3OTa47tKPrx5siwKIwhkjOvzAN82o4MzgUmqtfQ0yrntfrox2be5qzKx7U26aatS5G -JR/STHSjtoeKZn5FLMYysMJM00KMmMCQwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAg -QwCgYIKoZIzj0EAwMDaAAwZQIxAK5vbx5ZauD2RpeK2+v3u37cc9imCrMvF1JY4zbZ3ZZQ8UYa/HjnP -iB3pGd8whiA7wIwNiE2h4KKQEhF4Ory87EpxJCT39uXxVByr5TWQ89Ruj1rB2JSXU1psJ8GxlCkcBVD -anB1YmxpY19rZXn2aXVzZXJfZGF0YVhI4nBS+mXu8vE1TpAF0X8GLthggVJB44h4AnNzMvCtD3Qlagn -FYcA3/G9DXSk1uaaVLTm/O4nVtbjo4MaU8C2rqO94hvbTrml7ZW5vbmNlVAAAAAAAAAAAAAAAAAAAAA -AAAAAAWGDMVPwPgNQE0B4IvYVyzsWa6IguwPxu4RrKW7SzNkcv9b0RySXdAAPD071+Ju6Ic8Pr4EOyd -ac+wcqKQm4ZH3U5+yel2+YU33Tq/WvX1Ra2xmQsgQj3xqcL9XMBbdmNW8M=`, -} diff --git a/keysync_responder.go b/keysync_responder.go deleted file mode 100644 index c199119..0000000 --- a/keysync_responder.go +++ /dev/null @@ -1,128 +0,0 @@ -package main - -import ( - cryptoRand "crypto/rand" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/hf/nitrite" - "golang.org/x/crypto/nacl/box" -) - -var ( - errFailedNonce = errors.New("failed to create nonce") - errNoBase64 = errors.New("failed to Base64-decode attestation document") - errFailedVerify = errors.New("failed to verify attestation document") - errFailedRespBody = errors.New("failed to read response body") - errFailedPCR = errors.New("failed to get PCR values") - errFailedFindNonce = errors.New("could not find provided nonce") - errInvalidBoxKeys = errors.New("invalid box key material") - errPCRNotIdentical = errors.New("remote enclave's PCR values not identical") -) - -// nonceHandler returns a HandlerFunc that creates a new nonce and returns it -// to the client. -func nonceHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - nonce, err := newNonce() - if err != nil { - http.Error(w, errFailedNonce.Error(), http.StatusInternalServerError) - return - } - - e.nonceCache.Add(nonce.B64()) - fmt.Fprintln(w, nonce.B64()) - } -} - -// respSyncHandler returns a HandlerFunc that shares our secret key material -// with the requesting enclave -- after authentication, of course. -func respSyncHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ourNonce, theirNonce nonce - - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(maxReadLen))) - if err != nil { - http.Error(w, errFailedRespBody.Error(), http.StatusInternalServerError) - return - } - theirRawAttDoc, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(body))) - if err != nil { - http.Error(w, errNoBase64.Error(), http.StatusInternalServerError) - return - } - - // Verify the remote enclave's attestation document before touching it. - opts := nitrite.VerifyOptions{CurrentTime: currentTime()} - res, err := nitrite.Verify(theirRawAttDoc, opts) - if err != nil { - http.Error(w, errFailedVerify.Error(), http.StatusUnauthorized) - return - } - theirAttDoc := res.Document - - // Are the PCR values (i.e. image IDs) identical? - ourPCRs, err := getPCRValues() - if err != nil { - http.Error(w, errFailedPCR.Error(), http.StatusInternalServerError) - return - } - if !arePCRsIdentical(ourPCRs, theirAttDoc.PCRs) { - http.Error(w, errPCRNotIdentical.Error(), http.StatusUnauthorized) - return - } - - // Did we actually issue the nonce that the remote enclave provided? - copy(ourNonce[:], theirAttDoc.Nonce) - if !e.nonceCache.Exists(ourNonce.B64()) { - http.Error(w, errFailedFindNonce.Error(), http.StatusUnauthorized) - return - } - - // If we made it this far, we're convinced that we're talking to an - // identical enclave. Now get the remote enclave's nonce, which is in - // the attestation document's "user data" field. - copy(theirNonce[:], theirAttDoc.UserData) - - if len(theirAttDoc.PublicKey) != boxKeyLen { - http.Error(w, errInvalidBoxKeys.Error(), http.StatusBadRequest) - return - } - theirBoxPubKey := &[boxKeyLen]byte{} - copy(theirBoxPubKey[:], theirAttDoc.PublicKey[:]) - - // Encrypt our key material with the provided key. - jsonKeyMaterial, err := json.Marshal(e.keys) - if err != nil { - http.Error(w, "failed to marshal key material", http.StatusInternalServerError) - return - } - var encrypted []byte - if encrypted, err = box.SealAnonymous( - nil, - jsonKeyMaterial, - theirBoxPubKey, - cryptoRand.Reader, - ); err != nil { - http.Error(w, "failed to encrypt key material", http.StatusInternalServerError) - return - } - - // Encapsulate the remote enclave's nonce and the encrypted key - // material in an attestation document and send it back. - ourAttDoc, err := attest(theirNonce[:], encrypted, nil) - if err != nil { - http.Error(w, errFailedAttestation, http.StatusInternalServerError) - return - } - - b64AttDoc := base64.StdEncoding.EncodeToString(ourAttDoc) - fmt.Fprint(w, b64AttDoc) - } -} diff --git a/keysync_responder_test.go b/keysync_responder_test.go deleted file mode 100644 index b700a2e..0000000 --- a/keysync_responder_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package main - -import ( - "bytes" - "crypto/rand" - "encoding/base64" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "time" - - "testing" -) - -func queryHandler(handler http.HandlerFunc, path string, reader io.Reader) *http.Response { - req := httptest.NewRequest(http.MethodGet, path, reader) - rec := httptest.NewRecorder() - handler(rec, req) - res := rec.Result() - defer res.Body.Close() - return res -} - -func TestNonceHandler(t *testing.T) { - enclave := createEnclave(&defaultCfg) - res := queryHandler(nonceHandler(enclave), pathNonce, bytes.NewReader([]byte{})) - - // Did the operation succeed? - if res.StatusCode != http.StatusOK { - t.Fatalf("Expected status code %d but got %d.", http.StatusOK, res.StatusCode) - } - - // Did we get what looks like a nonce? - b64Nonce, err := io.ReadAll(res.Body) - failOnErr(t, err) - rawNonce, err := base64.StdEncoding.DecodeString(string(b64Nonce)) - if err != nil { - t.Fatalf("Failed to decode Base64-encoded nonce: %s", err) - } - if len(rawNonce) != nonceLen { - t.Fatalf("Expected nonce length %d but got %d.", nonceLen, len(rawNonce)) - } - - // Was the nonce added to the enclave's nonce cache? - if !enclave.nonceCache.Exists(strings.TrimSpace(string(b64Nonce))) { - t.Fatal("Nonce was not added to enclave's nonce cache.") - } -} - -func TestNonceHandlerIfErr(t *testing.T) { - cryptoRead = func(b []byte) (n int, err error) { - return 0, errors.New("not enough randomness") - } - defer func() { - cryptoRead = rand.Read - }() - - res := queryHandler( - nonceHandler(createEnclave(&defaultCfg)), - pathNonce, - bytes.NewReader([]byte{}), - ) - - // Did the operation fail? - if res.StatusCode != http.StatusInternalServerError { - t.Fatalf("Expected status code %d but got %d.", http.StatusInternalServerError, res.StatusCode) - } - - // Did we get the correct error string? - errMsg, err := io.ReadAll(res.Body) - failOnErr(t, err) - if strings.TrimSpace(string(errMsg)) != errFailedNonce.Error() { - t.Fatalf("Expected error message %q but got %q.", errFailedNonce.Error(), errMsg) - } -} - -func TestRespSyncHandlerForBadReqs(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - - // Send non-Base64 bogus data. - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader("foobar!")) - assertResponse(t, res, newResp(http.StatusInternalServerError, errNoBase64.Error())) - - // Send Base64-encoded bogus data. - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader("Zm9vYmFyCg==")) - assertResponse(t, res, newResp(http.StatusUnauthorized, errFailedVerify.Error())) -} - -func TestRespSyncHandler(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - enclave.nonceCache.Add(initAttInfo.nonce.B64()) - - // Mock functions for our tests to pass. - getPCRValues = func() (map[uint][]byte, error) { - return initAttInfo.pcr, nil - } - currentTime = func() time.Time { return initAttInfo.attDocTime } - - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader(initAttInfo.attDoc)) - // On a non-enclave platform, the responder code will get as far as to - // request its attestation document. - assertResponse(t, res, newResp(http.StatusInternalServerError, errFailedAttestation)) -} - -func TestRespSyncHandlerDoS(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - - // Send more data than the handler should be willing to read. - maxSize := base64.StdEncoding.EncodedLen(maxAttDocLen) - body := make([]byte, maxSize+1) - res = queryHandler(respSyncHandler(enclave), pathSync, bytes.NewReader(body)) - assertResponse(t, res, newResp(http.StatusInternalServerError, errFailedRespBody.Error())) -} - -var initAttInfo = &remoteAttInfo{ - pubKey: [boxKeyLen]byte{ - 213, 156, 108, 34, 179, 183, 69, 26, 209, 218, 58, 186, 9, 32, 237, - 253, 46, 80, 36, 200, 169, 239, 97, 200, 17, 188, 203, 99, 151, 40, - 10, 113, - }, - privKey: [boxKeyLen]byte{ - 74, 137, 121, 11, 209, 38, 48, 48, 167, 157, 184, 58, 2, 110, 9, 204, - 174, 148, 243, 154, 191, 74, 118, 90, 11, 240, 246, 131, 187, 157, - 157, 25, - }, - nonce: nonce{}, - pcr: map[uint][]byte{ - 0: { - 0xda, 0x54, 0x6f, 0x8d, 0xda, 0x37, 0x52, 0x19, 0x45, 0xdf, 0x4a, - 0x6d, 0x3e, 0x39, 0x70, 0x63, 0x58, 0x8c, 0xd5, 0xf8, 0x70, 0xaa, - 0xa0, 0x7a, 0x62, 0xe9, 0x67, 0xb2, 0x54, 0xd5, 0xf8, 0x17, 0x6d, - 0xaa, 0x96, 0xec, 0x83, 0xcd, 0xc5, 0x40, 0x2b, 0x0b, 0x52, 0x7a, - 0x16, 0x24, 0x72, 0xb5}, - 1: { - 0xbc, 0xdf, 0x05, 0xfe, 0xfc, 0xca, 0xa8, 0xe5, 0x5b, 0xf2, 0xc8, - 0xd6, 0xde, 0xe9, 0xe7, 0x9b, 0xbf, 0xf3, 0x1e, 0x34, 0xbf, 0x28, - 0xa9, 0x9a, 0xa1, 0x9e, 0x6b, 0x29, 0xc3, 0x7e, 0xe8, 0x0b, 0x21, - 0x4a, 0x41, 0x4b, 0x76, 0x07, 0x23, 0x6e, 0xdf, 0x26, 0xfc, 0xb7, - 0x86, 0x54, 0xe6, 0x3f}, - 2: { - 0x45, 0xaa, 0xd9, 0xf5, 0xc3, 0x9a, 0x90, 0x5b, 0x9f, 0xef, 0xac, - 0x05, 0x56, 0x87, 0x0a, 0x20, 0xd1, 0x6f, 0x3f, 0x3c, 0x21, 0xcf, - 0x93, 0x3e, 0x60, 0x64, 0xff, 0xf9, 0x24, 0xaf, 0x9c, 0x13, 0xed, - 0x26, 0xab, 0x6d, 0x56, 0x3e, 0x27, 0x2b, 0x85, 0xe7, 0xc3, 0x17, - 0x0f, 0x01, 0xac, 0xda}, - 3: null, - 4: { - 0xd8, 0xa8, 0xe8, 0xee, 0xe9, 0x6d, 0x81, 0xb7, 0x7a, 0x25, 0x14, - 0x10, 0xb7, 0xa9, 0xb1, 0x80, 0x78, 0x76, 0x53, 0xf1, 0x25, 0xd1, - 0xdb, 0xca, 0x79, 0x68, 0x5c, 0x93, 0xfb, 0x88, 0x5b, 0x33, 0x5e, - 0x0b, 0x8d, 0x17, 0x2c, 0x98, 0x21, 0xa8, 0x62, 0x51, 0x5a, 0x60, - 0x3c, 0xc3, 0x3a, 0xb2}, - 5: null, - 6: null, - 7: null, - 8: null, - 9: null, - 10: null, - 11: null, - 12: null, - 13: null, - 14: null, - 15: null, - }, - attDocTime: mustParse("2022-08-04T20:00:00Z"), - // The following attestation document was generated on 2022-08-04 and - // contains a nonce (set to all 0 bytes), user data (contains a nonce and - // is set to all 0 bytes), and a public key (contains an NaCl public key). - attDoc: ` -hEShATgioFkRB6lpbW9kdWxlX2lkeCdpLTA4MDk4NDk3MTBiZjFiNjFiLWVuYzAxODI2YTQ0OWEwMGI -xN2FmZGlnZXN0ZlNIQTM4NGl0aW1lc3RhbXAbAAABgmpE949kcGNyc7AAWDDaVG+N2jdSGUXfSm0+OX -BjWIzV+HCqoHpi6WeyVNX4F22qluyDzcVAKwtSehYkcrUBWDC83wX+/Mqo5VvyyNbe6eebv/MeNL8oq -Zqhnmspw37oCyFKQUt2ByNu3yb8t4ZU5j8CWDBFqtn1w5qQW5/vrAVWhwog0W8/PCHPkz5gZP/5JK+c -E+0mq21WPicrhefDFw8BrNoDWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAEWDDYqOju6W2Bt3olFBC3qbGAeHZT8SXR28p5aFyT+4hbM14LjRcsmCGoYlFaYDzDOr -IFWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGWDAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHWDAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAKWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAALWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAMWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWDAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOWDAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPWDAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABrY2VydGlmaWNhdGVZAoAwggJ8MIICAaADAgECAhAB -gmpEmgCxegAAAABi7BnHMAoGCCqGSM49BAMDMIGOMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGl -uZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxOTA3Bg -NVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsYXZlczAeFw0yM -jA4MDQxOTExMDBaFw0yMjA4MDQyMjExMDNaMIGTMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGlu -Z3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxPjA8BgN -VBAMMNWktMDgwOTg0OTcxMGJmMWI2MWItZW5jMDE4MjZhNDQ5YTAwYjE3YS51cy1lYXN0LTIuYXdzMH -YwEAYHKoZIzj0CAQYFK4EEACIDYgAEpy6eKLNsGy1mhV9SjR5Yj1Wn3wGmX87HinGw/jjpz/Ij3JsGO -HoF0Ve7wtVGgHxT0MjRh/1a45Zd39zpWMyc06tiN6ZM9S9GKws23tPr826TGE9PNB4jQhsNv8gHEJT3 -ox0wGzAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIGwDAKBggqhkjOPQQDAwNpADBmAjEA8SBbh3YYlv/ -XZPttIR9m43jTNkgUHkWyB9hxhWkVEjnfb3MDqAPFhMh5BFoArDD0AjEAj3XawBSe5AK9842TdW/mt+ -C0e/OSZpaFAJqvTAX9MNX3wSEm/Jron+wtoVb+DecTaGNhYnVuZGxlhFkCFTCCAhEwggGWoAMCAQICE -QD5MXVoG5Cv4R1GzLTk5/hWMAoGCCqGSM49BAMDMEkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF6 -b24xDDAKBgNVBAsMA0FXUzEbMBkGA1UEAwwSYXdzLm5pdHJvLWVuY2xhdmVzMB4XDTE5MTAyODEzMjg -wNVoXDTQ5MTAyODE0MjgwNVowSTELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpvbjEMMAoGA1UECw -wDQVdTMRswGQYDVQQDDBJhd3Mubml0cm8tZW5jbGF2ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAT8A -lTrpgjB82hw4prakL5GODKSc26JS//2ctmJREtQUeU0pLH22+PAvFgaMrexdgcO3hLWmj/qIRtm51LP -fdHdCV9vE3D0FwhD2dwQASHkz2MBKAlmRIfJeWKEME3FP/SjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQY -DVR0OBBYEFJAltQ3ZBUfnlsOW+nKdz5mp30uWMA4GA1UdDwEB/wQEAwIBhjAKBggqhkjOPQQDAwNpAD -BmAjEAo38vkaHJvV7nuGJ8FpjSVQOOHwND+VtjqWKMPTmAlUWhHry/LjtV2K7ucbTD1q3zAjEAovObF -gWycCil3UugabUBbmW0+96P4AYdalMZf5za9dlDvGH8K+sDy2/ujSMC89/2WQLBMIICvTCCAkSgAwIB -AgIQQpblfNs/3yOBCWXcu04/WDAKBggqhkjOPQQDAzBJMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1 -hem9uMQwwCgYDVQQLDANBV1MxGzAZBgNVBAMMEmF3cy5uaXRyby1lbmNsYXZlczAeFw0yMjA4MDQwNT -Q4MDhaFw0yMjA4MjQwNjQ4MDhaMGQxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF6b24xDDAKBgNVB -AsMA0FXUzE2MDQGA1UEAwwtOWUyOTllNTRmZTE2M2Q1YS51cy1lYXN0LTIuYXdzLm5pdHJvLWVuY2xh -dmVzMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEzjYIWIyVfPvNjCsLf8VS2P1R1lmaMox7vIOWVU5sfCp -/kyhzz1RLlKTPZLXRfpVZWT8F58ygN3AAzjqOfS8HzWRwfmH0kTsP9T/U2kLYIEYlEPv7Qw98U/1+Wx -VpP94zo4HVMIHSMBIGA1UdEwEB/wQIMAYBAf8CAQIwHwYDVR0jBBgwFoAUkCW1DdkFR+eWw5b6cp3Pm -anfS5YwHQYDVR0OBBYEFPjkq4ZrPqs7R5KtXk3YYASqnwhwMA4GA1UdDwEB/wQEAwIBhjBsBgNVHR8E -ZTBjMGGgX6BdhltodHRwOi8vYXdzLW5pdHJvLWVuY2xhdmVzLWNybC5zMy5hbWF6b25hd3MuY29tL2N -ybC9hYjQ5NjBjYy03ZDYzLTQyYmQtOWU5Zi01OTMzOGNiNjdmODQuY3JsMAoGCCqGSM49BAMDA2cAMG -QCMHXCswA211klCMLW+p3sD8sces9/WEEuIxeaQ1lwKfbCW9yWN7ynujRz01+W378qBgIwXoEP9UIQ2 -p7oC7+HJYp/GNiFrYg4mwEETh75CWX38CFEIyZtbx9abI8pb3bQ+zaZWQMZMIIDFTCCApugAwIBAgIR -ANSXsgCDg5YY9m4w3HzTIQ8wCgYIKoZIzj0EAwMwZDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXp -vbjEMMAoGA1UECwwDQVdTMTYwNAYDVQQDDC05ZTI5OWU1NGZlMTYzZDVhLnVzLWVhc3QtMi5hd3Mubm -l0cm8tZW5jbGF2ZXMwHhcNMjIwODA0MTU1MDI2WhcNMjIwODEwMTY1MDI1WjCBiTE8MDoGA1UEAwwzN -2MwM2Q3ZjMxM2IzZDdiOC56b25hbC51cy1lYXN0LTIuYXdzLm5pdHJvLWVuY2xhdmVzMQwwCgYDVQQL -DANBV1MxDzANBgNVBAoMBkFtYXpvbjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMRAwDgYDVQQHDAd -TZWF0dGxlMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEjh5qz+Cx8rHDYKvh1E7GqR+GDG/g5CzjPBiu+p -krFdYe8AN58lfwHv2+YN6i+lOmjpjFADefv6yBS7Va7Ddj6DB3cJWcOhOlKqIyDZpZ4yDeG5H2TvxGi -IR+1vFPFDKZo4HqMIHnMBIGA1UdEwEB/wQIMAYBAf8CAQEwHwYDVR0jBBgwFoAU+OSrhms+qztHkq1e -TdhgBKqfCHAwHQYDVR0OBBYEFGLAF3Hq2WorIwNXLsBHMR4s0uD1MA4GA1UdDwEB/wQEAwIBhjCBgAY -DVR0fBHkwdzB1oHOgcYZvaHR0cDovL2NybC11cy1lYXN0LTItYXdzLW5pdHJvLWVuY2xhdmVzLnMzLn -VzLWVhc3QtMi5hbWF6b25hd3MuY29tL2NybC83MjJlMzIxYy1mYTdmLTRjZjQtYjljMS00YzQ0YzFiN -2M3OWQuY3JsMAoGCCqGSM49BAMDA2gAMGUCMCC/N/6QnA+LQgtLMZhqSXcq8stbOQZ7PTZ6uOK6XcO2 -FC6huMamexK3bkjXQ9tUzgIxANwt5DWIAvBA1hfn1wBl7gQqz1bSlenLqz0ZFyxFW4sT0/rur4ui7OG -JCF5IG4P8zVkCgTCCAn0wggIEoAMCAQICFHPGskW4/ej8wLV8S9yhPGlYOibuMAoGCCqGSM49BAMDMI -GJMTwwOgYDVQQDDDM3YzAzZDdmMzEzYjNkN2I4LnpvbmFsLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jb -GF2ZXMxDDAKBgNVBAsMA0FXUzEPMA0GA1UECgwGQW1hem9uMQswCQYDVQQGEwJVUzELMAkGA1UECAwC -V0ExEDAOBgNVBAcMB1NlYXR0bGUwHhcNMjIwODA0MTcyMjMyWhcNMjIwODA1MTcyMjMyWjCBjjELMAk -GA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDzANBgNVBAoMBk -FtYXpvbjEMMAoGA1UECwwDQVdTMTkwNwYDVQQDDDBpLTA4MDk4NDk3MTBiZjFiNjFiLnVzLWVhc3QtM -i5hd3Mubml0cm8tZW5jbGF2ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAR2rjayOwUyMHjfs8D7fnHI -RJ87kcByyC0wB4dzk2uO7Sj68ebIsCiMIZIzr8wDfNqODM4FJqrX0NMq57X66Mdm3uasyse1NummrUu -RiUf0kx0o7aHimZ+RSzGMrDCTNNCjJjAkMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAg -IEMAoGCCqGSM49BAMDA2cAMGQCMCyFFdZ9EjHxnY9dnOnTevkwJFOYEmLsSQAzl2D6X64LpuuKunhnr -VGEE8wz7lxwZgIwaRQAmr0Ke/l2wNI5UcWoov7VVYoVgbA/VudK7x85KMoqRH+N/IRXcgQWAlw3pZv8 -anB1YmxpY19rZXlYINWcbCKzt0Ua0do6ugkg7f0uUCTIqe9hyBG8y2OXKApxaXVzZXJfZGF0YVQAAAA -AAAAAAAAAAAAAAAAAAAAAAGVub25jZVQAAAAAAAAAAAAAAAAAAAAAAAAAAFhgUx278a0ygjrmIGtSIe -WlqX/lNr5cx9VZAFb5rFJ6igkZsFxwmk764LQEcCE7sifYTKvf/4jpKGdTw+wwmu+Ekdqhi0rmm7dgG -PxIqEgb+JWZGN+Ke5HwVMMGoboAYCXw`, -} diff --git a/keysync_shared_test.go b/keysync_shared_test.go index 37d6249..073b69f 100644 --- a/keysync_shared_test.go +++ b/keysync_shared_test.go @@ -4,32 +4,8 @@ import ( "crypto/rand" "errors" "testing" - "time" ) -var ( - null = make([]byte, 48) // An empty PCR value. -) - -// remoteAttInfo contains everything that we need to verify a remote enclave's -// attestation information. -type remoteAttInfo struct { - pubKey [boxKeyLen]byte - privKey [boxKeyLen]byte - nonce nonce - pcr map[uint][]byte - attDocTime time.Time - attDoc string -} - -func mustParse(timeStr string) time.Time { - t, err := time.Parse(time.RFC3339, timeStr) - if err != nil { - panic(err) - } - return t -} - func failOnErr(t *testing.T, err error) { t.Helper() if err != nil { diff --git a/main.go b/main.go index caad6ba..91756f1 100644 --- a/main.go +++ b/main.go @@ -2,13 +2,11 @@ package main import ( "bufio" - "crypto/tls" "errors" "flag" "io" "log" "math" - "net/http" "net/url" "os" "os/exec" @@ -100,12 +98,6 @@ func main() { elog.Fatalf("-prometheus-namespace must be set when Prometheus is used.") } - // TODO: If we choose to abandon Let's Encrypt, we need to tell Go to - // forego certificate validation. We shouldn't do this globally though. - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - c := &Config{ FQDN: fqdn, FQDNLeader: fqdnLeader, diff --git a/metrics_test.go b/metrics_test.go index 299468c..0ccfbdc 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -18,7 +18,7 @@ func TestHandlerMetrics(t *testing.T) { // installed. c.PrometheusPort = 80 enclave := createEnclave(&c) - makeReq := makeRequestFor(enclave.pubSrv) + makeReq := makeRequestFor(enclave.extPubSrv) // GET /enclave/config assertResponse(t, diff --git a/sync_leader.go b/sync_leader.go new file mode 100644 index 0000000..6936efd --- /dev/null +++ b/sync_leader.go @@ -0,0 +1,107 @@ +package main + +import ( + cryptoRand "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "golang.org/x/crypto/nacl/box" +) + +// leaderSync holds the state and code that we need for a one-off sync with a +// worker enclave. +type leaderSync struct { + attester + keys *enclaveKeys +} + +// asLeader returns a new leaderSync struct. +func asLeader(keys *enclaveKeys) *leaderSync { + return &leaderSync{ + attester: &dummyAttester{}, + keys: keys, + } +} + +// syncWith makes the leader initiate key synchronization with the given worker +// enclave. +func (s *leaderSync) syncWith(worker *url.URL) error { + elog.Println("Initiating key synchronization with worker.") + + // Step 1: Create a nonce that the worker must embed in its attestation + // document, to prevent replay attacks. + nonce, err := newNonce() + if err != nil { + return err + } + + // Step 2: Request the worker's attestation document, and provide the + // previously-generated nonce. + reqURL := *worker + reqURL.RawQuery = fmt.Sprintf("nonce=%x", nonce) + resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + // Step 3: Verify the worker's attestation document and extract its + // auxiliary information. + b64Attstn, err := io.ReadAll(newLimitReader(resp.Body, maxAttDocLen)) + if err != nil { + return err + } + resp.Body.Close() + attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + if err != nil { + return err + } + workerAux, err := s.verifyAttstn(attstn, nonce) + if err != nil { + return err + } + + // Step 4: Encrypt the leader's enclave keys with the ephemeral public key + // that the worker put into its auxiliary information. + pubKey := &[boxKeyLen]byte{} + copy(pubKey[:], workerAux.(*workerAuxInfo).PublicKey[:]) + jsonKeys, err := json.Marshal(s.keys.get()) + if err != nil { + return err + } + var encrypted []byte + encrypted, err = box.SealAnonymous(nil, jsonKeys, pubKey, cryptoRand.Reader) + if err != nil { + return err + } + + // Step 5: Create the leader's auxiliary information, consisting of the + // worker's nonce and the encrypted enclave keys. + leaderAux := &leaderAuxInfo{ + WorkersNonce: workerAux.(*workerAuxInfo).WorkersNonce, + EnclaveKeys: encrypted, + } + attstn, err = s.createAttstn(leaderAux) + if err != nil { + return err + } + strAttstn := base64.StdEncoding.EncodeToString(attstn) + + // Step 6: Send the leader's attestation document to the worker. + resp, err = newUnauthenticatedHTTPClient().Post(worker.String(), "text/plain", strings.NewReader(strAttstn)) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + return nil +} diff --git a/sync_worker.go b/sync_worker.go new file mode 100644 index 0000000..f4388dd --- /dev/null +++ b/sync_worker.go @@ -0,0 +1,210 @@ +package main + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "golang.org/x/crypto/nacl/box" +) + +var ( + errBecameLeader = errors.New("became enclave leader") + errNonceRequired = errors.New("nonce is required") + errInProgress = errors.New("key sync already in progress") + errInvalidNonceLen = errors.New("invalid nonce length") + errDecrypting = errors.New("error decrypting enclave keys") +) + +// workerSync holds the state and code that we need for a one-off sync with a +// leader enclave. workerSync implements the http.Handler interface because the +// sync protocol requires two endpoints on the worker. +type workerSync struct { + attester + installKeys func(*enclaveKeys) error + ephemeralKeys chan *boxKey + nonce chan nonce + becameLeader chan struct{} +} + +// asWorker returns a new workerSync object. +func asWorker( + installKeys func(*enclaveKeys) error, + becameLeader chan struct{}, +) *workerSync { + return &workerSync{ + attester: &dummyAttester{}, + installKeys: installKeys, + becameLeader: becameLeader, + nonce: make(chan nonce, 1), + ephemeralKeys: make(chan *boxKey, 1), + } +} + +// registerWith registers the worker with the given leader enclave. +func (s *workerSync) registerWith(leader *url.URL) error { + elog.Println("Attempting to sync with leader.") + + errChan := make(chan error) + register := func(e chan error) { + resp, err := newUnauthenticatedHTTPClient().Post(leader.String(), "text/plain", nil) + if err != nil { + e <- err + return + } + if resp.StatusCode != http.StatusOK { + e <- fmt.Errorf("leader returned HTTP code %d", resp.StatusCode) + return + } + e <- nil + } + go register(errChan) + + // Keep on trying every five seconds, for a minute. + retry := time.NewTicker(5 * time.Second) + timeout := time.NewTicker(time.Minute) + for { + select { + case err := <-errChan: + if err == nil { + elog.Println("Successfully registered with leader.") + return nil + } + elog.Printf("Error registering with leader: %v", err) + case <-timeout.C: + return errors.New("timed out syncing with leader") + case <-s.becameLeader: + elog.Println("We became leader. Aborting key sync.") + return errBecameLeader + case <-retry.C: + go register(errChan) + } + } +} + +func (s *workerSync) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + s.initSync(w, r) + } else if r.Method == http.MethodPost { + s.finishSync(w, r) + } +} + +// initSync responds to the leader's request for initiating key synchronization. +func (s *workerSync) initSync(w http.ResponseWriter, r *http.Request) { + elog.Println("Received leader's request to initiate key sync.") + + // There must not be more than one key synchronization attempt at any given + // time. Abort if we get another request while key synchronization is still + // in progress. + if len(s.ephemeralKeys) > 0 { + http.Error(w, errInProgress.Error(), http.StatusTooManyRequests) + return + } + + // Extract the leader's nonce from the URL, which must look like this: + // https://example.com/enclave/sync?nonce=[HEX-ENCODED-NONCE] + hexNonce := r.URL.Query().Get("nonce") + if hexNonce == "" { + http.Error(w, errNonceRequired.Error(), http.StatusBadRequest) + return + } + nonceSlice, err := hex.DecodeString(hexNonce) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if len(nonceSlice) != nonceLen { + http.Error(w, errInvalidNonceLen.Error(), http.StatusBadRequest) + return + } + var leadersNonce nonce + copy(leadersNonce[:], nonceSlice) + + // Create the worker's nonce and store it in our channel, so we can later + // verify it. + workersNonce, err := newNonce() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + s.nonce <- workersNonce + + // Create an ephemeral key that the leader is going to use to encrypt + // its enclave keys. + boxKey, err := newBoxKey() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + s.ephemeralKeys <- boxKey + + // Create and return the worker's Base64-encoded attestation document. + aux := &workerAuxInfo{ + WorkersNonce: workersNonce, + LeadersNonce: leadersNonce, + PublicKey: boxKey.pubKey[:], + } + attstn, err := s.createAttstn(aux) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + fmt.Fprintln(w, base64.StdEncoding.EncodeToString(attstn)) +} + +// finishSync responds to the leader's final request before key synchronization +// is complete. +func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { + elog.Println("Received leader's request to complete key sync.") + + // Read the leader's Base64-encoded attestation document. + maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) + b64Attstn, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Decode Base64 to byte slice. + attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + aux, err := s.verifyAttstn(attstn, <-s.nonce) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + ephemeralKey := <-s.ephemeralKeys + // Decrypt the leader's enclave keys, which are encrypted with the + // public key that we provided earlier. + decrypted, ok := box.OpenAnonymous( + nil, + aux.(*leaderAuxInfo).EnclaveKeys, + ephemeralKey.pubKey, + ephemeralKey.privKey) + if !ok { + http.Error(w, errDecrypting.Error(), http.StatusBadRequest) + return + } + + // Install the leader's enclave keys. + var keys enclaveKeys + if err := json.Unmarshal(decrypted, &keys); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + s.installKeys(&keys) + + elog.Println("Successfully synced with leader.") +} diff --git a/sync_worker_test.go b/sync_worker_test.go new file mode 100644 index 0000000..082928f --- /dev/null +++ b/sync_worker_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "errors" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +// leaderKeys holds arbitrary keys that we use for testing. +var leaderKeys = &enclaveKeys{ + NitridingKey: []byte("NitridingTestKey"), + NitridingCert: []byte("NitridingTestCert"), + AppKeys: []byte("AppTestKeys"), +} + +func TestSuccessfulRegisterWith(t *testing.T) { + e := createEnclave(&defaultCfg) + hasRegistered := false + + srv := httptest.NewTLSServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + hasRegistered = true + w.WriteHeader(http.StatusOK) + }), + ) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("Error creating test server URL: %v", err) + } + + err = asWorker(e.installKeys, make(chan struct{})).registerWith(u) + if err != nil { + t.Fatalf("Error registering with leader: %v", err) + } + if !hasRegistered { + t.Fatal("Worker did not register with leader.") + } +} + +func TestAbortedRegisterWith(t *testing.T) { + e := createEnclave(&defaultCfg) + + // Provide a bogus URL that cannot be synced with. + bogusURL := &url.URL{ + Scheme: "https", + Host: "localhost:1", + } + abortChan := make(chan struct{}) + ret := make(chan error) + go func(ret chan error) { + ret <- asWorker(e.installKeys, abortChan).registerWith(bogusURL) + }(ret) + + // Designate the enclave as leader, after which registration should abort. + close(abortChan) + if err := <-ret; !errors.Is(err, errBecameLeader) { + t.Fatal("Enclave did not realize that it became leader.") + } +} + +func TestSuccessfulSync(t *testing.T) { + // Set up the worker. + worker := createEnclave(&defaultCfg) + srv := httptest.NewTLSServer( + asWorker(worker.installKeys, make(chan struct{})), + ) + workerURL, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("Error creating test server URL: %v", err) + } + + if err = asLeader(leaderKeys).syncWith(workerURL); err != nil { + t.Fatalf("Error syncing with leader: %v", err) + } + + // Make sure that the keys were synced correctly. + if !worker.keys.equal(leaderKeys) { + t.Fatalf("Keys differ between worker and leader:\n%v (worker)\n%v (leader)", + leaderKeys, worker.keys) + } +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..9ffc394 --- /dev/null +++ b/util.go @@ -0,0 +1,17 @@ +package main + +import ( + "crypto/tls" + "net/http" +) + +// newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS +// certificate validation. In the context of nitriding, this is fine because +// all we need is a *confidential* channel, and not an authenticated channel. +// Authentication is handled via attestation documents. +func newUnauthenticatedHTTPClient() *http.Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + return &http.Client{Transport: transport} +} diff --git a/workers.go b/workers.go index 74ff0e2..41ad601 100644 --- a/workers.go +++ b/workers.go @@ -21,6 +21,13 @@ func newWorkers(timeout time.Duration) *workers { } } +func (w *workers) length() int { + w.RLock() + defer w.RUnlock() + + return len(w.set) +} + func (w *workers) register(worker *url.URL) { w.Lock() defer w.Unlock() diff --git a/workers_test.go b/workers_test.go new file mode 100644 index 0000000..79da187 --- /dev/null +++ b/workers_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "net/url" + "testing" + "time" +) + +func TestWorkerRegistration(t *testing.T) { + w := newWorkers(time.Minute) + + // Identical URLs are only tracked once. + worker1 := url.URL{Host: "foo"} + w.register(&worker1) + w.register(&worker1) + assertEqual(t, w.length(), 1) + + worker2 := url.URL{Host: "bar"} + w.register(&worker2) + assertEqual(t, w.length(), 2) + + w.unregister(&worker1) + w.unregister(&worker2) + assertEqual(t, w.length(), 0) + + // Nothing should happen when attempting to unregister a non-existing + // worker. + w.unregister(&url.URL{Host: "does-not-exist"}) +} + +func TestUpdatingAndPruning(t *testing.T) { + // TODO +} From 2aa6d456130630cdfed5475ae78d5745ffead7c5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 15 Aug 2023 09:38:41 -0500 Subject: [PATCH 10/99] Fix proxy.go --- proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy.go b/proxy.go index de7937e..dfc02b4 100644 --- a/proxy.go +++ b/proxy.go @@ -23,7 +23,7 @@ var ( // runNetworking calls the function that sets up our networking environment. // If anything fails, we try again after a brief wait period. -func runNetworking(c *Config, stop chan bool) { +func runNetworking(c *Config, stop chan struct{}) { var err error for { if err = setupNetworking(c, stop); err == nil { @@ -42,9 +42,9 @@ func runNetworking(c *Config, stop chan bool) { // 3. Establish a connection with the proxy running on the host. // 4. Spawn goroutines to forward traffic between the TAP device and the proxy // running on the host. -func setupNetworking(c *Config, stop chan bool) error { elog.Println("Setting up networking between host and enclave.") defer elog.Println("Tearing down networking between host and enclave.") +func setupNetworking(c *Config, stop chan struct{}) error { // Establish connection with the proxy running on the EC2 host. endpoint := fmt.Sprintf("vsock://%d:%d/connect", parentCID, c.HostProxyPort) From bc8eb82c4e9f7ea58f078069dd9f3359b8e86416 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 15 Aug 2023 10:44:17 -0500 Subject: [PATCH 11/99] Add crude heartbeat mechanism. --- enclave.go | 37 +++++++++++++++++++++++++++++++++---- handlers.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/enclave.go b/enclave.go index 519dcf2..fb50380 100644 --- a/enclave.go +++ b/enclave.go @@ -21,6 +21,7 @@ import ( "net/http/httputil" _ "net/http/pprof" "net/url" + "strings" "sync" "time" @@ -90,8 +91,8 @@ type Enclave struct { metrics *metrics workers *workers keys *enclaveKeys - ready, stop, becameLeader chan struct{} httpsCert *certRetriever + ready, stop, becameLeader chan struct{} } // Config represents the configuration of our enclave service. @@ -295,7 +296,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(e)) - m.Get(pathHeartbeat, heartbeatHandler(e)) + m.Post(pathHeartbeat, heartbeatHandler(e)) m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader)) // Register enclave-internal HTTP API. @@ -369,18 +370,46 @@ func (e *Enclave) Start() error { } if e.cfg.isScalingEnabled() { - if err := asWorker(e.installKeys, e.becameLeader).registerWith(&url.URL{ + err := asWorker(e.installKeys, e.becameLeader).registerWith(&url.URL{ Scheme: "https", Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), Path: pathRegistration, - }); err != nil { + }) + if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } + go e.heartbeat() } return nil } +func (e *Enclave) getLeader(path string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), + Path: path, + } +} + +func (e *Enclave) heartbeat() { + // TODO: Use context to exit loop. + timer := time.NewTicker(time.Minute) + for range timer.C { + b64Hashes := base64.StdEncoding.EncodeToString(e.hashes.Serialize()) + _, err := newUnauthenticatedHTTPClient().Post( + e.getLeader(pathHeartbeat).String(), + "text/plain", + strings.NewReader(b64Hashes), + ) + if err != nil { + // TODO: what should we do if the leader is dead? + elog.Printf("Error sending heartbeat to leader: %v", err) + } + elog.Println("Sent heartbeat to leader.") + } +} + // Stop stops the enclave. func (e *Enclave) Stop() error { close(e.stop) diff --git a/handlers.go b/handlers.go index 8212a52..8535d40 100644 --- a/handlers.go +++ b/handlers.go @@ -18,6 +18,8 @@ const ( // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 + // The maximum length (in bytes) of the hash over our enclave keys. + maxEnclaveKeyHash = 128 // The HTML for the enclave's index page. indexPage = "This host runs inside an AWS Nitro Enclave.\n" ) @@ -240,8 +242,14 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // TODO: Use AttestationHashes instead. - fmt.Fprintln(w, e.keys.hashAndB64()) + // Read the worker's hashed key material. + body, err := io.ReadAll(newLimitReader(r.Body, maxEnclaveKeyHash)) + if err != nil { + http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + return + } + theirB64Hashes := string(body) + ourB64Hashes := base64.StdEncoding.EncodeToString(e.hashes.Serialize()) // Take note of the worker still being alive. worker, err := e.httpClientToSyncURL(r) @@ -250,5 +258,21 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { return } e.workers.updateAndPrune(worker) + + // Is the worker's key material outdated? If so, re-synchronize. + if ourB64Hashes != theirB64Hashes { + elog.Println("Worker's keys are outdated. Re-synchronizing.") + go func() { + worker, err := e.httpClientToSyncURL(r) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + if err := asLeader(e.keys.get()).syncWith(worker); err != nil { + elog.Printf("Error syncing with worker: %v", err) + return + } + elog.Println("Successfully re-synchronized with worker.") + }() + } } } From f65639373d84b78c141a18542fca1c0896e1e912 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 15 Aug 2023 10:57:59 -0500 Subject: [PATCH 12/99] Revise heartbeat mechanism. --- enclave.go | 20 ++++++-------------- enclave_keys.go | 11 +++++++++++ handlers.go | 6 +++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/enclave.go b/enclave.go index fb50380..1050852 100644 --- a/enclave.go +++ b/enclave.go @@ -9,7 +9,6 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/base64" "encoding/hex" "encoding/json" "encoding/pem" @@ -68,15 +67,6 @@ var ( errCfgMissingPort = errors.New("given config is missing port") ) -// hashAndB64 returns the Base64-encoded hash over our key material. The -// resulting string is not confidential as it's impractical to reverse the key -// material. -func (e *enclaveKeys) hashAndB64() string { - keys := append(append(e.NitridingCert, e.NitridingKey...), e.AppKeys...) - hash := sha256.Sum256(keys) - return base64.StdEncoding.EncodeToString(hash[:]) -} - // Enclave represents a service running inside an AWS Nitro Enclave. type Enclave struct { sync.RWMutex @@ -377,8 +367,9 @@ func (e *Enclave) Start() error { }) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) + } else if err == nil { + go e.heartbeat() } - go e.heartbeat() } return nil @@ -393,20 +384,21 @@ func (e *Enclave) getLeader(path string) *url.URL { } func (e *Enclave) heartbeat() { + elog.Println("Starting heartbeat loop.") // TODO: Use context to exit loop. timer := time.NewTicker(time.Minute) for range timer.C { - b64Hashes := base64.StdEncoding.EncodeToString(e.hashes.Serialize()) _, err := newUnauthenticatedHTTPClient().Post( e.getLeader(pathHeartbeat).String(), "text/plain", - strings.NewReader(b64Hashes), + strings.NewReader(e.keys.hashAndB64()), ) if err != nil { // TODO: what should we do if the leader is dead? elog.Printf("Error sending heartbeat to leader: %v", err) + } else { + elog.Println("Successfully sent heartbeat to leader.") } - elog.Println("Sent heartbeat to leader.") } } diff --git a/enclave_keys.go b/enclave_keys.go index 75d5f19..47e2c8d 100644 --- a/enclave_keys.go +++ b/enclave_keys.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/sha256" + "encoding/base64" "sync" ) @@ -70,3 +72,12 @@ func (e *enclaveKeys) getAppKeys() []byte { return e.AppKeys } + +// hashAndB64 returns the Base64-encoded hash over our key material. The +// resulting string is not confidential as it's impractical to reverse the key +// material. +func (e *enclaveKeys) hashAndB64() string { + keys := append(append(e.NitridingCert, e.NitridingKey...), e.AppKeys...) + hash := sha256.Sum256(keys) + return base64.StdEncoding.EncodeToString(hash[:]) +} diff --git a/handlers.go b/handlers.go index 8535d40..9e28491 100644 --- a/handlers.go +++ b/handlers.go @@ -248,8 +248,8 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - theirB64Hashes := string(body) - ourB64Hashes := base64.StdEncoding.EncodeToString(e.hashes.Serialize()) + theirB64Hash := string(body) + ourB64Hash := e.keys.hashAndB64() // Take note of the worker still being alive. worker, err := e.httpClientToSyncURL(r) @@ -260,7 +260,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { e.workers.updateAndPrune(worker) // Is the worker's key material outdated? If so, re-synchronize. - if ourB64Hashes != theirB64Hashes { + if ourB64Hash != theirB64Hash { elog.Println("Worker's keys are outdated. Re-synchronizing.") go func() { worker, err := e.httpClientToSyncURL(r) From 099e2c96072e4594d3e0da3f4cafab1266c8cc56 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 15 Aug 2023 12:22:00 -0500 Subject: [PATCH 13/99] Polish heartbeat mechanism. --- enclave.go | 16 ++++++++++------ enclave_test.go | 3 ++- handlers.go | 5 +++-- handlers_test.go | 3 ++- main.go | 4 +++- sync_worker.go | 2 +- workers.go | 20 ++++++++++++++++++++ 7 files changed, 41 insertions(+), 12 deletions(-) diff --git a/enclave.go b/enclave.go index 1050852..03f2b6a 100644 --- a/enclave.go +++ b/enclave.go @@ -218,7 +218,7 @@ func (c *Config) String() string { } // NewEnclave creates and returns a new enclave with the given config. -func NewEnclave(cfg *Config) (*Enclave, error) { +func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("failed to create enclave: %w", err) } @@ -252,6 +252,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { ready: make(chan struct{}), becameLeader: make(chan struct{}), } + go e.workers.monitor(ctx) // Increase the maximum number of idle connections per host. This is // critical to boosting the requests per second that our reverse proxy can @@ -286,7 +287,6 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(e)) - m.Post(pathHeartbeat, heartbeatHandler(e)) m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader)) // Register enclave-internal HTTP API. @@ -388,17 +388,21 @@ func (e *Enclave) heartbeat() { // TODO: Use context to exit loop. timer := time.NewTicker(time.Minute) for range timer.C { - _, err := newUnauthenticatedHTTPClient().Post( + resp, err := newUnauthenticatedHTTPClient().Post( e.getLeader(pathHeartbeat).String(), "text/plain", strings.NewReader(e.keys.hashAndB64()), ) + // TODO: what should we do if the leader is dead? if err != nil { - // TODO: what should we do if the leader is dead? elog.Printf("Error sending heartbeat to leader: %v", err) - } else { - elog.Println("Successfully sent heartbeat to leader.") + continue } + if resp.StatusCode != http.StatusOK { + elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) + continue + } + elog.Println("Successfully sent heartbeat to leader.") } } diff --git a/enclave_test.go b/enclave_test.go index 224e0bc..acdfda2 100644 --- a/enclave_test.go +++ b/enclave_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "testing" ) @@ -25,7 +26,7 @@ func assertEqual(t *testing.T, is, should interface{}) { } func createEnclave(cfg *Config) *Enclave { - e, err := NewEnclave(cfg) + e, err := NewEnclave(context.Background(), cfg) if err != nil { panic(err) } diff --git a/handlers.go b/handlers.go index 9e28491..e150763 100644 --- a/handlers.go +++ b/handlers.go @@ -214,8 +214,9 @@ func leaderHandler(e *Enclave) http.HandlerFunc { elog.Println("Designated enclave as leader.") close(e.becameLeader) // Signal to other parts of the code. + e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) - elog.Println("Set up worker registration endpoint.") + elog.Println("Set up worker registration and heartbeat endpoint.") w.WriteHeader(http.StatusOK) } @@ -261,7 +262,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { // Is the worker's key material outdated? If so, re-synchronize. if ourB64Hash != theirB64Hash { - elog.Println("Worker's keys are outdated. Re-synchronizing.") + elog.Printf("Worker's keys are outdated (ours=%s, theirs=%s).", ourB64Hash, theirB64Hash) go func() { worker, err := e.httpClientToSyncURL(r) if err != nil { diff --git a/handlers_test.go b/handlers_test.go index 7336e0d..d42610d 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "crypto/sha256" "crypto/tls" "encoding/base64" @@ -139,7 +140,7 @@ func TestProxyHandler(t *testing.T) { c := defaultCfg c.AppWebSrv = u - e, err := NewEnclave(&c) + e, err := NewEnclave(context.Background(), &c) if err != nil { t.Fatal(err) } diff --git a/main.go b/main.go index 91756f1..95968fb 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "context" "errors" "flag" "io" @@ -130,7 +131,8 @@ func main() { c.AppWebSrv = u } - enclave, err := NewEnclave(c) + ctx := context.Background() + enclave, err := NewEnclave(ctx, c) if err != nil { elog.Fatalf("Failed to create enclave: %v", err) } diff --git a/sync_worker.go b/sync_worker.go index f4388dd..af0d524 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -206,5 +206,5 @@ func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { } s.installKeys(&keys) - elog.Println("Successfully synced with leader.") + elog.Printf("Successfully synced keys %s with leader.", keys.hashAndB64()) } diff --git a/workers.go b/workers.go index 41ad601..c2ce102 100644 --- a/workers.go +++ b/workers.go @@ -1,6 +1,7 @@ package main import ( + "context" "net/url" "sync" "time" @@ -28,6 +29,20 @@ func (w *workers) length() int { return len(w.set) } +func (w *workers) monitor(ctx context.Context) { + elog.Println("Monitoring for defunct enclave workers.") + timer := time.NewTicker(time.Minute) + for { + select { + case <-timer.C: + w.pruneDefunctWorkers() + case <-ctx.Done(): + elog.Println("Stopping worker monitoring.") + return + } + } +} + func (w *workers) register(worker *url.URL) { w.Lock() defer w.Unlock() @@ -67,9 +82,14 @@ func (w *workers) pruneDefunctWorkers() { defer w.RUnlock() now := time.Now() + pruned := 0 for worker, lastSeen := range w.set { if now.Sub(lastSeen) > w.timeout { w.unregister(&worker) + pruned += 1 } } + if pruned > 0 { + elog.Printf("Pruned %d worker(s) from worker set.", pruned) + } } From 9f444ca76a92206a83b6475c72d0fb10df99136d Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 08:45:27 -0500 Subject: [PATCH 14/99] Make worker initiate re-synchronization. --- attestation_test.go | 3 +- enclave.go | 75 +++++++++++++++----------- handlers.go | 64 ++++++++++++++-------- handlers_test.go | 17 ++++-- main.go | 2 +- workers.go | 128 +++++++++++++++++++++++++++----------------- workers_test.go | 41 +++++++++++++- 7 files changed, 220 insertions(+), 110 deletions(-) diff --git a/attestation_test.go b/attestation_test.go index a13e29b..7228c40 100644 --- a/attestation_test.go +++ b/attestation_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "crypto/sha256" "encoding/base64" "net/http" @@ -39,7 +40,7 @@ func TestAttestationHashes(t *testing.T) { // Start the enclave. This is going to initialize the hash over the HTTPS // certificate. - if err := e.Start(); err != nil { + if err := e.Start(context.Background()); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck diff --git a/enclave.go b/enclave.go index 03f2b6a..2414d32 100644 --- a/enclave.go +++ b/enclave.go @@ -71,6 +71,7 @@ var ( type Enclave struct { sync.RWMutex attester + ctx context.Context cfg *Config extPubSrv, extPrivSrv *http.Server intSrv *http.Server @@ -226,6 +227,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ attester: &dummyAttester{}, + ctx: ctx, cfg: cfg, extPubSrv: &http.Server{ Handler: chi.NewRouter(), @@ -252,7 +254,6 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { ready: make(chan struct{}), becameLeader: make(chan struct{}), } - go e.workers.monitor(ctx) // Increase the maximum number of idle connections per host. This is // critical to boosting the requests per second that our reverse proxy can @@ -286,14 +287,13 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) - m.Get(pathLeader, leaderHandler(e)) + m.Get(pathLeader, leaderHandler(ctx, e)) m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) m.Get(pathReady, readyHandler(e)) m.Get(pathState, getStateHandler(e)) - m.Put(pathState, putStateHandler(e)) m.Post(pathHash, hashHandler(e)) // Configure our reverse proxy if the enclave application exposes an HTTP @@ -327,8 +327,11 @@ func (e *Enclave) installKeys(keys *enclaveKeys) error { // Start starts the Nitro Enclave. If something goes wrong, the function // returns an error. -func (e *Enclave) Start() error { - var err error +func (e *Enclave) Start(ctx context.Context) error { + var ( + err error + leader = e.getLeader(pathRegistration) + ) errPrefix := "failed to start Nitro Enclave" if inEnclave { @@ -360,15 +363,11 @@ func (e *Enclave) Start() error { } if e.cfg.isScalingEnabled() { - err := asWorker(e.installKeys, e.becameLeader).registerWith(&url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), - Path: pathRegistration, - }) + err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } else if err == nil { - go e.heartbeat() + go e.workerHeartbeat(ctx) } } @@ -383,26 +382,42 @@ func (e *Enclave) getLeader(path string) *url.URL { } } -func (e *Enclave) heartbeat() { - elog.Println("Starting heartbeat loop.") - // TODO: Use context to exit loop. - timer := time.NewTicker(time.Minute) - for range timer.C { - resp, err := newUnauthenticatedHTTPClient().Post( - e.getLeader(pathHeartbeat).String(), - "text/plain", - strings.NewReader(e.keys.hashAndB64()), - ) - // TODO: what should we do if the leader is dead? - if err != nil { - elog.Printf("Error sending heartbeat to leader: %v", err) - continue - } - if resp.StatusCode != http.StatusOK { - elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) - continue +func (e *Enclave) workerHeartbeat(ctx context.Context) { + elog.Println("Starting worker's heartbeat loop.") + defer elog.Println("Exiting worker's heartbeat loop.") + var ( + leader = e.getLeader(pathRegistration) + timer = time.NewTicker(time.Minute) + ) + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + resp, err := newUnauthenticatedHTTPClient().Post( + e.getLeader(pathHeartbeat).String(), + "text/plain", + strings.NewReader(e.keys.hashAndB64()), + ) + if err != nil { + elog.Printf("Error posting heartbeat to leader: %v", err) + continue + } + // Our key material is outdated. Asking for re-synchronization. + if resp.StatusCode == http.StatusConflict { + err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) + if err != nil && !errors.Is(err, errBecameLeader) { + elog.Fatalf("Error syncing with leader: %v", err) + } + continue + } + if resp.StatusCode != http.StatusOK { + elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) + continue + } + elog.Println("Successfully sent heartbeat to leader.") } - elog.Println("Successfully sent heartbeat to leader.") } } diff --git a/handlers.go b/handlers.go index e150763..596e49c 100644 --- a/handlers.go +++ b/handlers.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -93,8 +94,8 @@ func putStateHandler(e *Enclave) http.HandlerFunc { // The leader's application keys have changed. Re-synchronize the key // material with all registered workers. If synchronization fails for a // given worker, unregister it. - for worker := range e.workers.set { - go func(worker *url.URL) { + e.workers.forAll( + func(worker *url.URL) { if err := asLeader(e.keys.get()).syncWith(worker); err != nil { // TODO: Log in Prometheus. elog.Printf("Error re-syncing with worker %s: %v", worker.String(), err) @@ -102,8 +103,8 @@ func putStateHandler(e *Enclave) http.HandlerFunc { } else { elog.Printf("Successfully re-synced with worker %s.", worker.String()) } - }(&worker) - } + }, + ) } } @@ -209,11 +210,14 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl } } -func leaderHandler(e *Enclave) http.HandlerFunc { +func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { elog.Println("Designated enclave as leader.") close(e.becameLeader) // Signal to other parts of the code. + go e.workers.monitor(ctx) + // Make leader-specific endpoints available. + e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) elog.Println("Set up worker registration and heartbeat endpoint.") @@ -241,6 +245,11 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { } } +// heartbeatHandler exposes an endpoint that allows worker enclaves to send +// periodic heartbeats to the leader enclave. The heartbeat's body contains the +// worker's hashed key material. If the worker's hash is different from the +// leader's hash, the leader knows that the worker's key material is out of +// sync, which makes the leader re-synchronize its key material with the worker. func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Read the worker's hashed key material. @@ -249,31 +258,40 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - theirB64Hash := string(body) - ourB64Hash := e.keys.hashAndB64() + theirKeysHash := string(body) + ourKeysHash := e.keys.hashAndB64() - // Take note of the worker still being alive. + // Update the worker's "last seen" timestamp. worker, err := e.httpClientToSyncURL(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - e.workers.updateAndPrune(worker) + e.workers.updateHeartbeat(worker) - // Is the worker's key material outdated? If so, re-synchronize. - if ourB64Hash != theirB64Hash { - elog.Printf("Worker's keys are outdated (ours=%s, theirs=%s).", ourB64Hash, theirB64Hash) - go func() { - worker, err := e.httpClientToSyncURL(r) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - if err := asLeader(e.keys.get()).syncWith(worker); err != nil { - elog.Printf("Error syncing with worker: %v", err) - return - } - elog.Println("Successfully re-synchronized with worker.") - }() + // Let the worker know if their keys are outdated. + if ourKeysHash != theirKeysHash { + elog.Printf("Worker's keys are outdated (ours=%s, theirs=%s).", + ourKeysHash, theirKeysHash) + w.WriteHeader(http.StatusConflict) + } else { + w.WriteHeader(http.StatusOK) } + + // Is the worker's key material outdated? If so, re-synchronize. + // if ourKeysHash != theirKeysHash { + // go func() { + // worker, err := e.httpClientToSyncURL(r) + // if err != nil { + // // TODO: The client should actively re-register and + // // terminate if synchronization does not succeed. + // } + // if err := asLeader(e.keys.get()).syncWith(worker); err != nil { + // elog.Printf("Error syncing with worker: %v", err) + // return + // } + // elog.Println("Successfully re-synchronized with worker.") + // }() + // } } } diff --git a/handlers_test.go b/handlers_test.go index d42610d..32b9c4e 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -95,9 +95,18 @@ func signalReady(t *testing.T, e *Enclave) { } func TestStateHandlers(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).intSrv) + e := createEnclave(&defaultCfg) + + // First, designate the enclave as leader to make the "put state" endpoint + // available. + makeReq := makeRequestFor(e.extPrivSrv) + assertResponse(t, + makeReq(http.MethodGet, pathLeader, nil), + newResp(http.StatusOK, ""), + ) tooLargeKey := make([]byte, 1024*1024+1) + makeReq = makeRequestFor(e.intSrv) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(tooLargeKey)), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), @@ -145,7 +154,7 @@ func TestProxyHandler(t *testing.T) { t.Fatal(err) } e.revProxy = httputil.NewSingleHostReverseProxy(u) - if err := e.Start(); err != nil { + if err := e.Start(context.Background()); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck @@ -219,7 +228,7 @@ func TestReadiness(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = false e := createEnclave(&cfg) - if err := e.Start(); err != nil { + if err := e.Start(context.Background()); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck @@ -255,7 +264,7 @@ func TestReadyHandler(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = true e := createEnclave(&cfg) - if err := e.Start(); err != nil { + if err := e.Start(context.Background()); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck diff --git a/main.go b/main.go index 95968fb..6b7caf2 100644 --- a/main.go +++ b/main.go @@ -137,7 +137,7 @@ func main() { elog.Fatalf("Failed to create enclave: %v", err) } - if err := enclave.Start(); err != nil { + if err := enclave.Start(ctx); err != nil { elog.Fatalf("Enclave terminated: %v", err) } diff --git a/workers.go b/workers.go index c2ce102..e008aab 100644 --- a/workers.go +++ b/workers.go @@ -2,94 +2,122 @@ package main import ( "context" + "fmt" "net/url" "sync" "time" ) +type workerSet map[url.URL]time.Time + // workers represents a set of worker enclaves. The leader enclave keeps track // of workers. type workers struct { - sync.RWMutex - timeout time.Duration - set map[url.URL]time.Time + timeout time.Duration + reg, unreg, heartbeat chan *url.URL + len chan int + f chan func(*url.URL) } func newWorkers(timeout time.Duration) *workers { return &workers{ - set: make(map[url.URL]time.Time), - timeout: timeout, + timeout: timeout, + reg: make(chan *url.URL), + unreg: make(chan *url.URL), + heartbeat: make(chan *url.URL), + len: make(chan int), + f: make(chan func(*url.URL)), } } -func (w *workers) length() int { - w.RLock() - defer w.RUnlock() - - return len(w.set) -} - func (w *workers) monitor(ctx context.Context) { - elog.Println("Monitoring for defunct enclave workers.") - timer := time.NewTicker(time.Minute) + var ( + set = make(map[url.URL]time.Time) + timer = time.NewTicker(time.Minute) + ) + elog.Println("Starting worker event loop.") + defer elog.Println("Stopping worker event loop.") + for { select { - case <-timer.C: - w.pruneDefunctWorkers() case <-ctx.Done(): - elog.Println("Stopping worker monitoring.") return + + case <-timer.C: + go w.pruneDefunctWorkers(set) + + case worker := <-w.reg: + set[*worker] = time.Now() + elog.Printf("Registered worker %s; %d worker(s) now registered.", + worker.Host, len(set)) + + case worker := <-w.unreg: + delete(set, *worker) + elog.Printf("Unregistered worker %s; %d worker(s) left.", + worker.Host, len(set)) + + case worker := <-w.heartbeat: + _, exists := set[*worker] + if !exists { + elog.Printf("Updating heartbeat for previously-unregistered worker %s.", + worker.Host) + } + set[*worker] = time.Now() + + case f := <-w.f: + w.runForAll(f, set) + w.f <- nil // Signal to caller that we're done. + + case <-w.len: + w.len <- len(set) } } } -func (w *workers) register(worker *url.URL) { - w.Lock() - defer w.Unlock() +// runForAll blocks until the given function was run over all workers in our +// set. For key synchronization, this should never take more than a couple +// seconds. +func (w *workers) runForAll(f func(*url.URL), set workerSet) { + var wg sync.WaitGroup + fmt.Printf("# of workers: %d", len(set)) + for worker := range set { + wg.Add(1) + go func(wg *sync.WaitGroup, worker *url.URL) { + f(worker) + wg.Done() + }(&wg, &worker) + } + wg.Wait() +} - w.set[*worker] = time.Now() - elog.Printf("Registered worker %s; %d worker(s) now registered.", - worker.String(), len(w.set)) +func (w *workers) length() int { + w.len <- 0 // Signal to the event loop that we want the length. + return <-w.len } -func (w *workers) unregister(worker *url.URL) { - w.Lock() - defer w.Unlock() +func (w *workers) forAll(f func(*url.URL)) { + w.f <- f + <-w.f // Wait until the event loop is done running the given function. +} - delete(w.set, *worker) - elog.Printf("Unregistered worker %s; %d worker(s) left.", - worker.String(), len(w.set)) +func (w *workers) register(worker *url.URL) { + w.reg <- worker } -func (w *workers) updateAndPrune(worker *url.URL) { - w.updateHeartbeat(worker) - w.pruneDefunctWorkers() +func (w *workers) unregister(worker *url.URL) { + w.unreg <- worker } func (w *workers) updateHeartbeat(worker *url.URL) { - w.Lock() - defer w.Unlock() - - _, exists := w.set[*worker] - if !exists { - elog.Printf("Updating heartbeat for previously-unregistered worker %s.", worker) - } - w.set[*worker] = time.Now() + w.heartbeat <- worker } -func (w *workers) pruneDefunctWorkers() { - w.RLock() - defer w.RUnlock() - +func (w *workers) pruneDefunctWorkers(set workerSet) { now := time.Now() - pruned := 0 - for worker, lastSeen := range w.set { + for worker, lastSeen := range set { if now.Sub(lastSeen) > w.timeout { w.unregister(&worker) - pruned += 1 + elog.Printf("Pruned %s from worker set.", worker.Host) } } - if pruned > 0 { - elog.Printf("Pruned %d worker(s) from worker set.", pruned) - } } diff --git a/workers_test.go b/workers_test.go index 79da187..f144b00 100644 --- a/workers_test.go +++ b/workers_test.go @@ -1,13 +1,19 @@ package main import ( + "context" "net/url" "testing" "time" ) func TestWorkerRegistration(t *testing.T) { - w := newWorkers(time.Minute) + var ( + w = newWorkers(time.Minute) + ctx, cancel = context.WithCancel(context.Background()) + ) + go w.monitor(ctx) + defer cancel() // Identical URLs are only tracked once. worker1 := url.URL{Host: "foo"} @@ -28,6 +34,39 @@ func TestWorkerRegistration(t *testing.T) { w.unregister(&url.URL{Host: "does-not-exist"}) } +func TestForAll(t *testing.T) { + var ( + w = newWorkers(time.Minute) + ctx, cancel = context.WithCancel(context.Background()) + ) + go w.monitor(ctx) + defer cancel() + + w.register(&url.URL{Host: "foo"}) + w.register(&url.URL{Host: "bar"}) + assertEqual(t, w.length(), 2) + + total := 0 + w.forAll( + func(w *url.URL) { + total += 1 + }, + ) + assertEqual(t, total, 2) +} + +func TestIneffectiveForAll(t *testing.T) { + var ( + w = newWorkers(time.Minute) + ctx, cancel = context.WithCancel(context.Background()) + ) + go w.monitor(ctx) + defer cancel() + + // Make sure that forAll finishes for an empty worker set. + w.forAll(func(_ *url.URL) {}) +} + func TestUpdatingAndPruning(t *testing.T) { // TODO } From c72be54f71d91db5eadc921c5f19c65b7cb35fec Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 09:38:46 -0500 Subject: [PATCH 15/99] Refactoring. --- attester.go | 9 ++++----- enclave.go | 32 ++++++++++++++++++++------------ handlers.go | 31 ++++++++++++------------------- main.go | 2 +- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/attester.go b/attester.go index 0a9dc37..99b6f17 100644 --- a/attester.go +++ b/attester.go @@ -57,8 +57,10 @@ func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) { } func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { - var w workerAuxInfo - var l leaderAuxInfo + var ( + w workerAuxInfo + l leaderAuxInfo + ) // First, assume we're dealing with a worker's auxiliary information. if err := json.Unmarshal(doc, &w); err != nil { @@ -66,10 +68,8 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { } if len(w.WorkersNonce) == nonceLen && len(w.LeadersNonce) == nonceLen && w.PublicKey != nil { if n.B64() != w.LeadersNonce.B64() { - fmt.Printf("'%s' / '%s'", n.B64(), w.LeadersNonce.B64()) return nil, errors.New("leader nonce not in cache") } - elog.Println(w) return &w, nil } @@ -81,7 +81,6 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if n.B64() != l.WorkersNonce.B64() { return nil, errors.New("worker nonce not in cache") } - elog.Println(l) return &l, nil } diff --git a/enclave.go b/enclave.go index 2414d32..672768e 100644 --- a/enclave.go +++ b/enclave.go @@ -226,7 +226,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ - attester: &dummyAttester{}, + attester: &nitroAttester{}, ctx: ctx, cfg: cfg, extPubSrv: &http.Server{ @@ -263,6 +263,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { http.DefaultTransport.(*http.Transport).MaxIdleConns = 500 if cfg.Debug { + e.attester = &dummyAttester{} e.extPubSrv.Handler.(*chi.Mux).Use(middleware.Logger) e.extPrivSrv.Handler.(*chi.Mux).Use(middleware.Logger) e.intSrv.Handler.(*chi.Mux).Use(middleware.Logger) @@ -374,14 +375,10 @@ func (e *Enclave) Start(ctx context.Context) error { return nil } -func (e *Enclave) getLeader(path string) *url.URL { - return &url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), - Path: path, - } -} - +// workerHeartbeat periodically talks to the leader enclave to 1) let the leader +// know that we're still alive, and 2) to compare key material. If it turns out +// that the leader has different key material than the worker, the worker +// re-registers itself, which triggers key re-synchronization. func (e *Enclave) workerHeartbeat(ctx context.Context) { elog.Println("Starting worker's heartbeat loop.") defer elog.Println("Exiting worker's heartbeat loop.") @@ -396,7 +393,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { return case <-timer.C: resp, err := newUnauthenticatedHTTPClient().Post( - e.getLeader(pathHeartbeat).String(), + leader.String(), "text/plain", strings.NewReader(e.keys.hashAndB64()), ) @@ -404,12 +401,13 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { elog.Printf("Error posting heartbeat to leader: %v", err) continue } - // Our key material is outdated. Asking for re-synchronization. if resp.StatusCode == http.StatusConflict { + elog.Println("Our keys are outdated. Re-synchronizing.") err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } + elog.Println("Successfully re-synchronized with leader.") continue } if resp.StatusCode != http.StatusOK { @@ -651,7 +649,17 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { return nil } -func (e *Enclave) httpClientToSyncURL(r *http.Request) (*url.URL, error) { +// getLeader returns the leader enclave's URL. +func (e *Enclave) getLeader(path string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), + Path: path, + } +} + +// getWorker returns the worker enclave's URL from the given HTTP request. +func (e *Enclave) getWorker(r *http.Request) (*url.URL, error) { // Go's HTTP server sets RemoteAddr to IP:port: // https://pkg.go.dev/net/http#Request strIP, _, err := net.SplitHostPort(r.RemoteAddr) diff --git a/handlers.go b/handlers.go index 596e49c..8fb046a 100644 --- a/handlers.go +++ b/handlers.go @@ -210,6 +210,12 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl } } +// leaderHandler is called when the enclave is designated as leader enclave. +// If designated, we do the following: +// +// 1. Signal to other parts of the code that we became the leader. +// 2. Start the worker event loop, to keep track of worker enclaves. +// 3. Expose leader-specific endpoints. func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { elog.Println("Designated enclave as leader.") @@ -226,9 +232,12 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { } } +// workerRegistrationHandler allows worker to register themselves with the +// leader. Once a worker registered itself, the leader immediately proceeds to +// synchronize its key material with the worker. func workerRegistrationHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - worker, err := e.httpClientToSyncURL(r) + worker, err := e.getWorker(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -240,7 +249,7 @@ func workerRegistrationHandler(e *Enclave) http.HandlerFunc { return } e.workers.register(worker) - elog.Printf("Successfully registered and synced with worker %s.", worker.String()) + elog.Printf("Successfully registered and synced with worker %s.", worker.Host) }() } } @@ -262,7 +271,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { ourKeysHash := e.keys.hashAndB64() // Update the worker's "last seen" timestamp. - worker, err := e.httpClientToSyncURL(r) + worker, err := e.getWorker(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -277,21 +286,5 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { } else { w.WriteHeader(http.StatusOK) } - - // Is the worker's key material outdated? If so, re-synchronize. - // if ourKeysHash != theirKeysHash { - // go func() { - // worker, err := e.httpClientToSyncURL(r) - // if err != nil { - // // TODO: The client should actively re-register and - // // terminate if synchronization does not succeed. - // } - // if err := asLeader(e.keys.get()).syncWith(worker); err != nil { - // elog.Printf("Error syncing with worker: %v", err) - // return - // } - // elog.Println("Successfully re-synchronized with worker.") - // }() - // } } } diff --git a/main.go b/main.go index 6b7caf2..0c8e20f 100644 --- a/main.go +++ b/main.go @@ -72,7 +72,7 @@ func main() { flag.BoolVar(&waitForApp, "wait-for-app", false, "Start Internet-facing Web server only after application signals its readiness.") flag.BoolVar(&debug, "debug", false, - "Print debug messages.") + "Print extra debug messages and use dummy attester for testing outside enclaves.") flag.StringVar(&mockCertFp, "mock-cert-fp", "", "Mock certificate fingerprint to use in attestation documents (hexadecimal)") flag.Parse() From cb6b360706ff543c7d6cb3e6245373e444b263c9 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 09:39:10 -0500 Subject: [PATCH 16/99] Add scripts for testing sync outside of enclaves. --- scripts/README.md | 21 +++++++++++++++++++++ scripts/config.sh | 4 ++++ scripts/launch-leader.sh | 11 +++++++++++ scripts/make-leader.sh | 7 +++++++ scripts/update-app-keys.sh | 7 +++++++ 5 files changed, 50 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/config.sh create mode 100755 scripts/launch-leader.sh create mode 100755 scripts/make-leader.sh create mode 100755 scripts/update-app-keys.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..becf966 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,21 @@ +# Key synchronization test scripts + +This directory contains scripts that help with testing key synchronization +outside the context of Nitro Enclaves. The scripts assume that we have at least +two "enclaves" that run on separate IP addresses. + +To start the leader "enclave", run: + + ./launch-leader.sh + +To designate this "enclave" as the leader, run: + + ./make-leader.sh + +To start a worker "enclave", run: + + ./launch-worker.sh + +To update the leader's key material, run: + + ./update-app-keys.sh \ No newline at end of file diff --git a/scripts/config.sh b/scripts/config.sh new file mode 100644 index 0000000..e4c7ccc --- /dev/null +++ b/scripts/config.sh @@ -0,0 +1,4 @@ +leader="192.168.1.3" +ext_pub_port=8443 +ext_priv_port=8444 +int_port=8445 \ No newline at end of file diff --git a/scripts/launch-leader.sh b/scripts/launch-leader.sh new file mode 100755 index 0000000..0559cd2 --- /dev/null +++ b/scripts/launch-leader.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +nitriding-daemon \ + -debug \ + -fqdn localhost \ + -fqdn-leader "$leader" \ + -ext-pub-port "$ext_pub_port" \ + -ext-priv-port "$ext_priv_port" \ + -intport "$int_port" \ No newline at end of file diff --git a/scripts/make-leader.sh b/scripts/make-leader.sh new file mode 100755 index 0000000..ef03d7c --- /dev/null +++ b/scripts/make-leader.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --insecure \ + --include \ + "https://localhost:${ext_priv_port}/enclave/leader" \ No newline at end of file diff --git a/scripts/update-app-keys.sh b/scripts/update-app-keys.sh new file mode 100755 index 0000000..ca367a9 --- /dev/null +++ b/scripts/update-app-keys.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --request PUT \ + --include \ + "http://localhost:${int_port}/enclave/state" --data 'NewAppKeys' \ No newline at end of file From 060ca351f226463df186aa4f196da0976fcd4e6f Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 09:46:38 -0500 Subject: [PATCH 17/99] Update test scripts. --- scripts/README.md | 15 +++++++++------ scripts/config.sh | 2 +- scripts/{launch-leader.sh => launch-enclave.sh} | 0 3 files changed, 10 insertions(+), 7 deletions(-) rename scripts/{launch-leader.sh => launch-enclave.sh} (100%) diff --git a/scripts/README.md b/scripts/README.md index becf966..2ce914b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,18 +4,21 @@ This directory contains scripts that help with testing key synchronization outside the context of Nitro Enclaves. The scripts assume that we have at least two "enclaves" that run on separate IP addresses. -To start the leader "enclave", run: +First, change the `leader` variable inside config.sh to match the IP address of +your local leader "enclave". - ./launch-leader.sh +To start the leader "enclave", run on machine A: -To designate this "enclave" as the leader, run: + ./launch-enclave.sh + +To designate this "enclave" as the leader, run on machine A: ./make-leader.sh -To start a worker "enclave", run: +To start a worker "enclave", run on machine B: - ./launch-worker.sh + ./launch-enclave.sh -To update the leader's key material, run: +To update the leader's key material, run on machine A: ./update-app-keys.sh \ No newline at end of file diff --git a/scripts/config.sh b/scripts/config.sh index e4c7ccc..72c116e 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -1,4 +1,4 @@ -leader="192.168.1.3" +leader="192.168.1.3" # Change this to match your local leader "enclave" address. ext_pub_port=8443 ext_priv_port=8444 int_port=8445 \ No newline at end of file diff --git a/scripts/launch-leader.sh b/scripts/launch-enclave.sh similarity index 100% rename from scripts/launch-leader.sh rename to scripts/launch-enclave.sh From e9a4fb91b2ce3ed4c10cb9877cc4633858cad524 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 10:01:21 -0500 Subject: [PATCH 18/99] Remove unused constant. --- cache.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cache.go b/cache.go index 3e0d43e..5c461bf 100644 --- a/cache.go +++ b/cache.go @@ -5,10 +5,6 @@ import ( "time" ) -const ( - defaultItemExpiry = time.Minute -) - // cache implements a simple cache whose items expire. type cache struct { sync.RWMutex From 4c79050cb9a11768d37a850e87100d12814a0775 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 10:03:02 -0500 Subject: [PATCH 19/99] Terminate if enclave keys cannot be installed. --- sync_worker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sync_worker.go b/sync_worker.go index af0d524..ad8b7e5 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -204,7 +204,10 @@ func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - s.installKeys(&keys) + if err := s.installKeys(&keys); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + elog.Fatalf("Failed to install enclave keys: %v", err) + } elog.Printf("Successfully synced keys %s with leader.", keys.hashAndB64()) } From d306826f5c85049eb256d653dc5238df1a4baa7d Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 10:11:03 -0500 Subject: [PATCH 20/99] Remove annoying error messages. --- proxy.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/proxy.go b/proxy.go index dfc02b4..4149e4f 100644 --- a/proxy.go +++ b/proxy.go @@ -29,7 +29,6 @@ func runNetworking(c *Config, stop chan struct{}) { if err = setupNetworking(c, stop); err == nil { return } - elog.Printf("TAP tunnel to EC2 host failed: %v. Restarting.", err) time.Sleep(time.Second) } } @@ -42,10 +41,7 @@ func runNetworking(c *Config, stop chan struct{}) { // 3. Establish a connection with the proxy running on the host. // 4. Spawn goroutines to forward traffic between the TAP device and the proxy // running on the host. - elog.Println("Setting up networking between host and enclave.") - defer elog.Println("Tearing down networking between host and enclave.") func setupNetworking(c *Config, stop chan struct{}) error { - // Establish connection with the proxy running on the EC2 host. endpoint := fmt.Sprintf("vsock://%d:%d/connect", parentCID, c.HostProxyPort) conn, path, err := transport.Dial(endpoint) From be940b9be5659cafc300a28491fc841ad6a6b8a8 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 10:29:28 -0500 Subject: [PATCH 21/99] Address linter error. --- attester.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/attester.go b/attester.go index 99b6f17..f02478b 100644 --- a/attester.go +++ b/attester.go @@ -157,17 +157,41 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { return nil, fmt.Errorf("%s: nonce %s not in cache", errStr, b64Nonce) } + workersNonce, err := sliceToNonce(their.Document.Nonce) + if err != nil { + return nil, err + } + leadersNonce, err := sliceToNonce(their.Document.UserData) + if err != nil { + return nil, err + } // If the "public key" field is unset, we know that we're dealing with a // worker's auxiliary information. if their.Document.PublicKey != nil { return &workerAuxInfo{ - WorkersNonce: nonce(their.Document.Nonce), - LeadersNonce: nonce(their.Document.UserData), + WorkersNonce: workersNonce, + LeadersNonce: leadersNonce, PublicKey: their.Document.PublicKey, }, nil } + + workersNonce, err = sliceToNonce(their.Document.Nonce) + if err != nil { + return nil, err + } return &leaderAuxInfo{ - WorkersNonce: nonce(their.Document.Nonce), + WorkersNonce: workersNonce, EnclaveKeys: their.Document.UserData, }, nil } + +func sliceToNonce(s []byte) (nonce, error) { + var n nonce + + if len(s) != nonceLen { + return nonce{}, errors.New("slice is not of same length as nonce") + } + + copy(n[:], s[:nonceLen]) + return n, nil +} From 4cd624adb93957f02020ca2870e393a0be221a0f Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 10:59:14 -0500 Subject: [PATCH 22/99] Add log message. --- handlers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/handlers.go b/handlers.go index 8fb046a..8c7c80b 100644 --- a/handlers.go +++ b/handlers.go @@ -94,6 +94,8 @@ func putStateHandler(e *Enclave) http.HandlerFunc { // The leader's application keys have changed. Re-synchronize the key // material with all registered workers. If synchronization fails for a // given worker, unregister it. + elog.Printf("Application keys have changed. Re-synchronizing with %d workers.", + e.workers.length()) e.workers.forAll( func(worker *url.URL) { if err := asLeader(e.keys.get()).syncWith(worker); err != nil { From 6303f0303b5118d7337ca9e2a08077f1a593a730 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 11:28:33 -0500 Subject: [PATCH 23/99] Fix bug. --- enclave.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index 672768e..ca5aa79 100644 --- a/enclave.go +++ b/enclave.go @@ -331,7 +331,7 @@ func (e *Enclave) installKeys(keys *enclaveKeys) error { func (e *Enclave) Start(ctx context.Context) error { var ( err error - leader = e.getLeader(pathRegistration) + leader = e.getLeader(pathHeartbeat) ) errPrefix := "failed to start Nitro Enclave" From 269ec4dc538baf8d9b01a46a16bd89365d2ba62e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 11:34:35 -0500 Subject: [PATCH 24/99] Actually fix bug this time. --- enclave.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/enclave.go b/enclave.go index ca5aa79..2dad974 100644 --- a/enclave.go +++ b/enclave.go @@ -331,7 +331,7 @@ func (e *Enclave) installKeys(keys *enclaveKeys) error { func (e *Enclave) Start(ctx context.Context) error { var ( err error - leader = e.getLeader(pathHeartbeat) + leader = e.getLeader(pathRegistration) ) errPrefix := "failed to start Nitro Enclave" @@ -383,8 +383,9 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { elog.Println("Starting worker's heartbeat loop.") defer elog.Println("Exiting worker's heartbeat loop.") var ( - leader = e.getLeader(pathRegistration) - timer = time.NewTicker(time.Minute) + leaderHeartbeat = e.getLeader(pathHeartbeat) + leaderRegistration = e.getLeader(pathRegistration) + timer = time.NewTicker(time.Minute) ) for { @@ -393,7 +394,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { return case <-timer.C: resp, err := newUnauthenticatedHTTPClient().Post( - leader.String(), + leaderHeartbeat.String(), "text/plain", strings.NewReader(e.keys.hashAndB64()), ) @@ -403,7 +404,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { } if resp.StatusCode == http.StatusConflict { elog.Println("Our keys are outdated. Re-synchronizing.") - err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) + err := asWorker(e.installKeys, e.becameLeader).registerWith(leaderRegistration) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } From 3eb556a01ec7e7ea940d06fe40ad14f1b6789365 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 12:53:10 -0500 Subject: [PATCH 25/99] Improve test coverage. --- workers.go | 89 +++++++++++++++++++++++++++++++------------------ workers_test.go | 30 +++++++++++++---- 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/workers.go b/workers.go index e008aab..f4b12eb 100644 --- a/workers.go +++ b/workers.go @@ -2,38 +2,42 @@ package main import ( "context" - "fmt" "net/url" "sync" "time" ) -type workerSet map[url.URL]time.Time - -// workers represents a set of worker enclaves. The leader enclave keeps track -// of workers. -type workers struct { +// workerManager manages worker enclaves. +type workerManager struct { timeout time.Duration reg, unreg, heartbeat chan *url.URL len chan int - f chan func(*url.URL) + forAllFunc chan func(*url.URL) + afterTickFunc chan func() } -func newWorkers(timeout time.Duration) *workers { - return &workers{ - timeout: timeout, - reg: make(chan *url.URL), - unreg: make(chan *url.URL), - heartbeat: make(chan *url.URL), - len: make(chan int), - f: make(chan func(*url.URL)), +// workers maps worker enclaves to a timestamp that keeps track of when we last +// got a heartbeat from the enclave. +type workers map[url.URL]time.Time + +func newWorkerManager(timeout time.Duration) *workerManager { + return &workerManager{ + timeout: timeout, + reg: make(chan *url.URL), + unreg: make(chan *url.URL), + heartbeat: make(chan *url.URL), + len: make(chan int), + forAllFunc: make(chan func(*url.URL)), + afterTickFunc: make(chan func()), } } -func (w *workers) monitor(ctx context.Context) { +// start starts the worker manager's event loop. +func (w *workerManager) start(ctx context.Context) { var ( - set = make(map[url.URL]time.Time) - timer = time.NewTicker(time.Minute) + set = make(workers) + timer = time.NewTicker(w.timeout) + afterTickFunc = func() {} ) elog.Println("Starting worker event loop.") defer elog.Println("Stopping worker event loop.") @@ -43,8 +47,12 @@ func (w *workers) monitor(ctx context.Context) { case <-ctx.Done(): return + case f := <-w.afterTickFunc: + afterTickFunc = f + case <-timer.C: go w.pruneDefunctWorkers(set) + afterTickFunc() case worker := <-w.reg: set[*worker] = time.Now() @@ -64,9 +72,9 @@ func (w *workers) monitor(ctx context.Context) { } set[*worker] = time.Now() - case f := <-w.f: + case f := <-w.forAllFunc: w.runForAll(f, set) - w.f <- nil // Signal to caller that we're done. + w.forAllFunc <- nil // Signal to caller that we're done. case <-w.len: w.len <- len(set) @@ -77,42 +85,57 @@ func (w *workers) monitor(ctx context.Context) { // runForAll blocks until the given function was run over all workers in our // set. For key synchronization, this should never take more than a couple // seconds. -func (w *workers) runForAll(f func(*url.URL), set workerSet) { +func (w *workerManager) runForAll(f func(*url.URL), set workers) { var wg sync.WaitGroup - fmt.Printf("# of workers: %d", len(set)) for worker := range set { wg.Add(1) - go func(wg *sync.WaitGroup, worker *url.URL) { - f(worker) + go func(wg *sync.WaitGroup, worker url.URL) { + elog.Printf("Running function for worker %s.", worker.Host) + f(&worker) wg.Done() - }(&wg, &worker) + }(&wg, worker) } wg.Wait() } -func (w *workers) length() int { +// _afterTick runs the given function after the next event loop tick. This is +// only useful in unit tests. +func (w *workerManager) _afterTick(f func()) { + w.afterTickFunc <- f +} + +// length returns the number of workers that are currently registered. +func (w *workerManager) length() int { w.len <- 0 // Signal to the event loop that we want the length. return <-w.len } -func (w *workers) forAll(f func(*url.URL)) { - w.f <- f - <-w.f // Wait until the event loop is done running the given function. +// forAll runs the given function over all registered workers. This function +// blocks until the operation succeeded. +func (w *workerManager) forAll(f func(*url.URL)) { + w.forAllFunc <- f + <-w.forAllFunc // Wait until the event loop is done running the given function. } -func (w *workers) register(worker *url.URL) { +// register registers a new worker enclave. It is safe to repeatedly register +// the same worker enclave. +func (w *workerManager) register(worker *url.URL) { w.reg <- worker } -func (w *workers) unregister(worker *url.URL) { +// unregister unregisters the given worker enclave. +func (w *workerManager) unregister(worker *url.URL) { w.unreg <- worker } -func (w *workers) updateHeartbeat(worker *url.URL) { +// updateHeartbeat updates the "last seen" timestamp of the given worker. +func (w *workerManager) updateHeartbeat(worker *url.URL) { w.heartbeat <- worker } -func (w *workers) pruneDefunctWorkers(set workerSet) { +// pruneDefunctWorkers looks for and unregisters workers whose last heartbeat is +// older than our timeout. +func (w *workerManager) pruneDefunctWorkers(set workers) { now := time.Now() for worker, lastSeen := range set { if now.Sub(lastSeen) > w.timeout { diff --git a/workers_test.go b/workers_test.go index f144b00..2c373d3 100644 --- a/workers_test.go +++ b/workers_test.go @@ -9,10 +9,10 @@ import ( func TestWorkerRegistration(t *testing.T) { var ( - w = newWorkers(time.Minute) + w = newWorkerManager(time.Minute) ctx, cancel = context.WithCancel(context.Background()) ) - go w.monitor(ctx) + go w.start(ctx) defer cancel() // Identical URLs are only tracked once. @@ -27,6 +27,8 @@ func TestWorkerRegistration(t *testing.T) { w.unregister(&worker1) w.unregister(&worker2) + // It should be safe to unregister a non-existing worker. + w.unregister(&worker2) assertEqual(t, w.length(), 0) // Nothing should happen when attempting to unregister a non-existing @@ -36,10 +38,10 @@ func TestWorkerRegistration(t *testing.T) { func TestForAll(t *testing.T) { var ( - w = newWorkers(time.Minute) + w = newWorkerManager(time.Minute) ctx, cancel = context.WithCancel(context.Background()) ) - go w.monitor(ctx) + go w.start(ctx) defer cancel() w.register(&url.URL{Host: "foo"}) @@ -57,10 +59,10 @@ func TestForAll(t *testing.T) { func TestIneffectiveForAll(t *testing.T) { var ( - w = newWorkers(time.Minute) + w = newWorkerManager(time.Minute) ctx, cancel = context.WithCancel(context.Background()) ) - go w.monitor(ctx) + go w.start(ctx) defer cancel() // Make sure that forAll finishes for an empty worker set. @@ -68,5 +70,19 @@ func TestIneffectiveForAll(t *testing.T) { } func TestUpdatingAndPruning(t *testing.T) { - // TODO + var ( + w = newWorkerManager(time.Millisecond) + ctx, cancel = context.WithCancel(context.Background()) + ) + go w.start(ctx) + defer cancel() + + worker := &url.URL{Host: "foo"} + w.register(worker) + assertEqual(t, w.length(), 1) + + // Make sure that the worker got pruned after the next tick. + w._afterTick(func() { + assertEqual(t, w.length(), 0) + }) } From ca9c01c6402a5839b322cde40d4113fcda6878a5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 12:53:26 -0500 Subject: [PATCH 26/99] Improve log message. --- sync_leader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync_leader.go b/sync_leader.go index 6936efd..ab1eb76 100644 --- a/sync_leader.go +++ b/sync_leader.go @@ -31,7 +31,7 @@ func asLeader(keys *enclaveKeys) *leaderSync { // syncWith makes the leader initiate key synchronization with the given worker // enclave. func (s *leaderSync) syncWith(worker *url.URL) error { - elog.Println("Initiating key synchronization with worker.") + elog.Printf("Initiating key synchronization with worker %s.", worker.Host) // Step 1: Create a nonce that the worker must embed in its attestation // document, to prevent replay attacks. From d403d4bb1e8d1fa9493940e2cf9d6fb428c07dc5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 12:54:19 -0500 Subject: [PATCH 27/99] Use goroutines for execution. --- handlers.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/handlers.go b/handlers.go index 8c7c80b..a5ee373 100644 --- a/handlers.go +++ b/handlers.go @@ -96,7 +96,7 @@ func putStateHandler(e *Enclave) http.HandlerFunc { // given worker, unregister it. elog.Printf("Application keys have changed. Re-synchronizing with %d workers.", e.workers.length()) - e.workers.forAll( + go e.workers.forAll( func(worker *url.URL) { if err := asLeader(e.keys.get()).syncWith(worker); err != nil { // TODO: Log in Prometheus. @@ -222,8 +222,9 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { elog.Println("Designated enclave as leader.") close(e.becameLeader) // Signal to other parts of the code. + // TODO: Repeated calls make the goroutine panic. - go e.workers.monitor(ctx) + go e.workers.start(ctx) // Make leader-specific endpoints available. e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) From dbd88eaf4c970a7e3493d1e840384da9718b3692 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 12:54:37 -0500 Subject: [PATCH 28/99] Rename struct. --- enclave.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enclave.go b/enclave.go index 2dad974..6ae5f0b 100644 --- a/enclave.go +++ b/enclave.go @@ -80,7 +80,7 @@ type Enclave struct { hashes *AttestationHashes promRegistry *prometheus.Registry metrics *metrics - workers *workers + workers *workerManager keys *enclaveKeys httpsCert *certRetriever ready, stop, becameLeader chan struct{} @@ -249,7 +249,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { promRegistry: reg, metrics: newMetrics(reg, cfg.PrometheusNamespace), hashes: new(AttestationHashes), - workers: newWorkers(time.Minute), + workers: newWorkerManager(time.Minute), stop: make(chan struct{}), ready: make(chan struct{}), becameLeader: make(chan struct{}), From cc0636ef51b096f2085ba09e702856b1913d3147 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 13:44:56 -0500 Subject: [PATCH 29/99] Fix capitalization. --- enclave.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index 6ae5f0b..de59fc9 100644 --- a/enclave.go +++ b/enclave.go @@ -622,7 +622,7 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { if e.cfg.MockCertFp != "" { hash, err := hex.DecodeString(e.cfg.MockCertFp) if err != nil { - return errors.New("Failed to decode mock certificate fingerprint hex") + return errors.New("failed to decode mock certificate fingerprint hex") } copy(e.hashes.tlsKeyHash[:], hash) return nil From 0591be767b5c4ec66de1f3e6b44c2d66f533733e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 13:45:35 -0500 Subject: [PATCH 30/99] Tidy up dependencies. --- go.mod | 18 +- go.sum | 814 --------------------------------------------------------- 2 files changed, 1 insertion(+), 831 deletions(-) diff --git a/go.mod b/go.mod index 1be032d..b820fbe 100644 --- a/go.mod +++ b/go.mod @@ -4,53 +4,37 @@ go 1.20 require ( github.com/containers/gvisor-tap-vsock v0.5.0 + github.com/go-chi/chi v1.5.4 github.com/go-chi/chi/v5 v5.0.8 github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 github.com/mdlayher/vsock v1.2.1 github.com/milosgajdos/tenus v0.0.3 github.com/prometheus/client_golang v1.15.1 - github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/vishvananda/netlink v1.2.1-beta.2 golang.org/x/crypto v0.6.0 golang.org/x/sys v0.7.0 - gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec ) require ( - github.com/bazelbuild/rules_go v0.38.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cilium/ebpf v0.9.3 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/libcontainer v2.2.1+incompatible // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/go-chi/chi v1.5.4 // indirect - github.com/godbus/dbus/v5 v5.0.4 // indirect - github.com/gofrs/flock v0.8.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8 // indirect - github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1 // indirect github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/socket v0.4.1 // indirect - github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9 // indirect - github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.3.0 // indirect google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index c91772c..abf29cb 100644 --- a/go.sum +++ b/go.sum @@ -1,913 +1,99 @@ -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/bazelbuild/rules_go v0.27.0/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= -github.com/bazelbuild/rules_go v0.30.0 h1:kX4jVcstqrsRqKPJSn2mq2o+TI21edRzEJSrEOMQtr0= -github.com/bazelbuild/rules_go v0.30.0/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= -github.com/bazelbuild/rules_go v0.38.1 h1:YGNsLhWe18Ielebav7cClP3GMwBxBE+xEArLHtmXDx8= -github.com/bazelbuild/rules_go v0.38.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422 h1:8eZxmY1yvxGHzdzTEhI09npjMVGzNAdrqzruTX6jcK4= -github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.4.0 h1:QlHdikaxALkqWasW8hAC1mfR0jdmvbfaBdBPFmRSglA= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.9.3 h1:5KtxXZU+scyERvkJMEm16TbScVvuuMrlhPly78ZMbSc= -github.com/cilium/ebpf v0.9.3/go.mod h1:w27N4UjpaQ9X/DGrSugxUG+H+NhgntDuPb5lCzxCn8A= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.2.1/go.mod h1:wCYX+dRqZdImhGucXOqTQn05AhX6EUDaGEMUzTFFpLg= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containers/gvisor-tap-vsock v0.4.0 h1:sPu8OWiboCBJE/VJyxKOikslCI3sAApNfl3jPXdzBRw= -github.com/containers/gvisor-tap-vsock v0.4.0/go.mod h1:2daFkw9Qp3NTz7+SLEIeSEZsQbxvJVoPVF5WtRy7h/4= github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk= github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/stream-metadata-go v0.3.0/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libcontainer v2.2.1+incompatible h1:++SbbkCw+X8vAd4j2gOCzZ2Nn7s2xFALTf7LZKmM1/0= github.com/docker/libcontainer v2.2.1+incompatible/go.mod h1:osvj61pYsqhNCMLGX31xr7klUBhHb/ZBuXS0o1Fvwbw= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:PjfxuH4FZdUyfMdtBio2lsRr1AKEaVPwelzuHuh8Lqc= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= -github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8 h1:8nlgEAjIalk6uj/CGKCdOO8CQqTeysvcW4RFZ6HbkGM= -github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252/go.mod h1:DavVbd41y+b7ukKDmlnPR4nGYmkWXR6vHUkjQNiHPBs= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 h1:oTi0zYvHo1sfk5sevGc4LrfgpLYB6cIhP/HllCUGcZ8= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/insomniacslk/dhcp v0.0.0-20210812084645-decc701b3665/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= -github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= -github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= -github.com/jsimonetti/rtnetlink v0.0.0-20201110080708-d2c240429e6c/go.mod h1:huN4d1phzjhlOsNIjFsw2SVRbwIHj3fJDMEU2SDPTmg= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1 h1:zc0R6cOw98cMengLA0fvU55mqbnN7sd/tBMLzSejp+M= -github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= -github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= -github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= -github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= -github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= -github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= -github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= -github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= -github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= -github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= -github.com/mdlayher/vsock v1.1.1/go.mod h1:Y43jzcy7KM3QB+/FK15pfqGxDMCMzUXWegEfIbSM18U= -github.com/mdlayher/vsock v1.2.0 h1:klRY9lndjmg6k/QWbX/ucQ3e2JFRm1M7vfG9hijbQ0A= -github.com/mdlayher/vsock v1.2.0/go.mod h1:w4kdSTQB9p1l/WwGmAs0V62qQ869qRYoongwgN+Y1HE= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/milosgajdos/tenus v0.0.3 h1:jmaJzwaY1DUyYVD0lM4U+uvP2kkEg1VahDqRFxIkVBE= github.com/milosgajdos/tenus v0.0.3/go.mod h1:eIjx29vNeDOYWJuCnaHY2r4fq5egetV26ry3on7p8qY= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9 h1:Sha2bQdoWE5YQPTlJOL31rmce94/tYi113SlFo1xQ2c= -github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a h1:9iT75RHhYHWwWRlVWU7wnmtFulYcURCglzQOpT+cAF8= -github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w= -github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 h1:1zN6ImoqhSJhN8hGXFaJlSC8msLmIbX8bFqOfWLKw0w= -github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091/go.mod h1:N20Z5Y8oye9a7HmytmZ+tr8Q2vlP0tAHP13kTHzwvQY= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.1 h1:JDkWS7Axy5ziNM3svylLhpSgqjPDb+BgVUbXoDo+iPw= -github.com/vishvananda/netns v0.0.1/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.3 h1:WxY6MpgIdDMQX50UJ7bPIRJdBCOeUV6XtW8dZZja988= -github.com/vishvananda/netns v0.0.3/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.42.0-dev.0.20211020220737-f00baa6c3c84/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.2-0.20230118093459-a9481185b34d h1:qp0AnQCvRCMlu9jBjtdbTaaEmThIgZOrbVyDEOcmKhQ= -google.golang.org/protobuf v1.28.2-0.20230118093459-a9481185b34d/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gvisor.dev/gvisor v0.0.0-20220121190119-4f2d380c8b55/go.mod h1:vmN0Pug/s8TJmpnt30DvrEfZ5vDl52psGLU04tFuK2U= -gvisor.dev/gvisor v0.0.0-20221020013634-8e6a0b996cdf h1:odIYzLMj6x99QKT69c6O8Cc3BNas3oUZPPw74hUzsS0= -gvisor.dev/gvisor v0.0.0-20221020013634-8e6a0b996cdf/go.mod h1:D0iRe6RVONyvN6uEi/rqBtONyitX5GaHMDDbeMzwgiE= -gvisor.dev/gvisor v0.0.0-20230120050049-cc0dc87fa27d h1:AlEztVdR4/JpZ6H6jLn4emZjz0GxNtwZky6nlePV674= -gvisor.dev/gvisor v0.0.0-20230120050049-cc0dc87fa27d/go.mod h1:94x/o/BlxPAbw4phqHRac0/IzpcQRUP7ZQldDWV3TKU= -gvisor.dev/gvisor v0.0.0-20230120050912-b6da4fed55f0 h1:Yzd9ezp0lE3VtdTQUdkM7IJAeafJ/Ycpg+22jWOC2LA= -gvisor.dev/gvisor v0.0.0-20230120050912-b6da4fed55f0/go.mod h1:94x/o/BlxPAbw4phqHRac0/IzpcQRUP7ZQldDWV3TKU= -gvisor.dev/gvisor v0.0.0-20230218064137-f11778abbab6 h1:HFyFcXT7q8D4iklzDg7eLPojzeRgYZRfMfc53Vig8Gw= -gvisor.dev/gvisor v0.0.0-20230218064137-f11778abbab6/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= -gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec h1:C3Ox17AYBEjJx+FatCp61xBTxxWs4YFf6gFhb4nD/C4= -gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.16.13/go.mod h1:QWu8UWSTiuQZMMeYjwLs6ILu5O74qKSJ0c+4vrchDxs= -k8s.io/apimachinery v0.16.13/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ= -k8s.io/apimachinery v0.16.14-rc.0/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ= -k8s.io/client-go v0.16.13/go.mod h1:UKvVT4cajC2iN7DCjLgT0KVY/cbY6DGdUCyRiIfws5M= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 697f0331fc854a07e0bbca7514d670458f28b006 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 13:52:55 -0500 Subject: [PATCH 31/99] Merge heartbeat and registration handler. --- enclave.go | 35 +++++++++++++--------------- handlers.go | 62 +++++++++++++++----------------------------------- sync_leader.go | 10 ++++++-- workers.go | 24 ++++--------------- 4 files changed, 47 insertions(+), 84 deletions(-) diff --git a/enclave.go b/enclave.go index de59fc9..799e26e 100644 --- a/enclave.go +++ b/enclave.go @@ -34,7 +34,6 @@ import ( ) // TODO: Support Let's Encrypt (if we choose to). -// TODO: Handle the case of the leader restarting. Workers must not break. const ( acmeCertCacheDir = "cert-cache" @@ -45,18 +44,17 @@ const ( // https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-concepts.html parentCID = 3 // The following paths are handled by nitriding. - pathRoot = "/enclave" - pathNonce = "/enclave/nonce" - pathAttestation = "/enclave/attestation" - pathState = "/enclave/state" - pathSync = "/enclave/sync" - pathHash = "/enclave/hash" - pathReady = "/enclave/ready" - pathProfiling = "/enclave/debug" - pathConfig = "/enclave/config" - pathLeader = "/enclave/leader" - pathRegistration = "/enclave/registration" - pathHeartbeat = "/enclave/heartbeat" + pathRoot = "/enclave" + pathNonce = "/enclave/nonce" + pathAttestation = "/enclave/attestation" + pathState = "/enclave/state" + pathSync = "/enclave/sync" + pathHash = "/enclave/hash" + pathReady = "/enclave/ready" + pathProfiling = "/enclave/debug" + pathConfig = "/enclave/config" + pathLeader = "/enclave/leader" + pathHeartbeat = "/enclave/heartbeat" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" @@ -331,7 +329,7 @@ func (e *Enclave) installKeys(keys *enclaveKeys) error { func (e *Enclave) Start(ctx context.Context) error { var ( err error - leader = e.getLeader(pathRegistration) + leader = e.getLeader(pathHeartbeat) ) errPrefix := "failed to start Nitro Enclave" @@ -383,9 +381,8 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { elog.Println("Starting worker's heartbeat loop.") defer elog.Println("Exiting worker's heartbeat loop.") var ( - leaderHeartbeat = e.getLeader(pathHeartbeat) - leaderRegistration = e.getLeader(pathRegistration) - timer = time.NewTicker(time.Minute) + leader = e.getLeader(pathHeartbeat) + timer = time.NewTicker(time.Minute) ) for { @@ -394,7 +391,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { return case <-timer.C: resp, err := newUnauthenticatedHTTPClient().Post( - leaderHeartbeat.String(), + leader.String(), "text/plain", strings.NewReader(e.keys.hashAndB64()), ) @@ -404,7 +401,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { } if resp.StatusCode == http.StatusConflict { elog.Println("Our keys are outdated. Re-synchronizing.") - err := asWorker(e.installKeys, e.becameLeader).registerWith(leaderRegistration) + err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } diff --git a/handlers.go b/handlers.go index a5ee373..a16afae 100644 --- a/handlers.go +++ b/handlers.go @@ -94,16 +94,13 @@ func putStateHandler(e *Enclave) http.HandlerFunc { // The leader's application keys have changed. Re-synchronize the key // material with all registered workers. If synchronization fails for a // given worker, unregister it. - elog.Printf("Application keys have changed. Re-synchronizing with %d workers.", + elog.Printf("Application keys have changed. Re-synchronizing with %d worker(s).", e.workers.length()) go e.workers.forAll( func(worker *url.URL) { if err := asLeader(e.keys.get()).syncWith(worker); err != nil { // TODO: Log in Prometheus. - elog.Printf("Error re-syncing with worker %s: %v", worker.String(), err) e.workers.unregister(worker) - } else { - elog.Printf("Successfully re-synced with worker %s.", worker.String()) } }, ) @@ -228,66 +225,43 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { // Make leader-specific endpoints available. e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) - e.extPrivSrv.Handler.(*chi.Mux).Post(pathRegistration, workerRegistrationHandler(e)) elog.Println("Set up worker registration and heartbeat endpoint.") w.WriteHeader(http.StatusOK) } } -// workerRegistrationHandler allows worker to register themselves with the -// leader. Once a worker registered itself, the leader immediately proceeds to -// synchronize its key material with the worker. -func workerRegistrationHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - worker, err := e.getWorker(r) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - w.WriteHeader(http.StatusOK) - - go func() { - if err := asLeader(e.keys.get()).syncWith(worker); err != nil { - elog.Printf("Error syncing with worker: %v", err) - return - } - e.workers.register(worker) - elog.Printf("Successfully registered and synced with worker %s.", worker.Host) - }() - } -} - -// heartbeatHandler exposes an endpoint that allows worker enclaves to send -// periodic heartbeats to the leader enclave. The heartbeat's body contains the -// worker's hashed key material. If the worker's hash is different from the -// leader's hash, the leader knows that the worker's key material is out of -// sync, which makes the leader re-synchronize its key material with the worker. func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Read the worker's hashed key material. body, err := io.ReadAll(newLimitReader(r.Body, maxEnclaveKeyHash)) if err != nil { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - theirKeysHash := string(body) - ourKeysHash := e.keys.hashAndB64() - - // Update the worker's "last seen" timestamp. worker, err := e.getWorker(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - e.workers.updateHeartbeat(worker) - // Let the worker know if their keys are outdated. - if ourKeysHash != theirKeysHash { - elog.Printf("Worker's keys are outdated (ours=%s, theirs=%s).", - ourKeysHash, theirKeysHash) - w.WriteHeader(http.StatusConflict) + syncAndRegister := func(keys *enclaveKeys, worker *url.URL) { + if err := asLeader(keys.get()).syncWith(worker); err == nil { + e.workers.register(worker) + } + } + + if len(body) == 0 { + elog.Printf("Got heartbeat from uninitialized worker %s.", worker.Host) + go syncAndRegister(e.keys, worker) } else { - w.WriteHeader(http.StatusOK) + elog.Printf("Got heartbeat from initialized worker %s.", worker.Host) + ourKeysHash, theirKeysHash := e.keys.hashAndB64(), string(body) + if ourKeysHash != theirKeysHash { + go syncAndRegister(e.keys, worker) + } else { + e.workers.register(worker) + } } + w.WriteHeader(http.StatusOK) } } diff --git a/sync_leader.go b/sync_leader.go index ab1eb76..dc59561 100644 --- a/sync_leader.go +++ b/sync_leader.go @@ -30,8 +30,14 @@ func asLeader(keys *enclaveKeys) *leaderSync { // syncWith makes the leader initiate key synchronization with the given worker // enclave. -func (s *leaderSync) syncWith(worker *url.URL) error { - elog.Printf("Initiating key synchronization with worker %s.", worker.Host) +func (s *leaderSync) syncWith(worker *url.URL) (err error) { + defer func() { + if err == nil { + elog.Printf("Successfully synced with worker %s.", worker.Host) + } else { + elog.Printf("Error syncing with worker %s: %v", worker.Host, err) + } + }() // Step 1: Create a nonce that the worker must embed in its attestation // document, to prevent replay attacks. diff --git a/workers.go b/workers.go index f4b12eb..dec8c75 100644 --- a/workers.go +++ b/workers.go @@ -9,11 +9,11 @@ import ( // workerManager manages worker enclaves. type workerManager struct { - timeout time.Duration - reg, unreg, heartbeat chan *url.URL - len chan int - forAllFunc chan func(*url.URL) - afterTickFunc chan func() + timeout time.Duration + reg, unreg chan *url.URL + len chan int + forAllFunc chan func(*url.URL) + afterTickFunc chan func() } // workers maps worker enclaves to a timestamp that keeps track of when we last @@ -25,7 +25,6 @@ func newWorkerManager(timeout time.Duration) *workerManager { timeout: timeout, reg: make(chan *url.URL), unreg: make(chan *url.URL), - heartbeat: make(chan *url.URL), len: make(chan int), forAllFunc: make(chan func(*url.URL)), afterTickFunc: make(chan func()), @@ -64,14 +63,6 @@ func (w *workerManager) start(ctx context.Context) { elog.Printf("Unregistered worker %s; %d worker(s) left.", worker.Host, len(set)) - case worker := <-w.heartbeat: - _, exists := set[*worker] - if !exists { - elog.Printf("Updating heartbeat for previously-unregistered worker %s.", - worker.Host) - } - set[*worker] = time.Now() - case f := <-w.forAllFunc: w.runForAll(f, set) w.forAllFunc <- nil // Signal to caller that we're done. @@ -128,11 +119,6 @@ func (w *workerManager) unregister(worker *url.URL) { w.unreg <- worker } -// updateHeartbeat updates the "last seen" timestamp of the given worker. -func (w *workerManager) updateHeartbeat(worker *url.URL) { - w.heartbeat <- worker -} - // pruneDefunctWorkers looks for and unregisters workers whose last heartbeat is // older than our timeout. func (w *workerManager) pruneDefunctWorkers(set workers) { From 56a9745a25dedfd734c0e8d5f3dbad10691b0f1e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 18:20:37 -0500 Subject: [PATCH 32/99] Use self-signed certificates for tests to pass. --- enclave.go | 61 +++++------------------------------------- sync_worker_test.go | 13 +++++++++ util.go | 65 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 55 deletions(-) diff --git a/enclave.go b/enclave.go index 799e26e..7a64d4f 100644 --- a/enclave.go +++ b/enclave.go @@ -2,19 +2,14 @@ package main import ( "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" - "math/big" "net" "net/http" "net/http/httputil" @@ -495,67 +490,23 @@ func (e *Enclave) startWebServers() error { return nil } -// genSelfSignedCert creates and returns a self-signed TLS certificate based on -// the given FQDN. Some of the code below was taken from: -// https://eli.thegreenplace.net/2021/go-https-servers-with-tls/ +// genSelfSignedCert creates and installs a self-signed certificate. func (e *Enclave) genSelfSignedCert() error { - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + cert, key, err := createCertificate(e.cfg.FQDN) if err != nil { return err } - elog.Println("Generated private key for self-signed certificate.") - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { + if err := e.setCertFingerprint(cert); err != nil { return err } - elog.Println("Generated serial number for self-signed certificate.") + e.keys.setNitridingKeys(key, cert) - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{certificateOrg}, - }, - DNSNames: []string{e.cfg.FQDN}, - NotBefore: time.Now(), - NotAfter: time.Now().Add(certificateValidity), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + tlsCert, err := tls.X509KeyPair(cert, key) if err != nil { return err } - elog.Println("Created certificate from template.") - - pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - if pemCert == nil { - return errors.New("failed to encode certificate to PEM") - } - // Determine and set the certificate's fingerprint because we need to add - // the fingerprint to our Nitro attestation document. - if err := e.setCertFingerprint(pemCert); err != nil { - return err - } - - privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - elog.Fatalf("Unable to marshal private key: %v", err) - } - pemKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) - if pemKey == nil { - elog.Fatal("Failed to encode key to PEM.") - } - e.keys.setNitridingKeys(pemKey, pemCert) - - cert, err := tls.X509KeyPair(pemCert, pemKey) - if err != nil { - return err - } - e.httpsCert.set(&cert) + e.httpsCert.set(&tlsCert) e.extPubSrv.TLSConfig = &tls.Config{ GetCertificate: e.httpsCert.get, } diff --git a/sync_worker_test.go b/sync_worker_test.go index 082928f..e37ed03 100644 --- a/sync_worker_test.go +++ b/sync_worker_test.go @@ -15,6 +15,15 @@ var leaderKeys = &enclaveKeys{ AppKeys: []byte("AppTestKeys"), } +func initLeaderKeysCert(t *testing.T) { + t.Helper() + cert, key, err := createCertificate("example.com") + if err != nil { + t.Fatal(err) + } + leaderKeys.setNitridingKeys(key, cert) +} + func TestSuccessfulRegisterWith(t *testing.T) { e := createEnclave(&defaultCfg) hasRegistered := false @@ -61,6 +70,10 @@ func TestAbortedRegisterWith(t *testing.T) { } func TestSuccessfulSync(t *testing.T) { + // For key synchronization to be successful, we need actual certificates in + // the leader keys. + initLeaderKeysCert(t) + // Set up the worker. worker := createEnclave(&defaultCfg) srv := httptest.NewTLSServer( diff --git a/util.go b/util.go index 9ffc394..23fafc2 100644 --- a/util.go +++ b/util.go @@ -1,8 +1,17 @@ package main import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" "net/http" + "time" ) // newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS @@ -15,3 +24,59 @@ func newUnauthenticatedHTTPClient() *http.Client { } return &http.Client{Transport: transport} } + +// createCertificate creates a self-signed certificate and returns the +// PEM-encoded certificate and key. Some of the code below was taken from: +// https://eli.thegreenplace.net/2021/go-https-servers-with-tls/ +func createCertificate(fqdn string) (cert []byte, key []byte, err error) { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{certificateOrg}, + }, + DNSNames: []string{fqdn}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(certificateValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate( + rand.Reader, + &template, + &template, + &privateKey.PublicKey, + privateKey, + ) + if err != nil { + return nil, nil, err + } + + pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if pemCert == nil { + return nil, nil, errors.New("error encoding cert as PEM") + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + return nil, nil, err + } + pemKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + if pemKey == nil { + return nil, nil, errors.New("error encoding key as PEM") + } + + return pemCert, pemKey, nil +} From 4a83286510204e277b7a3e3fcf2a6b7e39098ca6 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 18:20:57 -0500 Subject: [PATCH 33/99] Delete unused code block. --- enclave.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/enclave.go b/enclave.go index 7a64d4f..01c215b 100644 --- a/enclave.go +++ b/enclave.go @@ -394,15 +394,6 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { elog.Printf("Error posting heartbeat to leader: %v", err) continue } - if resp.StatusCode == http.StatusConflict { - elog.Println("Our keys are outdated. Re-synchronizing.") - err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) - if err != nil && !errors.Is(err, errBecameLeader) { - elog.Fatalf("Error syncing with leader: %v", err) - } - elog.Println("Successfully re-synchronized with leader.") - continue - } if resp.StatusCode != http.StatusOK { elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) continue From ee8265380b2dfacd83a46ab406d14582bd023e28 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 16 Aug 2023 18:22:08 -0500 Subject: [PATCH 34/99] Don't block on forAll. --- workers.go | 20 ++++---------------- workers_test.go | 5 ++++- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/workers.go b/workers.go index dec8c75..d05f2a4 100644 --- a/workers.go +++ b/workers.go @@ -3,7 +3,6 @@ package main import ( "context" "net/url" - "sync" "time" ) @@ -65,7 +64,6 @@ func (w *workerManager) start(ctx context.Context) { case f := <-w.forAllFunc: w.runForAll(f, set) - w.forAllFunc <- nil // Signal to caller that we're done. case <-w.len: w.len <- len(set) @@ -73,20 +71,12 @@ func (w *workerManager) start(ctx context.Context) { } } -// runForAll blocks until the given function was run over all workers in our -// set. For key synchronization, this should never take more than a couple -// seconds. +// runForAll runs the given function over all workers in our set. For key +// synchronization, this should never take more than a couple seconds. func (w *workerManager) runForAll(f func(*url.URL), set workers) { - var wg sync.WaitGroup for worker := range set { - wg.Add(1) - go func(wg *sync.WaitGroup, worker url.URL) { - elog.Printf("Running function for worker %s.", worker.Host) - f(&worker) - wg.Done() - }(&wg, worker) + go f(&worker) } - wg.Wait() } // _afterTick runs the given function after the next event loop tick. This is @@ -101,11 +91,9 @@ func (w *workerManager) length() int { return <-w.len } -// forAll runs the given function over all registered workers. This function -// blocks until the operation succeeded. +// forAll runs the given function over all registered workers. func (w *workerManager) forAll(f func(*url.URL)) { w.forAllFunc <- f - <-w.forAllFunc // Wait until the event loop is done running the given function. } // register registers a new worker enclave. It is safe to repeatedly register diff --git a/workers_test.go b/workers_test.go index 2c373d3..bd76a52 100644 --- a/workers_test.go +++ b/workers_test.go @@ -54,7 +54,10 @@ func TestForAll(t *testing.T) { total += 1 }, ) - assertEqual(t, total, 2) + // Make sure that the worker got pruned after the next tick. + w._afterTick(func() { + assertEqual(t, total, 2) + }) } func TestIneffectiveForAll(t *testing.T) { From 165d947958e5e87d3cc51d9d20b8b6fe04eb132d Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 17 Aug 2023 08:53:38 -0500 Subject: [PATCH 35/99] Fix data race. --- enclave.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index 01c215b..9760aee 100644 --- a/enclave.go +++ b/enclave.go @@ -501,7 +501,8 @@ func (e *Enclave) genSelfSignedCert() error { e.extPubSrv.TLSConfig = &tls.Config{ GetCertificate: e.httpsCert.get, } - e.extPrivSrv.TLSConfig = e.extPubSrv.TLSConfig // Both servers share a TLS config. + // Both servers share a TLS config. + e.extPrivSrv.TLSConfig = e.extPubSrv.TLSConfig.Clone() return nil } From 359336b7ba6f62f2070430008498ef20e2fd95a3 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 10:58:51 -0500 Subject: [PATCH 36/99] Run the "designate leader" code only once. --- handlers.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/handlers.go b/handlers.go index a16afae..ff3036b 100644 --- a/handlers.go +++ b/handlers.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "strings" + "sync" "github.com/go-chi/chi/v5" ) @@ -212,21 +213,20 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl // leaderHandler is called when the enclave is designated as leader enclave. // If designated, we do the following: // -// 1. Signal to other parts of the code that we became the leader. +// 1. Signal to our leader registration goroutine that we're the leader. // 2. Start the worker event loop, to keep track of worker enclaves. // 3. Expose leader-specific endpoints. func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { + var once sync.Once return func(w http.ResponseWriter, r *http.Request) { - elog.Println("Designated enclave as leader.") - close(e.becameLeader) // Signal to other parts of the code. - // TODO: Repeated calls make the goroutine panic. - - go e.workers.start(ctx) - // Make leader-specific endpoints available. - e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) - e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) - elog.Println("Set up worker registration and heartbeat endpoint.") - + once.Do(func() { + e.becameLeader <- struct{}{} + go e.workers.start(ctx) + // Make leader-specific endpoints available. + e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) + e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) + elog.Println("Designated enclave as leader.") + }) w.WriteHeader(http.StatusOK) } } From e27377fa4b4fc22627e9143a6e58bd613c9e90ce Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 10:59:27 -0500 Subject: [PATCH 37/99] Make attestation handler use attester interface. --- handlers.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/handlers.go b/handlers.go index ff3036b..b9d2ade 100644 --- a/handlers.go +++ b/handlers.go @@ -176,7 +176,7 @@ func configHandler(cfg *Config) http.HandlerFunc { // subsequently asks its hypervisor for an attestation document that contains // both the nonce and the hashes in the given struct. The resulting // Base64-encoded attestation document is then returned to the requester. -func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.HandlerFunc { +func attestationHandler(useProfiling bool, hashes *AttestationHashes, a attester) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if useProfiling { http.Error(w, errProfilingSet, http.StatusServiceUnavailable) @@ -200,7 +200,16 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.Handl return } - rawDoc, err := attest(rawNonce, hashes.Serialize(), nil) + n, err := sliceToNonce(rawNonce) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rawDoc, err := a.createAttstn(&clientAuxInfo{ + clientNonce: n, + attestationHashes: hashes.Serialize(), + }) if err != nil { http.Error(w, errFailedAttestation, http.StatusInternalServerError) return From 33bcf393421e8b672b11514e6166a8301dbcf3f3 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 13:00:41 -0500 Subject: [PATCH 38/99] More refactoring and shuffling code around. --- attestation.go | 35 +---------------- attester.go | 42 +++++++++++--------- attester_test.go | 20 ++++++++++ enclave.go | 10 ++--- enclave_keys_test.go | 36 ++++++++++++------ handlers_test.go | 91 +++++++++++++++++++++++++++++++++++++++----- util.go | 37 +++++++++++++++++- util_test.go | 13 +++++++ workers.go | 59 ++++++++++------------------ workers_test.go | 35 ++++++----------- 10 files changed, 234 insertions(+), 144 deletions(-) create mode 100644 attester_test.go create mode 100644 util_test.go diff --git a/attestation.go b/attestation.go index 2dbd402..28b37a0 100644 --- a/attestation.go +++ b/attestation.go @@ -3,12 +3,9 @@ package main import ( "bytes" "crypto/sha256" - "errors" "fmt" "github.com/hf/nitrite" - "github.com/hf/nsm" - "github.com/hf/nsm/request" ) const ( @@ -55,7 +52,7 @@ func (a *AttestationHashes) Serialize() []byte { // _getPCRValues returns the enclave's platform configuration register (PCR) // values. func _getPCRValues() (map[uint][]byte, error) { - rawAttDoc, err := attest(nil, nil, nil) + rawAttDoc, err := newNitroAttester().createAttstn(nil) if err != nil { return nil, err } @@ -86,33 +83,3 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool { } return true } - -// attest takes as input a nonce, user-provided data and a public key, and then -// asks the Nitro hypervisor to return a signed attestation document that -// contains all three values. -func attest(nonce, userData, publicKey []byte) ([]byte, error) { - s, err := nsm.OpenDefaultSession() - if err != nil { - return nil, err - } - defer func() { - if err = s.Close(); err != nil { - elog.Printf("Attestation: Failed to close default NSM session: %s", err) - } - }() - - res, err := s.Send(&request.Attestation{ - Nonce: nonce, - UserData: userData, - PublicKey: publicKey, - }) - if err != nil { - return nil, err - } - - if res.Attestation == nil || res.Attestation.Document == nil { - return nil, errors.New("NSM device did not return an attestation") - } - - return res.Attestation.Document, nil -} diff --git a/attester.go b/attester.go index f02478b..5d0d382 100644 --- a/attester.go +++ b/attester.go @@ -21,6 +21,11 @@ type attester interface { type auxInfo interface{} +type clientAuxInfo struct { + clientNonce nonce + attestationHashes []byte +} + // workerAuxInfo holds the auxiliary information of the worker's attestation // document. type workerAuxInfo struct { @@ -87,10 +92,17 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { return nil, errors.New("invalid auxiliary information") } -// nitroAttester implements production functions for the creation and -// verification of attestation documents. +// nitroAttester implements the attester interface by drawing on the AWS Nitro +// Enclave hypervisor. type nitroAttester struct{} +// newNitroAttester returns a new nitroAttester. +func newNitroAttester() *nitroAttester { + return &nitroAttester{} +} + +// createAttstn asks the AWS Nitro Enclave hypervisor for an attestation +// document that contains the given auxiliary information. func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { var nonce, userData, publicKey []byte @@ -103,6 +115,9 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { case leaderAuxInfo: nonce = v.WorkersNonce[:] userData = v.EnclaveKeys + case clientAuxInfo: + nonce = v.clientNonce[:] + userData = v.attestationHashes } s, err := nsm.OpenDefaultSession() @@ -111,7 +126,7 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { } defer func() { if err = s.Close(); err != nil { - elog.Printf("Attestation: Failed to close default NSM session: %s", err) + elog.Printf("Error closing NSM session: %v", err) } }() @@ -130,6 +145,8 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { return res.Attestation.Document, nil } +// verifyAttstn verifies the given attestation document and, if successful, +// returns the document's auxiliary information. func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { errStr := "error verifying attestation document" // Verify the remote enclave's attestation document before doing anything @@ -137,24 +154,24 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { opts := nitrite.VerifyOptions{CurrentTime: currentTime()} their, err := nitrite.Verify(doc, opts) if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) + return nil, fmt.Errorf("%v: %w", errStr, err) } // Verify that the remote enclave's PCR values (e.g., the image ID) are // identical to ours. ourPCRs, err := getPCRValues() if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) + return nil, fmt.Errorf("%v: %w", errStr, err) } if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { - return nil, fmt.Errorf("%s: PCR values of remote enclave not identical to ours", errStr) + return nil, fmt.Errorf("%v: PCR values of remote enclave not identical to ours", errStr) } // Verify that the remote enclave's attestation document contains the nonce // that we asked it to embed. b64Nonce := base64.StdEncoding.EncodeToString(their.Document.Nonce) if n.B64() == b64Nonce { - return nil, fmt.Errorf("%s: nonce %s not in cache", errStr, b64Nonce) + return nil, fmt.Errorf("%v: nonce %s not in cache", errStr, b64Nonce) } workersNonce, err := sliceToNonce(their.Document.Nonce) @@ -184,14 +201,3 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { EnclaveKeys: their.Document.UserData, }, nil } - -func sliceToNonce(s []byte) (nonce, error) { - var n nonce - - if len(s) != nonceLen { - return nonce{}, errors.New("slice is not of same length as nonce") - } - - copy(n[:], s[:nonceLen]) - return n, nil -} diff --git a/attester_test.go b/attester_test.go new file mode 100644 index 0000000..ba7835e --- /dev/null +++ b/attester_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "errors" + "testing" + + "github.com/hf/nitrite" +) + +func TestVerifyNitroAttstn(t *testing.T) { + var n = newNitroAttester() + _, err := n.verifyAttstn([]byte("foobar"), nonce{}) + assertEqual(t, errors.Is(err, nitrite.ErrBadCOSESign1Structure), true) +} + +func TestCreateNitroAttstn(t *testing.T) { + var n = newNitroAttester() + _, err := n.createAttstn(nil) + assertEqual(t, err != nil, true) +} diff --git a/enclave.go b/enclave.go index 9760aee..f224a19 100644 --- a/enclave.go +++ b/enclave.go @@ -245,7 +245,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { workers: newWorkerManager(time.Minute), stop: make(chan struct{}), ready: make(chan struct{}), - becameLeader: make(chan struct{}), + becameLeader: make(chan struct{}, 1), } // Increase the maximum number of idle connections per host. This is @@ -275,7 +275,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external public HTTP API. m := e.extPubSrv.Handler.(*chi.Mux) - m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes)) + m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes, e.attester)) m.Get(pathRoot, rootHandler(e.cfg)) m.Get(pathConfig, configHandler(e.cfg)) @@ -607,9 +607,5 @@ func (e *Enclave) getWorker(r *http.Request) (*url.URL, error) { if err != nil { return nil, err } - return &url.URL{ - Scheme: "https", // Leader and workers use HTTPS to communicate. - Host: fmt.Sprintf("%s:%d", strIP, e.cfg.ExtPrivPort), - Path: pathSync, - }, nil + return getSyncURL(strIP, e.cfg.ExtPrivPort), nil } diff --git a/enclave_keys_test.go b/enclave_keys_test.go index c445534..c344cdd 100644 --- a/enclave_keys_test.go +++ b/enclave_keys_test.go @@ -5,17 +5,25 @@ import ( "testing" ) -// testKeys holds arbitrary keys that we use for testing. -var testKeys = &enclaveKeys{ - NitridingKey: []byte("NitridingTestKey"), - NitridingCert: []byte("NitridingTestCert"), - AppKeys: []byte("AppTestKeys"), +// newTestKeys returns arbitrary keys that we use for testing. +func newTestKeys(t *testing.T) *enclaveKeys { + t.Helper() + var testKeys = &enclaveKeys{ + AppKeys: []byte("AppTestKeys"), + } + cert, key, err := createCertificate("example.com") + if err != nil { + t.Fatal(err) + } + testKeys.setNitridingKeys(key, cert) + return testKeys } func TestSetKeys(t *testing.T) { var ( - keys enclaveKeys - appKeys = []byte("AppKeys") + keys enclaveKeys + appKeys = []byte("AppKeys") + testKeys = newTestKeys(t) ) // Ensure that the application keys are set correctly. @@ -45,8 +53,9 @@ func TestSetKeys(t *testing.T) { func TestGetKeys(t *testing.T) { var ( - appKeys = testKeys.getAppKeys() - keys = testKeys.get() + testKeys = newTestKeys(t) + appKeys = testKeys.getAppKeys() + keys = testKeys.get() ) // Ensure that the application key is retrieved correctly. @@ -61,12 +70,15 @@ func TestGetKeys(t *testing.T) { } func TestModifyCloneObject(t *testing.T) { - newKeys := testKeys.get() - newKeys.setAppKeys([]byte("foobar")) + var ( + keys = newTestKeys(t) + clonedKeys = keys.get() + ) // Make sure that setting the clone's application keys does not affect the // original object. - if bytes.Equal(newKeys.getAppKeys(), testKeys.getAppKeys()) { + keys.setAppKeys([]byte("foobar")) + if bytes.Equal(keys.getAppKeys(), clonedKeys.getAppKeys()) { t.Fatal("Cloned object must not affect original object.") } } diff --git a/handlers_test.go b/handlers_test.go index 32b9c4e..5795b4a 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "net/http/httputil" "net/url" + "sync" "syscall" "testing" "time" @@ -36,6 +37,17 @@ func newResp(status int, body string) *http.Response { } } +// designateLeader designates the enclave as a leader to make leader-specific +// endpoints available. +func designateLeader(t *testing.T, srv *http.Server) { + t.Helper() + makeReq := makeRequestFor(srv) + assertResponse(t, + makeReq(http.MethodGet, pathLeader, nil), + newResp(http.StatusOK, ""), + ) +} + // assertResponse ensures that the two given HTTP responses are (almost) // identical. We only check the HTTP status code and the response body. // If the expected response has no body, we only compare the status code. @@ -96,17 +108,10 @@ func signalReady(t *testing.T, e *Enclave) { func TestStateHandlers(t *testing.T) { e := createEnclave(&defaultCfg) - - // First, designate the enclave as leader to make the "put state" endpoint - // available. - makeReq := makeRequestFor(e.extPrivSrv) - assertResponse(t, - makeReq(http.MethodGet, pathLeader, nil), - newResp(http.StatusOK, ""), - ) + designateLeader(t, e.extPrivSrv) tooLargeKey := make([]byte, 1024*1024+1) - makeReq = makeRequestFor(e.intSrv) + makeReq := makeRequestFor(e.intSrv) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(tooLargeKey)), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), @@ -335,3 +340,71 @@ func TestConfigHandler(t *testing.T) { newResp(http.StatusOK, defaultCfg.String()), ) } + +func TestHeartbeatHandler(t *testing.T) { + var ( + e = createEnclave(&defaultCfg) + keys = newTestKeys(t) + makeReq = makeRequestFor(e.extPrivSrv) + ) + designateLeader(t, e.extPrivSrv) + e.keys.set(keys) + + tooLargeBuf := bytes.NewBuffer(make([]byte, maxEnclaveKeyHash+1)) + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, tooLargeBuf), + newResp(http.StatusInternalServerError, errFailedReqBody.Error()), + ) + + validKeys := bytes.NewBuffer([]byte(keys.hashAndB64())) + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, validKeys), + newResp(http.StatusOK, ""), + ) +} + +func TestHeartbeatHandlerWithSync(t *testing.T) { + var ( + wg = sync.WaitGroup{} + leaderEnclave = createEnclave(&defaultCfg) + makeReq = makeRequestFor(leaderEnclave.extPrivSrv) + workerKeys = newTestKeys(t) + setWorkerKeys = func(keys *enclaveKeys) error { + defer wg.Done() + workerKeys.set(keys) + return nil + } + worker = asWorker(setWorkerKeys, make(chan struct{})) + workerSrv = httptest.NewTLSServer(worker) + ) + defer workerSrv.Close() + leaderEnclave.Start(context.Background()) + designateLeader(t, leaderEnclave.extPrivSrv) + wg.Add(1) + + // Mock two functions to make the leader enclave talk to our test server. + newUnauthenticatedHTTPClient = workerSrv.Client + getSyncURL = func(host string, port uint16) *url.URL { + u, err := url.Parse(workerSrv.URL) + if err != nil { + t.Fatal(err) + } + return u + } + + assertEqual(t, leaderEnclave.workers.length(), 0) + + // Send a heartbeat to the leader. The heartbeat's keys don't match the + // leader's keys, + // which results in the leader initiating key synchronization. + invalidKeys := bytes.NewBuffer([]byte(workerKeys.hashAndB64())) + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, invalidKeys), + newResp(http.StatusOK, ""), + ) + + // Wait until the worker's keys were set and make sure that the keys were + // synchronized successfully. + wg.Wait() + assertEqual(t, leaderEnclave.keys.equal(workerKeys), true) +} diff --git a/util.go b/util.go index 23fafc2..b0744a7 100644 --- a/util.go +++ b/util.go @@ -9,16 +9,39 @@ import ( "crypto/x509/pkix" "encoding/pem" "errors" + "fmt" "math/big" "net/http" + "net/url" "time" ) +var ( + errBadSliceLen = errors.New("slice is not of same length as nonce") + newUnauthenticatedHTTPClient = func() *http.Client { + return _newUnauthenticatedHTTPClient() + } + getSyncURL = func(host string, port uint16) *url.URL { + return _getSyncURL(host, port) + } +) + +// _getSyncURL turns the given host and port into a URL that a leader enclave +// can sync with. +var _getSyncURL = func(host string, port uint16) *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", host, port), + Path: pathSync, + } +} + // newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS // certificate validation. In the context of nitriding, this is fine because // all we need is a *confidential* channel, and not an authenticated channel. // Authentication is handled via attestation documents. -func newUnauthenticatedHTTPClient() *http.Client { +func _newUnauthenticatedHTTPClient() *http.Client { + fmt.Println("ORIG CLIENT") transport := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } @@ -80,3 +103,15 @@ func createCertificate(fqdn string) (cert []byte, key []byte, err error) { return pemCert, pemKey, nil } + +// sliceToNonce copies the given slice into a nonce and returns the nonce. +func sliceToNonce(s []byte) (nonce, error) { + var n nonce + + if len(s) != nonceLen { + return nonce{}, errBadSliceLen + } + + copy(n[:], s[:nonceLen]) + return n, nil +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..0c834e3 --- /dev/null +++ b/util_test.go @@ -0,0 +1,13 @@ +package main + +import "testing" + +func TestSliceToNonce(t *testing.T) { + var err error + + _, err = sliceToNonce([]byte("foo")) + assertEqual(t, err, errBadSliceLen) + + _, err = sliceToNonce(make([]byte, nonceLen)) + assertEqual(t, err, nil) +} diff --git a/workers.go b/workers.go index d05f2a4..93336b7 100644 --- a/workers.go +++ b/workers.go @@ -8,34 +8,31 @@ import ( // workerManager manages worker enclaves. type workerManager struct { - timeout time.Duration - reg, unreg chan *url.URL - len chan int - forAllFunc chan func(*url.URL) - afterTickFunc chan func() + timeout time.Duration + reg, unreg chan *url.URL + len chan int + forAllFunc chan func(*url.URL) } -// workers maps worker enclaves to a timestamp that keeps track of when we last -// got a heartbeat from the enclave. +// workers maps worker enclaves (identified by a URL) to a timestamp that keeps +// track of when we last got a heartbeat from the worker. type workers map[url.URL]time.Time func newWorkerManager(timeout time.Duration) *workerManager { return &workerManager{ - timeout: timeout, - reg: make(chan *url.URL), - unreg: make(chan *url.URL), - len: make(chan int), - forAllFunc: make(chan func(*url.URL)), - afterTickFunc: make(chan func()), + timeout: timeout, + reg: make(chan *url.URL), + unreg: make(chan *url.URL), + len: make(chan int), + forAllFunc: make(chan func(*url.URL)), } } // start starts the worker manager's event loop. func (w *workerManager) start(ctx context.Context) { var ( - set = make(workers) - timer = time.NewTicker(w.timeout) - afterTickFunc = func() {} + set = make(workers) + timer = time.NewTicker(w.timeout) ) elog.Println("Starting worker event loop.") defer elog.Println("Stopping worker event loop.") @@ -45,12 +42,14 @@ func (w *workerManager) start(ctx context.Context) { case <-ctx.Done(): return - case f := <-w.afterTickFunc: - afterTickFunc = f - case <-timer.C: - go w.pruneDefunctWorkers(set) - afterTickFunc() + now := time.Now() + for worker, lastSeen := range set { + if now.Sub(lastSeen) > w.timeout { + delete(set, worker) + elog.Printf("Pruned %s from worker set.", worker.Host) + } + } case worker := <-w.reg: set[*worker] = time.Now() @@ -79,12 +78,6 @@ func (w *workerManager) runForAll(f func(*url.URL), set workers) { } } -// _afterTick runs the given function after the next event loop tick. This is -// only useful in unit tests. -func (w *workerManager) _afterTick(f func()) { - w.afterTickFunc <- f -} - // length returns the number of workers that are currently registered. func (w *workerManager) length() int { w.len <- 0 // Signal to the event loop that we want the length. @@ -106,15 +99,3 @@ func (w *workerManager) register(worker *url.URL) { func (w *workerManager) unregister(worker *url.URL) { w.unreg <- worker } - -// pruneDefunctWorkers looks for and unregisters workers whose last heartbeat is -// older than our timeout. -func (w *workerManager) pruneDefunctWorkers(set workers) { - now := time.Now() - for worker, lastSeen := range set { - if now.Sub(lastSeen) > w.timeout { - w.unregister(&worker) - elog.Printf("Pruned %s from worker set.", worker.Host) - } - } -} diff --git a/workers_test.go b/workers_test.go index bd76a52..5cba745 100644 --- a/workers_test.go +++ b/workers_test.go @@ -3,6 +3,7 @@ package main import ( "context" "net/url" + "sync" "testing" "time" ) @@ -38,8 +39,11 @@ func TestWorkerRegistration(t *testing.T) { func TestForAll(t *testing.T) { var ( - w = newWorkerManager(time.Minute) + w = newWorkerManager(time.Millisecond) ctx, cancel = context.WithCancel(context.Background()) + wg = sync.WaitGroup{} + mutex = sync.Mutex{} + total = 0 ) go w.start(ctx) defer cancel() @@ -48,16 +52,17 @@ func TestForAll(t *testing.T) { w.register(&url.URL{Host: "bar"}) assertEqual(t, w.length(), 2) - total := 0 + wg.Add(2) w.forAll( func(w *url.URL) { + mutex.Lock() + defer mutex.Unlock() + defer wg.Done() total += 1 }, ) - // Make sure that the worker got pruned after the next tick. - w._afterTick(func() { - assertEqual(t, total, 2) - }) + wg.Wait() + assertEqual(t, total, 2) } func TestIneffectiveForAll(t *testing.T) { @@ -71,21 +76,3 @@ func TestIneffectiveForAll(t *testing.T) { // Make sure that forAll finishes for an empty worker set. w.forAll(func(_ *url.URL) {}) } - -func TestUpdatingAndPruning(t *testing.T) { - var ( - w = newWorkerManager(time.Millisecond) - ctx, cancel = context.WithCancel(context.Background()) - ) - go w.start(ctx) - defer cancel() - - worker := &url.URL{Host: "foo"} - w.register(worker) - assertEqual(t, w.length(), 1) - - // Make sure that the worker got pruned after the next tick. - w._afterTick(func() { - assertEqual(t, w.length(), 0) - }) -} From d52948e07bf98ed181974ad772120d6d5ae1ef4d Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 13:16:49 -0500 Subject: [PATCH 39/99] Remove debug message. --- util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/util.go b/util.go index b0744a7..b65a204 100644 --- a/util.go +++ b/util.go @@ -41,7 +41,6 @@ var _getSyncURL = func(host string, port uint16) *url.URL { // all we need is a *confidential* channel, and not an authenticated channel. // Authentication is handled via attestation documents. func _newUnauthenticatedHTTPClient() *http.Client { - fmt.Println("ORIG CLIENT") transport := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } From 0557bda4fd6085bbb38b4acf7becaf7c71b21f78 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 16:09:50 -0500 Subject: [PATCH 40/99] Fix linter warning. --- handlers_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/handlers_test.go b/handlers_test.go index 5795b4a..59fac0c 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -378,7 +378,9 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { workerSrv = httptest.NewTLSServer(worker) ) defer workerSrv.Close() - leaderEnclave.Start(context.Background()) + if err := leaderEnclave.Start(context.Background()); err != nil { + t.Fatal(err) + } designateLeader(t, leaderEnclave.extPrivSrv) wg.Add(1) From 632f83a57090ef92fb024ee10c02d7985584c3d8 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 18 Aug 2023 16:41:57 -0500 Subject: [PATCH 41/99] Add work-in-progress documentation. --- README.md | 1 + doc/key-synchronization.md | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 doc/key-synchronization.md diff --git a/README.md b/README.md index 9068409..88e6855 100644 --- a/README.md +++ b/README.md @@ -55,4 +55,5 @@ system, take a look at our [research paper](https://arxiv.org/abs/2206.04123). * [How to use nitriding](doc/usage.md) * [System architecture](doc/architecture.md) +* [Horizontal scaling](doc/key-synchronization.md) * [Example application](example/) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md new file mode 100644 index 0000000..dae6cf8 --- /dev/null +++ b/doc/key-synchronization.md @@ -0,0 +1,82 @@ +# Enclave key synchronization + +Nitriding supports horizontal scaling, i.e., it allows for the synchronization +of key material among enclaves. Key material consists of both nitriding and +application keys: + +1. Nitriding's key material is the self-signed HTTPS certificate (both public + and private key) that provides the confidential channel between clients and + the enclave. +2. The application's key material is application-specific. Nitriding is + agnostic to the structure of this key material and treats it as arbitrary + bytes. + +For enclave key synchronization to work, there must be a single leader +enclave and one or more worker enclaves. The leader enclave's sole job is to +create key material and make itself available to synchronize this key material +with worker enclaves. + +To enable horizontal scaling, use the `-fqdn-leader` command line flag. + +```mermaid +sequenceDiagram + box rgba(100, 100, 100, .1) Leader enclave + participant leaderApp as Enclave application + participant leader as Leader enclave + end + participant leaderEC2 as Leader EC2 + participant workerEC2 as Worker EC2 + box rgba(100, 100, 100, .1) Worker enclave + participant worker as Worker enclave + participant workerApp as Enclave application + end + +leader->>leader: Generate HTTPS certificate +leaderApp->>leaderApp: Generate key material + +Note over leader,leaderEC2: Designating enclave as leader +leaderEC2->>+leader: GET /enclave/leader +leader->>leader: Expose leader-specific endpoints +leader-->>-leaderEC2: OK + +Note over leaderApp,leader: Application sets its key material +leaderApp->>+leader: PUT /enclave/state (key material) +leader->>leader: Save key material +leader-->>-leaderApp: OK + +Note over leader,worker: Worker announces itself to leader +worker->>+leader: POST /enclave/heartbeat +leader->>leader: Register new worker +leader-->>-worker: OK + +Note over leader,worker: Leader initiates key synchronization +leader->>leader: Create nonce +leader->>+worker: GET /enclave/sync (nonce_l) +worker->>worker: Create attestation, nonce, and ephemeral keys +worker-->>-leader: OK (Attestation(nonce_l, nonce_w, K_i)) + +leader->>leader: Verify & create attestation +leader->>+worker: POST /enclave/sync (Attestation(nonce_w, E(keys, K_i))) +worker->>worker: Verify attestation & install keys +worker-->>-leader: OK + +Note over worker,workerApp: Application retrieves key material +workerApp->>+worker: GET /enclave/state +worker->>worker: Retrieve key material +worker-->>-workerApp: OK (key material) +workerApp->>workerApp: Install key material + +Note over leader, worker: Worker starts heartbeat loop + +loop Heartbeat + worker->>+leader: POST /enclave/heartbeat (Hash(key material)) + leader-->>-worker: OK +end + +Note over leaderApp: Application updates its key material +leaderApp->>+leader: PUT /enclave/state (key material) +leader->>leader: Save key material +leader-->>-leaderApp: OK + +note over leader,worker: Leader initiates key re-synchronization as above +``` \ No newline at end of file From 25f54272b8dd6b51f155f77da3a9452fd3c4fb76 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 21 Aug 2023 08:30:23 -0500 Subject: [PATCH 42/99] Elaborate on key synchronization. --- doc/key-synchronization.md | 76 +++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index dae6cf8..e18e8c0 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -1,8 +1,8 @@ # Enclave key synchronization Nitriding supports horizontal scaling, i.e., it allows for the synchronization -of key material among enclaves. Key material consists of both nitriding and -application keys: +of key material among identical enclaves. Key material consists of both +_nitriding_ and _application_ keys: 1. Nitriding's key material is the self-signed HTTPS certificate (both public and private key) that provides the confidential channel between clients and @@ -11,12 +11,68 @@ application keys: agnostic to the structure of this key material and treats it as arbitrary bytes. -For enclave key synchronization to work, there must be a single leader -enclave and one or more worker enclaves. The leader enclave's sole job is to -create key material and make itself available to synchronize this key material -with worker enclaves. - -To enable horizontal scaling, use the `-fqdn-leader` command line flag. +All of the above must be synced among enclaves. + +For enclave key synchronization to work, there must be a _single leader +enclave_ and _one or more worker enclaves_. The leader's sole job is to +create key material and make itself available for synchronizing this key +material with worker enclaves. Worker enclaves do the actual work, i.e., +process user requests. Before doing any work though, workers must register +themselves with the leader, which triggers key synchronization. + +To set up key synchronization, several steps are necessary: + +* Use the `-fqdn-leader` command line flag on both the leader and the worker. + Note that the leader and worker images _must be identical_. The leader is + only willing to synchronize key material with _identical enclaves_. +* Practically speaking, the leader is meant to run in a separate k8s deployment + from the workers. + +## Protocol + +1. The leader creates a 20-byte nonce $\textrm{nonce}_l$ and sends it to the + worker as part of a `GET` request. +2. Upon receiving $\textrm{nonce}_l$, the worker creates its own 20-byte nonce + $\textrm{nonce}_w$ and an ephemeral asymmetric key pair $K_e = \(sk, pk\)$, + which we generate with Go's `crypto/nacl/box` package. The worker now asks + its hypervisor to create an attestation document $A_w$ containing + $\textrm{nonce}_l$, $\textrm{nonce}_w$, and $pk$. The worker responds to the + leader's `GET` request with $A_w$. +3. Having received $A_w$, the leader now verifies that... + 1. ...the attestation document is signed by the AWS Nitro Enclave hypervisor. + This stops attackers from sending spoofed attestation documents. + 2. ...the attestation document contains $\textrm{nonce}_l$. This stops + attackers from replaying old attestation documents. + 3. ...the attestation document's platform configuration registers are + identical to the leader's registers. This stops attackers from using + modified enclaves to extract the sensitive key material. +4. The leader is now convinced that it's dealing with an authentic worker + enclave. In the next and final interaction, the leader encrypts its sensitive + enclave keys $K_s$ using the worker's ephemeral public key $pk$, resulting in + $E = \textrm{Enc}(K_s, pk)$. The leader then asks its hypervisor to create an + attestation document $A_l$ containing $\textrm{nonce}_w$ and $E$. The leader + sends $A_l$ to the worker in a separate `POST` request. +6. Upon receiving $A_l$, the worker first verifies the attestation document + (same as above), and decrypts $E$ using $sk$, revealing in $K_s$, the + sensitive enclave keys. At this point, key synchronization is complete. + +## Security considerations + +The sensitive key material $K_s$ is protected as follows: + +* Communication between leader and worker enclaves happens over AWS's Virtual + Private Cloud (VPC). We therefore expose the endpoints for key + synchronization over a separate Web server that's not reachable over the + Internet. + +* Leader and worker enclaves use HTTPS as an underlying secure channel. Note + that the authenticity of our HTTPS certificates is rooted in the + hypervisor-signed attestation documents; not in a certificate authority. + +* Worker enclaves create an ephemeral key pair that's used to encrypt key + material using Go's `crypto/nacl/box` API. Even if an attacker can snoop on + the VPC network _and_ compromise the confidentiality of our HTTPS connection, + enclave keys are still protected by this ephemeral key pair. ```mermaid sequenceDiagram @@ -53,10 +109,10 @@ Note over leader,worker: Leader initiates key synchronization leader->>leader: Create nonce leader->>+worker: GET /enclave/sync (nonce_l) worker->>worker: Create attestation, nonce, and ephemeral keys -worker-->>-leader: OK (Attestation(nonce_l, nonce_w, K_i)) +worker-->>-leader: OK (Attestation(nonce_l, nonce_w, pk)) leader->>leader: Verify & create attestation -leader->>+worker: POST /enclave/sync (Attestation(nonce_w, E(keys, K_i))) +leader->>+worker: POST /enclave/sync (Attestation(nonce_w, E(keys, pk))) worker->>worker: Verify attestation & install keys worker-->>-leader: OK From bc27aa40db85b836d2dddccc086ab3b2d3fcb0df Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 21 Aug 2023 08:38:25 -0500 Subject: [PATCH 43/99] Elaborate on heartbeat mechanism. --- doc/key-synchronization.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index e18e8c0..dd1f210 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -55,6 +55,11 @@ To set up key synchronization, several steps are necessary: 6. Upon receiving $A_l$, the worker first verifies the attestation document (same as above), and decrypts $E$ using $sk$, revealing in $K_s$, the sensitive enclave keys. At this point, key synchronization is complete. +7. After key synchronization, workers send a periodic heartbeat to the leader in + a `POST` request. The request's body contains a Base64-encoded SHA-256 hash + over $K_s$. This allows the leader to verify if the worker's keys are still + up-to-date. If not, the leader initiates key-synchronization using the + protocol as above. ## Security considerations From a6d22c8a474fe2ed37be8b254f2de71bf8438afe Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 21 Aug 2023 08:52:22 -0500 Subject: [PATCH 44/99] Add installation of HTTPS certificate. --- doc/key-synchronization.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index dd1f210..9fb9f02 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -121,6 +121,8 @@ leader->>+worker: POST /enclave/sync (Attestation(nonce_w, E(keys, pk))) worker->>worker: Verify attestation & install keys worker-->>-leader: OK +worker->>worker: Install HTTPS certificate + Note over worker,workerApp: Application retrieves key material workerApp->>+worker: GET /enclave/state worker->>worker: Retrieve key material From ba7b3e363f30d04cfcdd87863f8aaac1159e737e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 22 Aug 2023 10:15:28 -0500 Subject: [PATCH 45/99] Shuffle variables around. --- attestation.go | 8 +++----- keysync_shared.go | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/attestation.go b/attestation.go index 28b37a0..7722513 100644 --- a/attestation.go +++ b/attestation.go @@ -9,11 +9,9 @@ import ( ) const ( - nonceLen = 20 // The size of a nonce in bytes. - nonceNumDigits = nonceLen * 2 // The number of hex digits in a nonce. - maxAttDocLen = 5000 // A (reasonable?) upper limit for attestation doc lengths. - hashPrefix = "sha256:" - hashSeparator = ";" + maxAttDocLen = 5000 // A (reasonable?) upper limit for attestation doc lengths. + hashPrefix = "sha256:" + hashSeparator = ";" ) var ( diff --git a/keysync_shared.go b/keysync_shared.go index d24d040..9d01c2a 100644 --- a/keysync_shared.go +++ b/keysync_shared.go @@ -9,7 +9,9 @@ import ( ) const ( - boxKeyLen = 32 // NaCl box's private and public key length. + nonceLen = 20 // The size of a nonce in bytes. + nonceNumDigits = nonceLen * 2 // The number of hex digits in a nonce. + boxKeyLen = 32 // NaCl box's private and public key length. ) var ( From b3cf4c22140af9ce519379a0c0afff58e0aaf816 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 22 Aug 2023 10:15:33 -0500 Subject: [PATCH 46/99] Remove TODO item. --- handlers.go | 1 - 1 file changed, 1 deletion(-) diff --git a/handlers.go b/handlers.go index b9d2ade..6a4308a 100644 --- a/handlers.go +++ b/handlers.go @@ -100,7 +100,6 @@ func putStateHandler(e *Enclave) http.HandlerFunc { go e.workers.forAll( func(worker *url.URL) { if err := asLeader(e.keys.get()).syncWith(worker); err != nil { - // TODO: Log in Prometheus. e.workers.unregister(worker) } }, From d67f0061c8be8defc045c3bc493fc9ff62393ae0 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 22 Aug 2023 13:12:15 -0500 Subject: [PATCH 47/99] Use AWS metadata service to get worker hostname. --- enclave.go | 52 +++++++++++++++++++++++++++++++++++++++++--------- go.mod | 12 ++++++++++++ go.sum | 32 +++++++++++++++++++++++++++++++ handlers.go | 14 +++++++++++--- sync_worker.go | 17 ++++++++++++++--- util.go | 30 +++++++++++++++++++++++++++++ 6 files changed, 142 insertions(+), 15 deletions(-) diff --git a/enclave.go b/enclave.go index f224a19..abfeda6 100644 --- a/enclave.go +++ b/enclave.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "crypto/sha256" "crypto/tls" @@ -10,12 +11,12 @@ import ( "encoding/pem" "errors" "fmt" + "io" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "net/url" - "strings" "sync" "time" @@ -357,11 +358,16 @@ func (e *Enclave) Start(ctx context.Context) error { } if e.cfg.isScalingEnabled() { - err := asWorker(e.installKeys, e.becameLeader).registerWith(leader) + workerHostname, err := getLocalEC2Hostname(e.cfg.ExtPrivPort) + if err != nil { + elog.Fatalf("Error determining instance hostname: %v", err) + } + worker := getSyncURL(workerHostname, e.cfg.ExtPrivPort) + err = asWorker(e.installKeys, e.becameLeader).registerWith(leader, worker) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } else if err == nil { - go e.workerHeartbeat(ctx) + go e.workerHeartbeat(ctx, worker) } } @@ -372,12 +378,15 @@ func (e *Enclave) Start(ctx context.Context) error { // know that we're still alive, and 2) to compare key material. If it turns out // that the leader has different key material than the worker, the worker // re-registers itself, which triggers key re-synchronization. -func (e *Enclave) workerHeartbeat(ctx context.Context) { +func (e *Enclave) workerHeartbeat(ctx context.Context, worker *url.URL) { elog.Println("Starting worker's heartbeat loop.") defer elog.Println("Exiting worker's heartbeat loop.") var ( leader = e.getLeader(pathHeartbeat) timer = time.NewTicker(time.Minute) + hbBody = heartbeatRequest{ + WorkerHostname: worker.Host, + } ) for { @@ -385,10 +394,17 @@ func (e *Enclave) workerHeartbeat(ctx context.Context) { case <-ctx.Done(): return case <-timer.C: + hbBody.HashedKeys = e.keys.hashAndB64() + body, err := json.Marshal(hbBody) + if err != nil { + elog.Printf("Error marshalling heartbeat request: %v", err) + continue + } + resp, err := newUnauthenticatedHTTPClient().Post( leader.String(), "text/plain", - strings.NewReader(e.keys.hashAndB64()), + bytes.NewReader(body), ) if err != nil { elog.Printf("Error posting heartbeat to leader: %v", err) @@ -601,11 +617,29 @@ func (e *Enclave) getLeader(path string) *url.URL { // getWorker returns the worker enclave's URL from the given HTTP request. func (e *Enclave) getWorker(r *http.Request) (*url.URL, error) { - // Go's HTTP server sets RemoteAddr to IP:port: - // https://pkg.go.dev/net/http#Request - strIP, _, err := net.SplitHostPort(r.RemoteAddr) + if e.cfg.Debug { + // Go's HTTP server sets RemoteAddr to IP:port: + // https://pkg.go.dev/net/http#Request + strIP, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return nil, err + } + elog.Printf("Worker's address from request source: %s", strIP) + return getSyncURL(strIP, e.cfg.ExtPrivPort), nil + } + + body, err := io.ReadAll(newLimitReader(r.Body, maxHeartbeatBody)) if err != nil { return nil, err } - return getSyncURL(strIP, e.cfg.ExtPrivPort), nil + defer r.Body.Close() + // Make the request's body readable again. + r.Body = io.NopCloser(bytes.NewBuffer(body)) + + var hb heartbeatRequest + if err := json.Unmarshal(body, &hb); err != nil { + return nil, err + } + elog.Printf("Worker's address from request body: %s", hb.WorkerHostname) + return getSyncURL(hb.WorkerHostname, e.cfg.ExtPrivPort), nil } diff --git a/go.mod b/go.mod index b820fbe..c5eda6d 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,18 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.36 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index abf29cb..94e94a7 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,34 @@ +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/config v1.18.36 h1:mLNA12PWU1Y+ueOO79QgQfKIPhc1MYKl44RmvASkJ7Q= +github.com/aws/aws-sdk-go-v2/config v1.18.36/go.mod h1:8AnEFxW9/XGKCbjYDCJy7iltVNyEI9Iu9qC21UzhhgQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk= github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/libcontainer v2.2.1+incompatible h1:++SbbkCw+X8vAd4j2gOCzZ2Nn7s2xFALTf7LZKmM1/0= @@ -21,11 +46,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 h1:oTi0zYvHo1sfk5sevGc4LrfgpLYB6cIhP/HllCUGcZ8= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -38,6 +66,7 @@ github.com/milosgajdos/tenus v0.0.3 h1:jmaJzwaY1DUyYVD0lM4U+uvP2kkEg1VahDqRFxIkV github.com/milosgajdos/tenus v0.0.3/go.mod h1:eIjx29vNeDOYWJuCnaHY2r4fq5egetV26ry3on7p8qY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= @@ -48,6 +77,7 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= @@ -97,3 +127,5 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index 6a4308a..5bce1ba 100644 --- a/handlers.go +++ b/handlers.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -20,8 +21,8 @@ const ( // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 - // The maximum length (in bytes) of the hash over our enclave keys. - maxEnclaveKeyHash = 128 + // The maximum length (in bytes) of a heartbeat's request body. + maxHeartbeatBody = 128 + 255 + 128 // The HTML for the enclave's index page. indexPage = "This host runs inside an AWS Nitro Enclave.\n" ) @@ -241,11 +242,18 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(newLimitReader(r.Body, maxEnclaveKeyHash)) + body, err := io.ReadAll(newLimitReader(r.Body, maxHeartbeatBody)) if err != nil { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } + + var hb heartbeatRequest + if err := json.Unmarshal(body, &hb); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Extract the worker's URL. worker, err := e.getWorker(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/sync_worker.go b/sync_worker.go index ad8b7e5..71090a1 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -47,13 +48,23 @@ func asWorker( } } -// registerWith registers the worker with the given leader enclave. -func (s *workerSync) registerWith(leader *url.URL) error { +type heartbeatRequest struct { + HashedKeys string `json:"hashed_keys"` + WorkerHostname string `json:"worker_hostname"` +} + +// registerWith registers the given worker with the given leader enclave. +func (s *workerSync) registerWith(leader, worker *url.URL) error { elog.Println("Attempting to sync with leader.") errChan := make(chan error) register := func(e chan error) { - resp, err := newUnauthenticatedHTTPClient().Post(leader.String(), "text/plain", nil) + body, err := json.Marshal(heartbeatRequest{WorkerHostname: worker.Host}) + if err != nil { + e <- err + return + } + resp, err := newUnauthenticatedHTTPClient().Post(leader.String(), "text/plain", bytes.NewBuffer(body)) if err != nil { e <- err return diff --git a/util.go b/util.go index b65a204..6b287ab 100644 --- a/util.go +++ b/util.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -10,10 +11,14 @@ import ( "encoding/pem" "errors" "fmt" + "io" "math/big" "net/http" "net/url" "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" ) var ( @@ -114,3 +119,28 @@ func sliceToNonce(s []byte) (nonce, error) { copy(n[:], s[:nonceLen]) return n, nil } + +// getLocalEC2Hostname returns the hostname of the EC2 instance or an error. +// The hostname is going to look like: ip-1-2-3-4.us-east-2.compute.internal +func getLocalEC2Hostname(port uint16) (string, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return "", err + } + client := imds.NewFromConfig(cfg) + output, err := client.GetMetadata( + context.Background(), + &imds.GetMetadataInput{ + Path: "local-hostname", + }, + ) + if err != nil { + return "", err + } + + rawHostname, err := io.ReadAll(output.Content) + if err != nil { + return "", err + } + return string(rawHostname), nil +} From b0a7b7666c6560f0359696793e15415100d3ccf2 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 22 Aug 2023 13:24:38 -0500 Subject: [PATCH 48/99] Fix conditional address extraction. --- enclave.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index abfeda6..298763c 100644 --- a/enclave.go +++ b/enclave.go @@ -617,7 +617,7 @@ func (e *Enclave) getLeader(path string) *url.URL { // getWorker returns the worker enclave's URL from the given HTTP request. func (e *Enclave) getWorker(r *http.Request) (*url.URL, error) { - if e.cfg.Debug { + if !inEnclave { // Go's HTTP server sets RemoteAddr to IP:port: // https://pkg.go.dev/net/http#Request strIP, _, err := net.SplitHostPort(r.RemoteAddr) From fb34e568aa58ff973d8c36ef85ca5f647d9e5754 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 08:27:44 -0500 Subject: [PATCH 49/99] Obtain hostname manually from IMDSv2. --- enclave.go | 41 +++++++++++++++++------------------------ go.mod | 12 ------------ go.sum | 32 -------------------------------- handlers.go | 42 ++++++++++++++++++++---------------------- util.go | 48 ++++++++++++++++++++++++++++++++---------------- 5 files changed, 69 insertions(+), 106 deletions(-) diff --git a/enclave.go b/enclave.go index 298763c..15a2654 100644 --- a/enclave.go +++ b/enclave.go @@ -11,7 +11,6 @@ import ( "encoding/pem" "errors" "fmt" - "io" "net" "net/http" "net/http/httputil" @@ -358,7 +357,7 @@ func (e *Enclave) Start(ctx context.Context) error { } if e.cfg.isScalingEnabled() { - workerHostname, err := getLocalEC2Hostname(e.cfg.ExtPrivPort) + workerHostname, err := getLocalEC2Hostname() if err != nil { elog.Fatalf("Error determining instance hostname: %v", err) } @@ -615,31 +614,25 @@ func (e *Enclave) getLeader(path string) *url.URL { } } -// getWorker returns the worker enclave's URL from the given HTTP request. -func (e *Enclave) getWorker(r *http.Request) (*url.URL, error) { +// getWorker takes as input the HTTP request and payload that were sent to the +// leader's heartbeat endpoint. The function extracts the worker's URL and +// returns it. +func (e *Enclave) getWorker(r *http.Request, hb *heartbeatRequest) (*url.URL, error) { + var ( + host string + err error + ) + host, _, err = net.SplitHostPort(hb.WorkerHostname) + // If we're testing the code outside of an enclave, simply use the worker's + // source address from the HTTP request. This doesn't work inside an + // enclave because nitriding receives incoming requests via a reverse + // proxy that does NAT. Nitriding therefore never sees the client's IP + // address. if !inEnclave { - // Go's HTTP server sets RemoteAddr to IP:port: - // https://pkg.go.dev/net/http#Request - strIP, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return nil, err - } - elog.Printf("Worker's address from request source: %s", strIP) - return getSyncURL(strIP, e.cfg.ExtPrivPort), nil + host, _, err = net.SplitHostPort(r.RemoteAddr) } - - body, err := io.ReadAll(newLimitReader(r.Body, maxHeartbeatBody)) if err != nil { return nil, err } - defer r.Body.Close() - // Make the request's body readable again. - r.Body = io.NopCloser(bytes.NewBuffer(body)) - - var hb heartbeatRequest - if err := json.Unmarshal(body, &hb); err != nil { - return nil, err - } - elog.Printf("Worker's address from request body: %s", hb.WorkerHostname) - return getSyncURL(hb.WorkerHostname, e.cfg.ExtPrivPort), nil + return getSyncURL(host, e.cfg.ExtPrivPort), nil } diff --git a/go.mod b/go.mod index c5eda6d..b820fbe 100644 --- a/go.mod +++ b/go.mod @@ -18,18 +18,6 @@ require ( ) require ( - github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.36 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect - github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 94e94a7..abf29cb 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,9 @@ -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= -github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= -github.com/aws/aws-sdk-go-v2/config v1.18.36 h1:mLNA12PWU1Y+ueOO79QgQfKIPhc1MYKl44RmvASkJ7Q= -github.com/aws/aws-sdk-go-v2/config v1.18.36/go.mod h1:8AnEFxW9/XGKCbjYDCJy7iltVNyEI9Iu9qC21UzhhgQ= -github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= -github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= -github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk= github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/libcontainer v2.2.1+incompatible h1:++SbbkCw+X8vAd4j2gOCzZ2Nn7s2xFALTf7LZKmM1/0= @@ -46,14 +21,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 h1:oTi0zYvHo1sfk5sevGc4LrfgpLYB6cIhP/HllCUGcZ8= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -66,7 +38,6 @@ github.com/milosgajdos/tenus v0.0.3 h1:jmaJzwaY1DUyYVD0lM4U+uvP2kkEg1VahDqRFxIkV github.com/milosgajdos/tenus v0.0.3/go.mod h1:eIjx29vNeDOYWJuCnaHY2r4fq5egetV26ry3on7p8qY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= @@ -77,7 +48,6 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= @@ -127,5 +97,3 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index 5bce1ba..e34a825 100644 --- a/handlers.go +++ b/handlers.go @@ -21,8 +21,10 @@ const ( // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 - // The maximum length (in bytes) of a heartbeat's request body. - maxHeartbeatBody = 128 + 255 + 128 + // The maximum length (in bytes) of a heartbeat's request body: + // 44 bytes for the Base64-encoded SHA-256 hash, 255 bytes for the domain + // name, and another 128 bytes for the port and the surrounding JSON. + maxHeartbeatBody = 44 + 255 + 128 // The HTML for the enclave's index page. indexPage = "This host runs inside an AWS Nitro Enclave.\n" ) @@ -242,41 +244,37 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var ( + hb heartbeatRequest + syncAndRegister = func(keys *enclaveKeys, worker *url.URL) { + if err := asLeader(keys.get()).syncWith(worker); err == nil { + e.workers.register(worker) + } + } + ) + body, err := io.ReadAll(newLimitReader(r.Body, maxHeartbeatBody)) if err != nil { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - - var hb heartbeatRequest if err := json.Unmarshal(body, &hb); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadRequest) return } - // Extract the worker's URL. - worker, err := e.getWorker(r) + worker, err := e.getWorker(r, &hb) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - syncAndRegister := func(keys *enclaveKeys, worker *url.URL) { - if err := asLeader(keys.get()).syncWith(worker); err == nil { - e.workers.register(worker) - } - } - - if len(body) == 0 { - elog.Printf("Got heartbeat from uninitialized worker %s.", worker.Host) + elog.Printf("Heartbeat from worker %s.", worker.Host) + ourKeysHash, theirKeysHash := e.keys.hashAndB64(), hb.HashedKeys + if ourKeysHash != theirKeysHash { + elog.Printf("Worker's keys are invalid. Re-synchronizing.") go syncAndRegister(e.keys, worker) } else { - elog.Printf("Got heartbeat from initialized worker %s.", worker.Host) - ourKeysHash, theirKeysHash := e.keys.hashAndB64(), string(body) - if ourKeysHash != theirKeysHash { - go syncAndRegister(e.keys, worker) - } else { - e.workers.register(worker) - } + e.workers.register(worker) } w.WriteHeader(http.StatusOK) } diff --git a/util.go b/util.go index 6b287ab..e12d30b 100644 --- a/util.go +++ b/util.go @@ -1,7 +1,6 @@ package main import ( - "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -16,9 +15,14 @@ import ( "net/http" "net/url" "time" +) - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" +const ( + // The endpoint of AWS's Instance Metadata Service, which allows an enclave + // to learn its internal hostname: + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html + metadataSvcToken = "http://169.254.169.254/latest/api/token" + metadataSvcInfo = "http://169.254.169.254/latest/meta-data/local-hostname" ) var ( @@ -120,27 +124,39 @@ func sliceToNonce(s []byte) (nonce, error) { return n, nil } -// getLocalEC2Hostname returns the hostname of the EC2 instance or an error. -// The hostname is going to look like: ip-1-2-3-4.us-east-2.compute.internal -func getLocalEC2Hostname(port uint16) (string, error) { - cfg, err := config.LoadDefaultConfig(context.Background()) +func getLocalEC2Hostname() (string, error) { + // IMDSv2, which we are using, is session-oriented (God knows why), so we + // first obtain a session token from the service. + req, err := http.NewRequest(http.MethodPut, metadataSvcToken, nil) if err != nil { return "", err } - client := imds.NewFromConfig(cfg) - output, err := client.GetMetadata( - context.Background(), - &imds.GetMetadataInput{ - Path: "local-hostname", - }, - ) + req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "10") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + body, err := io.ReadAll(resp.Body) if err != nil { return "", err } + token := string(body) + elog.Printf("session token: %s", token) - rawHostname, err := io.ReadAll(output.Content) + // Having obtained the session token, we can now make the actual metadata + // request. + req, err = http.NewRequest(http.MethodGet, metadataSvcInfo, nil) + if err != nil { + return "", err + } + req.Header.Set("X-aws-ec2-metadata-token", token) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return "", err + } + body, err = io.ReadAll(resp.Body) if err != nil { return "", err } - return string(rawHostname), nil + return string(body), nil } From 689ab2867c754c1f7ea7db188970ed44dc7cd22f Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 13:24:14 -0500 Subject: [PATCH 50/99] Add a way to determine hostname outside of enclaves. --- enclave.go | 7 ++----- util.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/enclave.go b/enclave.go index 15a2654..dd095d2 100644 --- a/enclave.go +++ b/enclave.go @@ -357,11 +357,8 @@ func (e *Enclave) Start(ctx context.Context) error { } if e.cfg.isScalingEnabled() { - workerHostname, err := getLocalEC2Hostname() - if err != nil { - elog.Fatalf("Error determining instance hostname: %v", err) - } - worker := getSyncURL(workerHostname, e.cfg.ExtPrivPort) + elog.Println("Obtaining worker's hostname.") + worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) err = asWorker(e.installKeys, e.becameLeader).registerWith(leader, worker) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) diff --git a/util.go b/util.go index e12d30b..8d25155 100644 --- a/util.go +++ b/util.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "math/big" + "net" "net/http" "net/url" "time" @@ -124,6 +125,51 @@ func sliceToNonce(s []byte) (nonce, error) { return n, nil } +// getHostnameOrDie returns the "enclave"'s hostname (or IP address) or dies +// trying. If inside an enclave, we query AWS's Instance Metadata Service. If +// outside an enclave, we pick whatever IP address the operating system would +// choose when talking to a public IP address. +func getHostnameOrDie() string { + var ( + err error + hostname string + ) + + if !inEnclave { + hostname = getLocalAddr() + elog.Printf("Test hostname: %s", hostname) + return hostname + } + + for i := 0; i < 5; i++ { + hostname, err = getLocalEC2Hostname() + if err == nil { + elog.Printf("EC2 hostname: %s", hostname) + return hostname + } + time.Sleep(time.Second) + } + if err != nil { + elog.Fatalf("Error obtaining hostname from IMDSv2: %v", err) + } + return "" +} + +func getLocalAddr() string { + const target = "1.1.1.1:53" + conn, err := net.Dial("udp", target) + if err != nil { + elog.Fatalf("Error dialing %s: %v", target, err) + } + defer conn.Close() + + host, _, err := net.SplitHostPort(conn.LocalAddr().String()) + if err != nil { + elog.Fatalf("Error extracing host: %v", err) + } + return host +} + func getLocalEC2Hostname() (string, error) { // IMDSv2, which we are using, is session-oriented (God knows why), so we // first obtain a session token from the service. From 3f806204eee505a776a75b3978832d2f81af21f4 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 13:37:29 -0500 Subject: [PATCH 51/99] Simplify getWorker. --- enclave.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/enclave.go b/enclave.go index dd095d2..2ebfa36 100644 --- a/enclave.go +++ b/enclave.go @@ -611,23 +611,14 @@ func (e *Enclave) getLeader(path string) *url.URL { } } -// getWorker takes as input the HTTP request and payload that were sent to the -// leader's heartbeat endpoint. The function extracts the worker's URL and -// returns it. -func (e *Enclave) getWorker(r *http.Request, hb *heartbeatRequest) (*url.URL, error) { +// getWorker takes as input the worker's heartbeat request payload and returns +// the worker's URL. +func (e *Enclave) getWorker(hb *heartbeatRequest) (*url.URL, error) { var ( host string err error ) host, _, err = net.SplitHostPort(hb.WorkerHostname) - // If we're testing the code outside of an enclave, simply use the worker's - // source address from the HTTP request. This doesn't work inside an - // enclave because nitriding receives incoming requests via a reverse - // proxy that does NAT. Nitriding therefore never sees the client's IP - // address. - if !inEnclave { - host, _, err = net.SplitHostPort(r.RemoteAddr) - } if err != nil { return nil, err } From 58fb3b5fb6b7540251a572962aab594ae64e645c Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 13:37:54 -0500 Subject: [PATCH 52/99] Log worker's hostname. --- util.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/util.go b/util.go index 8d25155..974af67 100644 --- a/util.go +++ b/util.go @@ -129,30 +129,32 @@ func sliceToNonce(s []byte) (nonce, error) { // trying. If inside an enclave, we query AWS's Instance Metadata Service. If // outside an enclave, we pick whatever IP address the operating system would // choose when talking to a public IP address. -func getHostnameOrDie() string { - var ( - err error - hostname string - ) +func getHostnameOrDie() (hostname string) { + defer func() { + elog.Printf("Determined our hostname: %s", hostname) + }() + var err error if !inEnclave { hostname = getLocalAddr() - elog.Printf("Test hostname: %s", hostname) - return hostname + return } - for i := 0; i < 5; i++ { + // We cannot easily tell when all components are in place to receive + // incoming connections. We therefore make five attempts to get our + // hostname from IMDS while waiting for one second in between attempts. + const retries = 5 + for i := 0; i < retries; i++ { hostname, err = getLocalEC2Hostname() if err == nil { - elog.Printf("EC2 hostname: %s", hostname) - return hostname + return } time.Sleep(time.Second) } if err != nil { elog.Fatalf("Error obtaining hostname from IMDSv2: %v", err) } - return "" + return } func getLocalAddr() string { From 4ce1d08c26959cff907cd2fb7b66aab8031192f2 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 13:40:19 -0500 Subject: [PATCH 53/99] Fix incorrect function invocation. --- handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers.go b/handlers.go index e34a825..305eef4 100644 --- a/handlers.go +++ b/handlers.go @@ -262,7 +262,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { http.Error(w, err.Error(), http.StatusBadRequest) return } - worker, err := e.getWorker(r, &hb) + worker, err := e.getWorker(&hb) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return From 9fa0f6170b89ea28a968789b8bdda1d3b72061d1 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 13:42:15 -0500 Subject: [PATCH 54/99] Remove debug message. --- util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/util.go b/util.go index 974af67..26958f0 100644 --- a/util.go +++ b/util.go @@ -189,7 +189,6 @@ func getLocalEC2Hostname() (string, error) { return "", err } token := string(body) - elog.Printf("session token: %s", token) // Having obtained the session token, we can now make the actual metadata // request. From 8b769ebffb3fcc2f6a983d228e7aa79be6896a67 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 14:00:43 -0500 Subject: [PATCH 55/99] Make log message more descriptive. --- workers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers.go b/workers.go index 93336b7..03167c6 100644 --- a/workers.go +++ b/workers.go @@ -53,7 +53,7 @@ func (w *workerManager) start(ctx context.Context) { case worker := <-w.reg: set[*worker] = time.Now() - elog.Printf("Registered worker %s; %d worker(s) now registered.", + elog.Printf("(Re-)registered worker %s; %d worker(s) now registered.", worker.Host, len(set)) case worker := <-w.unreg: From c888027cdfbfa9a99f347eb3ede2736043763f79 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 23 Aug 2023 14:48:44 -0500 Subject: [PATCH 56/99] Fix unit tests. --- handlers_test.go | 27 ++++++++++++++++++++------- sync_worker_test.go | 9 ++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/handlers_test.go b/handlers_test.go index 59fac0c..4c8961b 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "crypto/tls" "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -48,6 +49,20 @@ func designateLeader(t *testing.T, srv *http.Server) { ) } +// keysToHeartbeat turns the given keys into a Buffer that contains a heartbeat +// request. +func keysToHeartbeat(t *testing.T, keys *enclaveKeys) *bytes.Buffer { + t.Helper() + blob, err := json.Marshal(&heartbeatRequest{ + HashedKeys: keys.hashAndB64(), + WorkerHostname: "localhost:1234", + }) + if err != nil { + t.Fatal(err) + } + return bytes.NewBuffer(blob) +} + // assertResponse ensures that the two given HTTP responses are (almost) // identical. We only check the HTTP status code and the response body. // If the expected response has no body, we only compare the status code. @@ -350,15 +365,14 @@ func TestHeartbeatHandler(t *testing.T) { designateLeader(t, e.extPrivSrv) e.keys.set(keys) - tooLargeBuf := bytes.NewBuffer(make([]byte, maxEnclaveKeyHash+1)) + tooLargeBuf := bytes.NewBuffer(make([]byte, maxHeartbeatBody+1)) assertResponse(t, makeReq(http.MethodPost, pathHeartbeat, tooLargeBuf), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), ) - validKeys := bytes.NewBuffer([]byte(keys.hashAndB64())) assertResponse(t, - makeReq(http.MethodPost, pathHeartbeat, validKeys), + makeReq(http.MethodPost, pathHeartbeat, keysToHeartbeat(t, keys)), newResp(http.StatusOK, ""), ) } @@ -397,11 +411,10 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { assertEqual(t, leaderEnclave.workers.length(), 0) // Send a heartbeat to the leader. The heartbeat's keys don't match the - // leader's keys, - // which results in the leader initiating key synchronization. - invalidKeys := bytes.NewBuffer([]byte(workerKeys.hashAndB64())) + // leader's keys, which results in the leader initiating key + // synchronization. assertResponse(t, - makeReq(http.MethodPost, pathHeartbeat, invalidKeys), + makeReq(http.MethodPost, pathHeartbeat, keysToHeartbeat(t, workerKeys)), newResp(http.StatusOK, ""), ) diff --git a/sync_worker_test.go b/sync_worker_test.go index e37ed03..56f2ff9 100644 --- a/sync_worker_test.go +++ b/sync_worker_test.go @@ -34,12 +34,15 @@ func TestSuccessfulRegisterWith(t *testing.T) { w.WriteHeader(http.StatusOK) }), ) - u, err := url.Parse(srv.URL) + leader, err := url.Parse(srv.URL) if err != nil { t.Fatalf("Error creating test server URL: %v", err) } + worker := &url.URL{ + Host: "localhost", + } - err = asWorker(e.installKeys, make(chan struct{})).registerWith(u) + err = asWorker(e.installKeys, make(chan struct{})).registerWith(leader, worker) if err != nil { t.Fatalf("Error registering with leader: %v", err) } @@ -59,7 +62,7 @@ func TestAbortedRegisterWith(t *testing.T) { abortChan := make(chan struct{}) ret := make(chan error) go func(ret chan error) { - ret <- asWorker(e.installKeys, abortChan).registerWith(bogusURL) + ret <- asWorker(e.installKeys, abortChan).registerWith(bogusURL, bogusURL) }(ret) // Designate the enclave as leader, after which registration should abort. From e36b808da0cb2ae2ddf57423874c211a01bc1f8f Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 28 Aug 2023 16:50:27 -0500 Subject: [PATCH 57/99] Elaborate on image IDs. --- doc/key-synchronization.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index 9fb9f02..58520b6 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -42,7 +42,8 @@ To set up key synchronization, several steps are necessary: 1. ...the attestation document is signed by the AWS Nitro Enclave hypervisor. This stops attackers from sending spoofed attestation documents. 2. ...the attestation document contains $\textrm{nonce}_l$. This stops - attackers from replaying old attestation documents. + attackers from replaying old attestation documents. Note that attestation + documents [always contain the enclave's image ID](https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html). 3. ...the attestation document's platform configuration registers are identical to the leader's registers. This stops attackers from using modified enclaves to extract the sensitive key material. From 35d6911647cda6aef4a6e3e6d032cb2145390523 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 28 Aug 2023 17:39:06 -0500 Subject: [PATCH 58/99] Use nitro attester by default. --- enclave.go | 4 ++-- enclave_test.go | 2 +- handlers.go | 4 ++-- handlers_test.go | 6 ++++-- main.go | 3 +++ sync_leader.go | 4 ++-- sync_worker.go | 3 ++- sync_worker_test.go | 8 ++++---- 8 files changed, 20 insertions(+), 14 deletions(-) diff --git a/enclave.go b/enclave.go index 2ebfa36..4716216 100644 --- a/enclave.go +++ b/enclave.go @@ -282,7 +282,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(ctx, e)) - m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader)) + m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader, e.attester)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) @@ -359,7 +359,7 @@ func (e *Enclave) Start(ctx context.Context) error { if e.cfg.isScalingEnabled() { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) - err = asWorker(e.installKeys, e.becameLeader).registerWith(leader, worker) + err = asWorker(e.installKeys, e.becameLeader, e.attester).registerWith(leader, worker) if err != nil && !errors.Is(err, errBecameLeader) { elog.Fatalf("Error syncing with leader: %v", err) } else if err == nil { diff --git a/enclave_test.go b/enclave_test.go index acdfda2..701081a 100644 --- a/enclave_test.go +++ b/enclave_test.go @@ -12,7 +12,7 @@ var defaultCfg = Config{ IntPort: 50002, HostProxyPort: 1024, UseACME: false, - Debug: false, + Debug: true, FdCur: 1024, FdMax: 4096, WaitForApp: true, diff --git a/handlers.go b/handlers.go index 305eef4..4a35c08 100644 --- a/handlers.go +++ b/handlers.go @@ -102,7 +102,7 @@ func putStateHandler(e *Enclave) http.HandlerFunc { e.workers.length()) go e.workers.forAll( func(worker *url.URL) { - if err := asLeader(e.keys.get()).syncWith(worker); err != nil { + if err := asLeader(e.keys.get(), e.attester).syncWith(worker); err != nil { e.workers.unregister(worker) } }, @@ -247,7 +247,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { var ( hb heartbeatRequest syncAndRegister = func(keys *enclaveKeys, worker *url.URL) { - if err := asLeader(keys.get()).syncWith(worker); err == nil { + if err := asLeader(keys.get(), e.attester).syncWith(worker); err == nil { e.workers.register(worker) } } diff --git a/handlers_test.go b/handlers_test.go index 4c8961b..3ae06b3 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -320,7 +320,9 @@ func TestAttestationHandlerWhileProfiling(t *testing.T) { } func TestAttestationHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) + prodCfg := defaultCfg + prodCfg.Debug = false + makeReq := makeRequestFor(createEnclave(&prodCfg).extPubSrv) assertResponse(t, makeReq(http.MethodPost, pathAttestation, nil), @@ -388,7 +390,7 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { workerKeys.set(keys) return nil } - worker = asWorker(setWorkerKeys, make(chan struct{})) + worker = asWorker(setWorkerKeys, make(chan struct{}), &dummyAttester{}) workerSrv = httptest.NewTLSServer(worker) ) defer workerSrv.Close() diff --git a/main.go b/main.go index 0c8e20f..091b710 100644 --- a/main.go +++ b/main.go @@ -130,6 +130,9 @@ func main() { } c.AppWebSrv = u } + if debug { + elog.Println("WARNING: Using debug mode, which must not be enabled in production!") + } ctx := context.Background() enclave, err := NewEnclave(ctx, c) diff --git a/sync_leader.go b/sync_leader.go index dc59561..d3d4781 100644 --- a/sync_leader.go +++ b/sync_leader.go @@ -21,9 +21,9 @@ type leaderSync struct { } // asLeader returns a new leaderSync struct. -func asLeader(keys *enclaveKeys) *leaderSync { +func asLeader(keys *enclaveKeys, a attester) *leaderSync { return &leaderSync{ - attester: &dummyAttester{}, + attester: a, keys: keys, } } diff --git a/sync_worker.go b/sync_worker.go index 71090a1..b73d3d9 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -38,9 +38,10 @@ type workerSync struct { func asWorker( installKeys func(*enclaveKeys) error, becameLeader chan struct{}, + a attester, ) *workerSync { return &workerSync{ - attester: &dummyAttester{}, + attester: a, installKeys: installKeys, becameLeader: becameLeader, nonce: make(chan nonce, 1), diff --git a/sync_worker_test.go b/sync_worker_test.go index 56f2ff9..17fb7b7 100644 --- a/sync_worker_test.go +++ b/sync_worker_test.go @@ -42,7 +42,7 @@ func TestSuccessfulRegisterWith(t *testing.T) { Host: "localhost", } - err = asWorker(e.installKeys, make(chan struct{})).registerWith(leader, worker) + err = asWorker(e.installKeys, make(chan struct{}), &dummyAttester{}).registerWith(leader, worker) if err != nil { t.Fatalf("Error registering with leader: %v", err) } @@ -62,7 +62,7 @@ func TestAbortedRegisterWith(t *testing.T) { abortChan := make(chan struct{}) ret := make(chan error) go func(ret chan error) { - ret <- asWorker(e.installKeys, abortChan).registerWith(bogusURL, bogusURL) + ret <- asWorker(e.installKeys, abortChan, &dummyAttester{}).registerWith(bogusURL, bogusURL) }(ret) // Designate the enclave as leader, after which registration should abort. @@ -80,14 +80,14 @@ func TestSuccessfulSync(t *testing.T) { // Set up the worker. worker := createEnclave(&defaultCfg) srv := httptest.NewTLSServer( - asWorker(worker.installKeys, make(chan struct{})), + asWorker(worker.installKeys, make(chan struct{}), &dummyAttester{}), ) workerURL, err := url.Parse(srv.URL) if err != nil { t.Fatalf("Error creating test server URL: %v", err) } - if err = asLeader(leaderKeys).syncWith(workerURL); err != nil { + if err = asLeader(leaderKeys, &dummyAttester{}).syncWith(workerURL); err != nil { t.Fatalf("Error syncing with leader: %v", err) } From 0f42e6ba4df78993f7e05648e3fce76a965a08eb Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 08:21:24 -0500 Subject: [PATCH 59/99] Elaborate on security considerations. --- doc/key-synchronization.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index 58520b6..de52e9f 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -80,6 +80,9 @@ The sensitive key material $K_s$ is protected as follows: the VPC network _and_ compromise the confidentiality of our HTTPS connection, enclave keys are still protected by this ephemeral key pair. +The leader only synchronizes with workers that run _identical_ code. The leader +therefore has assurance that workers are not going to game the system. + ```mermaid sequenceDiagram box rgba(100, 100, 100, .1) Leader enclave From a1c15594cb07f52cb1c201dd06283883bbc8d995 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 12:48:06 -0500 Subject: [PATCH 60/99] Test leader designation via self-probing. --- attestation.go | 11 ++--- enclave.go | 100 +++++++++++++++++++++++++++++++++++--------- handlers.go | 25 ++--------- handlers_test.go | 10 ++--- sync_worker.go | 7 ---- sync_worker_test.go | 26 +----------- util.go | 25 +++++++++++ 7 files changed, 121 insertions(+), 83 deletions(-) diff --git a/attestation.go b/attestation.go index 7722513..8ebd544 100644 --- a/attestation.go +++ b/attestation.go @@ -3,6 +3,7 @@ package main import ( "bytes" "crypto/sha256" + "errors" "fmt" "github.com/hf/nitrite" @@ -15,11 +16,11 @@ const ( ) var ( - errBadForm = "failed to parse POST form data" - errNoNonce = "could not find nonce in URL query parameters" - errBadNonceFormat = fmt.Sprintf("unexpected nonce format; must be %d-digit hex string", nonceNumDigits) - errFailedAttestation = "failed to obtain attestation document from hypervisor" - errProfilingSet = "attestation disabled because profiling is enabled" + errBadForm = errors.New("failed to parse POST form data") + errNoNonce = errors.New("could not find nonce in URL query parameters") + errBadNonceFormat = fmt.Errorf("unexpected nonce format; must be %d-digit hex string", nonceNumDigits) + errFailedAttestation = errors.New("failed to obtain attestation document from hypervisor") + errProfilingSet = errors.New("attestation disabled because profiling is enabled") // getPCRValues is a variable pointing to a function that returns PCR // values. Using a variable allows us to easily mock the function in our diff --git a/enclave.go b/enclave.go index 4716216..ded85cb 100644 --- a/enclave.go +++ b/enclave.go @@ -64,19 +64,19 @@ var ( type Enclave struct { sync.RWMutex attester - ctx context.Context - cfg *Config - extPubSrv, extPrivSrv *http.Server - intSrv *http.Server - promSrv *http.Server - revProxy *httputil.ReverseProxy - hashes *AttestationHashes - promRegistry *prometheus.Registry - metrics *metrics - workers *workerManager - keys *enclaveKeys - httpsCert *certRetriever - ready, stop, becameLeader chan struct{} + ctx context.Context + cfg *Config + extPubSrv, extPrivSrv *http.Server + intSrv *http.Server + promSrv *http.Server + revProxy *httputil.ReverseProxy + hashes *AttestationHashes + promRegistry *prometheus.Registry + metrics *metrics + workers *workerManager + keys *enclaveKeys + httpsCert *certRetriever + ready, stop chan struct{} } // Config represents the configuration of our enclave service. @@ -245,7 +245,6 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { workers: newWorkerManager(time.Minute), stop: make(chan struct{}), ready: make(chan struct{}), - becameLeader: make(chan struct{}, 1), } // Increase the maximum number of idle connections per host. This is @@ -282,7 +281,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, leaderHandler(ctx, e)) - m.Handle(pathSync, asWorker(e.installKeys, e.becameLeader, e.attester)) + m.Handle(pathSync, asWorker(e.installKeys, e.attester)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) @@ -356,20 +355,81 @@ func (e *Enclave) Start(ctx context.Context) error { return fmt.Errorf("%s: %w", errPrefix, err) } - if e.cfg.isScalingEnabled() { + if !e.cfg.isScalingEnabled() { + return nil + } + + elog.Print("Now checking if we're the leader.") + // Check if we are the leader. + if weAreLeader(e) { + elog.Print("We are the leader.") + } else { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) - err = asWorker(e.installKeys, e.becameLeader, e.attester).registerWith(leader, worker) - if err != nil && !errors.Is(err, errBecameLeader) { + err = asWorker(e.installKeys, e.attester).registerWith(leader, worker) + if err != nil { elog.Fatalf("Error syncing with leader: %v", err) - } else if err == nil { - go e.workerHeartbeat(ctx, worker) } + go e.workerHeartbeat(ctx, worker) + } return nil } +func weAreLeader(e *Enclave) bool { + const pathCheckLeader = "/enclave/leader-check" + var ( + becameLeader = make(chan struct{}) + ) + + ourNonce, err := newNonce() + if err != nil { + elog.Fatalf("Error creating new nonce: %v", err) + } + + // Create a temporary endpoint that expects a random nonce. We subsequently + // call the endpoint and if we end up answering our own request, we know + // that we're the leader. + m := e.extPrivSrv.Handler.(*chi.Mux) + m.Get(pathCheckLeader, func(w http.ResponseWriter, r *http.Request) { + theirNonce, err := getNonceFromReq(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if ourNonce == theirNonce { + close(becameLeader) + } else { + elog.Print("Received nonce that does not match our own.") + } + }) + // Reset our ephemeral handler as it's no longer needed. + defer m.Get( + pathCheckLeader, + func(w http.ResponseWriter, r *http.Request) {}, + ) + + // TODO: Maybe try contacting endpoint repeatedly? + reqURL := e.getLeader(pathCheckLeader) + reqURL.RawQuery = fmt.Sprintf("nonce=%x", ourNonce[:]) + resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) + if err != nil { + elog.Fatalf("Error contacting ephemeral leader endpoint: %v", err) + } + if resp.StatusCode != http.StatusOK { + return false + } + + select { + case <-becameLeader: + return true + case <-time.After(5 * time.Second): + return false + } +} + // workerHeartbeat periodically talks to the leader enclave to 1) let the leader // know that we're still alive, and 2) to compare key material. If it turns out // that the leader has different key material than the worker, the worker diff --git a/handlers.go b/handlers.go index 4a35c08..cc4638d 100644 --- a/handlers.go +++ b/handlers.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -181,28 +180,11 @@ func configHandler(cfg *Config) http.HandlerFunc { func attestationHandler(useProfiling bool, hashes *AttestationHashes, a attester) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if useProfiling { - http.Error(w, errProfilingSet, http.StatusServiceUnavailable) - return - } - if err := r.ParseForm(); err != nil { - http.Error(w, errBadForm, http.StatusBadRequest) - return - } - - nonce := r.URL.Query().Get("nonce") - if nonce == "" { - http.Error(w, errNoNonce, http.StatusBadRequest) - return - } - nonce = strings.ToLower(nonce) - // Decode hex-encoded nonce. - rawNonce, err := hex.DecodeString(nonce) - if err != nil { - http.Error(w, errBadNonceFormat, http.StatusBadRequest) + http.Error(w, errProfilingSet.Error(), http.StatusServiceUnavailable) return } - n, err := sliceToNonce(rawNonce) + n, err := getNonceFromReq(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -213,7 +195,7 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes, a attester attestationHashes: hashes.Serialize(), }) if err != nil { - http.Error(w, errFailedAttestation, http.StatusInternalServerError) + http.Error(w, errFailedAttestation.Error(), http.StatusInternalServerError) return } b64Doc := base64.StdEncoding.EncodeToString(rawDoc) @@ -231,7 +213,6 @@ func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { var once sync.Once return func(w http.ResponseWriter, r *http.Request) { once.Do(func() { - e.becameLeader <- struct{}{} go e.workers.start(ctx) // Make leader-specific endpoints available. e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) diff --git a/handlers_test.go b/handlers_test.go index 3ae06b3..0a67f0b 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -315,7 +315,7 @@ func TestAttestationHandlerWhileProfiling(t *testing.T) { // Ensure that the attestation handler aborts if profiling is enabled. assertResponse(t, makeReq(http.MethodGet, pathAttestation, nil), - newResp(http.StatusServiceUnavailable, errProfilingSet), + newResp(http.StatusServiceUnavailable, errProfilingSet.Error()), ) } @@ -331,12 +331,12 @@ func TestAttestationHandler(t *testing.T) { assertResponse(t, makeReq(http.MethodGet, pathAttestation, nil), - newResp(http.StatusBadRequest, errNoNonce), + newResp(http.StatusBadRequest, errNoNonce.Error()), ) assertResponse(t, makeReq(http.MethodGet, pathAttestation+"?nonce=foobar", nil), - newResp(http.StatusBadRequest, errBadNonceFormat), + newResp(http.StatusBadRequest, errBadNonceFormat.Error()), ) // If we are not inside an enclave, attestation is going to result in an @@ -344,7 +344,7 @@ func TestAttestationHandler(t *testing.T) { if !inEnclave { assertResponse(t, makeReq(http.MethodGet, pathAttestation+"?nonce=0000000000000000000000000000000000000000", nil), - newResp(http.StatusInternalServerError, errFailedAttestation), + newResp(http.StatusInternalServerError, errFailedAttestation.Error()), ) } } @@ -390,7 +390,7 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { workerKeys.set(keys) return nil } - worker = asWorker(setWorkerKeys, make(chan struct{}), &dummyAttester{}) + worker = asWorker(setWorkerKeys, &dummyAttester{}) workerSrv = httptest.NewTLSServer(worker) ) defer workerSrv.Close() diff --git a/sync_worker.go b/sync_worker.go index b73d3d9..d201d04 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -16,7 +16,6 @@ import ( ) var ( - errBecameLeader = errors.New("became enclave leader") errNonceRequired = errors.New("nonce is required") errInProgress = errors.New("key sync already in progress") errInvalidNonceLen = errors.New("invalid nonce length") @@ -31,19 +30,16 @@ type workerSync struct { installKeys func(*enclaveKeys) error ephemeralKeys chan *boxKey nonce chan nonce - becameLeader chan struct{} } // asWorker returns a new workerSync object. func asWorker( installKeys func(*enclaveKeys) error, - becameLeader chan struct{}, a attester, ) *workerSync { return &workerSync{ attester: a, installKeys: installKeys, - becameLeader: becameLeader, nonce: make(chan nonce, 1), ephemeralKeys: make(chan *boxKey, 1), } @@ -91,9 +87,6 @@ func (s *workerSync) registerWith(leader, worker *url.URL) error { elog.Printf("Error registering with leader: %v", err) case <-timeout.C: return errors.New("timed out syncing with leader") - case <-s.becameLeader: - elog.Println("We became leader. Aborting key sync.") - return errBecameLeader case <-retry.C: go register(errChan) } diff --git a/sync_worker_test.go b/sync_worker_test.go index 17fb7b7..f69418f 100644 --- a/sync_worker_test.go +++ b/sync_worker_test.go @@ -1,7 +1,6 @@ package main import ( - "errors" "net/http" "net/http/httptest" "net/url" @@ -42,7 +41,7 @@ func TestSuccessfulRegisterWith(t *testing.T) { Host: "localhost", } - err = asWorker(e.installKeys, make(chan struct{}), &dummyAttester{}).registerWith(leader, worker) + err = asWorker(e.installKeys, &dummyAttester{}).registerWith(leader, worker) if err != nil { t.Fatalf("Error registering with leader: %v", err) } @@ -51,27 +50,6 @@ func TestSuccessfulRegisterWith(t *testing.T) { } } -func TestAbortedRegisterWith(t *testing.T) { - e := createEnclave(&defaultCfg) - - // Provide a bogus URL that cannot be synced with. - bogusURL := &url.URL{ - Scheme: "https", - Host: "localhost:1", - } - abortChan := make(chan struct{}) - ret := make(chan error) - go func(ret chan error) { - ret <- asWorker(e.installKeys, abortChan, &dummyAttester{}).registerWith(bogusURL, bogusURL) - }(ret) - - // Designate the enclave as leader, after which registration should abort. - close(abortChan) - if err := <-ret; !errors.Is(err, errBecameLeader) { - t.Fatal("Enclave did not realize that it became leader.") - } -} - func TestSuccessfulSync(t *testing.T) { // For key synchronization to be successful, we need actual certificates in // the leader keys. @@ -80,7 +58,7 @@ func TestSuccessfulSync(t *testing.T) { // Set up the worker. worker := createEnclave(&defaultCfg) srv := httptest.NewTLSServer( - asWorker(worker.installKeys, make(chan struct{}), &dummyAttester{}), + asWorker(worker.installKeys, &dummyAttester{}), ) workerURL, err := url.Parse(srv.URL) if err != nil { diff --git a/util.go b/util.go index 26958f0..0e4480e 100644 --- a/util.go +++ b/util.go @@ -7,6 +7,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/hex" "encoding/pem" "errors" "fmt" @@ -15,6 +16,7 @@ import ( "net" "net/http" "net/url" + "strings" "time" ) @@ -207,3 +209,26 @@ func getLocalEC2Hostname() (string, error) { } return string(body), nil } + +func getNonceFromReq(r *http.Request) (nonce, error) { + if err := r.ParseForm(); err != nil { + return nonce{}, errBadForm + } + + strNonce := r.URL.Query().Get("nonce") + if strNonce == "" { + return nonce{}, errNoNonce + } + strNonce = strings.ToLower(strNonce) + // Decode hex-encoded nonce. + rawNonce, err := hex.DecodeString(strNonce) + if err != nil { + return nonce{}, errBadNonceFormat + } + + n, err := sliceToNonce(rawNonce) + if err != nil { + return nonce{}, err + } + return n, nil +} From f3f91674cb61ad8545b8e496c32b4abfd2f44a9d Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 13:07:19 -0500 Subject: [PATCH 61/99] Cleaning up previous commit. --- enclave.go | 38 ++++++++++++++++++++++++-------------- handlers.go | 24 ------------------------ 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/enclave.go b/enclave.go index ded85cb..abfee1f 100644 --- a/enclave.go +++ b/enclave.go @@ -280,7 +280,6 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) - m.Get(pathLeader, leaderHandler(ctx, e)) m.Handle(pathSync, asWorker(e.installKeys, e.attester)) // Register enclave-internal HTTP API. @@ -361,9 +360,7 @@ func (e *Enclave) Start(ctx context.Context) error { elog.Print("Now checking if we're the leader.") // Check if we are the leader. - if weAreLeader(e) { - elog.Print("We are the leader.") - } else { + if !weAreLeader(ctx, e) { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) err = asWorker(e.installKeys, e.attester).registerWith(leader, worker) @@ -371,14 +368,12 @@ func (e *Enclave) Start(ctx context.Context) error { elog.Fatalf("Error syncing with leader: %v", err) } go e.workerHeartbeat(ctx, worker) - } return nil } -func weAreLeader(e *Enclave) bool { - const pathCheckLeader = "/enclave/leader-check" +func weAreLeader(ctx context.Context, e *Enclave) bool { var ( becameLeader = make(chan struct{}) ) @@ -392,7 +387,7 @@ func weAreLeader(e *Enclave) bool { // call the endpoint and if we end up answering our own request, we know // that we're the leader. m := e.extPrivSrv.Handler.(*chi.Mux) - m.Get(pathCheckLeader, func(w http.ResponseWriter, r *http.Request) { + m.Get(pathLeader, func(w http.ResponseWriter, r *http.Request) { theirNonce, err := getNonceFromReq(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -400,25 +395,32 @@ func weAreLeader(e *Enclave) bool { } if ourNonce == theirNonce { + e.setupLeader(ctx) close(becameLeader) } else { - elog.Print("Received nonce that does not match our own.") + elog.Println("Received nonce that does not match our own.") } }) - // Reset our ephemeral handler as it's no longer needed. - defer m.Get( - pathCheckLeader, - func(w http.ResponseWriter, r *http.Request) {}, + // Reset our ephemeral handler. + defer m.Get(pathLeader, + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGone) + }, ) // TODO: Maybe try contacting endpoint repeatedly? - reqURL := e.getLeader(pathCheckLeader) + reqURL := e.getLeader(pathLeader) reqURL.RawQuery = fmt.Sprintf("nonce=%x", ourNonce[:]) resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) if err != nil { elog.Fatalf("Error contacting ephemeral leader endpoint: %v", err) } + if resp.StatusCode == http.StatusGone { + // The leader already knows that it's the leader, and it's not us. + return false + } if resp.StatusCode != http.StatusOK { + elog.Printf("Leader returned status code %d.", resp.StatusCode) return false } @@ -430,6 +432,14 @@ func weAreLeader(e *Enclave) bool { } } +func (e *Enclave) setupLeader(ctx context.Context) { + go e.workers.start(ctx) + // Make leader-specific endpoints available. + e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) + e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) + elog.Println("Designated enclave as leader.") +} + // workerHeartbeat periodically talks to the leader enclave to 1) let the leader // know that we're still alive, and 2) to compare key material. If it turns out // that the leader has different key material than the worker, the worker diff --git a/handlers.go b/handlers.go index cc4638d..3f433bf 100644 --- a/handlers.go +++ b/handlers.go @@ -1,7 +1,6 @@ package main import ( - "context" "crypto/sha256" "encoding/base64" "encoding/json" @@ -11,9 +10,6 @@ import ( "net/http" "net/url" "strings" - "sync" - - "github.com/go-chi/chi/v5" ) const ( @@ -203,26 +199,6 @@ func attestationHandler(useProfiling bool, hashes *AttestationHashes, a attester } } -// leaderHandler is called when the enclave is designated as leader enclave. -// If designated, we do the following: -// -// 1. Signal to our leader registration goroutine that we're the leader. -// 2. Start the worker event loop, to keep track of worker enclaves. -// 3. Expose leader-specific endpoints. -func leaderHandler(ctx context.Context, e *Enclave) http.HandlerFunc { - var once sync.Once - return func(w http.ResponseWriter, r *http.Request) { - once.Do(func() { - go e.workers.start(ctx) - // Make leader-specific endpoints available. - e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) - e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) - elog.Println("Designated enclave as leader.") - }) - w.WriteHeader(http.StatusOK) - } -} - func heartbeatHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var ( From e60942258d23576434d1dee323009fe6bc58afa0 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 14:41:57 -0500 Subject: [PATCH 62/99] Fix deadlock and add log messages. --- enclave.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/enclave.go b/enclave.go index abfee1f..a191d2f 100644 --- a/enclave.go +++ b/enclave.go @@ -358,9 +358,8 @@ func (e *Enclave) Start(ctx context.Context) error { return nil } - elog.Print("Now checking if we're the leader.") // Check if we are the leader. - if !weAreLeader(ctx, e) { + if !e.weAreLeader(ctx) { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) err = asWorker(e.installKeys, e.attester).registerWith(leader, worker) @@ -373,22 +372,24 @@ func (e *Enclave) Start(ctx context.Context) error { return nil } -func weAreLeader(ctx context.Context, e *Enclave) bool { +func (e *Enclave) weAreLeader(ctx context.Context) bool { var ( - becameLeader = make(chan struct{}) + err error + becameLeader = make(chan struct{}, 1) + ourNonce, theirNonce nonce ) - ourNonce, err := newNonce() + ourNonce, err = newNonce() if err != nil { elog.Fatalf("Error creating new nonce: %v", err) } - // Create a temporary endpoint that expects a random nonce. We subsequently - // call the endpoint and if we end up answering our own request, we know - // that we're the leader. + // Create an ephemeral endpoint that expects a random nonce. We + // subsequently call the endpoint and if we end up answering our own + // request, we know that we're the leader. m := e.extPrivSrv.Handler.(*chi.Mux) m.Get(pathLeader, func(w http.ResponseWriter, r *http.Request) { - theirNonce, err := getNonceFromReq(r) + theirNonce, err = getNonceFromReq(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -396,8 +397,11 @@ func weAreLeader(ctx context.Context, e *Enclave) bool { if ourNonce == theirNonce { e.setupLeader(ctx) - close(becameLeader) + if len(becameLeader) == 0 { + becameLeader <- struct{}{} + } } else { + // We're probably the leader and got a request from a worker. elog.Println("Received nonce that does not match our own.") } }) @@ -417,6 +421,7 @@ func weAreLeader(ctx context.Context, e *Enclave) bool { } if resp.StatusCode == http.StatusGone { // The leader already knows that it's the leader, and it's not us. + elog.Println("Leader was designated. It's not us.") return false } if resp.StatusCode != http.StatusOK { @@ -426,8 +431,10 @@ func weAreLeader(ctx context.Context, e *Enclave) bool { select { case <-becameLeader: + elog.Println("We are the leader.") return true case <-time.After(5 * time.Second): + elog.Println("Request to leader timed out. Assuming we're a worker.") return false } } @@ -437,7 +444,7 @@ func (e *Enclave) setupLeader(ctx context.Context) { // Make leader-specific endpoints available. e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) - elog.Println("Designated enclave as leader.") + elog.Println("Set up leader endpoints and started worker event loop.") } // workerHeartbeat periodically talks to the leader enclave to 1) let the leader From 36ff04210151838bc7643ee6d69a8ba88b8f489e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 17:08:43 -0500 Subject: [PATCH 63/99] Re-attempt talking to leader designation endpoint. --- enclave.go | 77 ++++++++++++++++++------------------------------ handlers.go | 28 ++++++++++++++++++ handlers_test.go | 17 ++--------- util.go | 23 ++++++++++++++- 4 files changed, 81 insertions(+), 64 deletions(-) diff --git a/enclave.go b/enclave.go index a191d2f..d705b1a 100644 --- a/enclave.go +++ b/enclave.go @@ -372,70 +372,49 @@ func (e *Enclave) Start(ctx context.Context) error { return nil } -func (e *Enclave) weAreLeader(ctx context.Context) bool { +func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { var ( - err error - becameLeader = make(chan struct{}, 1) - ourNonce, theirNonce nonce + err error + ourNonce nonce + weAreLeader = make(chan struct{}, 1) + areWeLeader = make(chan bool) + errChan = make(chan error) + leader = e.getLeader(pathLeader) ) + defer func() { + elog.Printf("We are leader: %v", result) + }() ourNonce, err = newNonce() if err != nil { elog.Fatalf("Error creating new nonce: %v", err) } - // Create an ephemeral endpoint that expects a random nonce. We - // subsequently call the endpoint and if we end up answering our own - // request, we know that we're the leader. m := e.extPrivSrv.Handler.(*chi.Mux) - m.Get(pathLeader, func(w http.ResponseWriter, r *http.Request) { - theirNonce, err = getNonceFromReq(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if ourNonce == theirNonce { - e.setupLeader(ctx) - if len(becameLeader) == 0 { - becameLeader <- struct{}{} - } - } else { - // We're probably the leader and got a request from a worker. - elog.Println("Received nonce that does not match our own.") - } - }) - // Reset our ephemeral handler. + m.Get(pathLeader, getLeaderHandler(ourNonce, weAreLeader)) + // Reset the handler as we no longer have a need for it. defer m.Get(pathLeader, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusGone) }, ) - // TODO: Maybe try contacting endpoint repeatedly? - reqURL := e.getLeader(pathLeader) - reqURL.RawQuery = fmt.Sprintf("nonce=%x", ourNonce[:]) - resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) - if err != nil { - elog.Fatalf("Error contacting ephemeral leader endpoint: %v", err) - } - if resp.StatusCode == http.StatusGone { - // The leader already knows that it's the leader, and it's not us. - elog.Println("Leader was designated. It's not us.") - return false - } - if resp.StatusCode != http.StatusOK { - elog.Printf("Leader returned status code %d.", resp.StatusCode) - return false - } - - select { - case <-becameLeader: - elog.Println("We are the leader.") - return true - case <-time.After(5 * time.Second): - elog.Println("Request to leader timed out. Assuming we're a worker.") - return false + timeout := time.NewTicker(10 * time.Second) + for { + go makeLeaderRequest(leader, ourNonce, areWeLeader, errChan) + select { + case <-errChan: + elog.Println("Not yet able to talk to leader designation endpoint.") + continue + case result = <-areWeLeader: + return + case <-weAreLeader: + e.setupLeader(ctx) + result = true + return + case <-timeout.C: + elog.Fatal("Timed out talking to leader designation endpoint.") + } } } diff --git a/handlers.go b/handlers.go index 3f433bf..460aaa7 100644 --- a/handlers.go +++ b/handlers.go @@ -236,3 +236,31 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { w.WriteHeader(http.StatusOK) } } + +func getLeaderHandler(ourNonce nonce, weAreLeader chan struct{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + err error + theirNonce nonce + ) + theirNonce, err = getNonceFromReq(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if ourNonce == theirNonce { + if len(weAreLeader) == 0 { + weAreLeader <- struct{}{} + } + } else { + // We may end up in this branch for two reasons: + // 1. We're the leader and a worker beat us to talking to this + // endpoint. + // 2. We're a worker and some other entity in the private network is + // talking to this endpoint. That shouldn't happen. + elog.Println("Received nonce that does not match our own.") + } + w.WriteHeader(http.StatusOK) + } +} diff --git a/handlers_test.go b/handlers_test.go index 0a67f0b..e4ad66e 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -38,17 +38,6 @@ func newResp(status int, body string) *http.Response { } } -// designateLeader designates the enclave as a leader to make leader-specific -// endpoints available. -func designateLeader(t *testing.T, srv *http.Server) { - t.Helper() - makeReq := makeRequestFor(srv) - assertResponse(t, - makeReq(http.MethodGet, pathLeader, nil), - newResp(http.StatusOK, ""), - ) -} - // keysToHeartbeat turns the given keys into a Buffer that contains a heartbeat // request. func keysToHeartbeat(t *testing.T, keys *enclaveKeys) *bytes.Buffer { @@ -123,7 +112,7 @@ func signalReady(t *testing.T, e *Enclave) { func TestStateHandlers(t *testing.T) { e := createEnclave(&defaultCfg) - designateLeader(t, e.extPrivSrv) + e.setupLeader(context.Background()) tooLargeKey := make([]byte, 1024*1024+1) makeReq := makeRequestFor(e.intSrv) @@ -364,7 +353,7 @@ func TestHeartbeatHandler(t *testing.T) { keys = newTestKeys(t) makeReq = makeRequestFor(e.extPrivSrv) ) - designateLeader(t, e.extPrivSrv) + e.setupLeader(context.Background()) e.keys.set(keys) tooLargeBuf := bytes.NewBuffer(make([]byte, maxHeartbeatBody+1)) @@ -397,7 +386,7 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { if err := leaderEnclave.Start(context.Background()); err != nil { t.Fatal(err) } - designateLeader(t, leaderEnclave.extPrivSrv) + leaderEnclave.setupLeader(context.Background()) wg.Add(1) // Mock two functions to make the leader enclave talk to our test server. diff --git a/util.go b/util.go index 0e4480e..b864bff 100644 --- a/util.go +++ b/util.go @@ -56,7 +56,10 @@ func _newUnauthenticatedHTTPClient() *http.Client { transport := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } - return &http.Client{Transport: transport} + return &http.Client{ + Transport: transport, + Timeout: 3 * time.Second, + } } // createCertificate creates a self-signed certificate and returns the @@ -232,3 +235,21 @@ func getNonceFromReq(r *http.Request) (nonce, error) { } return n, nil } + +func makeLeaderRequest(leader *url.URL, ourNonce nonce, areWeLeader chan bool, errChan chan error) { + elog.Println("Attempting to talk to leader designation endpoint.") + + reqURL := *leader + reqURL.RawQuery = fmt.Sprintf("nonce=%x", ourNonce[:]) + resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) + if err != nil { + errChan <- err + return + } + if resp.StatusCode == http.StatusGone { + // The leader already knows that it's the leader, and it's not us. + areWeLeader <- false + return + } + errChan <- fmt.Errorf("leader designation endpoint returned %d", resp.StatusCode) +} From 2e0fba3807c9e26ca109e2828fa3d3e89dc65d4c Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 19:40:41 -0500 Subject: [PATCH 64/99] Raise read limit as 5K wasn't enough. --- attestation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attestation.go b/attestation.go index 8ebd544..7b72f1f 100644 --- a/attestation.go +++ b/attestation.go @@ -10,7 +10,7 @@ import ( ) const ( - maxAttDocLen = 5000 // A (reasonable?) upper limit for attestation doc lengths. + maxAttDocLen = 10000 // A (reasonable?) upper limit for attestation doc lengths. hashPrefix = "sha256:" hashSeparator = ";" ) From 62c3dd182a9a6bd71328166c2d699fd95e2900df Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 19:41:20 -0500 Subject: [PATCH 65/99] Back off for a second before re-trying. --- enclave.go | 1 + 1 file changed, 1 insertion(+) diff --git a/enclave.go b/enclave.go index d705b1a..b1efd7a 100644 --- a/enclave.go +++ b/enclave.go @@ -405,6 +405,7 @@ func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { select { case <-errChan: elog.Println("Not yet able to talk to leader designation endpoint.") + time.Sleep(time.Second) continue case result = <-areWeLeader: return From 72ed9c431d5a4eae20562a3c05e309babfe661b1 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Tue, 29 Aug 2023 21:00:25 -0500 Subject: [PATCH 66/99] Print differing PCR values. --- attester.go | 2 ++ util.go | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/attester.go b/attester.go index 5d0d382..75c6186 100644 --- a/attester.go +++ b/attester.go @@ -164,6 +164,8 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { return nil, fmt.Errorf("%v: %w", errStr, err) } if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { + elog.Printf("Our PCR values:\n%s", prettyFormat(ourPCRs)) + elog.Printf("Their PCR values:\n%s", prettyFormat(their.Document.PCRs)) return nil, fmt.Errorf("%v: PCR values of remote enclave not identical to ours", errStr) } diff --git a/util.go b/util.go index b864bff..9790d36 100644 --- a/util.go +++ b/util.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" + "encoding/json" "encoding/pem" "errors" "fmt" @@ -253,3 +254,11 @@ func makeLeaderRequest(leader *url.URL, ourNonce nonce, areWeLeader chan bool, e } errChan <- fmt.Errorf("leader designation endpoint returned %d", resp.StatusCode) } + +func prettyFormat(c any) string { + s, err := json.MarshalIndent(c, "", " ") + if err != nil { + return "" + } + return string(s) +} From 308f43e282ae0111fc8ff022f984e95f6fb0fb07 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 07:14:01 -0500 Subject: [PATCH 67/99] Start heartbeat loop after key synchronization. --- enclave.go | 36 +++++++++++++++++++++--------------- sync_worker.go | 8 ++++---- sync_worker_test.go | 4 ++-- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/enclave.go b/enclave.go index b1efd7a..5200472 100644 --- a/enclave.go +++ b/enclave.go @@ -280,7 +280,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register external but private HTTP API. m = e.extPrivSrv.Handler.(*chi.Mux) - m.Handle(pathSync, asWorker(e.installKeys, e.attester)) + m.Handle(pathSync, asWorker(e.setupWorkerPostSync, e.attester)) // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) @@ -305,18 +305,6 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { return e, nil } -// installKeys installs the given enclave keys. Worker enclaves do this after -// key synchronization. -func (e *Enclave) installKeys(keys *enclaveKeys) error { - e.keys.set(keys) - cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) - if err != nil { - return err - } - e.httpsCert.set(&cert) - return nil -} - // Start starts the Nitro Enclave. If something goes wrong, the function // returns an error. func (e *Enclave) Start(ctx context.Context) error { @@ -362,11 +350,10 @@ func (e *Enclave) Start(ctx context.Context) error { if !e.weAreLeader(ctx) { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) - err = asWorker(e.installKeys, e.attester).registerWith(leader, worker) + err = asWorker(e.setupWorkerPostSync, e.attester).registerWith(leader, worker) if err != nil { elog.Fatalf("Error syncing with leader: %v", err) } - go e.workerHeartbeat(ctx, worker) } return nil @@ -419,6 +406,25 @@ func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { } } +// setupWorkerPostSync performs necessary post-key synchronization tasks like +// installing the given enclave keys and starting the heartbeat loop. +func (e *Enclave) setupWorkerPostSync(keys *enclaveKeys) error { + e.keys.set(keys) + cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) + if err != nil { + return err + } + e.httpsCert.set(&cert) + + // Start our heartbeat. + worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) + go e.workerHeartbeat(context.Background(), worker) + + return nil +} + +// setupLeader performs necessary setup tasks like starting the worker event +// loop and installing leader-specific HTTP handlers. func (e *Enclave) setupLeader(ctx context.Context) { go e.workers.start(ctx) // Make leader-specific endpoints available. diff --git a/sync_worker.go b/sync_worker.go index d201d04..86f6875 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -27,19 +27,19 @@ var ( // sync protocol requires two endpoints on the worker. type workerSync struct { attester - installKeys func(*enclaveKeys) error + setupWorker func(*enclaveKeys) error ephemeralKeys chan *boxKey nonce chan nonce } // asWorker returns a new workerSync object. func asWorker( - installKeys func(*enclaveKeys) error, + setupWorker func(*enclaveKeys) error, a attester, ) *workerSync { return &workerSync{ attester: a, - installKeys: installKeys, + setupWorker: setupWorker, nonce: make(chan nonce, 1), ephemeralKeys: make(chan *boxKey, 1), } @@ -209,7 +209,7 @@ func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if err := s.installKeys(&keys); err != nil { + if err := s.setupWorker(&keys); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) elog.Fatalf("Failed to install enclave keys: %v", err) } diff --git a/sync_worker_test.go b/sync_worker_test.go index f69418f..8e2d575 100644 --- a/sync_worker_test.go +++ b/sync_worker_test.go @@ -41,7 +41,7 @@ func TestSuccessfulRegisterWith(t *testing.T) { Host: "localhost", } - err = asWorker(e.installKeys, &dummyAttester{}).registerWith(leader, worker) + err = asWorker(e.setupWorkerPostSync, &dummyAttester{}).registerWith(leader, worker) if err != nil { t.Fatalf("Error registering with leader: %v", err) } @@ -58,7 +58,7 @@ func TestSuccessfulSync(t *testing.T) { // Set up the worker. worker := createEnclave(&defaultCfg) srv := httptest.NewTLSServer( - asWorker(worker.installKeys, &dummyAttester{}), + asWorker(worker.setupWorkerPostSync, &dummyAttester{}), ) workerURL, err := url.Parse(srv.URL) if err != nil { From 374812bea56886ef6fcbbe6c22605a0e2f88193e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 07:55:42 -0500 Subject: [PATCH 68/99] Ignore PCR4 when comparing PCR values. --- attestation.go | 6 ++++++ attestation_test.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/attestation.go b/attestation.go index 7b72f1f..d2235bb 100644 --- a/attestation.go +++ b/attestation.go @@ -72,6 +72,12 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool { } for pcr, ourValue := range ourPCRs { + // PCR4 contains a hash over the parent's instance ID. Our enclaves run + // on different parent instances; PCR4 will therefore always differ: + // https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html + if pcr == 4 { + continue + } theirValue, exists := theirPCRs[pcr] if !exists { return false diff --git a/attestation_test.go b/attestation_test.go index 7228c40..4ce8fcd 100644 --- a/attestation_test.go +++ b/attestation_test.go @@ -21,6 +21,12 @@ func TestArePCRsIdentical(t *testing.T) { t.Fatal("Failed to recognize identical PCRs as such.") } + // PCR4 should be ignored. + pcr1[4], pcr2[4] = []byte("foo"), []byte("bar") + if !arePCRsIdentical(pcr1, pcr2) { + t.Fatal("Failed to recognize identical PCRs as such.") + } + // Add a new PCR value, so our two maps are no longer identical. pcr1[2] = []byte("barfoo") if arePCRsIdentical(pcr1, pcr2) { From 899092057e4371ea69e6561a90abaab15028d156 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 08:57:31 -0500 Subject: [PATCH 69/99] Dereference argument to fix bug. --- attester.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/attester.go b/attester.go index 75c6186..0f2ef2a 100644 --- a/attester.go +++ b/attester.go @@ -108,14 +108,14 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { // Prepare our auxiliary information. switch v := aux.(type) { - case workerAuxInfo: + case *workerAuxInfo: nonce = v.WorkersNonce[:] userData = v.LeadersNonce[:] publicKey = v.PublicKey - case leaderAuxInfo: + case *leaderAuxInfo: nonce = v.WorkersNonce[:] userData = v.EnclaveKeys - case clientAuxInfo: + case *clientAuxInfo: nonce = v.clientNonce[:] userData = v.attestationHashes } From d9605b0029c7612a7ee67de9e79a0cbbb81d4b44 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 09:10:49 -0500 Subject: [PATCH 70/99] Pad public key field if unused. --- attester.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/attester.go b/attester.go index 0f2ef2a..9763133 100644 --- a/attester.go +++ b/attester.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/base64" "encoding/json" "errors" @@ -11,6 +12,8 @@ import ( "github.com/hf/nsm/request" ) +var padding = []byte("dummy") + // attester defines functions for the creation and verification of attestation // documents. Making this an interface helps with testing: It allows us to // implement a dummy attester that works without the AWS Nitro hypervisor. @@ -106,7 +109,9 @@ func newNitroAttester() *nitroAttester { func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { var nonce, userData, publicKey []byte - // Prepare our auxiliary information. + // Prepare our auxiliary information. If the public key field is unused, we + // pad it with dummy bytes because the nitrite package (which we use to + // verify attestation documents) expects all three fields to be set. switch v := aux.(type) { case *workerAuxInfo: nonce = v.WorkersNonce[:] @@ -115,9 +120,11 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { case *leaderAuxInfo: nonce = v.WorkersNonce[:] userData = v.EnclaveKeys + publicKey = padding case *clientAuxInfo: nonce = v.clientNonce[:] userData = v.attestationHashes + publicKey = padding } s, err := nsm.OpenDefaultSession() @@ -184,9 +191,9 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if err != nil { return nil, err } - // If the "public key" field is unset, we know that we're dealing with a + // If the "public key" field is padded, we know that we're dealing with a // worker's auxiliary information. - if their.Document.PublicKey != nil { + if bytes.Equal(their.Document.PublicKey, padding) { return &workerAuxInfo{ WorkersNonce: workersNonce, LeadersNonce: leadersNonce, From 817cc818694daf02971a0bff63b35264fe278851 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 09:19:21 -0500 Subject: [PATCH 71/99] Fix if condition. --- attester.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/attester.go b/attester.go index 9763133..c51b42b 100644 --- a/attester.go +++ b/attester.go @@ -191,9 +191,9 @@ func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if err != nil { return nil, err } - // If the "public key" field is padded, we know that we're dealing with a - // worker's auxiliary information. - if bytes.Equal(their.Document.PublicKey, padding) { + // If the "public key" field does not contain padding, we know that we're + // dealing with a worker's auxiliary information. + if !bytes.Equal(their.Document.PublicKey, padding) { return &workerAuxInfo{ WorkersNonce: workersNonce, LeadersNonce: leadersNonce, From aaf3c72200d019ad68d1b5e5a6ab957f760d8997 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Wed, 30 Aug 2023 17:45:38 -0500 Subject: [PATCH 72/99] Transmit encrypted key material separately. There's not enough space in the auxiliary information. Use a hash over the encrypted key material in the aux info. --- attestation.go | 3 +- attester.go | 88 ++++++++++++++++++------------------------ keysync_shared.go | 56 --------------------------- keysync_shared_test.go | 64 ------------------------------ nonce.go | 32 +++++++++++++++ nonce_test.go | 51 ++++++++++++++++++++++++ sync_leader.go | 55 +++++++++++++++++++------- sync_shared.go | 47 ++++++++++++++++++++++ sync_shared_test.go | 22 +++++++++++ sync_worker.go | 73 +++++++++++++++++++++-------------- 10 files changed, 276 insertions(+), 215 deletions(-) delete mode 100644 keysync_shared.go delete mode 100644 keysync_shared_test.go create mode 100644 nonce.go create mode 100644 nonce_test.go create mode 100644 sync_shared.go create mode 100644 sync_shared_test.go diff --git a/attestation.go b/attestation.go index d2235bb..b86fce4 100644 --- a/attestation.go +++ b/attestation.go @@ -10,7 +10,6 @@ import ( ) const ( - maxAttDocLen = 10000 // A (reasonable?) upper limit for attestation doc lengths. hashPrefix = "sha256:" hashSeparator = ";" ) @@ -18,7 +17,7 @@ const ( var ( errBadForm = errors.New("failed to parse POST form data") errNoNonce = errors.New("could not find nonce in URL query parameters") - errBadNonceFormat = fmt.Errorf("unexpected nonce format; must be %d-digit hex string", nonceNumDigits) + errBadNonceFormat = fmt.Errorf("unexpected nonce format; must be %d-digit hex string", nonceLen*2) errFailedAttestation = errors.New("failed to obtain attestation document from hypervisor") errProfilingSet = errors.New("attestation disabled because profiling is enabled") diff --git a/attester.go b/attester.go index c51b42b..a081f2f 100644 --- a/attester.go +++ b/attester.go @@ -2,17 +2,20 @@ package main import ( "bytes" - "encoding/base64" "encoding/json" "errors" - "fmt" "github.com/hf/nitrite" "github.com/hf/nsm" "github.com/hf/nsm/request" ) -var padding = []byte("dummy") +var ( + errPCRMismatch = errors.New("PCR values differ") + errNonceMismatch = errors.New("nonce is unexpected") + errNoAttstnFromNSM = errors.New("NSM device did not return an attestation") + padding = []byte("dummy") +) // attester defines functions for the creation and verification of attestation // documents. Making this an interface helps with testing: It allows us to @@ -24,6 +27,8 @@ type attester interface { type auxInfo interface{} +// workerAuxInfo holds the auxilitary information of an attestation document +// requested by clients. type clientAuxInfo struct { clientNonce nonce attestationHashes []byte @@ -37,23 +42,11 @@ type workerAuxInfo struct { PublicKey []byte `json:"public_key"` } -func (w workerAuxInfo) String() string { - return fmt.Sprintf("Worker's auxiliary info:\n"+ - "Worker's nonce: %x\nLeader's nonce: %x\nPublic key: %x", - w.WorkersNonce, w.LeadersNonce, w.PublicKey) -} - // leaderAuxInfo holds the auxiliary information of the leader's attestation // document. type leaderAuxInfo struct { - WorkersNonce nonce `json:"workers_nonce"` - EnclaveKeys []byte `json:"enclave_keys"` -} - -func (l leaderAuxInfo) String() string { - return fmt.Sprintf("Leader's auxiliary info:\n"+ - "Worker's nonce: %x\nEnclave keys: %x", - l.WorkersNonce, l.EnclaveKeys) + WorkersNonce nonce `json:"workers_nonce"` + HashOfEncrypted []byte `json:"hash_of_encrypted"` } // dummyAttester helps with local testing. The interface simply turns @@ -75,7 +68,7 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { return nil, err } if len(w.WorkersNonce) == nonceLen && len(w.LeadersNonce) == nonceLen && w.PublicKey != nil { - if n.B64() != w.LeadersNonce.B64() { + if n.b64() != w.LeadersNonce.b64() { return nil, errors.New("leader nonce not in cache") } return &w, nil @@ -85,8 +78,8 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if err := json.Unmarshal(doc, &l); err != nil { return nil, err } - if len(l.WorkersNonce) == nonceLen && l.EnclaveKeys != nil { - if n.B64() != l.WorkersNonce.B64() { + if len(l.WorkersNonce) == nonceLen && l.HashOfEncrypted != nil { + if n.b64() != l.WorkersNonce.b64() { return nil, errors.New("worker nonce not in cache") } return &l, nil @@ -114,12 +107,12 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { // verify attestation documents) expects all three fields to be set. switch v := aux.(type) { case *workerAuxInfo: - nonce = v.WorkersNonce[:] - userData = v.LeadersNonce[:] + nonce = v.LeadersNonce[:] + userData = v.WorkersNonce[:] publicKey = v.PublicKey case *leaderAuxInfo: nonce = v.WorkersNonce[:] - userData = v.EnclaveKeys + userData = v.HashOfEncrypted publicKey = padding case *clientAuxInfo: nonce = v.clientNonce[:] @@ -146,7 +139,7 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { return nil, err } if res.Attestation == nil || res.Attestation.Document == nil { - return nil, errors.New("NSM device did not return an attestation") + return nil, errNoAttstnFromNSM } return res.Attestation.Document, nil @@ -154,59 +147,54 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { // verifyAttstn verifies the given attestation document and, if successful, // returns the document's auxiliary information. -func (*nitroAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { - errStr := "error verifying attestation document" - // Verify the remote enclave's attestation document before doing anything - // with it. +func (*nitroAttester) verifyAttstn(doc []byte, ourNonce nonce) (auxInfo, error) { + // First, verify the remote enclave's attestation document. opts := nitrite.VerifyOptions{CurrentTime: currentTime()} their, err := nitrite.Verify(doc, opts) if err != nil { - return nil, fmt.Errorf("%v: %w", errStr, err) + return nil, err } // Verify that the remote enclave's PCR values (e.g., the image ID) are // identical to ours. ourPCRs, err := getPCRValues() if err != nil { - return nil, fmt.Errorf("%v: %w", errStr, err) + return nil, err } if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { elog.Printf("Our PCR values:\n%s", prettyFormat(ourPCRs)) elog.Printf("Their PCR values:\n%s", prettyFormat(their.Document.PCRs)) - return nil, fmt.Errorf("%v: PCR values of remote enclave not identical to ours", errStr) + return nil, errPCRMismatch } // Verify that the remote enclave's attestation document contains the nonce // that we asked it to embed. - b64Nonce := base64.StdEncoding.EncodeToString(their.Document.Nonce) - if n.B64() == b64Nonce { - return nil, fmt.Errorf("%v: nonce %s not in cache", errStr, b64Nonce) - } - - workersNonce, err := sliceToNonce(their.Document.Nonce) + theirNonce, err := sliceToNonce(their.Document.Nonce) if err != nil { return nil, err } - leadersNonce, err := sliceToNonce(their.Document.UserData) - if err != nil { - return nil, err + if ourNonce != theirNonce { + return nil, errNonceMismatch } - // If the "public key" field does not contain padding, we know that we're - // dealing with a worker's auxiliary information. - if !bytes.Equal(their.Document.PublicKey, padding) { - return &workerAuxInfo{ - WorkersNonce: workersNonce, - LeadersNonce: leadersNonce, - PublicKey: their.Document.PublicKey, + + // If the "public key" field contains padding, we know that we're + // dealing with a leader's auxiliary information. + if bytes.Equal(their.Document.PublicKey, padding) { + elog.Println("Extracting leader's auxiliary information.") + return &leaderAuxInfo{ + WorkersNonce: theirNonce, + HashOfEncrypted: their.Document.UserData, }, nil } - workersNonce, err = sliceToNonce(their.Document.Nonce) + elog.Println("Extracting worker's auxiliary information.") + workersNonce, err := sliceToNonce(their.Document.UserData) if err != nil { return nil, err } - return &leaderAuxInfo{ + return &workerAuxInfo{ WorkersNonce: workersNonce, - EnclaveKeys: their.Document.UserData, + LeadersNonce: theirNonce, + PublicKey: their.Document.PublicKey, }, nil } diff --git a/keysync_shared.go b/keysync_shared.go deleted file mode 100644 index 9d01c2a..0000000 --- a/keysync_shared.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - cryptoRand "crypto/rand" - "encoding/base64" - "time" - - "golang.org/x/crypto/nacl/box" -) - -const ( - nonceLen = 20 // The size of a nonce in bytes. - nonceNumDigits = nonceLen * 2 // The number of hex digits in a nonce. - boxKeyLen = 32 // NaCl box's private and public key length. -) - -var ( - // Instead of using rand.Read or time.Now directly, we use the following - // variables to enable mocking as part of our unit tets. - cryptoRead = cryptoRand.Read - currentTime = func() time.Time { return time.Now().UTC() } -) - -// nonce represents a nonce that's used to prove the freshness of an enclave's -// attestation document. -type nonce [nonceLen]byte - -// boxKey represents key material for NaCl's box, i.e., a private and a public -// key. -type boxKey struct { - pubKey *[boxKeyLen]byte - privKey *[boxKeyLen]byte -} - -// newBoxKey creates and returns a key pair for use with box. -func newBoxKey() (*boxKey, error) { - pubKey, privKey, err := box.GenerateKey(cryptoRand.Reader) - if err != nil { - return nil, err - } - return &boxKey{pubKey: pubKey, privKey: privKey}, nil -} - -// newNonce creates and returns a cryptographically secure, random nonce. -func newNonce() (nonce, error) { - var n nonce - if _, err := cryptoRead(n[:]); err != nil { - return nonce{}, err - } - return n, nil -} - -// B64 returns a Base64-encoded representation of the nonce. -func (n *nonce) B64() string { - return base64.StdEncoding.EncodeToString(n[:]) -} diff --git a/keysync_shared_test.go b/keysync_shared_test.go deleted file mode 100644 index 073b69f..0000000 --- a/keysync_shared_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "crypto/rand" - "errors" - "testing" -) - -func failOnErr(t *testing.T, err error) { - t.Helper() - if err != nil { - t.Fatal(err) - } -} - -func TestBoxKeyRandomness(t *testing.T) { - k1, err := newBoxKey() - failOnErr(t, err) - k2, err := newBoxKey() - failOnErr(t, err) - - // It's notoriously difficult to test if something is truly random. Here, - // we simply make sure that two subsequently generated key pairs are not - // identical. That's a low bar to pass but better than nothing. - if k1.privKey == k2.privKey { - t.Error("Private keys of two separate box keys are identical.") - } - if k1.pubKey == k2.pubKey { - t.Error("Public keys of two separate box keys are identical.") - } -} - -func TestNonce(t *testing.T) { - n1, err := newNonce() - failOnErr(t, err) - n2, err := newNonce() - failOnErr(t, err) - - if n1 == n2 { - t.Error("Two separately generated nonces are identical.") - } - if n1.B64() == n2.B64() { - t.Error("Two separately generated Base64-encoded nonces are identical.") - } -} - -func TestErrors(t *testing.T) { - // Make cryptoRead always return an error, and check if functions propagate - // that error. - ourError := errors.New("not enough randomness") - cryptoRead = func(b []byte) (n int, err error) { - return 0, ourError - } - defer func() { - cryptoRead = rand.Read - }() - - if _, err := newNonce(); err == nil { - t.Error("Failed to return error") - if !errors.Is(err, ourError) { - t.Error("Propagated error does not contain expected error string.") - } - } -} diff --git a/nonce.go b/nonce.go new file mode 100644 index 0000000..7237fc6 --- /dev/null +++ b/nonce.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/base64" + "errors" +) + +const nonceLen = 20 // The size of a nonce in bytes. + +var errNotEnoughRead = errors.New("failed to read enough random bytes") + +// nonce represents a nonce that's used to prove the freshness of an enclave's +// attestation document. +type nonce [nonceLen]byte + +// newNonce returns a cryptographically secure, random nonce. +func newNonce() (nonce, error) { + var newNonce nonce + n, err := cryptoRead(newNonce[:]) + if err != nil { + return nonce{}, err + } + if n != nonceLen { + return nonce{}, errNotEnoughRead + } + return newNonce, nil +} + +// b64 returns a Base64-encoded representation of the nonce. +func (n *nonce) b64() string { + return base64.StdEncoding.EncodeToString(n[:]) +} diff --git a/nonce_test.go b/nonce_test.go new file mode 100644 index 0000000..1181104 --- /dev/null +++ b/nonce_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "crypto/rand" + "errors" + "testing" +) + +func failOnErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("Expected no error but got %v.", err) + } +} + +func TestNonce(t *testing.T) { + nonce1, err := newNonce() + failOnErr(t, err) + nonce2, err := newNonce() + failOnErr(t, err) + + if nonce1 == nonce2 { + t.Fatal("Two separate nonces should not be identical.") + } + if nonce1.b64() == nonce2.b64() { + t.Fatal("Two separate, Base64-encoded nonces should not be identical.") + } +} + +func TestNonceErrors(t *testing.T) { + defer func() { + cryptoRead = rand.Read + }() + + // Make cryptoRead return an error. + ourError := errors.New("not enough randomness") + cryptoRead = func(b []byte) (n int, err error) { + return 0, ourError + } + if _, err := newNonce(); !errors.Is(err, ourError) { + t.Fatal("Propagated error does not contain expected error string.") + } + + // Make cryptoRead return an insufficient number of random bytes. + cryptoRead = func(b []byte) (n int, err error) { + return nonceLen - 1, nil + } + if _, err := newNonce(); !errors.Is(err, errNotEnoughRead) { + t.Fatalf("Expected error %v but got %v.", errNotEnoughRead, err) + } +} diff --git a/sync_leader.go b/sync_leader.go index d3d4781..f213446 100644 --- a/sync_leader.go +++ b/sync_leader.go @@ -1,18 +1,24 @@ package main import ( + "bytes" cryptoRand "crypto/rand" + "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" - "strings" "golang.org/x/crypto/nacl/box" ) +var ( + errExpectedEmptyKeys = errors.New("expected encrypted keys to be unset") +) + // leaderSync holds the state and code that we need for a one-off sync with a // worker enclave. type leaderSync struct { @@ -31,6 +37,10 @@ func asLeader(keys *enclaveKeys, a attester) *leaderSync { // syncWith makes the leader initiate key synchronization with the given worker // enclave. func (s *leaderSync) syncWith(worker *url.URL) (err error) { + var ( + reqBody attstnBody + encrypted []byte + ) defer func() { if err == nil { elog.Printf("Successfully synced with worker %s.", worker.Host) @@ -60,48 +70,67 @@ func (s *leaderSync) syncWith(worker *url.URL) (err error) { // Step 3: Verify the worker's attestation document and extract its // auxiliary information. - b64Attstn, err := io.ReadAll(newLimitReader(resp.Body, maxAttDocLen)) + maxReadLen := base64.StdEncoding.EncodedLen(maxAttstnBodyLen) + jsonBody, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) if err != nil { return err } - resp.Body.Close() - attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + defer resp.Body.Close() + + if err := json.Unmarshal(jsonBody, &reqBody); err != nil { + return err + } + if len(reqBody.EncryptedKeys) != 0 { + return errExpectedEmptyKeys + } + attstnDoc, err := base64.StdEncoding.DecodeString(reqBody.Document) if err != nil { return err } - workerAux, err := s.verifyAttstn(attstn, nonce) + aux, err := s.verifyAttstn(attstnDoc, nonce) if err != nil { return err } + workerAux := aux.(*workerAuxInfo) // Step 4: Encrypt the leader's enclave keys with the ephemeral public key // that the worker put into its auxiliary information. pubKey := &[boxKeyLen]byte{} - copy(pubKey[:], workerAux.(*workerAuxInfo).PublicKey[:]) + copy(pubKey[:], workerAux.PublicKey[:]) jsonKeys, err := json.Marshal(s.keys.get()) if err != nil { return err } - var encrypted []byte encrypted, err = box.SealAnonymous(nil, jsonKeys, pubKey, cryptoRand.Reader) if err != nil { return err } // Step 5: Create the leader's auxiliary information, consisting of the - // worker's nonce and the encrypted enclave keys. + // worker's nonce and a hash of the encrypted enclave keys. + hash := sha256.Sum256(encrypted) leaderAux := &leaderAuxInfo{ - WorkersNonce: workerAux.(*workerAuxInfo).WorkersNonce, - EnclaveKeys: encrypted, + WorkersNonce: workerAux.WorkersNonce, + HashOfEncrypted: hash[:], } - attstn, err = s.createAttstn(leaderAux) + attstnDoc, err = s.createAttstn(leaderAux) if err != nil { return err } - strAttstn := base64.StdEncoding.EncodeToString(attstn) // Step 6: Send the leader's attestation document to the worker. - resp, err = newUnauthenticatedHTTPClient().Post(worker.String(), "text/plain", strings.NewReader(strAttstn)) + jsonBody, err = json.Marshal(&attstnBody{ + Document: base64.StdEncoding.EncodeToString(attstnDoc), + EncryptedKeys: base64.StdEncoding.EncodeToString(encrypted), + }) + if err != nil { + return err + } + resp, err = newUnauthenticatedHTTPClient().Post( + worker.String(), + "text/plain", + bytes.NewReader(jsonBody), + ) if err != nil { return err } diff --git a/sync_shared.go b/sync_shared.go new file mode 100644 index 0000000..719903b --- /dev/null +++ b/sync_shared.go @@ -0,0 +1,47 @@ +package main + +import ( + cryptoRand "crypto/rand" + "time" + + "golang.org/x/crypto/nacl/box" +) + +const ( + maxAttstnBodyLen = 1 << 14 // Upper limit for attestation body length. + boxKeyLen = 32 // NaCl box's private and public key length. +) + +var ( + // Instead of using rand.Read or time.Now directly, we use the following + // variables to enable mocking as part of our unit tets. + cryptoRead = cryptoRand.Read + currentTime = func() time.Time { return time.Now().UTC() } +) + +// attstnBody contains a JSON-formatted, Base64-encoded attestation document and +// encrypted key material. The leader and worker use this struct to exchange +// attestation documents. +type attstnBody struct { + Document string `json:"document"` + EncryptedKeys string `json:"encrypted_keys"` +} + +// boxKey represents key material for NaCl's box, i.e., a private and a public +// key. +type boxKey struct { + pubKey *[boxKeyLen]byte + privKey *[boxKeyLen]byte +} + +// newBoxKey returns a key pair for use with box. +func newBoxKey() (*boxKey, error) { + pubKey, privKey, err := box.GenerateKey(cryptoRand.Reader) + if err != nil { + return nil, err + } + return &boxKey{ + pubKey: pubKey, + privKey: privKey, + }, nil +} diff --git a/sync_shared_test.go b/sync_shared_test.go new file mode 100644 index 0000000..881e7fb --- /dev/null +++ b/sync_shared_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "testing" +) + +func TestBoxKeyRandomness(t *testing.T) { + k1, err := newBoxKey() + failOnErr(t, err) + k2, err := newBoxKey() + failOnErr(t, err) + + // It's notoriously difficult to test if something is truly random. Here, + // we simply make sure that two subsequently generated key pairs are not + // identical. That's a low bar to pass but better than nothing. + if k1.privKey == k2.privKey { + t.Error("Private keys of two separate box keys are identical.") + } + if k1.pubKey == k2.pubKey { + t.Error("Public keys of two separate box keys are identical.") + } +} diff --git a/sync_worker.go b/sync_worker.go index 86f6875..5a895f5 100644 --- a/sync_worker.go +++ b/sync_worker.go @@ -2,8 +2,8 @@ package main import ( "bytes" + "crypto/sha256" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -16,10 +16,9 @@ import ( ) var ( - errNonceRequired = errors.New("nonce is required") errInProgress = errors.New("key sync already in progress") - errInvalidNonceLen = errors.New("invalid nonce length") - errDecrypting = errors.New("error decrypting enclave keys") + errFailedToDecrypt = errors.New("error decrypting enclave keys") + errHashNotInAttstn = errors.New("hash of encrypted keys not in attestation document") ) // workerSync holds the state and code that we need for a one-off sync with a @@ -115,22 +114,10 @@ func (s *workerSync) initSync(w http.ResponseWriter, r *http.Request) { // Extract the leader's nonce from the URL, which must look like this: // https://example.com/enclave/sync?nonce=[HEX-ENCODED-NONCE] - hexNonce := r.URL.Query().Get("nonce") - if hexNonce == "" { - http.Error(w, errNonceRequired.Error(), http.StatusBadRequest) - return - } - nonceSlice, err := hex.DecodeString(hexNonce) + leadersNonce, err := getNonceFromReq(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) - return } - if len(nonceSlice) != nonceLen { - http.Error(w, errInvalidNonceLen.Error(), http.StatusBadRequest) - return - } - var leadersNonce nonce - copy(leadersNonce[:], nonceSlice) // Create the worker's nonce and store it in our channel, so we can later // verify it. @@ -151,60 +138,86 @@ func (s *workerSync) initSync(w http.ResponseWriter, r *http.Request) { s.ephemeralKeys <- boxKey // Create and return the worker's Base64-encoded attestation document. - aux := &workerAuxInfo{ + attstnDoc, err := s.createAttstn(&workerAuxInfo{ WorkersNonce: workersNonce, LeadersNonce: leadersNonce, PublicKey: boxKey.pubKey[:], - } - attstn, err := s.createAttstn(aux) + }) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - fmt.Fprintln(w, base64.StdEncoding.EncodeToString(attstn)) + respBody, err := json.Marshal(&attstnBody{ + Document: base64.StdEncoding.EncodeToString(attstnDoc), + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fmt.Fprintln(w, string(respBody)) } // finishSync responds to the leader's final request before key synchronization // is complete. func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { + var ( + reqBody attstnBody + keys enclaveKeys + ) elog.Println("Received leader's request to complete key sync.") // Read the leader's Base64-encoded attestation document. - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - b64Attstn, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) + maxReadLen := base64.StdEncoding.EncodedLen(maxAttstnBodyLen) + jsonBody, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - - // Decode Base64 to byte slice. - attstn, err := base64.StdEncoding.DecodeString(string(b64Attstn)) + if err := json.Unmarshal(jsonBody, &reqBody); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + attstnDoc, err := base64.StdEncoding.DecodeString(reqBody.Document) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - aux, err := s.verifyAttstn(attstn, <-s.nonce) + + // Verify attestation document and obtain its auxiliary information. + aux, err := s.verifyAttstn(attstnDoc, <-s.nonce) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + leaderAux := aux.(*leaderAuxInfo) + encrypted, err := base64.StdEncoding.DecodeString(reqBody.EncryptedKeys) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + // Make sure that the hash of the encrypted key material is present in the + // attestation document. + hash := sha256.Sum256(encrypted) + if !bytes.Equal(hash[:], leaderAux.HashOfEncrypted) { + http.Error(w, errHashNotInAttstn.Error(), http.StatusBadRequest) + return + } + ephemeralKey := <-s.ephemeralKeys // Decrypt the leader's enclave keys, which are encrypted with the // public key that we provided earlier. decrypted, ok := box.OpenAnonymous( nil, - aux.(*leaderAuxInfo).EnclaveKeys, + encrypted, ephemeralKey.pubKey, ephemeralKey.privKey) if !ok { - http.Error(w, errDecrypting.Error(), http.StatusBadRequest) + http.Error(w, errFailedToDecrypt.Error(), http.StatusBadRequest) return } // Install the leader's enclave keys. - var keys enclaveKeys if err := json.Unmarshal(decrypted, &keys); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return From 6e7cc50f0f9f4ae977000eef4753afa50601142c Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 09:07:09 -0500 Subject: [PATCH 73/99] Remove unused endpoint. --- enclave.go | 1 - 1 file changed, 1 deletion(-) diff --git a/enclave.go b/enclave.go index 5200472..9672081 100644 --- a/enclave.go +++ b/enclave.go @@ -40,7 +40,6 @@ const ( parentCID = 3 // The following paths are handled by nitriding. pathRoot = "/enclave" - pathNonce = "/enclave/nonce" pathAttestation = "/enclave/attestation" pathState = "/enclave/state" pathSync = "/enclave/sync" From f3099fb2ceabeead3ec0fbe9b28093ced32de5e5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 09:31:14 -0500 Subject: [PATCH 74/99] Implement Darnell's suggestion for state handlers. --- enclave.go | 37 ++++++++++++++++++++--- handlers.go | 86 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/enclave.go b/enclave.go index 9672081..bae23e3 100644 --- a/enclave.go +++ b/enclave.go @@ -52,6 +52,12 @@ const ( // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" + // The states the enclave can be in relating to key synchronization. + noSync = 0 // The enclave is not configured to synchronize keys. + inProgress = 1 // Leader designation is in progress. + isLeader = 2 // The enclave is the leader. + isWorker = 3 // The enclave is a worker. + ) var ( @@ -68,6 +74,7 @@ type Enclave struct { extPubSrv, extPrivSrv *http.Server intSrv *http.Server promSrv *http.Server + syncState int revProxy *httputil.ReverseProxy hashes *AttestationHashes promRegistry *prometheus.Registry @@ -270,6 +277,9 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { if cfg.DisableKeepAlives { e.extPubSrv.SetKeepAlivesEnabled(false) } + if cfg.isScalingEnabled() { + e.setSyncState(inProgress) + } // Register external public HTTP API. m := e.extPubSrv.Handler.(*chi.Mux) @@ -284,7 +294,8 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) m.Get(pathReady, readyHandler(e)) - m.Get(pathState, getStateHandler(e)) + m.Get(pathState, getStateHandler(e.getSyncState, e.keys)) + m.Put(pathState, putStateHandler(e)) m.Post(pathHash, hashHandler(e)) // Configure our reverse proxy if the enclave application exposes an HTTP @@ -358,6 +369,20 @@ func (e *Enclave) Start(ctx context.Context) error { return nil } +// getSyncState returns the enclave's key synchronization state. +func (e *Enclave) getSyncState() int { + e.RLock() + defer e.RUnlock() + return e.syncState +} + +// setSyncState sets the enclave's key synchronization state. +func (e *Enclave) setSyncState(state int) { + e.Lock() + defer e.Unlock() + e.syncState = state +} + func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { var ( err error @@ -369,6 +394,11 @@ func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { ) defer func() { elog.Printf("We are leader: %v", result) + if result { + e.setSyncState(isLeader) + } else { + e.setSyncState(isWorker) + } }() ourNonce, err = newNonce() @@ -426,10 +456,9 @@ func (e *Enclave) setupWorkerPostSync(keys *enclaveKeys) error { // loop and installing leader-specific HTTP handlers. func (e *Enclave) setupLeader(ctx context.Context) { go e.workers.start(ctx) - // Make leader-specific endpoints available. - e.intSrv.Handler.(*chi.Mux).Put(pathState, putStateHandler(e)) + // Make leader-specific endpoint available. e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) - elog.Println("Set up leader endpoints and started worker event loop.") + elog.Println("Set up leader endpoint and started worker event loop.") } // workerHeartbeat periodically talks to the leader enclave to 1) let the leader diff --git a/handlers.go b/handlers.go index 460aaa7..baee950 100644 --- a/handlers.go +++ b/handlers.go @@ -25,9 +25,11 @@ const ( ) var ( - errFailedReqBody = errors.New("failed to read request body") - errHashWrongSize = errors.New("given hash is of invalid size") - errNoBase64 = errors.New("no Base64 given") + errFailedReqBody = errors.New("failed to read request body") + errHashWrongSize = errors.New("given hash is of invalid size") + errNoBase64 = errors.New("no Base64 given") + errDesignationInProgress = errors.New("leader designation in progress") + errEndpointGone = errors.New("endpoint not meant to be used") ) func errNo200(code int) error { @@ -57,19 +59,26 @@ func rootHandler(cfg *Config) http.HandlerFunc { // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func getStateHandler(e *Enclave) http.HandlerFunc { +func getStateHandler(getSyncState func() int, keys *enclaveKeys) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - appKeys := e.keys.getAppKeys() - n, err := w.Write(appKeys) - if err != nil { - elog.Printf("Error writing state to client: %v", err) - return - } - expected := len(appKeys) - if n != expected { - elog.Printf("Only wrote %d out of %d-byte state to client.", n, expected) - return + switch getSyncState() { + case noSync: + fallthrough + case isLeader: + http.Error(w, errEndpointGone.Error(), http.StatusGone) + case inProgress: + http.Error(w, errDesignationInProgress.Error(), http.StatusServiceUnavailable) + case isWorker: + w.Header().Set("Content-Type", "application/octet-stream") + appKeys := keys.getAppKeys() + n, err := w.Write(appKeys) + if err != nil { + elog.Fatalf("Error writing state to client: %v", err) + } + expected := len(appKeys) + if n != expected { + elog.Fatalf("Only wrote %d out of %d-byte state to client.", n, expected) + } } } } @@ -82,26 +91,35 @@ func getStateHandler(e *Enclave) http.HandlerFunc { // trusted enclave application. func putStateHandler(e *Enclave) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - keys, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) - if err != nil { - http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) - return - } - e.keys.setAppKeys(keys) - w.WriteHeader(http.StatusOK) + switch e.getSyncState() { + case noSync: + fallthrough + case isWorker: + http.Error(w, errEndpointGone.Error(), http.StatusGone) + case inProgress: + http.Error(w, errDesignationInProgress.Error(), http.StatusServiceUnavailable) + case isLeader: + keys, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) + if err != nil { + http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + return + } + e.keys.setAppKeys(keys) + w.WriteHeader(http.StatusOK) - // The leader's application keys have changed. Re-synchronize the key - // material with all registered workers. If synchronization fails for a - // given worker, unregister it. - elog.Printf("Application keys have changed. Re-synchronizing with %d worker(s).", - e.workers.length()) - go e.workers.forAll( - func(worker *url.URL) { - if err := asLeader(e.keys.get(), e.attester).syncWith(worker); err != nil { - e.workers.unregister(worker) - } - }, - ) + // The leader's application keys have changed. Re-synchronize the key + // material with all registered workers. If synchronization fails for a + // given worker, unregister it. + elog.Printf("Application keys have changed. Re-synchronizing with %d worker(s).", + e.workers.length()) + go e.workers.forAll( + func(worker *url.URL) { + if err := asLeader(e.keys.get(), e.attester).syncWith(worker); err != nil { + e.workers.unregister(worker) + } + }, + ) + } } } From 18afc7a9c52388c8cb5eb703382305e4302dc6c8 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 09:32:21 -0500 Subject: [PATCH 75/99] Update tooling. --- scripts/get-app-keys.sh | 7 +++++++ scripts/{update-app-keys.sh => set-app-keys.sh} | 0 2 files changed, 7 insertions(+) create mode 100755 scripts/get-app-keys.sh rename scripts/{update-app-keys.sh => set-app-keys.sh} (100%) diff --git a/scripts/get-app-keys.sh b/scripts/get-app-keys.sh new file mode 100755 index 0000000..01d93d3 --- /dev/null +++ b/scripts/get-app-keys.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --request GET \ + --include \ + "http://localhost:${int_port}/enclave/state" diff --git a/scripts/update-app-keys.sh b/scripts/set-app-keys.sh similarity index 100% rename from scripts/update-app-keys.sh rename to scripts/set-app-keys.sh From d13fb9f686a3e6b07ca84f7dd678a606b758050b Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 11:41:30 -0500 Subject: [PATCH 76/99] Add tests for revised state handlers. --- enclave.go | 3 +- handlers.go | 19 +++++---- handlers_test.go | 106 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 106 insertions(+), 22 deletions(-) diff --git a/enclave.go b/enclave.go index bae23e3..90692f7 100644 --- a/enclave.go +++ b/enclave.go @@ -295,7 +295,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { m = e.intSrv.Handler.(*chi.Mux) m.Get(pathReady, readyHandler(e)) m.Get(pathState, getStateHandler(e.getSyncState, e.keys)) - m.Put(pathState, putStateHandler(e)) + m.Put(pathState, putStateHandler(e.attester, e.getSyncState, e.keys, e.workers)) m.Post(pathHash, hashHandler(e)) // Configure our reverse proxy if the enclave application exposes an HTTP @@ -383,6 +383,7 @@ func (e *Enclave) setSyncState(state int) { e.syncState = state } +// weAreLeader figures out if the enclave is the leader or worker. func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { var ( err error diff --git a/handlers.go b/handlers.go index baee950..ecd4d99 100644 --- a/handlers.go +++ b/handlers.go @@ -89,9 +89,14 @@ func getStateHandler(getSyncState func() int, keys *enclaveKeys) http.HandlerFun // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func putStateHandler(e *Enclave) http.HandlerFunc { +func putStateHandler( + a attester, + getSyncState func() int, + enclaveKeys *enclaveKeys, + workers *workerManager, +) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - switch e.getSyncState() { + switch getSyncState() { case noSync: fallthrough case isWorker: @@ -104,18 +109,18 @@ func putStateHandler(e *Enclave) http.HandlerFunc { http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) return } - e.keys.setAppKeys(keys) + enclaveKeys.setAppKeys(keys) w.WriteHeader(http.StatusOK) // The leader's application keys have changed. Re-synchronize the key // material with all registered workers. If synchronization fails for a // given worker, unregister it. elog.Printf("Application keys have changed. Re-synchronizing with %d worker(s).", - e.workers.length()) - go e.workers.forAll( + workers.length()) + go workers.forAll( func(worker *url.URL) { - if err := asLeader(e.keys.get(), e.attester).syncWith(worker); err != nil { - e.workers.unregister(worker) + if err := asLeader(enclaveKeys.get(), a).syncWith(worker); err != nil { + workers.unregister(worker) } }, ) diff --git a/handlers_test.go b/handlers_test.go index e4ad66e..c0b2e88 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -14,6 +14,7 @@ import ( "net/http/httptest" "net/http/httputil" "net/url" + "strings" "sync" "syscall" "testing" @@ -30,6 +31,15 @@ func makeRequestFor(srv *http.Server) func(method, path string, body io.Reader) } } +func makeReqForHandler(handler http.HandlerFunc) func(method, path string, body io.Reader) *http.Response { + return func(method, path string, body io.Reader) *http.Response { + req := httptest.NewRequest(method, path, body) + w := httptest.NewRecorder() + handler(w, req) + return w.Result() + } +} + // newResp is a helper function that creates an HTTP response. func newResp(status int, body string) *http.Response { return &http.Response{ @@ -38,6 +48,12 @@ func newResp(status int, body string) *http.Response { } } +func retState(state int) func() int { + return func() int { + return state + } +} + // keysToHeartbeat turns the given keys into a Buffer that contains a heartbeat // request. func keysToHeartbeat(t *testing.T, keys *enclaveKeys) *bytes.Buffer { @@ -110,36 +126,98 @@ func signalReady(t *testing.T, e *Enclave) { time.Sleep(100 * time.Millisecond) } -func TestStateHandlers(t *testing.T) { - e := createEnclave(&defaultCfg) - e.setupLeader(context.Background()) +func TestGetStateHandler(t *testing.T) { + var keys = newTestKeys(t) - tooLargeKey := make([]byte, 1024*1024+1) - makeReq := makeRequestFor(e.intSrv) + makeReq := makeReqForHandler(getStateHandler(retState(noSync), keys)) + assertResponse(t, + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusGone, errEndpointGone.Error()), + ) + + makeReq = makeReqForHandler(getStateHandler(retState(isLeader), keys)) + assertResponse(t, + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusGone, errEndpointGone.Error()), + ) + + makeReq = makeReqForHandler(getStateHandler(retState(isWorker), keys)) + assertResponse(t, + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusOK, string(keys.getAppKeys())), + ) + + makeReq = makeReqForHandler(getStateHandler(retState(inProgress), keys)) + assertResponse(t, + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), + ) +} + +func TestPutStateHandler(t *testing.T) { + var ( + tooLargeKey = make([]byte, maxKeyMaterialLen+1) + almostTooLargeKey = make([]byte, maxKeyMaterialLen) + a = &dummyAttester{} + keys = newTestKeys(t) + ctx, cancel = context.WithCancel(context.Background()) + workers = newWorkerManager(time.Second) + ) + go workers.start(ctx) + defer cancel() + + makeReq := makeReqForHandler(putStateHandler(a, retState(noSync), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusGone, errEndpointGone.Error()), + ) + + makeReq = makeReqForHandler(putStateHandler(a, retState(isWorker), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusGone, errEndpointGone.Error()), + ) + + makeReq = makeReqForHandler(putStateHandler(a, retState(inProgress), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), + ) + + makeReq = makeReqForHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(tooLargeKey)), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), ) - - // As long as we don't hit our (generous) upload limit, we always expect an - // HTTP 200 response. - almostTooLargeKey := make([]byte, 1024*1024) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(almostTooLargeKey)), newResp(http.StatusOK, ""), ) +} + +func TestGetPutStateHandlers(t *testing.T) { + var ( + a = &dummyAttester{} + keys = newTestKeys(t) + appKeys = "application keys" + ctx, cancel = context.WithCancel(context.Background()) + workers = newWorkerManager(time.Second) + ) + go workers.start(ctx) + defer cancel() - // Subsequent calls to the endpoint overwrite the previous call. - expected := []byte("foobar") + // Set application state. + makeReq := makeReqForHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, - makeReq(http.MethodPut, pathState, bytes.NewReader(expected)), + makeReq(http.MethodPut, pathState, strings.NewReader(appKeys)), newResp(http.StatusOK, ""), ) - // Now retrieve the state and make sure that it's what we sent earlier. + // Retrieve previously-set application state. + makeReq = makeReqForHandler(getStateHandler(retState(isWorker), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), - newResp(http.StatusOK, string(expected)), + newResp(http.StatusOK, appKeys), ) } From ae0fd4cf25d0a3debccac757b133b583a2c38fbd Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 11:43:04 -0500 Subject: [PATCH 77/99] Rename helper functions for clarity. --- handlers_test.go | 41 ++++++++++++++++++++--------------------- metrics_test.go | 4 ++-- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/handlers_test.go b/handlers_test.go index c0b2e88..fe6ae34 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -21,8 +21,7 @@ import ( "time" ) -// makeRequestFor is a helper function that creates an HTTP request. -func makeRequestFor(srv *http.Server) func(method, path string, body io.Reader) *http.Response { +func makeReqToSrv(srv *http.Server) func(method, path string, body io.Reader) *http.Response { return func(method, path string, body io.Reader) *http.Response { req := httptest.NewRequest(method, path, body) rec := httptest.NewRecorder() @@ -31,7 +30,7 @@ func makeRequestFor(srv *http.Server) func(method, path string, body io.Reader) } } -func makeReqForHandler(handler http.HandlerFunc) func(method, path string, body io.Reader) *http.Response { +func makeReqToHandler(handler http.HandlerFunc) func(method, path string, body io.Reader) *http.Response { return func(method, path string, body io.Reader) *http.Response { req := httptest.NewRequest(method, path, body) w := httptest.NewRecorder() @@ -100,7 +99,7 @@ func assertResponse(t *testing.T, actual, expected *http.Response) { } func TestRootHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) + makeReq := makeReqToSrv(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathRoot, nil), @@ -112,7 +111,7 @@ func TestRootHandler(t *testing.T) { // instructing it to spin up its Internet-facing Web server. func signalReady(t *testing.T, e *Enclave) { t.Helper() - makeReq := makeRequestFor(e.intSrv) + makeReq := makeReqToSrv(e.intSrv) assertResponse(t, makeReq(http.MethodGet, pathReady, nil), @@ -129,25 +128,25 @@ func signalReady(t *testing.T, e *Enclave) { func TestGetStateHandler(t *testing.T) { var keys = newTestKeys(t) - makeReq := makeReqForHandler(getStateHandler(retState(noSync), keys)) + makeReq := makeReqToHandler(getStateHandler(retState(noSync), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), newResp(http.StatusGone, errEndpointGone.Error()), ) - makeReq = makeReqForHandler(getStateHandler(retState(isLeader), keys)) + makeReq = makeReqToHandler(getStateHandler(retState(isLeader), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), newResp(http.StatusGone, errEndpointGone.Error()), ) - makeReq = makeReqForHandler(getStateHandler(retState(isWorker), keys)) + makeReq = makeReqToHandler(getStateHandler(retState(isWorker), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), newResp(http.StatusOK, string(keys.getAppKeys())), ) - makeReq = makeReqForHandler(getStateHandler(retState(inProgress), keys)) + makeReq = makeReqToHandler(getStateHandler(retState(inProgress), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), @@ -166,25 +165,25 @@ func TestPutStateHandler(t *testing.T) { go workers.start(ctx) defer cancel() - makeReq := makeReqForHandler(putStateHandler(a, retState(noSync), keys, workers)) + makeReq := makeReqToHandler(putStateHandler(a, retState(noSync), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), newResp(http.StatusGone, errEndpointGone.Error()), ) - makeReq = makeReqForHandler(putStateHandler(a, retState(isWorker), keys, workers)) + makeReq = makeReqToHandler(putStateHandler(a, retState(isWorker), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), newResp(http.StatusGone, errEndpointGone.Error()), ) - makeReq = makeReqForHandler(putStateHandler(a, retState(inProgress), keys, workers)) + makeReq = makeReqToHandler(putStateHandler(a, retState(inProgress), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), ) - makeReq = makeReqForHandler(putStateHandler(a, retState(isLeader), keys, workers)) + makeReq = makeReqToHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(tooLargeKey)), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), @@ -207,14 +206,14 @@ func TestGetPutStateHandlers(t *testing.T) { defer cancel() // Set application state. - makeReq := makeReqForHandler(putStateHandler(a, retState(isLeader), keys, workers)) + makeReq := makeReqToHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, strings.NewReader(appKeys)), newResp(http.StatusOK, ""), ) // Retrieve previously-set application state. - makeReq = makeReqForHandler(getStateHandler(retState(isWorker), keys)) + makeReq = makeReqToHandler(getStateHandler(retState(isWorker), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), newResp(http.StatusOK, appKeys), @@ -272,7 +271,7 @@ func TestHashHandler(t *testing.T) { validHash := [sha256.Size]byte{} validHashB64 := base64.StdEncoding.EncodeToString(validHash[:]) e := createEnclave(&defaultCfg) - makeReq := makeRequestFor(e.intSrv) + makeReq := makeReqToSrv(e.intSrv) // Send invalid Base64. assertResponse(t, @@ -377,7 +376,7 @@ func TestReadyHandler(t *testing.T) { func TestAttestationHandlerWhileProfiling(t *testing.T) { cfg := defaultCfg cfg.UseProfiling = true - makeReq := makeRequestFor(createEnclave(&cfg).extPubSrv) + makeReq := makeReqToSrv(createEnclave(&cfg).extPubSrv) // Ensure that the attestation handler aborts if profiling is enabled. assertResponse(t, @@ -389,7 +388,7 @@ func TestAttestationHandlerWhileProfiling(t *testing.T) { func TestAttestationHandler(t *testing.T) { prodCfg := defaultCfg prodCfg.Debug = false - makeReq := makeRequestFor(createEnclave(&prodCfg).extPubSrv) + makeReq := makeReqToSrv(createEnclave(&prodCfg).extPubSrv) assertResponse(t, makeReq(http.MethodPost, pathAttestation, nil), @@ -417,7 +416,7 @@ func TestAttestationHandler(t *testing.T) { } func TestConfigHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).extPubSrv) + makeReq := makeReqToSrv(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathConfig, nil), @@ -429,7 +428,7 @@ func TestHeartbeatHandler(t *testing.T) { var ( e = createEnclave(&defaultCfg) keys = newTestKeys(t) - makeReq = makeRequestFor(e.extPrivSrv) + makeReq = makeReqToSrv(e.extPrivSrv) ) e.setupLeader(context.Background()) e.keys.set(keys) @@ -450,7 +449,7 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { var ( wg = sync.WaitGroup{} leaderEnclave = createEnclave(&defaultCfg) - makeReq = makeRequestFor(leaderEnclave.extPrivSrv) + makeReq = makeReqToSrv(leaderEnclave.extPrivSrv) workerKeys = newTestKeys(t) setWorkerKeys = func(keys *enclaveKeys) error { defer wg.Done() diff --git a/metrics_test.go b/metrics_test.go index 0ccfbdc..3cf6320 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -18,7 +18,7 @@ func TestHandlerMetrics(t *testing.T) { // installed. c.PrometheusPort = 80 enclave := createEnclave(&c) - makeReq := makeRequestFor(enclave.extPubSrv) + makeReq := makeReqToSrv(enclave.extPubSrv) // GET /enclave/config assertResponse(t, @@ -50,7 +50,7 @@ func TestHandlerMetrics(t *testing.T) { ), float64(1)) // POST /enclave/hash - makeReq = makeRequestFor(enclave.intSrv) + makeReq = makeReqToSrv(enclave.intSrv) assertResponse(t, makeReq(http.MethodPost, pathHash, bytes.NewBufferString("foo")), newResp(http.StatusBadRequest, errNoBase64.Error()), From 436ed32ec412bc5bb901335c1c7ae6631cf39f90 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 12:10:36 -0500 Subject: [PATCH 78/99] Add unit test. --- handlers_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/handlers_test.go b/handlers_test.go index fe6ae34..0e85b22 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -491,3 +491,33 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { wg.Wait() assertEqual(t, leaderEnclave.keys.equal(workerKeys), true) } + +func TestGetLeaderHandler(t *testing.T) { + var ( + weAreLeader = make(chan struct{}) + unexpectedSuffix = "?nonce=0000000000000000000000000000000000000000" + ) + nonce, err := newNonce() + failOnErr(t, err) + + // Don't provide the expected nonce. + makeReq := makeReqToHandler(getLeaderHandler(nonce, weAreLeader)) + assertResponse(t, + makeReq(http.MethodGet, pathLeader, nil), + newResp(http.StatusBadRequest, errNoNonce.Error()), + ) + + // Send an unexpected nonce. + assertResponse(t, + makeReq(http.MethodGet, pathLeader+unexpectedSuffix, nil), + newResp(http.StatusOK, ""), + ) + + // Send the expected nonce. + go func() { <-weAreLeader }() + suffix := fmt.Sprintf("?nonce=%x", nonce) + assertResponse(t, + makeReq(http.MethodGet, pathLeader+suffix, nil), + newResp(http.StatusOK, ""), + ) +} From 277491e3740e3b977b80664fb2039260dd7dea96 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 12:17:02 -0500 Subject: [PATCH 79/99] Delete unused context. --- enclave.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/enclave.go b/enclave.go index 90692f7..9a168c2 100644 --- a/enclave.go +++ b/enclave.go @@ -69,7 +69,6 @@ var ( type Enclave struct { sync.RWMutex attester - ctx context.Context cfg *Config extPubSrv, extPrivSrv *http.Server intSrv *http.Server @@ -226,7 +225,6 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ attester: &nitroAttester{}, - ctx: ctx, cfg: cfg, extPubSrv: &http.Server{ Handler: chi.NewRouter(), From 193b2bdfc1b6de7037dbe022249e55daccaf9394 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 12:37:58 -0500 Subject: [PATCH 80/99] Replace context with stop channel. --- attestation_test.go | 3 +-- enclave.go | 22 ++++++++++++---------- enclave_test.go | 3 +-- handlers_test.go | 35 +++++++++++++++++------------------ main.go | 6 ++---- workers.go | 5 ++--- workers_test.go | 31 +++++++++++++++---------------- 7 files changed, 50 insertions(+), 55 deletions(-) diff --git a/attestation_test.go b/attestation_test.go index 4ce8fcd..efe7da7 100644 --- a/attestation_test.go +++ b/attestation_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "crypto/sha256" "encoding/base64" "net/http" @@ -46,7 +45,7 @@ func TestAttestationHashes(t *testing.T) { // Start the enclave. This is going to initialize the hash over the HTTPS // certificate. - if err := e.Start(context.Background()); err != nil { + if err := e.Start(); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck diff --git a/enclave.go b/enclave.go index 9a168c2..305747d 100644 --- a/enclave.go +++ b/enclave.go @@ -217,7 +217,7 @@ func (c *Config) String() string { } // NewEnclave creates and returns a new enclave with the given config. -func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { +func NewEnclave(cfg *Config) (*Enclave, error) { if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("failed to create enclave: %w", err) } @@ -315,7 +315,7 @@ func NewEnclave(ctx context.Context, cfg *Config) (*Enclave, error) { // Start starts the Nitro Enclave. If something goes wrong, the function // returns an error. -func (e *Enclave) Start(ctx context.Context) error { +func (e *Enclave) Start() error { var ( err error leader = e.getLeader(pathHeartbeat) @@ -355,7 +355,7 @@ func (e *Enclave) Start(ctx context.Context) error { } // Check if we are the leader. - if !e.weAreLeader(ctx) { + if !e.weAreLeader() { elog.Println("Obtaining worker's hostname.") worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) err = asWorker(e.setupWorkerPostSync, e.attester).registerWith(leader, worker) @@ -382,7 +382,7 @@ func (e *Enclave) setSyncState(state int) { } // weAreLeader figures out if the enclave is the leader or worker. -func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { +func (e *Enclave) weAreLeader() (result bool) { var ( err error ourNonce nonce @@ -418,6 +418,8 @@ func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { for { go makeLeaderRequest(leader, ourNonce, areWeLeader, errChan) select { + case <-e.stop: + return case <-errChan: elog.Println("Not yet able to talk to leader designation endpoint.") time.Sleep(time.Second) @@ -425,7 +427,7 @@ func (e *Enclave) weAreLeader(ctx context.Context) (result bool) { case result = <-areWeLeader: return case <-weAreLeader: - e.setupLeader(ctx) + e.setupLeader() result = true return case <-timeout.C: @@ -446,15 +448,15 @@ func (e *Enclave) setupWorkerPostSync(keys *enclaveKeys) error { // Start our heartbeat. worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) - go e.workerHeartbeat(context.Background(), worker) + go e.workerHeartbeat(worker) return nil } // setupLeader performs necessary setup tasks like starting the worker event // loop and installing leader-specific HTTP handlers. -func (e *Enclave) setupLeader(ctx context.Context) { - go e.workers.start(ctx) +func (e *Enclave) setupLeader() { + go e.workers.start(e.stop) // Make leader-specific endpoint available. e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) elog.Println("Set up leader endpoint and started worker event loop.") @@ -464,7 +466,7 @@ func (e *Enclave) setupLeader(ctx context.Context) { // know that we're still alive, and 2) to compare key material. If it turns out // that the leader has different key material than the worker, the worker // re-registers itself, which triggers key re-synchronization. -func (e *Enclave) workerHeartbeat(ctx context.Context, worker *url.URL) { +func (e *Enclave) workerHeartbeat(worker *url.URL) { elog.Println("Starting worker's heartbeat loop.") defer elog.Println("Exiting worker's heartbeat loop.") var ( @@ -477,7 +479,7 @@ func (e *Enclave) workerHeartbeat(ctx context.Context, worker *url.URL) { for { select { - case <-ctx.Done(): + case <-e.stop: return case <-timer.C: hbBody.HashedKeys = e.keys.hashAndB64() diff --git a/enclave_test.go b/enclave_test.go index 701081a..624411e 100644 --- a/enclave_test.go +++ b/enclave_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "testing" ) @@ -26,7 +25,7 @@ func assertEqual(t *testing.T, is, should interface{}) { } func createEnclave(cfg *Config) *Enclave { - e, err := NewEnclave(context.Background(), cfg) + e, err := NewEnclave(cfg) if err != nil { panic(err) } diff --git a/handlers_test.go b/handlers_test.go index 0e85b22..6064901 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "crypto/sha256" "crypto/tls" "encoding/base64" @@ -159,11 +158,11 @@ func TestPutStateHandler(t *testing.T) { almostTooLargeKey = make([]byte, maxKeyMaterialLen) a = &dummyAttester{} keys = newTestKeys(t) - ctx, cancel = context.WithCancel(context.Background()) + stop = make(chan struct{}) workers = newWorkerManager(time.Second) ) - go workers.start(ctx) - defer cancel() + go workers.start(stop) + defer close(stop) makeReq := makeReqToHandler(putStateHandler(a, retState(noSync), keys, workers)) assertResponse(t, @@ -196,14 +195,14 @@ func TestPutStateHandler(t *testing.T) { func TestGetPutStateHandlers(t *testing.T) { var ( - a = &dummyAttester{} - keys = newTestKeys(t) - appKeys = "application keys" - ctx, cancel = context.WithCancel(context.Background()) - workers = newWorkerManager(time.Second) + a = &dummyAttester{} + keys = newTestKeys(t) + appKeys = "application keys" + stop = make(chan struct{}) + workers = newWorkerManager(time.Second) ) - go workers.start(ctx) - defer cancel() + go workers.start(stop) + defer close(stop) // Set application state. makeReq := makeReqToHandler(putStateHandler(a, retState(isLeader), keys, workers)) @@ -235,12 +234,12 @@ func TestProxyHandler(t *testing.T) { c := defaultCfg c.AppWebSrv = u - e, err := NewEnclave(context.Background(), &c) + e, err := NewEnclave(&c) if err != nil { t.Fatal(err) } e.revProxy = httputil.NewSingleHostReverseProxy(u) - if err := e.Start(context.Background()); err != nil { + if err := e.Start(); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck @@ -314,7 +313,7 @@ func TestReadiness(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = false e := createEnclave(&cfg) - if err := e.Start(context.Background()); err != nil { + if err := e.Start(); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck @@ -350,7 +349,7 @@ func TestReadyHandler(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = true e := createEnclave(&cfg) - if err := e.Start(context.Background()); err != nil { + if err := e.Start(); err != nil { t.Fatal(err) } defer e.Stop() //nolint:errcheck @@ -430,7 +429,7 @@ func TestHeartbeatHandler(t *testing.T) { keys = newTestKeys(t) makeReq = makeReqToSrv(e.extPrivSrv) ) - e.setupLeader(context.Background()) + e.setupLeader() e.keys.set(keys) tooLargeBuf := bytes.NewBuffer(make([]byte, maxHeartbeatBody+1)) @@ -460,10 +459,10 @@ func TestHeartbeatHandlerWithSync(t *testing.T) { workerSrv = httptest.NewTLSServer(worker) ) defer workerSrv.Close() - if err := leaderEnclave.Start(context.Background()); err != nil { + if err := leaderEnclave.Start(); err != nil { t.Fatal(err) } - leaderEnclave.setupLeader(context.Background()) + leaderEnclave.setupLeader() wg.Add(1) // Mock two functions to make the leader enclave talk to our test server. diff --git a/main.go b/main.go index 091b710..23f314b 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "context" "errors" "flag" "io" @@ -134,13 +133,12 @@ func main() { elog.Println("WARNING: Using debug mode, which must not be enabled in production!") } - ctx := context.Background() - enclave, err := NewEnclave(ctx, c) + enclave, err := NewEnclave(c) if err != nil { elog.Fatalf("Failed to create enclave: %v", err) } - if err := enclave.Start(ctx); err != nil { + if err := enclave.Start(); err != nil { elog.Fatalf("Enclave terminated: %v", err) } diff --git a/workers.go b/workers.go index 03167c6..39935e7 100644 --- a/workers.go +++ b/workers.go @@ -1,7 +1,6 @@ package main import ( - "context" "net/url" "time" ) @@ -29,7 +28,7 @@ func newWorkerManager(timeout time.Duration) *workerManager { } // start starts the worker manager's event loop. -func (w *workerManager) start(ctx context.Context) { +func (w *workerManager) start(stop chan struct{}) { var ( set = make(workers) timer = time.NewTicker(w.timeout) @@ -39,7 +38,7 @@ func (w *workerManager) start(ctx context.Context) { for { select { - case <-ctx.Done(): + case <-stop: return case <-timer.C: diff --git a/workers_test.go b/workers_test.go index 5cba745..bf489dc 100644 --- a/workers_test.go +++ b/workers_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "net/url" "sync" "testing" @@ -10,11 +9,11 @@ import ( func TestWorkerRegistration(t *testing.T) { var ( - w = newWorkerManager(time.Minute) - ctx, cancel = context.WithCancel(context.Background()) + w = newWorkerManager(time.Minute) + stop = make(chan struct{}) ) - go w.start(ctx) - defer cancel() + go w.start(stop) + defer close(stop) // Identical URLs are only tracked once. worker1 := url.URL{Host: "foo"} @@ -39,14 +38,14 @@ func TestWorkerRegistration(t *testing.T) { func TestForAll(t *testing.T) { var ( - w = newWorkerManager(time.Millisecond) - ctx, cancel = context.WithCancel(context.Background()) - wg = sync.WaitGroup{} - mutex = sync.Mutex{} - total = 0 + w = newWorkerManager(time.Millisecond) + stop = make(chan struct{}) + wg = sync.WaitGroup{} + mutex = sync.Mutex{} + total = 0 ) - go w.start(ctx) - defer cancel() + go w.start(stop) + defer close(stop) w.register(&url.URL{Host: "foo"}) w.register(&url.URL{Host: "bar"}) @@ -67,11 +66,11 @@ func TestForAll(t *testing.T) { func TestIneffectiveForAll(t *testing.T) { var ( - w = newWorkerManager(time.Minute) - ctx, cancel = context.WithCancel(context.Background()) + w = newWorkerManager(time.Minute) + stop = make(chan struct{}) ) - go w.start(ctx) - defer cancel() + go w.start(stop) + defer close(stop) // Make sure that forAll finishes for an empty worker set. w.forAll(func(_ *url.URL) {}) From 3bad177dea4029682a9f8674e717baf136f65182 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Thu, 31 Aug 2023 14:35:41 -0500 Subject: [PATCH 81/99] Add draft of HTTP API docs. --- README.md | 1 + doc/http-api.md | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 doc/http-api.md diff --git a/README.md b/README.md index 88e6855..48b2ec0 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,6 @@ system, take a look at our [research paper](https://arxiv.org/abs/2206.04123). * [How to use nitriding](doc/usage.md) * [System architecture](doc/architecture.md) +* [HTTP API](doc/http-api.md) * [Horizontal scaling](doc/key-synchronization.md) * [Example application](example/) diff --git a/doc/http-api.md b/doc/http-api.md new file mode 100644 index 0000000..2780fc9 --- /dev/null +++ b/doc/http-api.md @@ -0,0 +1,101 @@ +# Nitriding's HTTP API + +## External endpoints, reachable to the Internet + +* `GET /enclave` Returns an index page informing the visitor that this code runs + inside an enclave. + The enclave responds with status code `200`. + +* `GET /enclave/attestation?nonce={nonce}` Returns an attestation document + containing the given nonce. + The attestation document is encoded with Base64. + If all goes well, the enclave responds with status code `200`. + +* `GET /enclave/config` Returns nitriding's configuration. + The enclave responds with status code `200`. + +## External endpoints, reachable to other enclaves + +* `GET /enclave/sync?nonce={nonce}` Exposed by workers, the leader talks to this endpoint to initiate key synchronization. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + If all goes well, the worker responds with status code `200` and the following JSON-formatted body: + ``` + { + "document": "{Base64-encoded attestation document}", + } + ``` + +* `POST /enclave/sync` Exposed by workers, the leader talks to this endpoint to + complete key synchronization. + + The request must contain the following JSON-formatted body: + ``` + { + "document": "{Base64-encoded attestation document}", + "encrypted_keys": "{Base64-encoded, encrypted enclave keys}", + } + ``` + If all goes well, the worker responds with status code `200`. + +* `POST /enclave/heartbeat` Exposed by the leader, workers periodically send a heartbeat to this endpoint. + The request must contain the following JSON-formatted body: + ``` + { + "hashed_keys": "{hashed_keys}", + "worker_hostname": "{worker_hostname}", + } + ``` + `worker_hostname` contains the worker's EC2-internal hostname, e.g., `ip-12-34-56-78.us-east-2.compute.internal`. + `hashed_keys` contains the Base64-encoded SHA-256 hash over the worker's enclave key material. + If all goes well, the leader responds with status code `200`. + +* `GET /enclave/leader?nonce={nonce}` Exposed by all enclaves, this endpoint + helps enclaves figure out who the leader is. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + All enclaves create a random `nonce` and send it to the leader's endpoint. + If the leader notices that it's talking to itself (by comparing the received nonce to its previously-generated nonce), + it designated itself as the leader. + After that, the leader responds with status code `410`. + Before that, the leader responds with status code `200`. + While workers expose this endpoint too, they should never receive any requests. + +## Internal endpoints, reachable to the application + +* `GET /enclave/ready` Invoked by the application to signal its readiness. + When nitriding is invoked with the command line argument `-wait-for-app`, + it refrains from starting its external Web servers until the application + signals its readiness by calling this endpoint, after which nitriding starts + the external Web servers. + +* `GET /enclave/state` Returns the application's state in the response body. + This endpoint allows an application to retrieve state + (e.g., confidential key material) that was previously set by the "leader" application. + If synchronization is not enabled via the `-fqdn-leader` command line + argument, the endpoint responds with status code `410`. + If synchronization is enabled but leader designation is currently in progress, + the endpoint responds with status code `503`. + If synchronization is enabled and the enclave is the leader, + the endpoint responds with status code `410`. + Finally, if synchronization is enabled _and_ the enclave is a worker, + the endpoint returns the application's state in the response body and + responds with status code `200`. + +* `PUT /enclave/state` Sets the application's state. + This endpoint allows the "leader" application to set state that is + subsequently synchronized with worker enclaves. + If synchronization is not enabled via the `-fqdn-leader` command line + argument, the endpoint responds with status code `410`. + If synchronization is enabled but leader designation is currently in progress, + the endpoint responds with status code `503`. + If synchronization is enabled and the enclave is a worker, + the endpoint responds with status code `410`. + Finally, if synchronization is enabled _and_ the enclave is the leader, + the endpoint saves the state that's set in the request body and + responds with status code `200`. + +* `POST /enclave/hash` Allows the application to set a hash that's included in + attestation documents. + The enclave application can invoke this endpoint to submit a SHA-256 hash that + nitriding is subsequently going to include in attestation documents. + The Base64-encoded SHA-256 hash must be given in the request body. + If all goes well, the endpoint responds with status code `200`. \ No newline at end of file From a206598042f1b1731c96f2c7ea0fc7b1705048dc Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Fri, 1 Sep 2023 07:59:21 -0500 Subject: [PATCH 82/99] Replace status code 410 with 403. --- doc/http-api.md | 4 ++-- handlers.go | 5 +++-- handlers_test.go | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/http-api.md b/doc/http-api.md index 2780fc9..d69db27 100644 --- a/doc/http-api.md +++ b/doc/http-api.md @@ -71,7 +71,7 @@ This endpoint allows an application to retrieve state (e.g., confidential key material) that was previously set by the "leader" application. If synchronization is not enabled via the `-fqdn-leader` command line - argument, the endpoint responds with status code `410`. + argument, the endpoint responds with status code `403`. If synchronization is enabled but leader designation is currently in progress, the endpoint responds with status code `503`. If synchronization is enabled and the enclave is the leader, @@ -84,7 +84,7 @@ This endpoint allows the "leader" application to set state that is subsequently synchronized with worker enclaves. If synchronization is not enabled via the `-fqdn-leader` command line - argument, the endpoint responds with status code `410`. + argument, the endpoint responds with status code `403`. If synchronization is enabled but leader designation is currently in progress, the endpoint responds with status code `503`. If synchronization is enabled and the enclave is a worker, diff --git a/handlers.go b/handlers.go index ecd4d99..c09de8d 100644 --- a/handlers.go +++ b/handlers.go @@ -30,6 +30,7 @@ var ( errNoBase64 = errors.New("no Base64 given") errDesignationInProgress = errors.New("leader designation in progress") errEndpointGone = errors.New("endpoint not meant to be used") + errKeySyncDisabled = errors.New("key synchronization is disabled") ) func errNo200(code int) error { @@ -63,7 +64,7 @@ func getStateHandler(getSyncState func() int, keys *enclaveKeys) http.HandlerFun return func(w http.ResponseWriter, r *http.Request) { switch getSyncState() { case noSync: - fallthrough + http.Error(w, errKeySyncDisabled.Error(), http.StatusForbidden) case isLeader: http.Error(w, errEndpointGone.Error(), http.StatusGone) case inProgress: @@ -98,7 +99,7 @@ func putStateHandler( return func(w http.ResponseWriter, r *http.Request) { switch getSyncState() { case noSync: - fallthrough + http.Error(w, errKeySyncDisabled.Error(), http.StatusForbidden) case isWorker: http.Error(w, errEndpointGone.Error(), http.StatusGone) case inProgress: diff --git a/handlers_test.go b/handlers_test.go index 6064901..48bffb9 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -130,7 +130,7 @@ func TestGetStateHandler(t *testing.T) { makeReq := makeReqToHandler(getStateHandler(retState(noSync), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), - newResp(http.StatusGone, errEndpointGone.Error()), + newResp(http.StatusForbidden, errKeySyncDisabled.Error()), ) makeReq = makeReqToHandler(getStateHandler(retState(isLeader), keys)) @@ -167,7 +167,7 @@ func TestPutStateHandler(t *testing.T) { makeReq := makeReqToHandler(putStateHandler(a, retState(noSync), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), - newResp(http.StatusGone, errEndpointGone.Error()), + newResp(http.StatusForbidden, errKeySyncDisabled.Error()), ) makeReq = makeReqToHandler(putStateHandler(a, retState(isWorker), keys, workers)) From 2a8f019d534937449295f0c4107611b0e7cdfc7a Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:38:01 -0500 Subject: [PATCH 83/99] Add unit test. --- attester_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/attester_test.go b/attester_test.go index ba7835e..68d1ced 100644 --- a/attester_test.go +++ b/attester_test.go @@ -1,12 +1,38 @@ package main import ( + "bytes" "errors" "testing" "github.com/hf/nitrite" ) +func TestDummyAttestation(t *testing.T) { + var ( + d = newDummyAttester() + workersNonce = nonce{1, 2, 3} + hashOfEncrypted = []byte("this is a hash") + ) + + attstn, err := d.createAttstn(&leaderAuxInfo{ + WorkersNonce: workersNonce, + HashOfEncrypted: hashOfEncrypted, + }) + failOnErr(t, err) + + aux, err := d.verifyAttstn(attstn, workersNonce) + failOnErr(t, err) + + leaderAux := aux.(*leaderAuxInfo) + if leaderAux.WorkersNonce != workersNonce { + t.Fatal("Extracted unexpected workers nonce.") + } + if !bytes.Equal(leaderAux.HashOfEncrypted, hashOfEncrypted) { + t.Fatalf("Extracted unexpected hash over encrypted keys.") + } +} + func TestVerifyNitroAttstn(t *testing.T) { var n = newNitroAttester() _, err := n.verifyAttstn([]byte("foobar"), nonce{}) From e5245c9c48ef2f25e248bc930b9febeb965118c5 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:38:16 -0500 Subject: [PATCH 84/99] Clean up and delete unnecessary code. --- attester.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/attester.go b/attester.go index a081f2f..2180d15 100644 --- a/attester.go +++ b/attester.go @@ -53,6 +53,11 @@ type leaderAuxInfo struct { // auxiliary information into JSON, and does not do any cryptography. type dummyAttester struct{} +// newDummyAttester returns a new dummyAttester. +func newDummyAttester() *dummyAttester { + return new(dummyAttester) +} + func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) { return json.Marshal(aux) } @@ -67,9 +72,9 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if err := json.Unmarshal(doc, &w); err != nil { return nil, err } - if len(w.WorkersNonce) == nonceLen && len(w.LeadersNonce) == nonceLen && w.PublicKey != nil { + if w.PublicKey != nil { if n.b64() != w.LeadersNonce.b64() { - return nil, errors.New("leader nonce not in cache") + return nil, errNonceMismatch } return &w, nil } @@ -78,9 +83,9 @@ func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { if err := json.Unmarshal(doc, &l); err != nil { return nil, err } - if len(l.WorkersNonce) == nonceLen && l.HashOfEncrypted != nil { + if l.HashOfEncrypted != nil { if n.b64() != l.WorkersNonce.b64() { - return nil, errors.New("worker nonce not in cache") + return nil, errNonceMismatch } return &l, nil } @@ -94,7 +99,7 @@ type nitroAttester struct{} // newNitroAttester returns a new nitroAttester. func newNitroAttester() *nitroAttester { - return &nitroAttester{} + return new(nitroAttester) } // createAttstn asks the AWS Nitro Enclave hypervisor for an attestation @@ -124,11 +129,7 @@ func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { if err != nil { return nil, err } - defer func() { - if err = s.Close(); err != nil { - elog.Printf("Error closing NSM session: %v", err) - } - }() + defer s.Close() res, err := s.Send(&request.Attestation{ Nonce: nonce, @@ -162,8 +163,6 @@ func (*nitroAttester) verifyAttstn(doc []byte, ourNonce nonce) (auxInfo, error) return nil, err } if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { - elog.Printf("Our PCR values:\n%s", prettyFormat(ourPCRs)) - elog.Printf("Their PCR values:\n%s", prettyFormat(their.Document.PCRs)) return nil, errPCRMismatch } @@ -180,14 +179,12 @@ func (*nitroAttester) verifyAttstn(doc []byte, ourNonce nonce) (auxInfo, error) // If the "public key" field contains padding, we know that we're // dealing with a leader's auxiliary information. if bytes.Equal(their.Document.PublicKey, padding) { - elog.Println("Extracting leader's auxiliary information.") return &leaderAuxInfo{ WorkersNonce: theirNonce, HashOfEncrypted: their.Document.UserData, }, nil } - elog.Println("Extracting worker's auxiliary information.") workersNonce, err := sliceToNonce(their.Document.UserData) if err != nil { return nil, err From f66886cc1945fd5be216e3837e1ebb659bd4f682 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:53:59 -0500 Subject: [PATCH 85/99] Add test and clean up code. --- certcache.go | 16 +++++++++------- certcache_test.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/certcache.go b/certcache.go index a871935..e233e5a 100644 --- a/certcache.go +++ b/certcache.go @@ -9,13 +9,15 @@ import ( "golang.org/x/crypto/acme/autocert" ) +var errUninitializedCert = errors.New("certificate not yet initialized") + // certRetriever stores an HTTPS certificate and implements the GetCertificate // function signature, which allows our Web servers to retrieve the // certificate when clients connect: // https://pkg.go.dev/crypto/tls#Config type certRetriever struct { - sync.RWMutex - cert *tls.Certificate + sync.Mutex // Guards cert. + cert *tls.Certificate } func (c *certRetriever) set(cert *tls.Certificate) { @@ -26,19 +28,19 @@ func (c *certRetriever) set(cert *tls.Certificate) { } func (c *certRetriever) get(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { - c.RLock() - defer c.RUnlock() + c.Lock() + defer c.Unlock() if c.cert == nil { - return nil, errors.New("certificate not yet initialized") + return nil, errUninitializedCert } return c.cert, nil } // certCache implements the autocert.Cache interface. type certCache struct { - sync.RWMutex - cache map[string][]byte + sync.RWMutex // Guards cache. + cache map[string][]byte } func newCertCache() *certCache { diff --git a/certcache_test.go b/certcache_test.go index a591bc8..0b386df 100644 --- a/certcache_test.go +++ b/certcache_test.go @@ -3,12 +3,23 @@ package main import ( "bytes" "context" + "crypto/tls" "errors" "testing" "golang.org/x/crypto/acme/autocert" ) +func TestPrematureGet(t *testing.T) { + var ( + err error + r = new(certRetriever) + ) + + _, err = r.get(new(tls.ClientHelloInfo)) + assertEqual(t, err, errUninitializedCert) +} + func TestGet(t *testing.T) { var err error var key = "foo" From 2552574822a2ff2a77485dc0af9d4393eb00f646 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:55:53 -0500 Subject: [PATCH 86/99] Improve clarity and remove unnecessary function calls. --- enclave_keys.go | 31 +++++++++++++++---------------- enclave_keys_test.go | 4 ++-- handlers.go | 4 ++-- sync_leader.go | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/enclave_keys.go b/enclave_keys.go index 47e2c8d..24560c3 100644 --- a/enclave_keys.go +++ b/enclave_keys.go @@ -14,17 +14,17 @@ import ( // implements getters and setters that allow for thread-safe setting and getting // of members. type enclaveKeys struct { - sync.RWMutex + sync.Mutex NitridingKey []byte `json:"nitriding_key"` NitridingCert []byte `json:"nitriding_cert"` AppKeys []byte `json:"app_keys"` } func (e1 *enclaveKeys) equal(e2 *enclaveKeys) bool { - e1.RLock() - e2.RLock() - defer e1.RUnlock() - defer e2.RUnlock() + e1.Lock() + e2.Lock() + defer e1.Unlock() + defer e2.Unlock() return bytes.Equal(e1.NitridingCert, e2.NitridingCert) && bytes.Equal(e1.NitridingKey, e2.NitridingKey) && @@ -47,17 +47,13 @@ func (e *enclaveKeys) setNitridingKeys(key, cert []byte) { } func (e *enclaveKeys) set(newKeys *enclaveKeys) { - e.Lock() - defer e.Unlock() - - e.NitridingKey = newKeys.NitridingKey - e.NitridingCert = newKeys.NitridingCert - e.AppKeys = newKeys.AppKeys + e.setAppKeys(newKeys.AppKeys) + e.setNitridingKeys(newKeys.NitridingKey, newKeys.NitridingCert) } -func (e *enclaveKeys) get() *enclaveKeys { - e.RLock() - defer e.RUnlock() +func (e *enclaveKeys) copy() *enclaveKeys { + e.Lock() + defer e.Unlock() return &enclaveKeys{ NitridingKey: e.NitridingKey, @@ -67,8 +63,8 @@ func (e *enclaveKeys) get() *enclaveKeys { } func (e *enclaveKeys) getAppKeys() []byte { - e.RLock() - defer e.RUnlock() + e.Lock() + defer e.Unlock() return e.AppKeys } @@ -77,6 +73,9 @@ func (e *enclaveKeys) getAppKeys() []byte { // resulting string is not confidential as it's impractical to reverse the key // material. func (e *enclaveKeys) hashAndB64() string { + e.Lock() + defer e.Unlock() + keys := append(append(e.NitridingCert, e.NitridingKey...), e.AppKeys...) hash := sha256.Sum256(keys) return base64.StdEncoding.EncodeToString(hash[:]) diff --git a/enclave_keys_test.go b/enclave_keys_test.go index c344cdd..28952e7 100644 --- a/enclave_keys_test.go +++ b/enclave_keys_test.go @@ -55,7 +55,7 @@ func TestGetKeys(t *testing.T) { var ( testKeys = newTestKeys(t) appKeys = testKeys.getAppKeys() - keys = testKeys.get() + keys = testKeys.copy() ) // Ensure that the application key is retrieved correctly. @@ -72,7 +72,7 @@ func TestGetKeys(t *testing.T) { func TestModifyCloneObject(t *testing.T) { var ( keys = newTestKeys(t) - clonedKeys = keys.get() + clonedKeys = keys.copy() ) // Make sure that setting the clone's application keys does not affect the diff --git a/handlers.go b/handlers.go index c09de8d..aead3d7 100644 --- a/handlers.go +++ b/handlers.go @@ -120,7 +120,7 @@ func putStateHandler( workers.length()) go workers.forAll( func(worker *url.URL) { - if err := asLeader(enclaveKeys.get(), a).syncWith(worker); err != nil { + if err := asLeader(enclaveKeys, a).syncWith(worker); err != nil { workers.unregister(worker) } }, @@ -228,7 +228,7 @@ func heartbeatHandler(e *Enclave) http.HandlerFunc { var ( hb heartbeatRequest syncAndRegister = func(keys *enclaveKeys, worker *url.URL) { - if err := asLeader(keys.get(), e.attester).syncWith(worker); err == nil { + if err := asLeader(keys, e.attester).syncWith(worker); err == nil { e.workers.register(worker) } } diff --git a/sync_leader.go b/sync_leader.go index f213446..32812c9 100644 --- a/sync_leader.go +++ b/sync_leader.go @@ -97,7 +97,7 @@ func (s *leaderSync) syncWith(worker *url.URL) (err error) { // that the worker put into its auxiliary information. pubKey := &[boxKeyLen]byte{} copy(pubKey[:], workerAux.PublicKey[:]) - jsonKeys, err := json.Marshal(s.keys.get()) + jsonKeys, err := json.Marshal(s.keys.copy()) if err != nil { return err } From 8638ab655e71f59e0420396b87719750be5295f7 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:56:51 -0500 Subject: [PATCH 87/99] Use sync.Once and change return code after first run. --- handlers.go | 16 +++++++++------- handlers_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/handlers.go b/handlers.go index aead3d7..3018a21 100644 --- a/handlers.go +++ b/handlers.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strings" + "sync" ) const ( @@ -168,17 +169,18 @@ func hashHandler(e *Enclave) http.HandlerFunc { // signal that it's ready, instructing nitriding to start its Internet-facing // Web server. We initially gate access to the Internet-facing API to avoid // the issuance of unexpected attestation documents that lack the application's -// hash because the application couldn't register it in time. The downside is -// that state synchronization among enclaves does not work until the -// application signalled its readiness. While not ideal, we chose to ignore -// this for now. +// hash because the application couldn't register it in time. // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func readyHandler(e *Enclave) http.HandlerFunc { +func readyHandler(ready chan struct{}) http.HandlerFunc { + var once sync.Once return func(w http.ResponseWriter, r *http.Request) { - close(e.ready) - w.WriteHeader(http.StatusOK) + once.Do(func() { + close(ready) + w.WriteHeader(http.StatusOK) + }) + w.WriteHeader(http.StatusGone) } } diff --git a/handlers_test.go b/handlers_test.go index 48bffb9..01e0776 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -345,7 +345,7 @@ func TestReadiness(t *testing.T) { }(t, u) } -func TestReadyHandler(t *testing.T) { +func TestReadinessWithWaitForUp(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = true e := createEnclave(&cfg) @@ -372,6 +372,31 @@ func TestReadyHandler(t *testing.T) { } } +func TestReadyHandler(t *testing.T) { + var ( + ready = make(chan struct{}) + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + defer wg.Done() + <-ready + }() + + makeReq := makeReqToHandler(readyHandler(ready)) + assertResponse(t, + makeReq(http.MethodGet, pathReady, nil), + newResp(http.StatusOK, ""), + ) + wg.Wait() + + // Subsequent calls should return 410 Gone. + assertResponse(t, + makeReq(http.MethodGet, pathReady, nil), + newResp(http.StatusGone, ""), + ) +} + func TestAttestationHandlerWhileProfiling(t *testing.T) { cfg := defaultCfg cfg.UseProfiling = true From fba9cb73a1dc50136c0a9e3f580de778961af3cb Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Sun, 10 Sep 2023 21:57:27 -0500 Subject: [PATCH 88/99] Remove unused function. --- util.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/util.go b/util.go index 9790d36..b864bff 100644 --- a/util.go +++ b/util.go @@ -8,7 +8,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" - "encoding/json" "encoding/pem" "errors" "fmt" @@ -254,11 +253,3 @@ func makeLeaderRequest(leader *url.URL, ourNonce nonce, areWeLeader chan bool, e } errChan <- fmt.Errorf("leader designation endpoint returned %d", resp.StatusCode) } - -func prettyFormat(c any) string { - s, err := json.MarshalIndent(c, "", " ") - if err != nil { - return "" - } - return string(s) -} From fe8205335f5d4a3b73997b5f50b6eda023cb9762 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:11:51 -0500 Subject: [PATCH 89/99] Remove comment. --- enclave.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/enclave.go b/enclave.go index 305747d..9b04273 100644 --- a/enclave.go +++ b/enclave.go @@ -28,8 +28,6 @@ import ( "golang.org/x/crypto/acme/autocert" ) -// TODO: Support Let's Encrypt (if we choose to). - const ( acmeCertCacheDir = "cert-cache" certificateOrg = "AWS Nitro enclave application" From 20910101f2cafcbbc4146dbe7ddf8a94802cdc25 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:12:05 -0500 Subject: [PATCH 90/99] Fix comment. --- util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util.go b/util.go index b864bff..393ba21 100644 --- a/util.go +++ b/util.go @@ -48,10 +48,10 @@ var _getSyncURL = func(host string, port uint16) *url.URL { } } -// newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS +// _newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS // certificate validation. In the context of nitriding, this is fine because -// all we need is a *confidential* channel, and not an authenticated channel. -// Authentication is handled via attestation documents. +// all we need is a *confidential* channel; not an authenticated channel. +// Authentication is handled on the next layer, using attestation documents. func _newUnauthenticatedHTTPClient() *http.Client { transport := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, From cd09ceef4213e7941bcf259551723984f68653c0 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:13:07 -0500 Subject: [PATCH 91/99] Use sync.Mutex instead. --- enclave.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/enclave.go b/enclave.go index 9b04273..6874f05 100644 --- a/enclave.go +++ b/enclave.go @@ -65,13 +65,13 @@ var ( // Enclave represents a service running inside an AWS Nitro Enclave. type Enclave struct { - sync.RWMutex attester + sync.Mutex // Guard syncState. cfg *Config + syncState int extPubSrv, extPrivSrv *http.Server intSrv *http.Server promSrv *http.Server - syncState int revProxy *httputil.ReverseProxy hashes *AttestationHashes promRegistry *prometheus.Registry @@ -367,8 +367,8 @@ func (e *Enclave) Start() error { // getSyncState returns the enclave's key synchronization state. func (e *Enclave) getSyncState() int { - e.RLock() - defer e.RUnlock() + e.Lock() + defer e.Unlock() return e.syncState } From 44487e66f2d66222be5712029e77e65ef44b17f0 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:13:49 -0500 Subject: [PATCH 92/99] Only expose endpoint if necessary. --- enclave.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/enclave.go b/enclave.go index 6874f05..4e95339 100644 --- a/enclave.go +++ b/enclave.go @@ -289,7 +289,9 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Register enclave-internal HTTP API. m = e.intSrv.Handler.(*chi.Mux) - m.Get(pathReady, readyHandler(e)) + if cfg.WaitForApp { + m.Get(pathReady, readyHandler(e.ready)) + } m.Get(pathState, getStateHandler(e.getSyncState, e.keys)) m.Put(pathState, putStateHandler(e.attester, e.getSyncState, e.keys, e.workers)) m.Post(pathHash, hashHandler(e)) From 138580f6747b841de072e9f760a42e4334353bf1 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:15:10 -0500 Subject: [PATCH 93/99] Expose Prometheus metrics for heartbeats. --- enclave.go | 4 ++++ metrics.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/enclave.go b/enclave.go index 4e95339..1308b6e 100644 --- a/enclave.go +++ b/enclave.go @@ -486,6 +486,7 @@ func (e *Enclave) workerHeartbeat(worker *url.URL) { body, err := json.Marshal(hbBody) if err != nil { elog.Printf("Error marshalling heartbeat request: %v", err) + e.metrics.heartbeats.With(badHb(err)).Inc() continue } @@ -496,13 +497,16 @@ func (e *Enclave) workerHeartbeat(worker *url.URL) { ) if err != nil { elog.Printf("Error posting heartbeat to leader: %v", err) + e.metrics.heartbeats.With(badHb(err)).Inc() continue } if resp.StatusCode != http.StatusOK { + e.metrics.heartbeats.With(badHb(fmt.Errorf("got status code %d", resp.StatusCode))).Inc() elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) continue } elog.Println("Successfully sent heartbeat to leader.") + e.metrics.heartbeats.With(goodHb).Inc() } } } diff --git a/metrics.go b/metrics.go index 3889362..38cbc34 100644 --- a/metrics.go +++ b/metrics.go @@ -18,10 +18,22 @@ const ( notAvailable = "n/a" ) +var ( + goodHb = prometheus.Labels{ + respErr: notAvailable, + } + badHb = func(err error) prometheus.Labels { + return prometheus.Labels{ + respErr: err.Error(), + } + } +) + // metrics contains our Prometheus metrics. type metrics struct { reqs *prometheus.CounterVec proxiedReqs *prometheus.CounterVec + heartbeats *prometheus.CounterVec } // newMetrics initializes our Prometheus metrics. @@ -44,6 +56,14 @@ func newMetrics(reg prometheus.Registerer, namespace string) *metrics { }, []string{reqPath, reqMethod, respStatus, respErr}, ), + heartbeats: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "heartbeats", + Help: "Heartbeats sent to the leader enclave", + }, + []string{respErr}, + ), } reg.MustRegister(m.proxiedReqs) reg.MustRegister(m.reqs) From 1d6f59fdb98cf8c3b49f8ff0f79b9f10399b3ce7 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:34:05 -0500 Subject: [PATCH 94/99] Minor improvements to clarity. --- doc/http-api.md | 53 +++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/doc/http-api.md b/doc/http-api.md index d69db27..47fe911 100644 --- a/doc/http-api.md +++ b/doc/http-api.md @@ -2,23 +2,29 @@ ## External endpoints, reachable to the Internet -* `GET /enclave` Returns an index page informing the visitor that this code runs +* `GET /enclave` Returns an index page explaining that this code runs inside an enclave. - The enclave responds with status code `200`. + The enclave responds with status code `200 OK`. * `GET /enclave/attestation?nonce={nonce}` Returns an attestation document containing the given nonce. - The attestation document is encoded with Base64. - If all goes well, the enclave responds with status code `200`. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + The attestation document is encoded using Base64. + If all goes well, the enclave responds with status code `200 OK`. * `GET /enclave/config` Returns nitriding's configuration. - The enclave responds with status code `200`. + The enclave responds with status code `200 OK`. + +* `GET /enclave/debug` If enabled, returns profiling information. + If nitriding is invoked with the `-debug` command line flag, + it exposes this endpoint to make profiling information available. + If all goes well, the enclave responds with status code `200 OK`. ## External endpoints, reachable to other enclaves * `GET /enclave/sync?nonce={nonce}` Exposed by workers, the leader talks to this endpoint to initiate key synchronization. `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. - If all goes well, the worker responds with status code `200` and the following JSON-formatted body: + If all goes well, the worker responds with status code `200 OK` and the following JSON-formatted body: ``` { "document": "{Base64-encoded attestation document}", @@ -35,7 +41,7 @@ "encrypted_keys": "{Base64-encoded, encrypted enclave keys}", } ``` - If all goes well, the worker responds with status code `200`. + If all goes well, the worker responds with status code `200 OK`. * `POST /enclave/heartbeat` Exposed by the leader, workers periodically send a heartbeat to this endpoint. The request must contain the following JSON-formatted body: @@ -47,55 +53,60 @@ ``` `worker_hostname` contains the worker's EC2-internal hostname, e.g., `ip-12-34-56-78.us-east-2.compute.internal`. `hashed_keys` contains the Base64-encoded SHA-256 hash over the worker's enclave key material. - If all goes well, the leader responds with status code `200`. + If all goes well, the leader responds with status code `200 OK`. * `GET /enclave/leader?nonce={nonce}` Exposed by all enclaves, this endpoint helps enclaves figure out who the leader is. `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. All enclaves create a random `nonce` and send it to the leader's endpoint. If the leader notices that it's talking to itself (by comparing the received nonce to its previously-generated nonce), - it designated itself as the leader. - After that, the leader responds with status code `410`. - Before that, the leader responds with status code `200`. + it designates itself as the leader. + After that, the leader responds with status code `410 Gone`. + Workers know that they are workers when they receive status code `410 Gone`. + Before that, the leader responds with status code `200 OK`. While workers expose this endpoint too, they should never receive any requests. ## Internal endpoints, reachable to the application -* `GET /enclave/ready` Invoked by the application to signal its readiness. +* `GET /enclave/ready` Used by the enclave application to signal its readiness. When nitriding is invoked with the command line argument `-wait-for-app`, it refrains from starting its external Web servers until the application signals its readiness by calling this endpoint, after which nitriding starts the external Web servers. + The first invocation of this endpoint returns status code `200 OK`. + Subsequent invocations return status code `410 Gone`. * `GET /enclave/state` Returns the application's state in the response body. This endpoint allows an application to retrieve state (e.g., confidential key material) that was previously set by the "leader" application. If synchronization is not enabled via the `-fqdn-leader` command line - argument, the endpoint responds with status code `403`. + argument, the endpoint responds with status code `403 Forbidden`. If synchronization is enabled but leader designation is currently in progress, - the endpoint responds with status code `503`. + the endpoint responds with status code `503 Service Unavailable`. If synchronization is enabled and the enclave is the leader, - the endpoint responds with status code `410`. + the endpoint responds with status code `410 Gone`. Finally, if synchronization is enabled _and_ the enclave is a worker, the endpoint returns the application's state in the response body and - responds with status code `200`. + responds with status code `200 OK`. + The application's state is returned without encoding, + using the `application/octet-stream` content type. * `PUT /enclave/state` Sets the application's state. This endpoint allows the "leader" application to set state that is subsequently synchronized with worker enclaves. If synchronization is not enabled via the `-fqdn-leader` command line - argument, the endpoint responds with status code `403`. + argument, the endpoint responds with status code `403 Forbidden`. If synchronization is enabled but leader designation is currently in progress, - the endpoint responds with status code `503`. + the endpoint responds with status code `503 Service Unavailable`. If synchronization is enabled and the enclave is a worker, - the endpoint responds with status code `410`. + the endpoint responds with status code `410 Gone`. Finally, if synchronization is enabled _and_ the enclave is the leader, the endpoint saves the state that's set in the request body and - responds with status code `200`. + responds with status code `200 OK`. * `POST /enclave/hash` Allows the application to set a hash that's included in attestation documents. The enclave application can invoke this endpoint to submit a SHA-256 hash that nitriding is subsequently going to include in attestation documents. The Base64-encoded SHA-256 hash must be given in the request body. - If all goes well, the endpoint responds with status code `200`. \ No newline at end of file + If all goes well, the endpoint responds with status code `200 OK`. \ No newline at end of file From 0fe09226b011b7ed925bf9fd4367c074f4889797 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 07:34:20 -0500 Subject: [PATCH 95/99] Remove unnecessary newline. --- enclave.go | 1 - 1 file changed, 1 deletion(-) diff --git a/enclave.go b/enclave.go index 1308b6e..06857ea 100644 --- a/enclave.go +++ b/enclave.go @@ -55,7 +55,6 @@ const ( inProgress = 1 // Leader designation is in progress. isLeader = 2 // The enclave is the leader. isWorker = 3 // The enclave is a worker. - ) var ( From f1594eb7103e29f59cdb6cc007b86a31c78f1f01 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 09:01:22 -0500 Subject: [PATCH 96/99] Update docs to match protocol. --- doc/key-synchronization.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md index de52e9f..2a6f137 100644 --- a/doc/key-synchronization.md +++ b/doc/key-synchronization.md @@ -51,8 +51,8 @@ To set up key synchronization, several steps are necessary: enclave. In the next and final interaction, the leader encrypts its sensitive enclave keys $K_s$ using the worker's ephemeral public key $pk$, resulting in $E = \textrm{Enc}(K_s, pk)$. The leader then asks its hypervisor to create an - attestation document $A_l$ containing $\textrm{nonce}_w$ and $E$. The leader - sends $A_l$ to the worker in a separate `POST` request. + attestation document $A_l$ containing $\textrm{nonce}_w$ and SHA-256($E$). + The leader sends $A_l$ and $E$ to the worker in a separate `POST` request. 6. Upon receiving $A_l$, the worker first verifies the attestation document (same as above), and decrypts $E$ using $sk$, revealing in $K_s$, the sensitive enclave keys. At this point, key synchronization is complete. @@ -89,8 +89,6 @@ sequenceDiagram participant leaderApp as Enclave application participant leader as Leader enclave end - participant leaderEC2 as Leader EC2 - participant workerEC2 as Worker EC2 box rgba(100, 100, 100, .1) Worker enclave participant worker as Worker enclave participant workerApp as Enclave application @@ -99,10 +97,12 @@ sequenceDiagram leader->>leader: Generate HTTPS certificate leaderApp->>leaderApp: Generate key material -Note over leader,leaderEC2: Designating enclave as leader -leaderEC2->>+leader: GET /enclave/leader -leader->>leader: Expose leader-specific endpoints -leader-->>-leaderEC2: OK +Note over leader,worker: Enclaves designate the leader +worker->>+leader: GET /enclave/leader (nonce_w) +leader-->>-worker: OK +worker->>worker: Did not call itself: worker +leader->>leader: GET /enclave/leader (nonce_l) +leader->>leader: Did call itself: leader Note over leaderApp,leader: Application sets its key material leaderApp->>+leader: PUT /enclave/state (key material) @@ -121,7 +121,7 @@ worker->>worker: Create attestation, nonce, and ephemeral keys worker-->>-leader: OK (Attestation(nonce_l, nonce_w, pk)) leader->>leader: Verify & create attestation -leader->>+worker: POST /enclave/sync (Attestation(nonce_w, E(keys, pk))) +leader->>+worker: POST /enclave/sync (Attestation(nonce_w, SHA-256(E(keys, pk))), E(keys, pk)) worker->>worker: Verify attestation & install keys worker-->>-leader: OK From eb379302ead31b7b89c1538f3609e0af8b4a541e Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 11 Sep 2023 09:07:48 -0500 Subject: [PATCH 97/99] Limit the # of bytes we're willing to read. --- util.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/util.go b/util.go index 393ba21..8478e3c 100644 --- a/util.go +++ b/util.go @@ -178,6 +178,10 @@ func getLocalAddr() string { } func getLocalEC2Hostname() (string, error) { + const ( + maxTokenLen = 100 + maxHostnameLen = 255 + ) // IMDSv2, which we are using, is session-oriented (God knows why), so we // first obtain a session token from the service. req, err := http.NewRequest(http.MethodPut, metadataSvcToken, nil) @@ -189,7 +193,7 @@ func getLocalEC2Hostname() (string, error) { if err != nil { return "", err } - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(newLimitReader(resp.Body, maxTokenLen)) if err != nil { return "", err } @@ -206,7 +210,7 @@ func getLocalEC2Hostname() (string, error) { if err != nil { return "", err } - body, err = io.ReadAll(resp.Body) + body, err = io.ReadAll(newLimitReader(resp.Body, maxHostnameLen)) if err != nil { return "", err } From 94a9d73af4cbab13db06446c33f4eb0b6f840db7 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 25 Sep 2023 12:36:36 -0500 Subject: [PATCH 98/99] Register heartbeat metrics with Prometheus. --- metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/metrics.go b/metrics.go index 38cbc34..68a2643 100644 --- a/metrics.go +++ b/metrics.go @@ -67,6 +67,7 @@ func newMetrics(reg prometheus.Registerer, namespace string) *metrics { } reg.MustRegister(m.proxiedReqs) reg.MustRegister(m.reqs) + reg.MustRegister(m.heartbeats) reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ Namespace: namespace, From a5fd697f5458ab01f57223d144b89538a6b98bc2 Mon Sep 17 00:00:00 2001 From: Philipp Winter Date: Mon, 25 Sep 2023 12:38:32 -0500 Subject: [PATCH 99/99] Also run 'go vet' and govulncheck. --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index f7ad015..e2010c8 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ all: lint test $(binary) .PHONY: lint lint: $(godeps) golangci-lint run + go vet ./... + govulncheck ./... .PHONY: test test: $(godeps)