Implement key synchronization.

Makefile

@@ -8,6 +8,8 @@ all: lint test $(binary)
 .PHONY: lint
 lint: $(godeps)
 	golangci-lint run
+	go vet ./...
+	govulncheck ./...
 
 .PHONY: test
 test: $(godeps)

README.md

@@ -55,4 +55,6 @@ system, take a look at our [research paper](https://arxiv.org/abs/2206.04123).
 
 * [How to use nitriding](doc/usage.md)
 * [System architecture](doc/architecture.md)
+* [HTTP API](doc/http-api.md)
+* [Horizontal scaling](doc/key-synchronization.md)
 * [Example application](example/)

attestation.go

@@ -7,24 +7,19 @@ import (
 	"fmt"
 
 	"github.com/hf/nitrite"
-	"github.com/hf/nsm"
-	"github.com/hf/nsm/request"
 )
 
 const (
-	nonceLen       = 20           // The size of a nonce in bytes.
-	nonceNumDigits = nonceLen * 2 // The number of hex digits in a nonce.
-	maxAttDocLen   = 5000         // A (reasonable?) upper limit for attestation doc lengths.
-	hashPrefix     = "sha256:"
-	hashSeparator  = ";"
+	hashPrefix    = "sha256:"
+	hashSeparator = ";"
 )
 
 var (
-	errBadForm           = "failed to parse POST form data"
-	errNoNonce           = "could not find nonce in URL query parameters"
-	errBadNonceFormat    = fmt.Sprintf("unexpected nonce format; must be %d-digit hex string", nonceNumDigits)
-	errFailedAttestation = "failed to obtain attestation document from hypervisor"
-	errProfilingSet      = "attestation disabled because profiling is enabled"
+	errBadForm           = errors.New("failed to parse POST form data")
+	errNoNonce           = errors.New("could not find nonce in URL query parameters")
+	errBadNonceFormat    = fmt.Errorf("unexpected nonce format; must be %d-digit hex string", nonceLen*2)
+	errFailedAttestation = errors.New("failed to obtain attestation document from hypervisor")
+	errProfilingSet      = errors.New("attestation disabled because profiling is enabled")
 
 	// getPCRValues is a variable pointing to a function that returns PCR
 	// values.  Using a variable allows us to easily mock the function in our
@@ -55,7 +50,7 @@ func (a *AttestationHashes) Serialize() []byte {
 // _getPCRValues returns the enclave's platform configuration register (PCR)
 // values.
 func _getPCRValues() (map[uint][]byte, error) {
-	rawAttDoc, err := attest(nil, nil, nil)
+	rawAttDoc, err := newNitroAttester().createAttstn(nil)
 	if err != nil {
 		return nil, err
 	}
@@ -76,6 +71,12 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool {
 	}
 
 	for pcr, ourValue := range ourPCRs {
+		// PCR4 contains a hash over the parent's instance ID.  Our enclaves run
+		// on different parent instances; PCR4 will therefore always differ:
+		// https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html
+		if pcr == 4 {
+			continue
+		}
 		theirValue, exists := theirPCRs[pcr]
 		if !exists {
 			return false
@@ -86,33 +87,3 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool {
 	}
 	return true
 }
-
-// attest takes as input a nonce, user-provided data and a public key, and then
-// asks the Nitro hypervisor to return a signed attestation document that
-// contains all three values.
-func attest(nonce, userData, publicKey []byte) ([]byte, error) {
-	s, err := nsm.OpenDefaultSession()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err = s.Close(); err != nil {
-			elog.Printf("Attestation: Failed to close default NSM session: %s", err)
-		}
-	}()
-
-	res, err := s.Send(&request.Attestation{
-		Nonce:     nonce,
-		UserData:  userData,
-		PublicKey: publicKey,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if res.Attestation == nil || res.Attestation.Document == nil {
-		return nil, errors.New("NSM device did not return an attestation")
-	}
-
-	return res.Attestation.Document, nil
-}

attestation_test.go

@@ -20,6 +20,12 @@ func TestArePCRsIdentical(t *testing.T) {
 		t.Fatal("Failed to recognize identical PCRs as such.")
 	}
 
+	// PCR4 should be ignored.
+	pcr1[4], pcr2[4] = []byte("foo"), []byte("bar")
+	if !arePCRsIdentical(pcr1, pcr2) {
+		t.Fatal("Failed to recognize identical PCRs as such.")
+	}
+
 	// Add a new PCR value, so our two maps are no longer identical.
 	pcr1[2] = []byte("barfoo")
 	if arePCRsIdentical(pcr1, pcr2) {
@@ -49,7 +55,7 @@ func TestAttestationHashes(t *testing.T) {
 	rec := httptest.NewRecorder()
 	buf := bytes.NewBufferString(base64.StdEncoding.EncodeToString(appKeyHash[:]))
 	req := httptest.NewRequest(http.MethodPost, pathHash, buf)
-	e.privSrv.Handler.ServeHTTP(rec, req)
+	e.intSrv.Handler.ServeHTTP(rec, req)
 
 	s := e.hashes.Serialize()
 	expectedLen := sha256.Size*2 + len(hashPrefix)*2 + len(hashSeparator)

attester.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/hf/nitrite"
+	"github.com/hf/nsm"
+	"github.com/hf/nsm/request"
+)
+
+var (
+	errPCRMismatch     = errors.New("PCR values differ")
+	errNonceMismatch   = errors.New("nonce is unexpected")
+	errNoAttstnFromNSM = errors.New("NSM device did not return an attestation")
+	padding            = []byte("dummy")
+)
+
+// attester defines functions for the creation and verification of attestation
+// documents.  Making this an interface helps with testing: It allows us to
+// implement a dummy attester that works without the AWS Nitro hypervisor.
+type attester interface {
+	createAttstn(auxInfo) ([]byte, error)
+	verifyAttstn([]byte, nonce) (auxInfo, error)
+}
+
+type auxInfo interface{}
+
+// workerAuxInfo holds the auxilitary information of an attestation document
+// requested by clients.
+type clientAuxInfo struct {
+	clientNonce       nonce
+	attestationHashes []byte
+}
+
+// workerAuxInfo holds the auxiliary information of the worker's attestation
+// document.
+type workerAuxInfo struct {
+	WorkersNonce nonce  `json:"workers_nonce"`
+	LeadersNonce nonce  `json:"leaders_nonce"`
+	PublicKey    []byte `json:"public_key"`
+}
+
+// leaderAuxInfo holds the auxiliary information of the leader's attestation
+// document.
+type leaderAuxInfo struct {
+	WorkersNonce    nonce  `json:"workers_nonce"`
+	HashOfEncrypted []byte `json:"hash_of_encrypted"`
+}
+
+// dummyAttester helps with local testing.  The interface simply turns
+// auxiliary information into JSON, and does not do any cryptography.
+type dummyAttester struct{}
+
+// newDummyAttester returns a new dummyAttester.
+func newDummyAttester() *dummyAttester {
+	return new(dummyAttester)
+}
+
+func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) {
+	return json.Marshal(aux)
+}
+
+func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) {
+	var (
+		w workerAuxInfo
+		l leaderAuxInfo
+	)
+
+	// First, assume we're dealing with a worker's auxiliary information.
+	if err := json.Unmarshal(doc, &w); err != nil {
+		return nil, err
+	}
+	if w.PublicKey != nil {
+		if n.b64() != w.LeadersNonce.b64() {
+			return nil, errNonceMismatch
+		}
+		return &w, nil
+	}
+
+	// Next, let's assume it's a leader.
+	if err := json.Unmarshal(doc, &l); err != nil {
+		return nil, err
+	}
+	if l.HashOfEncrypted != nil {
+		if n.b64() != l.WorkersNonce.b64() {
+			return nil, errNonceMismatch
+		}
+		return &l, nil
+	}
+
+	return nil, errors.New("invalid auxiliary information")
+}
+
+// nitroAttester implements the attester interface by drawing on the AWS Nitro
+// Enclave hypervisor.
+type nitroAttester struct{}
+
+// newNitroAttester returns a new nitroAttester.
+func newNitroAttester() *nitroAttester {
+	return new(nitroAttester)
+}
+
+// createAttstn asks the AWS Nitro Enclave hypervisor for an attestation
+// document that contains the given auxiliary information.
+func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) {
+	var nonce, userData, publicKey []byte
+
+	// Prepare our auxiliary information.  If the public key field is unused, we
+	// pad it with dummy bytes because the nitrite package (which we use to
+	// verify attestation documents) expects all three fields to be set.
+	switch v := aux.(type) {
+	case *workerAuxInfo:
+		nonce = v.LeadersNonce[:]
+		userData = v.WorkersNonce[:]
+		publicKey = v.PublicKey
+	case *leaderAuxInfo:
+		nonce = v.WorkersNonce[:]
+		userData = v.HashOfEncrypted
+		publicKey = padding
+	case *clientAuxInfo:
+		nonce = v.clientNonce[:]
+		userData = v.attestationHashes
+		publicKey = padding
+	}
+
+	s, err := nsm.OpenDefaultSession()
+	if err != nil {
+		return nil, err
+	}
+	defer s.Close()
+
+	res, err := s.Send(&request.Attestation{
+		Nonce:     nonce,
+		UserData:  userData,
+		PublicKey: publicKey,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if res.Attestation == nil || res.Attestation.Document == nil {
+		return nil, errNoAttstnFromNSM
+	}
+
+	return res.Attestation.Document, nil
+}
+
+// verifyAttstn verifies the given attestation document and, if successful,
+// returns the document's auxiliary information.
+func (*nitroAttester) verifyAttstn(doc []byte, ourNonce nonce) (auxInfo, error) {
+	// First, verify the remote enclave's attestation document.
+	opts := nitrite.VerifyOptions{CurrentTime: currentTime()}
+	their, err := nitrite.Verify(doc, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Verify that the remote enclave's PCR values (e.g., the image ID) are
+	// identical to ours.
+	ourPCRs, err := getPCRValues()
+	if err != nil {
+		return nil, err
+	}
+	if !arePCRsIdentical(ourPCRs, their.Document.PCRs) {
+		return nil, errPCRMismatch
+	}
+
+	// Verify that the remote enclave's attestation document contains the nonce
+	// that we asked it to embed.
+	theirNonce, err := sliceToNonce(their.Document.Nonce)
+	if err != nil {
+		return nil, err
+	}
+	if ourNonce != theirNonce {
+		return nil, errNonceMismatch
+	}
+
+	// If the "public key" field contains padding, we know that we're
+	// dealing with a leader's auxiliary information.
+	if bytes.Equal(their.Document.PublicKey, padding) {
+		return &leaderAuxInfo{
+			WorkersNonce:    theirNonce,
+			HashOfEncrypted: their.Document.UserData,
+		}, nil
+	}
+
+	workersNonce, err := sliceToNonce(their.Document.UserData)
+	if err != nil {
+		return nil, err
+	}
+	return &workerAuxInfo{
+		WorkersNonce: workersNonce,
+		LeadersNonce: theirNonce,
+		PublicKey:    their.Document.PublicKey,
+	}, nil
+}

attester_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/hf/nitrite"
+)
+
+func TestDummyAttestation(t *testing.T) {
+	var (
+		d               = newDummyAttester()
+		workersNonce    = nonce{1, 2, 3}
+		hashOfEncrypted = []byte("this is a hash")
+	)
+
+	attstn, err := d.createAttstn(&leaderAuxInfo{
+		WorkersNonce:    workersNonce,
+		HashOfEncrypted: hashOfEncrypted,
+	})
+	failOnErr(t, err)
+
+	aux, err := d.verifyAttstn(attstn, workersNonce)
+	failOnErr(t, err)
+
+	leaderAux := aux.(*leaderAuxInfo)
+	if leaderAux.WorkersNonce != workersNonce {
+		t.Fatal("Extracted unexpected workers nonce.")
+	}
+	if !bytes.Equal(leaderAux.HashOfEncrypted, hashOfEncrypted) {
+		t.Fatalf("Extracted unexpected hash over encrypted keys.")
+	}
+}
+
+func TestVerifyNitroAttstn(t *testing.T) {
+	var n = newNitroAttester()
+	_, err := n.verifyAttstn([]byte("foobar"), nonce{})
+	assertEqual(t, errors.Is(err, nitrite.ErrBadCOSESign1Structure), true)
+}
+
+func TestCreateNitroAttstn(t *testing.T) {
+	var n = newNitroAttester()
+	_, err := n.createAttstn(nil)
+	assertEqual(t, err != nil, true)
+}

cache.go

@@ -5,10 +5,6 @@ import (
 	"time"
 )
 
-const (
-	defaultItemExpiry = time.Minute
-)
-
 // cache implements a simple cache whose items expire.
 type cache struct {
 	sync.RWMutex