From fa1154b0c1044c1644b59fe1236163247c541179 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Mon, 19 Aug 2024 16:43:30 +0000 Subject: [PATCH 01/15] Fix contenttype Need to initialize the writer before setting the content type --- storage/gcp/gcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/gcp/gcp.go b/storage/gcp/gcp.go index 47797379..46b95dbd 100644 --- a/storage/gcp/gcp.go +++ b/storage/gcp/gcp.go @@ -669,13 +669,13 @@ func (s *gcsStorage) setObject(ctx context.Context, objName string, data []byte, obj := bkt.Object(objName) var w *gcs.Writer - w.ObjectAttrs.ContentType = contType if cond == nil { w = obj.NewWriter(ctx) } else { w = obj.If(*cond).NewWriter(ctx) } + w.ObjectAttrs.ContentType = contType if _, err := w.Write(data); err != nil { return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket, err) } From 810e51decb0227866e56513665eb8b64b73ec980 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Fri, 9 Aug 2024 19:09:11 +0000 Subject: [PATCH 02/15] implement a map on GCS --- personalities/sctfe/storage/gcp/map.go | 114 +++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 personalities/sctfe/storage/gcp/map.go diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/map.go new file mode 100644 index 00000000..0c03f749 --- /dev/null +++ b/personalities/sctfe/storage/gcp/map.go @@ -0,0 +1,114 @@ +// Copyright 2016 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gcp implements SCTFE storage systems for issuers and deduplication. +// +// The interfaces are defined in sctfe/storage.go +package gcp + +import ( + "context" + "encoding/hex" + "fmt" + "net/http" + "path" + + gcs "cloud.google.com/go/storage" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "k8s.io/klog/v2" +) + +// GCSStorage is a map backed by GCS on GCP. +type GCSStorage struct { + bucket *gcs.BucketHandle + prefix string +} + +// NewGCSStorage creates a new GCSStorage. +// +// The specified bucket must exist or an error will be returned. +func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix string) (*GCSStorage, error) { + c, err := gcs.NewClient(ctx, gcs.WithJSONReads()) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %v", err) + } + + it := c.Buckets(ctx, projectID) + for { + bAttrs, err := it.Next() + if err == iterator.Done { + return nil, fmt.Errorf("bucket %q does not exist, please create it", bucket) + } + if err != nil { + return nil, fmt.Errorf("error scanning buckets: %v", err) + } + if bAttrs.Name == bucket { + break + } + } + r := &GCSStorage{ + bucket: c.Bucket(bucket), + prefix: prefix, + } + + return r, nil +} + +// keyToObjName converts bytes to a GCS object name. +func (s *GCSStorage) keyToObjName(key [32]byte) string { + return path.Join(s.prefix, hex.EncodeToString(key[:])) +} + +// Exists checks whether an object is stored under key. +func (s *GCSStorage) Exists(ctx context.Context, key [32]byte) (bool, error) { + objName := s.keyToObjName(key) + obj := s.bucket.Object(objName) + _, err := obj.Attrs(ctx) + if err == gcs.ErrObjectNotExist { + return false, nil + } + if err != nil { + return false, fmt.Errorf("error fetching attributes for %q :%v", objName, err) + } + klog.V(2).Infof("Exists: object %q already exists in bucket %q", objName, s.bucket.BucketName()) + return true, nil +} + +// Add stores the provided data under key. +// +// If there is already an object under that key, it does not override it, and returns. +// TODO(phboneff): consider reading the object to make sure it's identical +func (s *GCSStorage) Add(ctx context.Context, key [32]byte, data []byte) error { + objName := s.keyToObjName(key) + obj := s.bucket.Object(objName) + + // Don't overwrite if it already exists + w := obj.If(gcs.Conditions{DoesNotExist: true}).NewWriter(ctx) + + if _, err := w.Write(data); err != nil { + return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket.BucketName(), err) + } + + if err := w.Close(); err != nil { + // If we run into a precondition failure error, it means that the object already exists. + if ee, ok := err.(*googleapi.Error); ok && ee.Code == http.StatusPreconditionFailed { + klog.V(2).Infof("Add: object %q already exists in bucket %q, continuing", objName, s.bucket.BucketName()) + return nil + } + + return fmt.Errorf("failed to close write on %q: %v", objName, err) + } + return nil +} From 5abff2b5e839c39c805855af12df12f09065b111 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 8 Aug 2024 20:00:21 +0000 Subject: [PATCH 03/15] add AddIssuerChain to the SCTFE Storage interface --- personalities/sctfe/storage.go | 51 ++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index 8bf78b5c..91f5301d 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -16,34 +16,75 @@ package sctfe import ( "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "github.com/google/certificate-transparency-go/x509" tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/ctonly" + "golang.org/x/sync/errgroup" ) // Storage provides all the storage primitives necessary to write to a ct-static-api log. type Storage interface { - // Add assign an index to the provided Entry, stages the entry for integration, and return it the assigned index. + // Add assign an index to the provided Entry, stages the entry for integration, and return it the assigned index. Add(context.Context, *ctonly.Entry) (uint64, error) + // AddIssuerChain stores all certificates in the chain in a content-addressable store under their sha256 hash. + AddIssuerChain(context.Context, []*x509.Certificate) error +} + +type IssuerStorage interface { + Exists(ctx context.Context, key [32]byte) (bool, error) + Add(ctx context.Context, key [32]byte, data []byte) error } // CTStorage implements Storage. type CTStorage struct { storeData func(context.Context, *ctonly.Entry) (uint64, error) - // TODO(phboneff): add storeExtraData - // TODO(phboneff): add dedupe + issuers IssuerStorage } // NewCTStorage instantiates a CTStorage object. -func NewCTSTorage(logStorage tessera.Storage) (*CTStorage, error) { +func NewCTSTorage(logStorage tessera.Storage, issuerStorage IssuerStorage) (*CTStorage, error) { ctStorage := &CTStorage{ storeData: tessera.NewCertificateTransparencySequencedWriter(logStorage), + issuers: issuerStorage, } return ctStorage, nil } // Add stores CT entries. -func (cts CTStorage) Add(ctx context.Context, entry *ctonly.Entry) (uint64, error) { +func (cts *CTStorage) Add(ctx context.Context, entry *ctonly.Entry) (uint64, error) { // TODO(phboneff): add deduplication and chain storage return cts.storeData(ctx, entry) } + +// AddIssuerChain stores every certificate in the chain under its sha256. +// If an object is already stored under this hash, continues. +func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certificate) error { + errG := errgroup.Group{} + for _, c := range chain { + errG.Go(func() error { + key := sha256.Sum256(c.Raw) + // We first try and see if this issuer cert has already been stored since reads + // are cheaper than writes. + // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns + ok, err := cts.issuers.Exists(ctx, key) + if err != nil { + return fmt.Errorf("error checking if issuer %q exists: %s", hex.EncodeToString(key[:]), err) + } + if !ok { + if err = cts.issuers.Add(ctx, key, c.Raw); err != nil { + return fmt.Errorf("error adding certificate for issuer %q: %v", hex.EncodeToString(key[:]), err) + + } + } + return nil + }) + } + if err := errG.Wait(); err != nil { + return err + } + return nil +} From df4a829b3327fbbc557163080d2b35efc1d35ecc Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 8 Aug 2024 20:27:25 +0000 Subject: [PATCH 04/15] connect issuer storage service to sctfe --- personalities/sctfe/ct_server_gcp/main.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/personalities/sctfe/ct_server_gcp/main.go b/personalities/sctfe/ct_server_gcp/main.go index 1ef7bf30..57817a13 100644 --- a/personalities/sctfe/ct_server_gcp/main.go +++ b/personalities/sctfe/ct_server_gcp/main.go @@ -43,7 +43,8 @@ import ( tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/personalities/sctfe" "github.com/transparency-dev/trillian-tessera/personalities/sctfe/configpb" - "github.com/transparency-dev/trillian-tessera/storage/gcp" + gcpMap "github.com/transparency-dev/trillian-tessera/personalities/sctfe/storage/gcp" + gcpTessera "github.com/transparency-dev/trillian-tessera/storage/gcp" "golang.org/x/mod/sumdb/note" "google.golang.org/protobuf/proto" "k8s.io/klog/v2" @@ -268,14 +269,19 @@ func setupAndRegister(ctx context.Context, deadline time.Duration, vCfg *sctfe.V func newGCPStorage(ctx context.Context, vCfg *sctfe.ValidatedLogConfig, signer note.Signer) (*sctfe.CTStorage, error) { cfg := vCfg.Config.GetGcp() - gcpCfg := gcp.Config{ + gcpCfg := gcpTessera.Config{ ProjectID: cfg.ProjectId, Bucket: cfg.Bucket, Spanner: cfg.SpannerDbPath, } - storage, err := gcp.New(ctx, gcpCfg, tessera.WithCheckpointSignerVerifier(signer, nil)) + tesseraStorage, err := gcpTessera.New(ctx, gcpCfg, tessera.WithCheckpointSignerVerifier(signer, nil)) if err != nil { - return nil, fmt.Errorf("Failed to initialize GCP storage: %v", err) + return nil, fmt.Errorf("Failed to initialize GCP Tesserea storage: %v", err) } - return sctfe.NewCTSTorage(storage) + + issuerStorage, err := gcpMap.NewGCSStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/") + if err != nil { + return nil, fmt.Errorf("Failed to initialize GCP issuer storage: %v", err) + } + return sctfe.NewCTSTorage(tesseraStorage, issuerStorage) } From fa6a54309d22c468d64377b543efe06b883f972d Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 8 Aug 2024 20:29:14 +0000 Subject: [PATCH 05/15] update mock_ct_storage --- .../sctfe/mockstorage/mock_ct_storage.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/personalities/sctfe/mockstorage/mock_ct_storage.go b/personalities/sctfe/mockstorage/mock_ct_storage.go index f1f285df..373556f0 100644 --- a/personalities/sctfe/mockstorage/mock_ct_storage.go +++ b/personalities/sctfe/mockstorage/mock_ct_storage.go @@ -9,6 +9,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + x509 "github.com/google/certificate-transparency-go/x509" ctonly "github.com/transparency-dev/trillian-tessera/ctonly" ) @@ -49,3 +50,17 @@ func (mr *MockStorageMockRecorder) Add(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockStorage)(nil).Add), arg0, arg1) } + +// AddIssuerChain mocks base method. +func (m *MockStorage) AddIssuerChain(arg0 context.Context, arg1 []*x509.Certificate) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddIssuerChain", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddIssuerChain indicates an expected call of AddIssuerChain. +func (mr *MockStorageMockRecorder) AddIssuerChain(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIssuerChain", reflect.TypeOf((*MockStorage)(nil).AddIssuerChain), arg0, arg1) +} From 2ca97c21857ee199e3c79d4db743587fef174f5e Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 8 Aug 2024 18:23:44 +0000 Subject: [PATCH 06/15] store the issuer chain --- personalities/sctfe/handlers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/personalities/sctfe/handlers.go b/personalities/sctfe/handlers.go index eea7a656..4efe0001 100644 --- a/personalities/sctfe/handlers.go +++ b/personalities/sctfe/handlers.go @@ -321,6 +321,13 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err) } + // TODO(phboneff): refactor entryFromChain to avoid recomputing hashes in AddIssuerChain + if len(chain) > 1 { + if err := li.storage.AddIssuerChain(ctx, chain[1:]); err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to store issuer chain: %s", err) + } + } + klog.V(2).Infof("%s: %s => storage.Add", li.LogOrigin, method) idx, err := li.storage.Add(ctx, entry) if err != nil { From 1b7de209b9e0759956c67ff58d2327ea12e241f3 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 8 Aug 2024 21:04:41 +0000 Subject: [PATCH 07/15] add content type --- personalities/sctfe/ct_server_gcp/main.go | 2 +- personalities/sctfe/storage/gcp/map.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/personalities/sctfe/ct_server_gcp/main.go b/personalities/sctfe/ct_server_gcp/main.go index 57817a13..90a8c34d 100644 --- a/personalities/sctfe/ct_server_gcp/main.go +++ b/personalities/sctfe/ct_server_gcp/main.go @@ -279,7 +279,7 @@ func newGCPStorage(ctx context.Context, vCfg *sctfe.ValidatedLogConfig, signer n return nil, fmt.Errorf("Failed to initialize GCP Tesserea storage: %v", err) } - issuerStorage, err := gcpMap.NewGCSStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/") + issuerStorage, err := gcpMap.NewGCSStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/", "application/pkix-cert") if err != nil { return nil, fmt.Errorf("Failed to initialize GCP issuer storage: %v", err) } diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/map.go index 0c03f749..9c35389b 100644 --- a/personalities/sctfe/storage/gcp/map.go +++ b/personalities/sctfe/storage/gcp/map.go @@ -32,14 +32,15 @@ import ( // GCSStorage is a map backed by GCS on GCP. type GCSStorage struct { - bucket *gcs.BucketHandle - prefix string + bucket *gcs.BucketHandle + prefix string + contentType string } // NewGCSStorage creates a new GCSStorage. // // The specified bucket must exist or an error will be returned. -func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix string) (*GCSStorage, error) { +func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix string, contentType string) (*GCSStorage, error) { c, err := gcs.NewClient(ctx, gcs.WithJSONReads()) if err != nil { return nil, fmt.Errorf("failed to create GCS client: %v", err) @@ -59,8 +60,9 @@ func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix } } r := &GCSStorage{ - bucket: c.Bucket(bucket), - prefix: prefix, + bucket: c.Bucket(bucket), + prefix: prefix, + contentType: contentType, } return r, nil @@ -96,6 +98,7 @@ func (s *GCSStorage) Add(ctx context.Context, key [32]byte, data []byte) error { // Don't overwrite if it already exists w := obj.If(gcs.Conditions{DoesNotExist: true}).NewWriter(ctx) + w.ObjectAttrs.ContentType = s.contentType if _, err := w.Write(data); err != nil { return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket.BucketName(), err) From 3d7106393b6c25b3e8ddfe718c22be0ff77f0399 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Fri, 9 Aug 2024 12:34:25 +0000 Subject: [PATCH 08/15] add calls to AddIssuerChain to tests --- personalities/sctfe/handlers_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/personalities/sctfe/handlers_test.go b/personalities/sctfe/handlers_test.go index bf6649ce..e8f629ae 100644 --- a/personalities/sctfe/handlers_test.go +++ b/personalities/sctfe/handlers_test.go @@ -228,7 +228,7 @@ func TestAddChainWhitespace(t *testing.T) { chunk2 := "\"MIIDnTCCAoWgAwIBAgIIQoIqW4Zvv+swDQYJKoZIhvcNAQELBQAwcTELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMQ8wDQYDVQQKDAZHb29nbGUxDDAKBgNVBAsMA0VuZzEhMB8GA1UEAwwYRmFrZUNlcnRpZmljYXRlQXV0aG9yaXR5MB4XDTE2MDUxMzE0MjY0NFoXDTE5MDcxMjE0MjY0NFowcjELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMQ8wDQYDVQQKDAZHb29nbGUxDDAKBgNVBAsMA0VuZzEiMCAGA1UEAwwZRmFrZUludGVybWVkaWF0ZUF1dGhvcml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMqkDHpt6SYi1GcZyClAxr3LRDnn+oQBHbMEFUg3+lXVmEsq/xQO1s4naynV6I05676XvlMh0qPyJ+9GaBxvhHeFtGh4etQ9UEmJj55rSs50wA/IaDh+roKukQxthyTESPPgjqg+DPjh6H+h3Sn00Os6sjh3DxpOphTEsdtb7fmk8J0e2KjQQCjW/GlECzc359b9KbBwNkcAiYFayVHPLaCAdvzYVyiHgXHkEEs5FlHyhe2gNEG/81Io8c3E3DH5JhT9tmVRL3bpgpT8Kr4aoFhU2LXe45YIB1A9DjUm5TrHZ+iNtvE0YfYMR9L9C1HPppmX1CahEhTdog7laE1198UCAwEAAaM4MDYwDwYDVR0jBAgwBoAEAQIDBDASBgNVHRMBAf8ECDAGAQH/AgEAMA8GA1UdDwEB/wQFAwMH/4AwDQYJKoZIhvcNAQELBQADggEBAAHiOgwAvEzhrNMQVAz8a+SsyMIABXQ5P8WbJeHjkIipE4+5ZpkrZVXq9p8wOdkYnOHx4WNi9PVGQbLG9Iufh9fpk8cyyRWDi+V20/CNNtawMq3ClV3dWC98Tj4WX/BXDCeY2jK4jYGV+ds43HYV0ToBmvvrccq/U7zYMGFcQiKBClz5bTE+GMvrZWcO5A/Lh38i2YSF1i8SfDVnAOBlAgZmllcheHpGsWfSnduIllUvTsRvEIsaaqfVLl5QpRXBOq8tbjK85/2g6ear1oxPhJ1w9hds+WTFXkmHkWvKJebY13t3OfSjAyhaRSt8hdzDzHTFwjPjHT8h6dU7/hMdkUg=\"" epilog := "]}\n" - req, _ := parseChain(t, false, pemChain, info.roots.RawCertificates()[0]) + req, leafChain := parseChain(t, false, pemChain, info.roots.RawCertificates()[0]) rsp := uint64(0) var tests = []struct { @@ -266,6 +266,7 @@ func TestAddChainWhitespace(t *testing.T) { for _, test := range tests { t.Run(test.descr, func(t *testing.T) { if test.want == http.StatusOK { + info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(rsp, nil) } @@ -337,8 +338,9 @@ func TestAddChain(t *testing.T) { pool := loadCertsIntoPoolOrDie(t, test.chain) chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { - req, _ := parseChain(t, false, test.chain, info.roots.RawCertificates()[0]) + req, leafChain := parseChain(t, false, test.chain, info.roots.RawCertificates()[0]) rsp := uint64(0) + info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(rsp, test.err) } @@ -425,9 +427,9 @@ func TestAddPrechain(t *testing.T) { pool := loadCertsIntoPoolOrDie(t, test.chain) chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { - req, _ := parseChain(t, true, test.chain, info.roots.RawCertificates()[0]) + req, leafChain := parseChain(t, true, test.chain, info.roots.RawCertificates()[0]) rsp := uint64(0) - + info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(rsp, test.err) } From 2d97d9da0640c883775caadb67da75e6684ae186 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Mon, 12 Aug 2024 14:31:59 +0000 Subject: [PATCH 09/15] typo fix --- personalities/sctfe/storage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index 91f5301d..fe3753ab 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -28,9 +28,9 @@ import ( // Storage provides all the storage primitives necessary to write to a ct-static-api log. type Storage interface { - // Add assign an index to the provided Entry, stages the entry for integration, and return it the assigned index. + // Add assigns an index to the provided Entry, stages the entry for integration, and return it the assigned index. Add(context.Context, *ctonly.Entry) (uint64, error) - // AddIssuerChain stores all certificates in the chain in a content-addressable store under their sha256 hash. + // AddIssuerChain stores every the chain certificate in a content-addressable store under their sha256 hash. AddIssuerChain(context.Context, []*x509.Certificate) error } @@ -60,7 +60,7 @@ func (cts *CTStorage) Add(ctx context.Context, entry *ctonly.Entry) (uint64, err return cts.storeData(ctx, entry) } -// AddIssuerChain stores every certificate in the chain under its sha256. +// AddIssuerChain stores every chain certificate under its sha256. // If an object is already stored under this hash, continues. func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certificate) error { errG := errgroup.Group{} From 1c1b0219ddad69b20616d9496404d2af0564c0ae Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Thu, 15 Aug 2024 14:13:01 +0000 Subject: [PATCH 10/15] change interface for []byte instead of [32]byte --- personalities/sctfe/storage.go | 12 +++++++----- personalities/sctfe/storage/gcp/map.go | 9 ++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index fe3753ab..8cf60382 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -35,8 +35,8 @@ type Storage interface { } type IssuerStorage interface { - Exists(ctx context.Context, key [32]byte) (bool, error) - Add(ctx context.Context, key [32]byte, data []byte) error + Exists(ctx context.Context, key []byte) (bool, error) + Add(ctx context.Context, key []byte, data []byte) error } // CTStorage implements Storage. @@ -66,17 +66,19 @@ func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certific errG := errgroup.Group{} for _, c := range chain { errG.Go(func() error { - key := sha256.Sum256(c.Raw) + id := sha256.Sum256(c.Raw) + key := make([]byte, 32) + _ = hex.Encode(key, id[:]) // We first try and see if this issuer cert has already been stored since reads // are cheaper than writes. // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns ok, err := cts.issuers.Exists(ctx, key) if err != nil { - return fmt.Errorf("error checking if issuer %q exists: %s", hex.EncodeToString(key[:]), err) + return fmt.Errorf("error checking if issuer %q exists: %s", string(key), err) } if !ok { if err = cts.issuers.Add(ctx, key, c.Raw); err != nil { - return fmt.Errorf("error adding certificate for issuer %q: %v", hex.EncodeToString(key[:]), err) + return fmt.Errorf("error adding certificate for issuer %q: %v", string(key), err) } } diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/map.go index 9c35389b..9ed37e2e 100644 --- a/personalities/sctfe/storage/gcp/map.go +++ b/personalities/sctfe/storage/gcp/map.go @@ -19,7 +19,6 @@ package gcp import ( "context" - "encoding/hex" "fmt" "net/http" "path" @@ -69,12 +68,12 @@ func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix } // keyToObjName converts bytes to a GCS object name. -func (s *GCSStorage) keyToObjName(key [32]byte) string { - return path.Join(s.prefix, hex.EncodeToString(key[:])) +func (s *GCSStorage) keyToObjName(key []byte) string { + return path.Join(s.prefix, string(key)) } // Exists checks whether an object is stored under key. -func (s *GCSStorage) Exists(ctx context.Context, key [32]byte) (bool, error) { +func (s *GCSStorage) Exists(ctx context.Context, key []byte) (bool, error) { objName := s.keyToObjName(key) obj := s.bucket.Object(objName) _, err := obj.Attrs(ctx) @@ -92,7 +91,7 @@ func (s *GCSStorage) Exists(ctx context.Context, key [32]byte) (bool, error) { // // If there is already an object under that key, it does not override it, and returns. // TODO(phboneff): consider reading the object to make sure it's identical -func (s *GCSStorage) Add(ctx context.Context, key [32]byte, data []byte) error { +func (s *GCSStorage) Add(ctx context.Context, key []byte, data []byte) error { objName := s.keyToObjName(key) obj := s.bucket.Object(objName) From e8efb461872e8ad2c1cfe423a3a79dd57c2b3289 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Mon, 19 Aug 2024 15:54:15 +0000 Subject: [PATCH 11/15] Typo fixes --- personalities/sctfe/ct_server_gcp/main.go | 2 +- personalities/sctfe/storage/gcp/map.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/personalities/sctfe/ct_server_gcp/main.go b/personalities/sctfe/ct_server_gcp/main.go index 90a8c34d..4646ecc3 100644 --- a/personalities/sctfe/ct_server_gcp/main.go +++ b/personalities/sctfe/ct_server_gcp/main.go @@ -276,7 +276,7 @@ func newGCPStorage(ctx context.Context, vCfg *sctfe.ValidatedLogConfig, signer n } tesseraStorage, err := gcpTessera.New(ctx, gcpCfg, tessera.WithCheckpointSignerVerifier(signer, nil)) if err != nil { - return nil, fmt.Errorf("Failed to initialize GCP Tesserea storage: %v", err) + return nil, fmt.Errorf("Failed to initialize GCP Tessera storage: %v", err) } issuerStorage, err := gcpMap.NewGCSStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/", "application/pkix-cert") diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/map.go index 9ed37e2e..9bead710 100644 --- a/personalities/sctfe/storage/gcp/map.go +++ b/personalities/sctfe/storage/gcp/map.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google LLC. All Rights Reserved. +// Copyright 2024 The Tessera authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ import ( "k8s.io/klog/v2" ) -// GCSStorage is a map backed by GCS on GCP. +// GCSStorage is a key value store backed by GCS on GCP. type GCSStorage struct { bucket *gcs.BucketHandle prefix string From 12c2b32acaccd08bf05d75767e8ee44298512c72 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Mon, 19 Aug 2024 16:35:31 +0000 Subject: [PATCH 12/15] fix key encoding --- personalities/sctfe/storage.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index 8cf60382..12d34247 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -67,8 +67,7 @@ func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certific for _, c := range chain { errG.Go(func() error { id := sha256.Sum256(c.Raw) - key := make([]byte, 32) - _ = hex.Encode(key, id[:]) + key := []byte(hex.EncodeToString(id[:])) // We first try and see if this issuer cert has already been stored since reads // are cheaper than writes. // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns From 4f849ef915be0d71d2ccfb496a696d3f975501a1 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Wed, 21 Aug 2024 10:58:31 +0000 Subject: [PATCH 13/15] Multiple writes at a time --- personalities/sctfe/storage.go | 45 +++++++++++++------------- personalities/sctfe/storage/gcp/map.go | 12 +++++++ 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index 12d34247..a3cf75ca 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -23,7 +23,6 @@ import ( "github.com/google/certificate-transparency-go/x509" tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/ctonly" - "golang.org/x/sync/errgroup" ) // Storage provides all the storage primitives necessary to write to a ct-static-api log. @@ -34,9 +33,14 @@ type Storage interface { AddIssuerChain(context.Context, []*x509.Certificate) error } +type KV struct { + K []byte + V []byte +} + type IssuerStorage interface { Exists(ctx context.Context, key []byte) (bool, error) - Add(ctx context.Context, key []byte, data []byte) error + AddMultiple(ctx context.Context, kv []KV) error } // CTStorage implements Storage. @@ -63,29 +67,24 @@ func (cts *CTStorage) Add(ctx context.Context, entry *ctonly.Entry) (uint64, err // AddIssuerChain stores every chain certificate under its sha256. // If an object is already stored under this hash, continues. func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certificate) error { - errG := errgroup.Group{} + kvs := []KV{} for _, c := range chain { - errG.Go(func() error { - id := sha256.Sum256(c.Raw) - key := []byte(hex.EncodeToString(id[:])) - // We first try and see if this issuer cert has already been stored since reads - // are cheaper than writes. - // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns - ok, err := cts.issuers.Exists(ctx, key) - if err != nil { - return fmt.Errorf("error checking if issuer %q exists: %s", string(key), err) - } - if !ok { - if err = cts.issuers.Add(ctx, key, c.Raw); err != nil { - return fmt.Errorf("error adding certificate for issuer %q: %v", string(key), err) - - } - } - return nil - }) + id := sha256.Sum256(c.Raw) + key := []byte(hex.EncodeToString(id[:])) + // We first try and see if this issuer cert has already been stored since reads + // are cheaper than writes. + // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns + ok, err := cts.issuers.Exists(ctx, key) + if err != nil { + return fmt.Errorf("error checking if issuer %q exists: %s", string(key), err) + } + if !ok { + kvs = append(kvs, KV{K: key, V: c.Raw}) + } } - if err := errG.Wait(); err != nil { - return err + if err := cts.issuers.AddMultiple(ctx, kvs); err != nil { + return fmt.Errorf("error storing intermediates: %v", err) + } return nil } diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/map.go index 9bead710..95614115 100644 --- a/personalities/sctfe/storage/gcp/map.go +++ b/personalities/sctfe/storage/gcp/map.go @@ -24,6 +24,7 @@ import ( "path" gcs "cloud.google.com/go/storage" + "github.com/transparency-dev/trillian-tessera/personalities/sctfe" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "k8s.io/klog/v2" @@ -114,3 +115,14 @@ func (s *GCSStorage) Add(ctx context.Context, key []byte, data []byte) error { } return nil } + +func (s *GCSStorage) AddMultiple(ctx context.Context, kv []sctfe.KV) error { + // TODO(phboneff): add parallel writes + for _, kv := range kv { + err := s.Add(ctx, kv.K, kv.V) + if err != nil { + return fmt.Errorf("error storing value under key %q: %v", string(kv.K), err) + } + } + return nil +} From 6b3ebe3e67b076557a95676f8e5e5a4671e9bf03 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Wed, 21 Aug 2024 16:18:49 +0000 Subject: [PATCH 14/15] Rename and push down the Exists logic --- personalities/sctfe/ct_server_gcp/main.go | 2 +- personalities/sctfe/storage.go | 16 +--- .../sctfe/storage/gcp/{map.go => issuers.go} | 73 ++++++++++--------- 3 files changed, 43 insertions(+), 48 deletions(-) rename personalities/sctfe/storage/gcp/{map.go => issuers.go} (56%) diff --git a/personalities/sctfe/ct_server_gcp/main.go b/personalities/sctfe/ct_server_gcp/main.go index 4646ecc3..13316197 100644 --- a/personalities/sctfe/ct_server_gcp/main.go +++ b/personalities/sctfe/ct_server_gcp/main.go @@ -279,7 +279,7 @@ func newGCPStorage(ctx context.Context, vCfg *sctfe.ValidatedLogConfig, signer n return nil, fmt.Errorf("Failed to initialize GCP Tessera storage: %v", err) } - issuerStorage, err := gcpMap.NewGCSStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/", "application/pkix-cert") + issuerStorage, err := gcpMap.NewIssuerStorage(ctx, cfg.ProjectId, cfg.Bucket, "fingerprints/", "application/pkix-cert") if err != nil { return nil, fmt.Errorf("Failed to initialize GCP issuer storage: %v", err) } diff --git a/personalities/sctfe/storage.go b/personalities/sctfe/storage.go index a3cf75ca..b01b1a08 100644 --- a/personalities/sctfe/storage.go +++ b/personalities/sctfe/storage.go @@ -40,7 +40,7 @@ type KV struct { type IssuerStorage interface { Exists(ctx context.Context, key []byte) (bool, error) - AddMultiple(ctx context.Context, kv []KV) error + AddIssuers(ctx context.Context, kv []KV) error } // CTStorage implements Storage. @@ -71,20 +71,10 @@ func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certific for _, c := range chain { id := sha256.Sum256(c.Raw) key := []byte(hex.EncodeToString(id[:])) - // We first try and see if this issuer cert has already been stored since reads - // are cheaper than writes. - // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns - ok, err := cts.issuers.Exists(ctx, key) - if err != nil { - return fmt.Errorf("error checking if issuer %q exists: %s", string(key), err) - } - if !ok { - kvs = append(kvs, KV{K: key, V: c.Raw}) - } + kvs = append(kvs, KV{K: key, V: c.Raw}) } - if err := cts.issuers.AddMultiple(ctx, kvs); err != nil { + if err := cts.issuers.AddIssuers(ctx, kvs); err != nil { return fmt.Errorf("error storing intermediates: %v", err) - } return nil } diff --git a/personalities/sctfe/storage/gcp/map.go b/personalities/sctfe/storage/gcp/issuers.go similarity index 56% rename from personalities/sctfe/storage/gcp/map.go rename to personalities/sctfe/storage/gcp/issuers.go index 95614115..6a508213 100644 --- a/personalities/sctfe/storage/gcp/map.go +++ b/personalities/sctfe/storage/gcp/issuers.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package gcp implements SCTFE storage systems for issuers and deduplication. +// Package gcp implements SCTFE storage systems for issuers. // // The interfaces are defined in sctfe/storage.go package gcp @@ -30,17 +30,17 @@ import ( "k8s.io/klog/v2" ) -// GCSStorage is a key value store backed by GCS on GCP. -type GCSStorage struct { +// IssuersStorage is a key value store backed by GCS on GCP to store issuer chains. +type IssuersStorage struct { bucket *gcs.BucketHandle prefix string contentType string } -// NewGCSStorage creates a new GCSStorage. +// NewIssuerStorage creates a new GCSStorage. // // The specified bucket must exist or an error will be returned. -func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix string, contentType string) (*GCSStorage, error) { +func NewIssuerStorage(ctx context.Context, projectID string, bucket string, prefix string, contentType string) (*IssuersStorage, error) { c, err := gcs.NewClient(ctx, gcs.WithJSONReads()) if err != nil { return nil, fmt.Errorf("failed to create GCS client: %v", err) @@ -59,7 +59,7 @@ func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix break } } - r := &GCSStorage{ + r := &IssuersStorage{ bucket: c.Bucket(bucket), prefix: prefix, contentType: contentType, @@ -69,12 +69,12 @@ func NewGCSStorage(ctx context.Context, projectID string, bucket string, prefix } // keyToObjName converts bytes to a GCS object name. -func (s *GCSStorage) keyToObjName(key []byte) string { +func (s *IssuersStorage) keyToObjName(key []byte) string { return path.Join(s.prefix, string(key)) } // Exists checks whether an object is stored under key. -func (s *GCSStorage) Exists(ctx context.Context, key []byte) (bool, error) { +func (s *IssuersStorage) Exists(ctx context.Context, key []byte) (bool, error) { objName := s.keyToObjName(key) obj := s.bucket.Object(objName) _, err := obj.Attrs(ctx) @@ -88,40 +88,45 @@ func (s *GCSStorage) Exists(ctx context.Context, key []byte) (bool, error) { return true, nil } -// Add stores the provided data under key. +// AddIssuers stores all Issuers values under Key // // If there is already an object under that key, it does not override it, and returns. // TODO(phboneff): consider reading the object to make sure it's identical -func (s *GCSStorage) Add(ctx context.Context, key []byte, data []byte) error { - objName := s.keyToObjName(key) - obj := s.bucket.Object(objName) - - // Don't overwrite if it already exists - w := obj.If(gcs.Conditions{DoesNotExist: true}).NewWriter(ctx) - w.ObjectAttrs.ContentType = s.contentType - - if _, err := w.Write(data); err != nil { - return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket.BucketName(), err) +func (s *IssuersStorage) AddIssuers(ctx context.Context, kv []sctfe.KV) error { + // We first try and see if this issuer cert has already been stored since reads + // are cheaper than writes. + // TODO(phboneff): monitor usage, eventually write directly depending on usage patterns + toStore := []sctfe.KV{} + for _, kv := range kv { + ok, err := s.Exists(ctx, kv.K) + if err != nil { + return fmt.Errorf("error checking if issuer %q exists: %s", string(kv.K), err) + } + if !ok { + toStore = append(toStore, kv) + } } + // TODO(phboneff): add parallel writes + for _, kv := range toStore { + objName := s.keyToObjName(kv.K) + obj := s.bucket.Object(objName) + + // Don't overwrite if it already exists + w := obj.If(gcs.Conditions{DoesNotExist: true}).NewWriter(ctx) + w.ObjectAttrs.ContentType = s.contentType - if err := w.Close(); err != nil { - // If we run into a precondition failure error, it means that the object already exists. - if ee, ok := err.(*googleapi.Error); ok && ee.Code == http.StatusPreconditionFailed { - klog.V(2).Infof("Add: object %q already exists in bucket %q, continuing", objName, s.bucket.BucketName()) - return nil + if _, err := w.Write(kv.V); err != nil { + return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket.BucketName(), err) } - return fmt.Errorf("failed to close write on %q: %v", objName, err) - } - return nil -} + if err := w.Close(); err != nil { + // If we run into a precondition failure error, it means that the object already exists. + if ee, ok := err.(*googleapi.Error); ok && ee.Code == http.StatusPreconditionFailed { + klog.V(2).Infof("Add: object %q already exists in bucket %q, continuing", objName, s.bucket.BucketName()) + return nil + } -func (s *GCSStorage) AddMultiple(ctx context.Context, kv []sctfe.KV) error { - // TODO(phboneff): add parallel writes - for _, kv := range kv { - err := s.Add(ctx, kv.K, kv.V) - if err != nil { - return fmt.Errorf("error storing value under key %q: %v", string(kv.K), err) + return fmt.Errorf("failed to close write on %q: %v", objName, err) } } return nil From fdd5de7f940a95c5b0d0eafeb54569149bfb3f46 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Wed, 21 Aug 2024 16:40:05 +0000 Subject: [PATCH 15/15] fix a few strings --- personalities/sctfe/storage/gcp/issuers.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/personalities/sctfe/storage/gcp/issuers.go b/personalities/sctfe/storage/gcp/issuers.go index 6a508213..7ab5e92a 100644 --- a/personalities/sctfe/storage/gcp/issuers.go +++ b/personalities/sctfe/storage/gcp/issuers.go @@ -73,7 +73,7 @@ func (s *IssuersStorage) keyToObjName(key []byte) string { return path.Join(s.prefix, string(key)) } -// Exists checks whether an object is stored under key. +// Exists checks whether a value is stored under key. func (s *IssuersStorage) Exists(ctx context.Context, key []byte) (bool, error) { objName := s.keyToObjName(key) obj := s.bucket.Object(objName) @@ -88,10 +88,9 @@ func (s *IssuersStorage) Exists(ctx context.Context, key []byte) (bool, error) { return true, nil } -// AddIssuers stores all Issuers values under Key +// AddIssuers stores all Issuers values under their Key. // -// If there is already an object under that key, it does not override it, and returns. -// TODO(phboneff): consider reading the object to make sure it's identical +// If there is already an object under a given key, it does not override it. func (s *IssuersStorage) AddIssuers(ctx context.Context, kv []sctfe.KV) error { // We first try and see if this issuer cert has already been stored since reads // are cheaper than writes. @@ -112,6 +111,7 @@ func (s *IssuersStorage) AddIssuers(ctx context.Context, kv []sctfe.KV) error { obj := s.bucket.Object(objName) // Don't overwrite if it already exists + // TODO(phboneff): consider reading the object to make sure it's identical w := obj.If(gcs.Conditions{DoesNotExist: true}).NewWriter(ctx) w.ObjectAttrs.ContentType = s.contentType