From 8a4b322aef4a391420dc3a2b5fc8ca1ba3c41b7e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 13 Jul 2024 22:56:28 +0200 Subject: [PATCH 01/88] Update protobuf --- go.mod | 2 +- net/pb/Makefile | 14 +- net/pb/net.pb.go | 32 ++--- net/pb/net_grpc.pb.go | 27 ++-- net/pb/net_vtproto.pb.go | 274 ++++++++++++--------------------------- 5 files changed, 130 insertions(+), 219 deletions(-) diff --git a/go.mod b/go.mod index 6af60bcdf1..14dc3ed450 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 - github.com/pelletier/go-toml v1.9.5 + github.com/planetscale/vtprotobuf v0.6.0 github.com/sourcenetwork/acp_core v0.0.0-20240607160510-47a5306b2ad2 github.com/sourcenetwork/badger/v4 v4.2.1-0.20231113215945-a63444ca5276 github.com/sourcenetwork/corelog v0.0.8 diff --git a/net/pb/Makefile b/net/pb/Makefile index 233665c334..db8b72e810 100644 --- a/net/pb/Makefile +++ b/net/pb/Makefile @@ -1,6 +1,10 @@ PB = $(wildcard *.proto) GO = $(PB:.proto=.pb.go) +PROTOC_GEN_GO := $(shell which protoc-gen-go) +PROTOC_GEN_GO_GRPC := $(shell which protoc-gen-go-grpc) +PROTOC_GEN_GO_VTPROTO := $(shell which protoc-gen-go-vtproto) + all: $(GO) deps: @@ -10,14 +14,14 @@ deps: %.pb.go: %.proto protoc \ - --go_out=. --plugin protoc-gen-go="${GOBIN}/protoc-gen-go" \ - --go-grpc_out=. --plugin protoc-gen-go-grpc="${GOBIN}/protoc-gen-go-grpc" \ - --go-vtproto_out=. --plugin protoc-gen-go-vtproto="${GOBIN}/protoc-gen-go-vtproto" \ + --go_out=. --plugin protoc-gen-go="$(PROTOC_GEN_GO)" \ + --go-grpc_out=. --plugin protoc-gen-go-grpc="$(PROTOC_GEN_GO_GRPC)" \ + --go-vtproto_out=. --plugin protoc-gen-go-vtproto="$(PROTOC_GEN_GO_VTPROTO)" \ --go-vtproto_opt=features=marshal+unmarshal+size \ - $< + $< # This line specifies the input file clean: rm -f *.pb.go rm -f *pb_test.go -.PHONY: clean \ No newline at end of file +.PHONY: clean deps debug \ No newline at end of file diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index a9b5a2162d..595a33f2c5 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v4.25.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: net.proto package net_pb @@ -672,7 +672,7 @@ func file_net_proto_rawDescGZIP() []byte { } var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 13) -var file_net_proto_goTypes = []interface{}{ +var file_net_proto_goTypes = []any{ (*Document)(nil), // 0: net.pb.Document (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest (*GetDocGraphReply)(nil), // 2: net.pb.GetDocGraphReply @@ -713,7 +713,7 @@ func file_net_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_net_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Document); i { case 0: return &v.state @@ -725,7 +725,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*GetDocGraphRequest); i { case 0: return &v.state @@ -737,7 +737,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*GetDocGraphReply); i { case 0: return &v.state @@ -749,7 +749,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*PushDocGraphRequest); i { case 0: return &v.state @@ -761,7 +761,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*PushDocGraphReply); i { case 0: return &v.state @@ -773,7 +773,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*GetLogRequest); i { case 0: return &v.state @@ -785,7 +785,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*GetLogReply); i { case 0: return &v.state @@ -797,7 +797,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest); i { case 0: return &v.state @@ -809,7 +809,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state @@ -821,7 +821,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*PushLogReply); i { case 0: return &v.state @@ -833,7 +833,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogReply); i { case 0: return &v.state @@ -845,7 +845,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*Document_Log); i { case 0: return &v.state @@ -857,7 +857,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state diff --git a/net/pb/net_grpc.pb.go b/net/pb/net_grpc.pb.go index 75ae790ab6..84564d6bec 100644 --- a/net/pb/net_grpc.pb.go +++ b/net/pb/net_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v4.25.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v5.27.1 // source: net.proto package net_pb @@ -15,8 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( Service_GetDocGraph_FullMethodName = "/net.pb.Service/GetDocGraph" @@ -29,6 +29,8 @@ const ( // ServiceClient is the client API for Service service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Service is the peer-to-peer network API for document sync type ServiceClient interface { // GetDocGraph from this peer. GetDocGraph(ctx context.Context, in *GetDocGraphRequest, opts ...grpc.CallOption) (*GetDocGraphReply, error) @@ -51,8 +53,9 @@ func NewServiceClient(cc grpc.ClientConnInterface) ServiceClient { } func (c *serviceClient) GetDocGraph(ctx context.Context, in *GetDocGraphRequest, opts ...grpc.CallOption) (*GetDocGraphReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetDocGraphReply) - err := c.cc.Invoke(ctx, Service_GetDocGraph_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Service_GetDocGraph_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -60,8 +63,9 @@ func (c *serviceClient) GetDocGraph(ctx context.Context, in *GetDocGraphRequest, } func (c *serviceClient) PushDocGraph(ctx context.Context, in *PushDocGraphRequest, opts ...grpc.CallOption) (*PushDocGraphReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PushDocGraphReply) - err := c.cc.Invoke(ctx, Service_PushDocGraph_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Service_PushDocGraph_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -69,8 +73,9 @@ func (c *serviceClient) PushDocGraph(ctx context.Context, in *PushDocGraphReques } func (c *serviceClient) GetLog(ctx context.Context, in *GetLogRequest, opts ...grpc.CallOption) (*GetLogReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetLogReply) - err := c.cc.Invoke(ctx, Service_GetLog_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Service_GetLog_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -78,8 +83,9 @@ func (c *serviceClient) GetLog(ctx context.Context, in *GetLogRequest, opts ...g } func (c *serviceClient) PushLog(ctx context.Context, in *PushLogRequest, opts ...grpc.CallOption) (*PushLogReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PushLogReply) - err := c.cc.Invoke(ctx, Service_PushLog_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Service_PushLog_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -87,8 +93,9 @@ func (c *serviceClient) PushLog(ctx context.Context, in *PushLogRequest, opts .. } func (c *serviceClient) GetHeadLog(ctx context.Context, in *GetHeadLogRequest, opts ...grpc.CallOption) (*GetHeadLogReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetHeadLogReply) - err := c.cc.Invoke(ctx, Service_GetHeadLog_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Service_GetHeadLog_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -98,6 +105,8 @@ func (c *serviceClient) GetHeadLog(ctx context.Context, in *GetHeadLogRequest, o // ServiceServer is the server API for Service service. // All implementations must embed UnimplementedServiceServer // for forward compatibility +// +// Service is the peer-to-peer network API for document sync type ServiceServer interface { // GetDocGraph from this peer. GetDocGraph(context.Context, *GetDocGraphRequest) (*GetDocGraphReply, error) diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 2bae8f83f3..34a256ee4e 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -1,14 +1,14 @@ // Code generated by protoc-gen-go-vtproto. DO NOT EDIT. -// protoc-gen-go-vtproto version: v0.5.0 +// protoc-gen-go-vtproto version: v0.6.0 // source: net.proto package net_pb import ( fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" protoimpl "google.golang.org/protobuf/runtime/protoimpl" io "io" - bits "math/bits" ) const ( @@ -51,7 +51,7 @@ func (m *Document_Log) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if len(m.Block) > 0 { i -= len(m.Block) copy(dAtA[i:], m.Block) - i = encodeVarint(dAtA, i, uint64(len(m.Block))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Block))) i-- dAtA[i] = 0xa } @@ -91,14 +91,14 @@ func (m *Document) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if len(m.Head) > 0 { i -= len(m.Head) copy(dAtA[i:], m.Head) - i = encodeVarint(dAtA, i, uint64(len(m.Head))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Head))) i-- dAtA[i] = 0x22 } if len(m.DocID) > 0 { i -= len(m.DocID) copy(dAtA[i:], m.DocID) - i = encodeVarint(dAtA, i, uint64(len(m.DocID))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) i-- dAtA[i] = 0xa } @@ -339,35 +339,35 @@ func (m *PushLogRequest_Body) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return 0, err } i -= size - i = encodeVarint(dAtA, i, uint64(size)) + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- dAtA[i] = 0x32 } if len(m.Creator) > 0 { i -= len(m.Creator) copy(dAtA[i:], m.Creator) - i = encodeVarint(dAtA, i, uint64(len(m.Creator))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Creator))) i-- dAtA[i] = 0x22 } if len(m.SchemaRoot) > 0 { i -= len(m.SchemaRoot) copy(dAtA[i:], m.SchemaRoot) - i = encodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) i-- dAtA[i] = 0x1a } if len(m.Cid) > 0 { i -= len(m.Cid) copy(dAtA[i:], m.Cid) - i = encodeVarint(dAtA, i, uint64(len(m.Cid))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) i-- dAtA[i] = 0x12 } if len(m.DocID) > 0 { i -= len(m.DocID) copy(dAtA[i:], m.DocID) - i = encodeVarint(dAtA, i, uint64(len(m.DocID))) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) i-- dAtA[i] = 0xa } @@ -410,7 +410,7 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return 0, err } i -= size - i = encodeVarint(dAtA, i, uint64(size)) + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- dAtA[i] = 0xa } @@ -516,17 +516,6 @@ func (m *GetHeadLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarint(dAtA []byte, offset int, v uint64) int { - offset -= sov(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} func (m *Document_Log) SizeVT() (n int) { if m == nil { return 0 @@ -535,7 +524,7 @@ func (m *Document_Log) SizeVT() (n int) { _ = l l = len(m.Block) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n @@ -549,11 +538,11 @@ func (m *Document) SizeVT() (n int) { _ = l l = len(m.DocID) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } l = len(m.Head) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n @@ -627,23 +616,23 @@ func (m *PushLogRequest_Body) SizeVT() (n int) { _ = l l = len(m.DocID) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } l = len(m.Cid) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } l = len(m.SchemaRoot) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } l = len(m.Creator) if l > 0 { - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } if m.Log != nil { l = m.Log.SizeVT() - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n @@ -657,7 +646,7 @@ func (m *PushLogRequest) SizeVT() (n int) { _ = l if m.Body != nil { l = m.Body.SizeVT() - n += 1 + l + sov(uint64(l)) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n @@ -693,12 +682,6 @@ func (m *GetHeadLogReply) SizeVT() (n int) { return n } -func sov(x uint64) (n int) { - return (bits.Len64(x|1) + 6) / 7 -} -func soz(x uint64) (n int) { - return sov(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} func (m *Document_Log) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -707,7 +690,7 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -735,7 +718,7 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -748,11 +731,11 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -764,12 +747,12 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -792,7 +775,7 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -820,7 +803,7 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -833,11 +816,11 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -854,7 +837,7 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -867,11 +850,11 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -883,12 +866,12 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -911,7 +894,7 @@ func (m *GetDocGraphRequest) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -934,12 +917,12 @@ func (m *GetDocGraphRequest) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -962,7 +945,7 @@ func (m *GetDocGraphReply) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -985,12 +968,12 @@ func (m *GetDocGraphReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1013,7 +996,7 @@ func (m *PushDocGraphRequest) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1036,12 +1019,12 @@ func (m *PushDocGraphRequest) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1064,7 +1047,7 @@ func (m *PushDocGraphReply) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1087,12 +1070,12 @@ func (m *PushDocGraphReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1115,7 +1098,7 @@ func (m *GetLogRequest) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1138,12 +1121,12 @@ func (m *GetLogRequest) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1166,7 +1149,7 @@ func (m *GetLogReply) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1189,12 +1172,12 @@ func (m *GetLogReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1217,7 +1200,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1245,7 +1228,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1258,11 +1241,11 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1279,7 +1262,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1292,11 +1275,11 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1313,7 +1296,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1326,11 +1309,11 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { } } if byteLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + byteLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1347,7 +1330,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1361,11 +1344,11 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1379,7 +1362,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1392,11 +1375,11 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1410,12 +1393,12 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1438,7 +1421,7 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1466,7 +1449,7 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1479,11 +1462,11 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF @@ -1497,12 +1480,12 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1525,7 +1508,7 @@ func (m *GetHeadLogRequest) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1548,12 +1531,12 @@ func (m *GetHeadLogRequest) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1576,7 +1559,7 @@ func (m *PushLogReply) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1599,12 +1582,12 @@ func (m *PushLogReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1627,7 +1610,7 @@ func (m *GetHeadLogReply) UnmarshalVT(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflow + return protohelpers.ErrIntOverflow } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1650,12 +1633,12 @@ func (m *GetHeadLogReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { default: iNdEx = preIndex - skippy, err := skip(dAtA[iNdEx:]) + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLength + return protohelpers.ErrInvalidLength } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1670,88 +1653,3 @@ func (m *GetHeadLogReply) UnmarshalVT(dAtA []byte) error { } return nil } - -func skip(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflow - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflow - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflow - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLength - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroup - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLength - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLength = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflow = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroup = fmt.Errorf("proto: unexpected end of group") -) From 8c81b46f951452ee16657dc2d3ad3b482a9a7fd6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 17 Jul 2024 08:37:12 +0200 Subject: [PATCH 02/88] Fix after rebase --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 14dc3ed450..3447a74e72 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 + github.com/pelletier/go-toml v1.9.5 github.com/planetscale/vtprotobuf v0.6.0 github.com/sourcenetwork/acp_core v0.0.0-20240607160510-47a5306b2ad2 github.com/sourcenetwork/badger/v4 v4.2.1-0.20231113215945-a63444ca5276 diff --git a/go.sum b/go.sum index 3d191a245f..edaba460a8 100644 --- a/go.sum +++ b/go.sum @@ -1265,6 +1265,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA= +github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From b6ef757349123ac4f7e14f487326ee3733424794 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 24 Jul 2024 19:13:17 +0200 Subject: [PATCH 03/88] Enable very naive key exchange --- cli/collection_create.go | 2 +- client/db.go | 5 + datastore/multi.go | 3 +- http/client.go | 4 + internal/core/key.go | 14 +- internal/db/context.go | 2 +- internal/db/db.go | 8 +- internal/db/fetcher/dag.go | 2 +- internal/db/fetcher/versioned.go | 3 +- internal/db/merge.go | 141 +++- internal/db/messages.go | 25 + internal/encryption/context.go | 5 +- internal/encryption/encryptor.go | 36 + internal/encryption/event.go | 70 ++ internal/merkle/clock/clock.go | 20 +- internal/merkle/clock/clock_test.go | 2 +- internal/merkle/clock/heads_test.go | 2 +- internal/merkle/crdt/merklecrdt.go | 8 +- net/client.go | 2 +- net/dialer_test.go | 6 + net/pb/net.pb.go | 389 +++++++---- net/pb/net.proto | 35 +- net/pb/net_grpc.pb.go | 50 +- net/pb/net_vtproto.pb.go | 658 +++++++++++++----- net/peer.go | 36 +- net/peer_test.go | 20 +- net/server.go | 125 +++- net/server_test.go | 4 +- node/node.go | 2 +- tests/clients/cli/wrapper.go | 4 + tests/clients/http/wrapper.go | 4 + .../integration/encryption/peer_share_test.go | 69 ++ tests/integration/encryption/peer_test.go | 4 +- tests/integration/events.go | 30 + tests/integration/p2p.go | 11 +- tests/integration/state.go | 17 +- tests/integration/utils2.go | 6 +- 37 files changed, 1400 insertions(+), 424 deletions(-) create mode 100644 internal/encryption/event.go create mode 100644 tests/integration/encryption/peer_share_test.go diff --git a/cli/collection_create.go b/cli/collection_create.go index eecdfef2d8..0bdb5014b7 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -122,7 +122,7 @@ func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptF } ctx := cmd.Context() if txn != nil { - ctx = encryption.ContextWithStore(ctx, txn) + ctx = encryption.ContextWithStore(ctx, txn.Encstore()) } ctx = encryption.SetContextConfigFromParams(ctx, shouldEncryptDoc, encryptFields) cmd.SetContext(ctx) diff --git a/client/db.go b/client/db.go index ad2229cdb0..d3ea9e63f9 100644 --- a/client/db.go +++ b/client/db.go @@ -50,6 +50,11 @@ type DB interface { // It sits within the rootstore returned by [Root]. Blockstore() datastore.Blockstore + // Encstore returns the store, that contains all known encryption keys for documents and their fields. + // + // It sits within the rootstore returned by [Root]. + Encstore() datastore.DSReaderWriter + // Peerstore returns the peerstore where known host information is stored. // // It sits within the rootstore returned by [Root]. diff --git a/datastore/multi.go b/datastore/multi.go index f863924d5d..bcf0e7191f 100644 --- a/datastore/multi.go +++ b/datastore/multi.go @@ -33,8 +33,7 @@ type multistore struct { head DSReaderWriter peer DSBatching system DSReaderWriter - // block DSReaderWriter - dag Blockstore + dag Blockstore } var _ MultiStore = (*multistore)(nil) diff --git a/http/client.go b/http/client.go index 6e5cc21276..83bb57a99a 100644 --- a/http/client.go +++ b/http/client.go @@ -447,6 +447,10 @@ func (c *Client) Blockstore() datastore.Blockstore { panic("client side database") } +func (c *Client) Encstore() datastore.DSReaderWriter { + panic("client side database") +} + func (c *Client) Peerstore() datastore.DSBatching { panic("client side database") } diff --git a/internal/core/key.go b/internal/core/key.go index 8f0ab3fd4e..6450540ce2 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -106,7 +106,7 @@ var _ Key = (*PrimaryDataStoreKey)(nil) type HeadStoreKey struct { DocID string - FieldId string //can be 'C' + FieldID string //can be 'C' Cid cid.Cid } @@ -253,7 +253,7 @@ func NewHeadStoreKey(key string) (HeadStoreKey, error) { return HeadStoreKey{ // elements[0] is empty (key has leading '/') DocID: elements[1], - FieldId: elements[2], + FieldID: elements[2], Cid: cid, }, nil } @@ -417,7 +417,7 @@ func (k DataStoreKey) WithFieldId(fieldId string) DataStoreKey { func (k DataStoreKey) ToHeadStoreKey() HeadStoreKey { return HeadStoreKey{ DocID: k.DocID, - FieldId: k.FieldID, + FieldID: k.FieldID, } } @@ -433,9 +433,9 @@ func (k HeadStoreKey) WithCid(c cid.Cid) HeadStoreKey { return newKey } -func (k HeadStoreKey) WithFieldId(fieldId string) HeadStoreKey { +func (k HeadStoreKey) WithFieldID(fieldID string) HeadStoreKey { newKey := k - newKey.FieldId = fieldId + newKey.FieldID = fieldID return newKey } @@ -723,8 +723,8 @@ func (k HeadStoreKey) ToString() string { if k.DocID != "" { result = result + "/" + k.DocID } - if k.FieldId != "" { - result = result + "/" + k.FieldId + if k.FieldID != "" { + result = result + "/" + k.FieldID } if k.Cid.Defined() { result = result + "/" + k.Cid.String() diff --git a/internal/db/context.go b/internal/db/context.go index 8ad51c86ce..3149bd6205 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -63,7 +63,7 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if err != nil { return nil, txn, err } - ctx = encryption.ContextWithStore(ctx, txn) + ctx = encryption.ContextWithStore(ctx, txn.Encstore()) return SetContextTxn(ctx, txn), txn, nil } diff --git a/internal/db/db.go b/internal/db/db.go index 81ec48e199..07bc118750 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -31,6 +31,7 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/request/graphql" ) @@ -130,7 +131,7 @@ func newDB( return nil, err } - sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName) + sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName, encryption.KeyRetrievedEventName) if err != nil { return nil, err } @@ -161,6 +162,11 @@ func (db *db) Blockstore() datastore.Blockstore { return db.multistore.Blockstore() } +// Encstore returns the internal enc store which contains encryption key for documents and their fields. +func (db *db) Encstore() datastore.DSReaderWriter { + return db.multistore.Encstore() +} + // Peerstore returns the internal DAG store which contains IPLD blocks. func (db *db) Peerstore() datastore.DSBatching { return db.multistore.Peerstore() diff --git a/internal/db/fetcher/dag.go b/internal/db/fetcher/dag.go index cec1121827..3d3a6dd85e 100644 --- a/internal/db/fetcher/dag.go +++ b/internal/db/fetcher/dag.go @@ -92,7 +92,7 @@ func (hf *HeadFetcher) FetchNext() (*cid.Cid, error) { return nil, err } - if hf.fieldId.HasValue() && hf.fieldId.Value() != headStoreKey.FieldId { + if hf.fieldId.HasValue() && hf.fieldId.Value() != headStoreKey.FieldID { // FieldIds do not match, continue to next row return hf.FetchNext() } diff --git a/internal/db/fetcher/versioned.go b/internal/db/fetcher/versioned.go index 0ff58c4eeb..a81e105714 100644 --- a/internal/db/fetcher/versioned.go +++ b/internal/db/fetcher/versioned.go @@ -415,8 +415,7 @@ func (vf *VersionedFetcher) processBlock( vf.mCRDTs[crdtIndex] = mcrdt } - err = mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink, false) - return err + return mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink, false, false) } func (vf *VersionedFetcher) getDAGBlock(c cid.Cid) (*coreblock.Block, error) { diff --git a/internal/db/merge.go b/internal/db/merge.go index e588cb60a4..2c6116f9ce 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -28,6 +28,7 @@ import ( "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" "github.com/sourcenetwork/defradb/internal/db/base" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/merkle/clock" merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt" ) @@ -63,12 +64,12 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.loadComposites(ctx, dagMerge.Cid, mt) + err = mp.loadComposites(ctx, dagMerge.Cid, mt, false) if err != nil { return err } - err = mp.mergeComposites(ctx) + err = mp.mergeComposites(ctx, false) if err != nil { return err } @@ -88,6 +89,60 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return nil } +func (db *db) mergeEncryptedBlock(ctx context.Context, keyEvent encryption.KeyRetrievedEvent) error { + ctx, txn, err := ensureContextTxn(ctx, db, false) + if err != nil { + return err + } + defer txn.Discard(ctx) + + col, err := getCollectionFromRootSchema(ctx, db, keyEvent.SchemaRoot) + if err != nil { + return err + } + + ls := cidlink.DefaultLinkSystem() + ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) + + docID, err := client.NewDocIDFromString(keyEvent.DocID) + if err != nil { + return err + } + dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(col.Description(), docID.String()) + + mp, err := db.newMergeProcessor(txn, ls, col, dsKey) + if err != nil { + return err + } + + mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey) + if err != nil { + return err + } + + err = mp.loadComposites(ctx, keyEvent.Cid, mt, true) + if err != nil { + return err + } + + err = mp.mergeComposites(ctx, true) + if err != nil { + return err + } + + err = syncIndexedDoc(ctx, docID, col) + if err != nil { + return err + } + + err = txn.Commit(ctx) + if err != nil { + return err + } + + return nil +} + // mergeQueue is synchronization source to ensure that concurrent // document merges do not cause transaction conflicts. type mergeQueue struct { @@ -170,10 +225,15 @@ func (mp *mergeProcessor) loadComposites( ctx context.Context, blockCid cid.Cid, mt mergeTarget, + willDecrypt bool, ) error { - if _, ok := mt.heads[blockCid]; ok { - // We've already processed this block. - return nil + if b, ok := mt.heads[blockCid]; ok { + // the head is already known, but the block might be encrypted + // if this time we try to decrypt it, we load the block + if b.IsEncrypted == nil || !*b.IsEncrypted || !willDecrypt { + // We've already processed this block. + return nil + } } nd, err := mp.lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, coreblock.SchemaPrototype) @@ -193,7 +253,7 @@ func (mp *mergeProcessor) loadComposites( mp.composites.PushFront(block) for _, link := range block.Links { if link.Name == core.HEAD { - err := mp.loadComposites(ctx, link.Cid, mt) + err := mp.loadComposites(ctx, link.Cid, mt, willDecrypt) if err != nil { return err } @@ -219,23 +279,19 @@ func (mp *mergeProcessor) loadComposites( } } } - return mp.loadComposites(ctx, blockCid, newMT) + return mp.loadComposites(ctx, blockCid, newMT, willDecrypt) } return nil } -func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { +func (mp *mergeProcessor) mergeComposites(ctx context.Context, skipHeads bool) error { for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) - var onlyHeads bool - if block.IsEncrypted != nil && *block.IsEncrypted { - onlyHeads = true - } link, err := block.GenerateLink() if err != nil { return err } - err = mp.processBlock(ctx, block, link, onlyHeads) + err = mp.processBlock(ctx, block, link, skipHeads) if err != nil { return err } @@ -247,11 +303,32 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { // If onlyHeads is true, it will skip merging and update only the heads. func (mp *mergeProcessor) processBlock( ctx context.Context, - block *coreblock.Block, + dagBlock *coreblock.Block, blockLink cidlink.Link, - onlyHeads bool, + skipHeads bool, ) error { - crdt, err := mp.initCRDTForType(block.Delta.GetFieldName()) + block := dagBlock + var skipMerge bool + if dagBlock.IsEncrypted != nil && *dagBlock.IsEncrypted { + plainTextBlock, err := decryptBlock(ctx, dagBlock) + if err != nil { + return err + } + if plainTextBlock != nil { + block = plainTextBlock + } else { + skipMerge = true + block = dagBlock + // if we weren't able to decrypt the block we request the encryption key + if dagBlock.Delta.IsComposite() { + docID := string(dagBlock.Delta.GetDocID()) + schemaRoot := mp.col.SchemaRoot() + mp.col.db.events.Publish(encryption.NewRequestKeyMessage(docID, blockLink.Cid, schemaRoot)) + } + } + } + + crdt, err := mp.initCRDTForType(dagBlock.Delta.GetFieldName()) if err != nil { return err } @@ -262,12 +339,12 @@ func (mp *mergeProcessor) processBlock( return nil } - err = crdt.Clock().ProcessBlock(ctx, block, blockLink, onlyHeads) + err = crdt.Clock().ProcessBlock(ctx, block, blockLink, skipMerge, skipHeads) if err != nil { return err } - for _, link := range block.Links { + for _, link := range dagBlock.Links { if link.Name == core.HEAD { continue } @@ -282,7 +359,7 @@ func (mp *mergeProcessor) processBlock( return err } - if err := mp.processBlock(ctx, childBlock, link.Link, onlyHeads); err != nil { + if err := mp.processBlock(ctx, childBlock, link.Link, skipHeads); err != nil { return err } } @@ -290,6 +367,32 @@ func (mp *mergeProcessor) processBlock( return nil } +func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { + if block.Delta.IsComposite() { + // for composite blocks there is nothing to decrypt + // so we just check if we have the encryption key for child blocks + bytes, err := encryption.GetDocKey(ctx, string(block.Delta.GetDocID())) + if err != nil { + return nil, err + } + if len(bytes) == 0 { + return nil, nil + } + return block, nil + } + clonedCRDT := block.Delta.Clone() + bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), + clonedCRDT.GetFieldName(), clonedCRDT.GetData()) + if err != nil { + return nil, err + } + if len(bytes) == 0 { + return nil, nil + } + clonedCRDT.SetData(bytes) + return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil +} + func (mp *mergeProcessor) initCRDTForType( field string, ) (merklecrdt.MerkleCRDT, error) { diff --git a/internal/db/messages.go b/internal/db/messages.go index 04b81cd210..976f82d8ca 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -19,6 +19,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/internal/encryption" ) func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { @@ -78,6 +79,30 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { log.ErrorContextE(ctx, "Failed to load replicators", err) } }) + + case encryption.KeyRetrievedEvent: + go func() { + ctx = encryption.ContextWithStore(ctx, db.Encstore()) + err := encryption.SaveDocKey(ctx, evt.DocID, evt.Key) + + if err != nil { + log.ErrorContextE( + ctx, + "Failed to save doc encryption key", + err, + corelog.Any("Event", evt)) + } + + err = db.mergeEncryptedBlock(ctx, evt) + + if err != nil { + log.ErrorContextE( + ctx, + "Failed to merge encrypted block", + err, + corelog.Any("Event", evt)) + } + }() } } } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index 96e90a7e0c..e1ccb033b0 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -36,6 +36,7 @@ func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { func setConfig(ctx context.Context, enc *DocEncryptor) { enc.SetConfig(GetContextConfig(ctx)) + enc.ctx = ctx } func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { @@ -49,9 +50,9 @@ func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncrypto // ContextWithStore sets the store on the doc encryptor in the context. // If the doc encryptor is not present, it will be created. -func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { +func ContextWithStore(ctx context.Context, encstore datastore.DSReaderWriter) context.Context { ctx, encryptor := ensureContextWithDocEnc(ctx) - encryptor.SetStore(txn.Encstore()) + encryptor.SetStore(encstore) return ctx } diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 9a6cb8f6f0..3cae2d33be 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -176,6 +176,26 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName string) ([]byt return encryptionKey, nil } +func (d *DocEncryptor) GetDocKey(docID string) ([]byte, error) { + if d.store == nil { + return nil, ErrNoStorageProvided + } + storeKey := core.NewEncStoreDocKey(docID, "") + encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) + if err != nil && !errors.Is(err, ds.ErrNotFound) { + return nil, err + } + return encryptionKey, nil +} + +func (d *DocEncryptor) SaveDocKey(docID string, encryptionKey []byte) error { + if d.store == nil { + return ErrNoStorageProvided + } + storeKey := core.NewEncStoreDocKey(docID, "") + return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) +} + // EncryptDoc encrypts the given plainText that is associated with the given docID and fieldName with // encryptor in the context. // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. @@ -202,3 +222,19 @@ func DecryptDoc(ctx context.Context, docID string, fieldName string, cipherText func ShouldEncryptField(ctx context.Context, fieldName string) bool { return shouldEncryptField(GetContextConfig(ctx), fieldName) } + +func SaveDocKey(ctx context.Context, docID string, encryptionKey []byte) error { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return nil + } + return enc.SaveDocKey(docID, encryptionKey) +} + +func GetDocKey(ctx context.Context, docID string) ([]byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return nil, nil + } + return enc.GetDocKey(docID) +} diff --git a/internal/encryption/event.go b/internal/encryption/event.go new file mode 100644 index 0000000000..7a81209cc2 --- /dev/null +++ b/internal/encryption/event.go @@ -0,0 +1,70 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "github.com/ipfs/go-cid" + "github.com/sourcenetwork/defradb/event" +) + +const RequestKeyEventName = event.Name("enc-key-request") +const KeyRetrievedEventName = event.Name("enc-key-retrieved") + +// RequestKeyEvent represents a request of a node to fetch an encryption key for a specific +// docID/field +// +// It must only contain public elements not protected by ACP. +type RequestKeyEvent struct { + // DocID is the unique immutable identifier of the document that was updated. + DocID string + + // Cid is the id of the composite commit that formed this update in the DAG. + Cid cid.Cid + + // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. + SchemaRoot string +} + +// KeyRetrievedEvent represents a key that was retrieved. +type KeyRetrievedEvent struct { + // DocID is the unique immutable identifier of the document that was updated. + DocID string + + // Cid is the id of the composite commit that formed this update in the DAG. + Cid cid.Cid + + // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. + SchemaRoot string + + // TODO: should be encrypted + // Key is the encryption key that was retrieved. + Key []byte +} + +// NewRequestKeyMessage creates a new event message for a request of a node to fetch an encryption key +// for a specific docID/field +func NewRequestKeyMessage(docID string, cid cid.Cid, schemaRoot string) event.Message { + return event.NewMessage(RequestKeyEventName, RequestKeyEvent{ + DocID: docID, + Cid: cid, + SchemaRoot: schemaRoot, + }) +} + +// NewKeyRetrievedMessage creates a new event message for a key that was retrieved +func NewKeyRetrievedMessage(docID string, cid cid.Cid, schemaRoot string, key []byte) event.Message { + return event.NewMessage(KeyRetrievedEventName, KeyRetrievedEvent{ + DocID: docID, + Cid: cid, + SchemaRoot: schemaRoot, + Key: key, + }) +} diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index b5b5f2631c..ce6ba8db2b 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -109,12 +109,7 @@ func (mc *MerkleClock) AddDelta( } // merge the delta and update the state - err = mc.ProcessBlock( - ctx, - block, - link, - false, - ) + err = mc.ProcessBlock(ctx, block, link, false, false) if err != nil { return cidlink.Link{}, nil, err } @@ -166,21 +161,26 @@ func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } // ProcessBlock merges the delta CRDT and updates the state accordingly. -// If onlyHeads is true, it will skip merging and update only the heads. +// If skipMerge is true, it will skip merging and update only the heads. +// If skipHeads is true, it will skip updating the heads. func (mc *MerkleClock) ProcessBlock( ctx context.Context, block *coreblock.Block, blockLink cidlink.Link, - onlyHeads bool, + skipMerge bool, + skipHeads bool, ) error { - if !onlyHeads { + if !skipMerge { err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) if err != nil { return NewErrMergingDelta(blockLink.Cid, err) } } - return mc.updateHeads(ctx, block, blockLink) + if !skipHeads { + return mc.updateHeads(ctx, block, blockLink) + } + return nil } func (mc *MerkleClock) updateHeads( diff --git a/internal/merkle/clock/clock_test.go b/internal/merkle/clock/clock_test.go index 7effc02fef..dbff19bc09 100644 --- a/internal/merkle/clock/clock_test.go +++ b/internal/merkle/clock/clock_test.go @@ -37,7 +37,7 @@ func newTestMerkleClock() *MerkleClock { return NewMerkleClock( multistore.Headstore(), multistore.Blockstore(), - core.HeadStoreKey{DocID: request.DocIDArgName, FieldId: "1"}, + core.HeadStoreKey{DocID: request.DocIDArgName, FieldID: "1"}, reg, ) } diff --git a/internal/merkle/clock/heads_test.go b/internal/merkle/clock/heads_test.go index 94680569a8..0eb7acdd0e 100644 --- a/internal/merkle/clock/heads_test.go +++ b/internal/merkle/clock/heads_test.go @@ -45,7 +45,7 @@ func newHeadSet() *heads { return NewHeadSet( datastore.AsDSReaderWriter(s), - core.HeadStoreKey{}.WithDocID("myDocID").WithFieldId("1"), + core.HeadStoreKey{}.WithDocID("myDocID").WithFieldID("1"), ) } diff --git a/internal/merkle/crdt/merklecrdt.go b/internal/merkle/crdt/merklecrdt.go index c7733be778..2c99c5f23d 100644 --- a/internal/merkle/crdt/merklecrdt.go +++ b/internal/merkle/crdt/merklecrdt.go @@ -48,9 +48,11 @@ type MerkleClock interface { links ...coreblock.DAGLink, ) (cidlink.Link, []byte, error) // ProcessBlock processes a block and updates the CRDT state. - // The bool argument indicates whether only heads need to be updated. It is needed in case - // merge should be skipped for example if the block is encrypted. - ProcessBlock(context.Context, *coreblock.Block, cidlink.Link, bool) error + // skipMerge argument indicates whether merging of block's delta should be skipped. It is needed in case + // the block is encrypted and there is not encryption key to decrypt it. + // skipHeads argument indicates whether updating of heads should be skipped. It is needed in case + // the block's head was already updated when the encrypted block was processed. + ProcessBlock(ctx context.Context, block *coreblock.Block, cid cidlink.Link, skipMerge, skipHeads bool) error } // baseMerkleCRDT handles the MerkleCRDT overhead functions that aren't CRDT specific like the mutations and state diff --git a/net/client.go b/net/client.go index 9930710891..b0e957efbf 100644 --- a/net/client.go +++ b/net/client.go @@ -37,7 +37,7 @@ func (s *server) pushLog(ctx context.Context, evt event.Update, pid peer.ID) err Cid: evt.Cid.Bytes(), SchemaRoot: []byte(evt.SchemaRoot), Creator: s.peer.host.ID().String(), - Log: &pb.Document_Log{ + Log: &pb.Log{ Block: evt.Block, }, } diff --git a/net/dialer_test.go b/net/dialer_test.go index 479d4d7e63..913df60545 100644 --- a/net/dialer_test.go +++ b/net/dialer_test.go @@ -30,6 +30,7 @@ func TestDial_WithConnectedPeer_NoError(t *testing.T) { ctx, db1.Rootstore(), db1.Blockstore(), + db1.Encstore(), db1.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -39,6 +40,7 @@ func TestDial_WithConnectedPeer_NoError(t *testing.T) { ctx, db2.Rootstore(), db2.Blockstore(), + db1.Encstore(), db2.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -63,6 +65,7 @@ func TestDial_WithConnectedPeerAndSecondConnection_NoError(t *testing.T) { ctx, db1.Rootstore(), db1.Blockstore(), + db1.Encstore(), db1.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -72,6 +75,7 @@ func TestDial_WithConnectedPeerAndSecondConnection_NoError(t *testing.T) { ctx, db2.Rootstore(), db2.Blockstore(), + db1.Encstore(), db2.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -99,6 +103,7 @@ func TestDial_WithConnectedPeerAndSecondConnectionWithConnectionShutdown_Closing ctx, db1.Rootstore(), db1.Blockstore(), + db1.Encstore(), db1.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -108,6 +113,7 @@ func TestDial_WithConnectedPeerAndSecondConnectionWithConnectionShutdown_Closing ctx, db2.Rootstore(), db2.Blockstore(), + db1.Encstore(), db2.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 595a33f2c5..768e7ddb0e 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -21,19 +21,17 @@ const ( ) // Log represents a thread log. -type Document struct { +type Log struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // ID of the document. - DocID []byte `protobuf:"bytes,1,opt,name=docID,proto3" json:"docID,omitempty"` - // head of the log. - Head []byte `protobuf:"bytes,4,opt,name=head,proto3" json:"head,omitempty"` + // block is the top-level node's raw data as an ipld.Block. + Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` } -func (x *Document) Reset() { - *x = Document{} +func (x *Log) Reset() { + *x = Log{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -41,13 +39,13 @@ func (x *Document) Reset() { } } -func (x *Document) String() string { +func (x *Log) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Document) ProtoMessage() {} +func (*Log) ProtoMessage() {} -func (x *Document) ProtoReflect() protoreflect.Message { +func (x *Log) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -59,21 +57,14 @@ func (x *Document) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Document.ProtoReflect.Descriptor instead. -func (*Document) Descriptor() ([]byte, []int) { +// Deprecated: Use Log.ProtoReflect.Descriptor instead. +func (*Log) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{0} } -func (x *Document) GetDocID() []byte { - if x != nil { - return x.DocID - } - return nil -} - -func (x *Document) GetHead() []byte { +func (x *Log) GetBlock() []byte { if x != nil { - return x.Head + return x.Block } return nil } @@ -353,14 +344,21 @@ func (x *PushLogRequest) GetBody() *PushLogRequest_Body { return nil } -type GetHeadLogRequest struct { +type FetchEncryptionKeyRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // docID is the ID of the document that is affected by the log. + DocID []byte `protobuf:"bytes,1,opt,name=docID,proto3" json:"docID,omitempty"` + // cid is the CID of the composite of the document. + Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` + // schemaRoot is the SchemaRoot of the collection that the document resides in. + SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` } -func (x *GetHeadLogRequest) Reset() { - *x = GetHeadLogRequest{} +func (x *FetchEncryptionKeyRequest) Reset() { + *x = FetchEncryptionKeyRequest{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -368,13 +366,13 @@ func (x *GetHeadLogRequest) Reset() { } } -func (x *GetHeadLogRequest) String() string { +func (x *FetchEncryptionKeyRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetHeadLogRequest) ProtoMessage() {} +func (*FetchEncryptionKeyRequest) ProtoMessage() {} -func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { +func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -386,19 +384,40 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. -func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. +func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{8} } -type PushLogReply struct { +func (x *FetchEncryptionKeyRequest) GetDocID() []byte { + if x != nil { + return x.DocID + } + return nil +} + +func (x *FetchEncryptionKeyRequest) GetCid() []byte { + if x != nil { + return x.Cid + } + return nil +} + +func (x *FetchEncryptionKeyRequest) GetSchemaRoot() []byte { + if x != nil { + return x.SchemaRoot + } + return nil +} + +type GetHeadLogRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *PushLogReply) Reset() { - *x = PushLogReply{} +func (x *GetHeadLogRequest) Reset() { + *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -406,13 +425,13 @@ func (x *PushLogReply) Reset() { } } -func (x *PushLogReply) String() string { +func (x *GetHeadLogRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PushLogReply) ProtoMessage() {} +func (*GetHeadLogRequest) ProtoMessage() {} -func (x *PushLogReply) ProtoReflect() protoreflect.Message { +func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -424,19 +443,19 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. -func (*PushLogReply) Descriptor() ([]byte, []int) { +// Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. +func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{9} } -type GetHeadLogReply struct { +type PushLogReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *GetHeadLogReply) Reset() { - *x = GetHeadLogReply{} +func (x *PushLogReply) Reset() { + *x = PushLogReply{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -444,13 +463,13 @@ func (x *GetHeadLogReply) Reset() { } } -func (x *GetHeadLogReply) String() string { +func (x *PushLogReply) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetHeadLogReply) ProtoMessage() {} +func (*PushLogReply) ProtoMessage() {} -func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { +func (x *PushLogReply) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -462,23 +481,25 @@ func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. -func (*GetHeadLogReply) Descriptor() ([]byte, []int) { +// Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. +func (*PushLogReply) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{10} } -// Record is a thread record containing link data. -type Document_Log struct { +type FetchEncryptionKeyReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // block is the top-level node's raw data as an ipld.Block. - Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` + EncryptionKey []byte `protobuf:"bytes,1,opt,name=encryptionKey,proto3" json:"encryptionKey,omitempty"` + // cid is the CID of the composite of the document. + Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` + // schemaRoot is the SchemaRoot of the collection that the document resides in. + SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` } -func (x *Document_Log) Reset() { - *x = Document_Log{} +func (x *FetchEncryptionKeyReply) Reset() { + *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -486,13 +507,13 @@ func (x *Document_Log) Reset() { } } -func (x *Document_Log) String() string { +func (x *FetchEncryptionKeyReply) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Document_Log) ProtoMessage() {} +func (*FetchEncryptionKeyReply) ProtoMessage() {} -func (x *Document_Log) ProtoReflect() protoreflect.Message { +func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -504,18 +525,70 @@ func (x *Document_Log) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Document_Log.ProtoReflect.Descriptor instead. -func (*Document_Log) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{0, 0} +// Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. +func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { + return file_net_proto_rawDescGZIP(), []int{11} } -func (x *Document_Log) GetBlock() []byte { +func (x *FetchEncryptionKeyReply) GetEncryptionKey() []byte { if x != nil { - return x.Block + return x.EncryptionKey + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetCid() []byte { + if x != nil { + return x.Cid + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { + if x != nil { + return x.SchemaRoot } return nil } +type GetHeadLogReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetHeadLogReply) Reset() { + *x = GetHeadLogReply{} + if protoimpl.UnsafeEnabled { + mi := &file_net_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetHeadLogReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetHeadLogReply) ProtoMessage() {} + +func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { + mi := &file_net_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. +func (*GetHeadLogReply) Descriptor() ([]byte, []int) { + return file_net_proto_rawDescGZIP(), []int{12} +} + type PushLogRequest_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -530,13 +603,13 @@ type PushLogRequest_Body struct { // creator is the PeerID of the peer that created the log. Creator string `protobuf:"bytes,4,opt,name=creator,proto3" json:"creator,omitempty"` // log hold the block that represent version of the document. - Log *Document_Log `protobuf:"bytes,6,opt,name=log,proto3" json:"log,omitempty"` + Log *Log `protobuf:"bytes,6,opt,name=log,proto3" json:"log,omitempty"` } func (x *PushLogRequest_Body) Reset() { *x = PushLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -549,7 +622,7 @@ func (x *PushLogRequest_Body) String() string { func (*PushLogRequest_Body) ProtoMessage() {} func (x *PushLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -593,7 +666,7 @@ func (x *PushLogRequest_Body) GetCreator() string { return "" } -func (x *PushLogRequest_Body) GetLog() *Document_Log { +func (x *PushLogRequest_Body) GetLog() *Log { if x != nil { return x.Log } @@ -604,59 +677,74 @@ var File_net_proto protoreflect.FileDescriptor var file_net_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x22, 0x51, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x12, - 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x1a, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, - 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, - 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, - 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xd4, 0x01, 0x0a, - 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x1a, 0x90, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, + 0x2e, 0x70, 0x62, 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, + 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, + 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x63, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, + 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x71, + 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x65, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, - 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x26, 0x0a, 0x03, 0x6c, - 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, - 0x62, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x03, - 0x6c, 0x6f, 0x67, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, - 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xd1, 0x02, 0x0a, 0x07, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, - 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, - 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, - 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, - 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, - 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, - 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, - 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x74, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, + 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, + 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, + 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, + 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, + 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, + 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, + 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, + 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -671,37 +759,40 @@ func file_net_proto_rawDescGZIP() []byte { return file_net_proto_rawDescData } -var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_net_proto_goTypes = []any{ - (*Document)(nil), // 0: net.pb.Document - (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest - (*GetDocGraphReply)(nil), // 2: net.pb.GetDocGraphReply - (*PushDocGraphRequest)(nil), // 3: net.pb.PushDocGraphRequest - (*PushDocGraphReply)(nil), // 4: net.pb.PushDocGraphReply - (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest - (*GetLogReply)(nil), // 6: net.pb.GetLogReply - (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest - (*GetHeadLogRequest)(nil), // 8: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 9: net.pb.PushLogReply - (*GetHeadLogReply)(nil), // 10: net.pb.GetHeadLogReply - (*Document_Log)(nil), // 11: net.pb.Document.Log - (*PushLogRequest_Body)(nil), // 12: net.pb.PushLogRequest.Body + (*Log)(nil), // 0: net.pb.Log + (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest + (*GetDocGraphReply)(nil), // 2: net.pb.GetDocGraphReply + (*PushDocGraphRequest)(nil), // 3: net.pb.PushDocGraphRequest + (*PushDocGraphReply)(nil), // 4: net.pb.PushDocGraphReply + (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest + (*GetLogReply)(nil), // 6: net.pb.GetLogReply + (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest + (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest + (*GetHeadLogRequest)(nil), // 9: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 10: net.pb.PushLogReply + (*FetchEncryptionKeyReply)(nil), // 11: net.pb.FetchEncryptionKeyReply + (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply + (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body } var file_net_proto_depIdxs = []int32{ - 12, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 11, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Document.Log + 13, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body + 0, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log 1, // 2: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest 3, // 3: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 8, // 6: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 7: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 8: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 9: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 9, // 10: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 10, // 11: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 7, // [7:12] is the sub-list for method output_type - 2, // [2:7] is the sub-list for method input_type + 8, // 6: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest + 9, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 8: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 9: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 10: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 10, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 11, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 12, // 13: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 8, // [8:14] is the sub-list for method output_type + 2, // [2:8] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name @@ -714,7 +805,7 @@ func file_net_proto_init() { } if !protoimpl.UnsafeEnabled { file_net_proto_msgTypes[0].Exporter = func(v any, i int) any { - switch v := v.(*Document); i { + switch v := v.(*Log); i { case 0: return &v.state case 1: @@ -810,7 +901,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogRequest); i { + switch v := v.(*FetchEncryptionKeyRequest); i { case 0: return &v.state case 1: @@ -822,7 +913,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*PushLogReply); i { + switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state case 1: @@ -834,7 +925,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogReply); i { + switch v := v.(*PushLogReply); i { case 0: return &v.state case 1: @@ -846,7 +937,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*Document_Log); i { + switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state case 1: @@ -858,6 +949,18 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { + switch v := v.(*GetHeadLogReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state @@ -876,7 +979,7 @@ func file_net_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_net_proto_rawDesc, NumEnums: 0, - NumMessages: 13, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/net/pb/net.proto b/net/pb/net.proto index 5b0ee35dfb..e39ab9fa91 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -4,17 +4,9 @@ package net.pb; option go_package = "/;net_pb"; // Log represents a thread log. -message Document { - // ID of the document. - bytes docID = 1; - // head of the log. - bytes head = 4; - - // Record is a thread record containing link data. - message Log { - // block is the top-level node's raw data as an ipld.Block. - bytes block = 1; - } +message Log { + // block is the top-level node's raw data as an ipld.Block. + bytes block = 1; } message GetDocGraphRequest {} @@ -42,14 +34,31 @@ message PushLogRequest { // creator is the PeerID of the peer that created the log. string creator = 4; // log hold the block that represent version of the document. - Document.Log log = 6; + Log log = 6; } } +message FetchEncryptionKeyRequest { + // docID is the ID of the document that is affected by the log. + bytes docID = 1; + // cid is the CID of the composite of the document. + bytes cid = 2; + // schemaRoot is the SchemaRoot of the collection that the document resides in. + bytes schemaRoot = 3; +} + message GetHeadLogRequest {} message PushLogReply {} +message FetchEncryptionKeyReply { + bytes encryptionKey = 1; + // cid is the CID of the composite of the document. + bytes cid = 2; + // schemaRoot is the SchemaRoot of the collection that the document resides in. + bytes schemaRoot = 3; +} + message GetHeadLogReply {} // Service is the peer-to-peer network API for document sync @@ -62,6 +71,8 @@ service Service { rpc GetLog(GetLogRequest) returns (GetLogReply) {} // PushLog to this peer. rpc PushLog(PushLogRequest) returns (PushLogReply) {} + // TryGenEncryptionKey from this peer. + rpc TryGenEncryptionKey(FetchEncryptionKeyRequest) returns (FetchEncryptionKeyReply) {} // GetHeadLog from this peer rpc GetHeadLog(GetHeadLogRequest) returns (GetHeadLogReply) {} } diff --git a/net/pb/net_grpc.pb.go b/net/pb/net_grpc.pb.go index 84564d6bec..8c0e2eab05 100644 --- a/net/pb/net_grpc.pb.go +++ b/net/pb/net_grpc.pb.go @@ -19,11 +19,12 @@ import ( const _ = grpc.SupportPackageIsVersion8 const ( - Service_GetDocGraph_FullMethodName = "/net.pb.Service/GetDocGraph" - Service_PushDocGraph_FullMethodName = "/net.pb.Service/PushDocGraph" - Service_GetLog_FullMethodName = "/net.pb.Service/GetLog" - Service_PushLog_FullMethodName = "/net.pb.Service/PushLog" - Service_GetHeadLog_FullMethodName = "/net.pb.Service/GetHeadLog" + Service_GetDocGraph_FullMethodName = "/net.pb.Service/GetDocGraph" + Service_PushDocGraph_FullMethodName = "/net.pb.Service/PushDocGraph" + Service_GetLog_FullMethodName = "/net.pb.Service/GetLog" + Service_PushLog_FullMethodName = "/net.pb.Service/PushLog" + Service_TryGenEncryptionKey_FullMethodName = "/net.pb.Service/TryGenEncryptionKey" + Service_GetHeadLog_FullMethodName = "/net.pb.Service/GetHeadLog" ) // ServiceClient is the client API for Service service. @@ -40,6 +41,8 @@ type ServiceClient interface { GetLog(ctx context.Context, in *GetLogRequest, opts ...grpc.CallOption) (*GetLogReply, error) // PushLog to this peer. PushLog(ctx context.Context, in *PushLogRequest, opts ...grpc.CallOption) (*PushLogReply, error) + // TryGenEncryptionKey from this peer. + TryGenEncryptionKey(ctx context.Context, in *FetchEncryptionKeyRequest, opts ...grpc.CallOption) (*FetchEncryptionKeyReply, error) // GetHeadLog from this peer GetHeadLog(ctx context.Context, in *GetHeadLogRequest, opts ...grpc.CallOption) (*GetHeadLogReply, error) } @@ -92,6 +95,16 @@ func (c *serviceClient) PushLog(ctx context.Context, in *PushLogRequest, opts .. return out, nil } +func (c *serviceClient) TryGenEncryptionKey(ctx context.Context, in *FetchEncryptionKeyRequest, opts ...grpc.CallOption) (*FetchEncryptionKeyReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FetchEncryptionKeyReply) + err := c.cc.Invoke(ctx, Service_TryGenEncryptionKey_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *serviceClient) GetHeadLog(ctx context.Context, in *GetHeadLogRequest, opts ...grpc.CallOption) (*GetHeadLogReply, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetHeadLogReply) @@ -116,6 +129,8 @@ type ServiceServer interface { GetLog(context.Context, *GetLogRequest) (*GetLogReply, error) // PushLog to this peer. PushLog(context.Context, *PushLogRequest) (*PushLogReply, error) + // TryGenEncryptionKey from this peer. + TryGenEncryptionKey(context.Context, *FetchEncryptionKeyRequest) (*FetchEncryptionKeyReply, error) // GetHeadLog from this peer GetHeadLog(context.Context, *GetHeadLogRequest) (*GetHeadLogReply, error) mustEmbedUnimplementedServiceServer() @@ -137,6 +152,9 @@ func (UnimplementedServiceServer) GetLog(context.Context, *GetLogRequest) (*GetL func (UnimplementedServiceServer) PushLog(context.Context, *PushLogRequest) (*PushLogReply, error) { return nil, status.Errorf(codes.Unimplemented, "method PushLog not implemented") } +func (UnimplementedServiceServer) TryGenEncryptionKey(context.Context, *FetchEncryptionKeyRequest) (*FetchEncryptionKeyReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method TryGenEncryptionKey not implemented") +} func (UnimplementedServiceServer) GetHeadLog(context.Context, *GetHeadLogRequest) (*GetHeadLogReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetHeadLog not implemented") } @@ -225,6 +243,24 @@ func _Service_PushLog_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _Service_TryGenEncryptionKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FetchEncryptionKeyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServiceServer).TryGenEncryptionKey(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Service_TryGenEncryptionKey_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServiceServer).TryGenEncryptionKey(ctx, req.(*FetchEncryptionKeyRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Service_GetHeadLog_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetHeadLogRequest) if err := dec(in); err != nil { @@ -266,6 +302,10 @@ var Service_ServiceDesc = grpc.ServiceDesc{ MethodName: "PushLog", Handler: _Service_PushLog_Handler, }, + { + MethodName: "TryGenEncryptionKey", + Handler: _Service_TryGenEncryptionKey_Handler, + }, { MethodName: "GetHeadLog", Handler: _Service_GetHeadLog_Handler, diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 34a256ee4e..4456228e6e 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -18,7 +18,7 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -func (m *Document_Log) MarshalVT() (dAtA []byte, err error) { +func (m *Log) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -31,12 +31,12 @@ func (m *Document_Log) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Document_Log) MarshalToVT(dAtA []byte) (int, error) { +func (m *Log) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *Document_Log) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *Log) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -58,53 +58,6 @@ func (m *Document_Log) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *Document) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil - } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Document) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *Document) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) - } - if len(m.Head) > 0 { - i -= len(m.Head) - copy(dAtA[i:], m.Head) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Head))) - i-- - dAtA[i] = 0x22 - } - if len(m.DocID) > 0 { - i -= len(m.DocID) - copy(dAtA[i:], m.DocID) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *GetDocGraphRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -417,6 +370,60 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FetchEncryptionKeyRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FetchEncryptionKeyRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.SchemaRoot) > 0 { + i -= len(m.SchemaRoot) + copy(dAtA[i:], m.SchemaRoot) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) + i-- + dAtA[i] = 0x1a + } + if len(m.Cid) > 0 { + i -= len(m.Cid) + copy(dAtA[i:], m.Cid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + i-- + dAtA[i] = 0x12 + } + if len(m.DocID) > 0 { + i -= len(m.DocID) + copy(dAtA[i:], m.DocID) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *GetHeadLogRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -483,7 +490,7 @@ func (m *PushLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *GetHeadLogReply) MarshalVT() (dAtA []byte, err error) { +func (m *FetchEncryptionKeyReply) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -496,12 +503,12 @@ func (m *GetHeadLogReply) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *GetHeadLogReply) MarshalToVT(dAtA []byte) (int, error) { +func (m *FetchEncryptionKeyReply) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *GetHeadLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -513,34 +520,70 @@ func (m *GetHeadLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.SchemaRoot) > 0 { + i -= len(m.SchemaRoot) + copy(dAtA[i:], m.SchemaRoot) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) + i-- + dAtA[i] = 0x1a + } + if len(m.Cid) > 0 { + i -= len(m.Cid) + copy(dAtA[i:], m.Cid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + i-- + dAtA[i] = 0x12 + } + if len(m.EncryptionKey) > 0 { + i -= len(m.EncryptionKey) + copy(dAtA[i:], m.EncryptionKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptionKey))) + i-- + dAtA[i] = 0xa + } return len(dAtA) - i, nil } -func (m *Document_Log) SizeVT() (n int) { +func (m *GetHeadLogReply) MarshalVT() (dAtA []byte, err error) { if m == nil { - return 0 + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GetHeadLogReply) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *GetHeadLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil } + i := len(dAtA) + _ = i var l int _ = l - l = len(m.Block) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) } - n += len(m.unknownFields) - return n + return len(dAtA) - i, nil } -func (m *Document) SizeVT() (n int) { +func (m *Log) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.DocID) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - l = len(m.Head) + l = len(m.Block) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } @@ -652,6 +695,28 @@ func (m *PushLogRequest) SizeVT() (n int) { return n } +func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.DocID) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Cid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.SchemaRoot) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *GetHeadLogRequest) SizeVT() (n int) { if m == nil { return 0 @@ -672,6 +737,28 @@ func (m *PushLogReply) SizeVT() (n int) { return n } +func (m *FetchEncryptionKeyReply) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.EncryptionKey) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Cid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.SchemaRoot) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *GetHeadLogReply) SizeVT() (n int) { if m == nil { return 0 @@ -682,7 +769,7 @@ func (m *GetHeadLogReply) SizeVT() (n int) { return n } -func (m *Document_Log) UnmarshalVT(dAtA []byte) error { +func (m *Log) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -705,10 +792,10 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Document_Log: wiretype end group for non-group") + return fmt.Errorf("proto: Log: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Document_Log: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Log: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -767,7 +854,7 @@ func (m *Document_Log) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *Document) UnmarshalVT(dAtA []byte) error { +func (m *GetDocGraphRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -790,129 +877,10 @@ func (m *Document) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Document: wiretype end group for non-group") + return fmt.Errorf("proto: GetDocGraphRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Document: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DocID", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.DocID = append(m.DocID[:0], dAtA[iNdEx:postIndex]...) - if m.DocID == nil { - m.DocID = []byte{} - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Head", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Head = append(m.Head[:0], dAtA[iNdEx:postIndex]...) - if m.Head == nil { - m.Head = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *GetDocGraphRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: GetDocGraphRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: GetDocGraphRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: GetDocGraphRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: @@ -1385,7 +1353,7 @@ func (m *PushLogRequest_Body) UnmarshalVT(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Log == nil { - m.Log = &Document_Log{} + m.Log = &Log{} } if err := m.Log.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err @@ -1500,6 +1468,159 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FetchEncryptionKeyRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FetchEncryptionKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DocID", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DocID = append(m.DocID[:0], dAtA[iNdEx:postIndex]...) + if m.DocID == nil { + m.DocID = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cid = append(m.Cid[:0], dAtA[iNdEx:postIndex]...) + if m.Cid == nil { + m.Cid = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SchemaRoot = append(m.SchemaRoot[:0], dAtA[iNdEx:postIndex]...) + if m.SchemaRoot == nil { + m.SchemaRoot = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GetHeadLogRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1602,6 +1723,159 @@ func (m *PushLogReply) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FetchEncryptionKeyReply: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FetchEncryptionKeyReply: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EncryptionKey = append(m.EncryptionKey[:0], dAtA[iNdEx:postIndex]...) + if m.EncryptionKey == nil { + m.EncryptionKey = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cid = append(m.Cid[:0], dAtA[iNdEx:postIndex]...) + if m.Cid == nil { + m.Cid = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SchemaRoot = append(m.SchemaRoot[:0], dAtA[iNdEx:postIndex]...) + if m.SchemaRoot == nil { + m.SchemaRoot = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GetHeadLogReply) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/net/peer.go b/net/peer.go index 7222d7cf9f..5ffce49bb0 100644 --- a/net/peer.go +++ b/net/peer.go @@ -53,6 +53,7 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" corenet "github.com/sourcenetwork/defradb/internal/core/net" + "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -60,6 +61,7 @@ import ( // to the underlying DefraDB instance. type Peer struct { blockstore datastore.Blockstore + encstore datastore.DSReaderWriter bus *event.Bus updateSub *event.Subscription @@ -85,10 +87,11 @@ func NewPeer( ctx context.Context, rootstore datastore.Rootstore, blockstore datastore.Blockstore, + encstore datastore.DSReaderWriter, bus *event.Bus, opts ...NodeOpt, ) (p *Peer, err error) { - if rootstore == nil || blockstore == nil { + if rootstore == nil || blockstore == nil || encstore == nil { return nil, ErrNilDB } @@ -214,6 +217,7 @@ func NewPeer( dht: ddht, ps: ps, blockstore: blockstore, + encstore: encstore, bus: bus, p2pRPC: grpc.NewServer(options.GRPCServerOptions...), ctx: ctx, @@ -259,7 +263,7 @@ func (p *Peer) Start() error { } if p.ps != nil { - sub, err := p.bus.Subscribe(event.UpdateName, event.P2PTopicName, event.ReplicatorName) + sub, err := p.bus.Subscribe(event.UpdateName, event.P2PTopicName, event.ReplicatorName, encryption.RequestKeyEventName) if err != nil { return err } @@ -281,6 +285,11 @@ func (p *Peer) Start() error { } }() + err = p.server.addPubSubEncryptionTopic() + if err != nil { + return err + } + p.bus.Publish(event.NewMessage(event.PeerInfoName, event.PeerInfo{Info: p.PeerInfo()})) return nil @@ -353,6 +362,11 @@ func (p *Peer) handleMessageLoop() { case event.Replicator: p.server.updateReplicators(evt) + case encryption.RequestKeyEvent: + err := p.handleEncryptionKeyRequest(evt) + if err != nil { + log.ErrorContextE(p.ctx, "Error while handling broadcast log", err) + } default: // ignore other events continue @@ -386,7 +400,7 @@ func (p *Peer) RegisterNewDocument( Cid: c.Bytes(), SchemaRoot: []byte(schemaRoot), Creator: p.host.ID().String(), - Log: &pb.Document_Log{ + Log: &pb.Log{ Block: rawBlock, }, }, @@ -424,7 +438,7 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { Cid: evt.Cid.Bytes(), SchemaRoot: []byte(evt.SchemaRoot), Creator: p.host.ID().String(), - Log: &pb.Document_Log{ + Log: &pb.Log{ Block: evt.Block, }, } @@ -446,6 +460,20 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { return nil } +func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeyEvent) error { + req := &pb.FetchEncryptionKeyRequest{ + DocID: []byte(evt.DocID), + Cid: evt.Cid.Bytes(), + SchemaRoot: []byte(evt.SchemaRoot), + } + + if err := p.server.requestEncryptionKey(p.ctx, req); err != nil { + return NewErrPublishingToDocIDTopic(err, evt.Cid.String(), evt.DocID) + } + + return nil +} + func (p *Peer) pushLogToReplicators(lg event.Update) { // let the exchange know we have this block // this should speed up the dag sync process diff --git a/net/peer_test.go b/net/peer_test.go index cb8c8eab44..efd5503ca1 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -77,6 +77,7 @@ func newTestPeer(ctx context.Context, t *testing.T) (client.DB, *Peer) { ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses(randomMultiaddr), ) @@ -91,14 +92,14 @@ func TestNewPeer_NoError(t *testing.T) { db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) defer db.Close() - p, err := NewPeer(ctx, db.Rootstore(), db.Blockstore(), db.Events()) + p, err := NewPeer(ctx, db.Rootstore(), db.Blockstore(), db.Encstore(), db.Events()) require.NoError(t, err) p.Close() } func TestNewPeer_NoDB_NilDBError(t *testing.T) { ctx := context.Background() - _, err := NewPeer(ctx, nil, nil, nil) + _, err := NewPeer(ctx, nil, nil, nil, nil) require.ErrorIs(t, err, ErrNilDB) } @@ -128,6 +129,7 @@ func TestStart_WithKnownPeer_NoError(t *testing.T) { ctx, db1.Rootstore(), db1.Blockstore(), + db1.Encstore(), db1.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -137,6 +139,7 @@ func TestStart_WithKnownPeer_NoError(t *testing.T) { ctx, db2.Rootstore(), db2.Blockstore(), + db1.Encstore(), db2.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -169,6 +172,7 @@ func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { ctx, db1.Rootstore(), db1.Blockstore(), + db1.Encstore(), db1.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -178,6 +182,7 @@ func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { ctx, db2.Rootstore(), db2.Blockstore(), + db1.Encstore(), db2.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -477,6 +482,7 @@ func TestNewPeer_WithEnableRelay_NoError(t *testing.T) { context.Background(), db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithEnableRelay(true), ) @@ -496,6 +502,7 @@ func TestNewPeer_WithDBClosed_NoError(t *testing.T) { context.Background(), db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), ) require.ErrorContains(t, err, "datastore closed") @@ -512,6 +519,7 @@ func TestNewPeer_NoPubSub_NoError(t *testing.T) { context.Background(), db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithEnablePubSub(false), ) @@ -531,6 +539,7 @@ func TestNewPeer_WithEnablePubSub_NoError(t *testing.T) { ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithEnablePubSub(true), ) @@ -551,6 +560,7 @@ func TestNodeClose_NoError(t *testing.T) { context.Background(), db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), ) require.NoError(t, err) @@ -568,6 +578,7 @@ func TestNewPeer_BootstrapWithNoPeer_NoError(t *testing.T) { ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -586,6 +597,7 @@ func TestNewPeer_BootstrapWithOnePeer_NoError(t *testing.T) { ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -595,6 +607,7 @@ func TestNewPeer_BootstrapWithOnePeer_NoError(t *testing.T) { ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -618,6 +631,7 @@ func TestNewPeer_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -627,6 +641,7 @@ func TestNewPeer_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing ctx, db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) @@ -653,6 +668,7 @@ func TestListenAddrs_WithListenAddresses_NoError(t *testing.T) { context.Background(), db.Rootstore(), db.Blockstore(), + db.Encstore(), db.Events(), WithListenAddresses("/ip4/0.0.0.0/tcp/0"), ) diff --git a/net/server.go b/net/server.go index 0e36eb7e3c..8dd65b6c5e 100644 --- a/net/server.go +++ b/net/server.go @@ -32,9 +32,12 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" coreblock "github.com/sourcenetwork/defradb/internal/core/block" + "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) +const encryptionTopic = "encryption" + // Server is the request/response instance for all P2P RPC communication. // Implements gRPC server. See net/pb/net.proto for corresponding service definitions. // @@ -145,7 +148,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL corelog.Any("DocID", docID.String())) // Once processed, subscribe to the DocID topic on the pubsub network unless we already - // suscribe to the collection. + // subscribed to the collection. if !s.hasPubSubTopic(string(req.Body.SchemaRoot)) { err = s.addPubSubTopic(docID.String(), true) if err != nil { @@ -164,6 +167,32 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return &pb.PushLogReply{}, nil } +func (s *server) TryGenEncryptionKey( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) (*pb.FetchEncryptionKeyReply, error) { + docID, err := client.NewDocIDFromString(string(req.DocID)) + if err != nil { + return nil, err + } + + encKey, err := encryption.GetDocKey(encryption.ContextWithStore(ctx, s.peer.encstore), docID.String()) + + if err != nil { + return nil, err + } + + if len(encKey) == 0 { + return nil, nil + } + + res := &pb.FetchEncryptionKeyReply{ + EncryptionKey: encKey, + } + + return res, nil +} + // GetHeadLog receives a get head log request func (s *server) GetHeadLog( ctx context.Context, @@ -210,6 +239,30 @@ func (s *server) addPubSubTopic(topic string, subscribe bool) error { return nil } +// addPubSubEncryptionTopic subscribes to a topic on the pubsub network +func (s *server) addPubSubEncryptionTopic() error { + if s.peer.ps == nil { + return nil + } + + t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), encryptionTopic, true) + if err != nil { + return err + } + + t.SetEventHandler(s.pubSubEventHandler) + t.SetMessageHandler(s.pubSubEncryptionMessageHandler) + + s.mu.Lock() + defer s.mu.Unlock() + + s.topics[encryptionTopic] = pubsubTopic{ + Topic: t, + subscribed: true, + } + return nil +} + // hasPubSubTopic checks if we are subscribed to a topic. func (s *server) hasPubSubTopic(topic string) bool { s.mu.Lock() @@ -288,6 +341,57 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe return nil } +// requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network +func (s *server) requestEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) error { + if s.peer.ps == nil { // skip if we aren't running with a pubsub net + return nil + } + s.mu.Lock() + t, ok := s.topics[encryptionTopic] + s.mu.Unlock() + if !ok { + err := s.addPubSubTopic(encryptionTopic, false) + if err != nil { + return errors.Wrap(fmt.Sprintf("failed to created single use topic %s", encryptionTopic), err) + } + return s.requestEncryptionKey(ctx, req) + } + + data, err := req.MarshalVT() + if err != nil { + return errors.Wrap("failed marshling pubsub message", err) + } + + respChan, err := t.Publish(ctx, data) + if err != nil { + return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) + } + go func() { + s.handleFetchEncryptionKeyResponse(<-respChan, req) + }() + return nil +} + +// handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages +func (s *server) handleFetchEncryptionKeyResponse( + resp rpc.Response, + req *pb.FetchEncryptionKeyRequest, +) { + var keyResp pb.FetchEncryptionKeyReply + err := proto.Unmarshal(resp.Data, &keyResp) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to unmarshal encryption key response", err) + return + } + cid, err := cid.Cast(req.Cid) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to parse CID", err) + return + } + s.peer.bus.Publish(encryption.NewKeyRetrievedMessage( + string(req.DocID), cid, string(req.SchemaRoot), keyResp.EncryptionKey)) +} + // pubSubMessageHandler handles incoming PushLog messages from the pubsub network. func (s *server) pubSubMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { log.InfoContext(s.peer.ctx, "Received new pubsub event", @@ -309,6 +413,25 @@ func (s *server) pubSubMessageHandler(from libpeer.ID, topic string, msg []byte) return nil, nil } +// pubSubEncryptionMessageHandler handles incoming FetchEncryptionKeyRequest messages from the pubsub network. +func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { + req := new(pb.FetchEncryptionKeyRequest) + if err := proto.Unmarshal(msg, req); err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to unmarshal pubsub message %s", err) + return nil, err + } + + ctx := grpcpeer.NewContext(s.peer.ctx, &grpcpeer.Peer{ + Addr: addr{from}, + }) + res, err := s.TryGenEncryptionKey(ctx, req) + if err != nil { + return nil, errors.Wrap(fmt.Sprintf("Failed pushing log for doc %s", topic), err) + } + return res.MarshalVT() + //return proto.Marshal(res) +} + // pubSubEventHandler logs events from the subscribed DocID topics. func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { log.InfoContext(s.peer.ctx, "Received new pubsub event", diff --git a/net/server_test.go b/net/server_test.go index 1ac178a2d1..3e557ba24f 100644 --- a/net/server_test.go +++ b/net/server_test.go @@ -126,7 +126,7 @@ func TestGetHeadLog(t *testing.T) { } func getHead(ctx context.Context, db client.DB, docID client.DocID) (cid.Cid, error) { - prefix := core.DataStoreKeyFromDocID(docID).ToHeadStoreKey().WithFieldId(core.COMPOSITE_NAMESPACE).ToString() + prefix := core.DataStoreKeyFromDocID(docID).ToHeadStoreKey().WithFieldID(core.COMPOSITE_NAMESPACE).ToString() results, err := db.Headstore().Query(ctx, query.Query{Prefix: prefix}) if err != nil { return cid.Undef, err @@ -185,7 +185,7 @@ func TestPushLog(t *testing.T) { Cid: headCID.Bytes(), SchemaRoot: []byte(col.SchemaRoot()), Creator: p.PeerID().String(), - Log: &net_pb.Document_Log{ + Log: &net_pb.Log{ Block: b, }, }, diff --git a/node/node.go b/node/node.go index 6dad35f593..72b6471f5b 100644 --- a/node/node.go +++ b/node/node.go @@ -141,7 +141,7 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { var peer *net.Peer if !options.disableP2P { // setup net node - peer, err = net.NewPeer(ctx, db.Rootstore(), db.Blockstore(), db.Events(), netOpts...) + peer, err = net.NewPeer(ctx, db.Rootstore(), db.Blockstore(), db.Encstore(), db.Events(), netOpts...) if err != nil { return nil, err } diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 14e4df7cc4..4b4bc3880b 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -510,6 +510,10 @@ func (w *Wrapper) Rootstore() datastore.Rootstore { return w.node.DB.Rootstore() } +func (w *Wrapper) Encstore() datastore.DSReaderWriter { + return w.node.DB.Encstore() +} + func (w *Wrapper) Blockstore() datastore.Blockstore { return w.node.DB.Blockstore() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 76e31d9cbd..29f87d88fe 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -203,6 +203,10 @@ func (w *Wrapper) Rootstore() datastore.Rootstore { return w.node.DB.Rootstore() } +func (w *Wrapper) Encstore() datastore.DSReaderWriter { + return w.node.DB.Encstore() +} + func (w *Wrapper) Blockstore() datastore.Blockstore { return w.node.DB.Blockstore() } diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go new file mode 100644 index 0000000000..4ae3b339e2 --- /dev/null +++ b/tests/integration/encryption/peer_share_test.go @@ -0,0 +1,69 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + "github.com/sourcenetwork/immutable" + + "github.com/sourcenetwork/defradb/internal/encryption" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeyRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + age + } + }`, + Results: []map[string]any{ + { + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 4eb0ddc19a..cc3ea7b1d0 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -18,7 +18,7 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { +/*func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { test := testUtils.TestCase{ Actions: []any{ testUtils.RandomNetworkingConfig(), @@ -51,7 +51,7 @@ func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { } testUtils.ExecuteTestCase(t, test) -} +}*/ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { test := testUtils.TestCase{ diff --git a/tests/integration/events.go b/tests/integration/events.go index c2db144d93..b950e5ac54 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -12,6 +12,7 @@ package tests import ( "encoding/json" + "slices" "time" "github.com/sourcenetwork/immutable" @@ -19,6 +20,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/internal/encryption" ) // eventTimeout is the amount of time to wait @@ -348,3 +350,31 @@ func getEventsForCreateDoc(s *state, action CreateDoc) map[string]struct{} { return expect } + +func waitForKeyRetrievedEvent(s *state, nodeIDs []int) { + for nodeID := 0; nodeID < len(s.nodes); nodeID++ { + if len(nodeIDs) > 0 && !slices.Contains(nodeIDs, nodeID) { + continue + } + + select { + case _, ok := <-s.nodeEvents[nodeID].encKeyRetrieved.Message(): + if !ok { + require.Fail(s.t, "subscription closed waiting for key retrieved event") + } + + case <-time.After(eventTimeout): + require.Fail(s.t, "timeout waiting for key retrieved event") + } + } +} + +func waitForSync(s *state, action WaitForSync) { + if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { + waitForMergeEvents(s) + } else if action.Event.Value() == encryption.KeyRetrievedEventName { + waitForKeyRetrievedEvent(s, action.NodeIDs) + } else { + require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) + } +} diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index b1b79982cf..17b374016d 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -14,7 +14,9 @@ import ( "time" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" + "github.com/sourcenetwork/immutable" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/corelog" @@ -134,7 +136,14 @@ type GetAllP2PCollections struct { // // For example you will likely wish to `WaitForSync` after creating a document in node 0 before querying // node 1 to see if it has been replicated. -type WaitForSync struct{} +type WaitForSync struct { + // Event is the name of the event to wait for. + // If is not provided then the action will wait for MergeComplete event. + Event immutable.Option[event.Name] + // NodeIDs are the node IDs (indexes) of the nodes to wait for sync on. + // If not provided then the action will wait for sync on all nodes. + NodeIDs []int +} // connectPeers connects two existing, started, nodes as peers. It returns a channel // that will receive an empty struct upon sync completion of all expected peer-sync events. diff --git a/tests/integration/state.go b/tests/integration/state.go index 0ea720b09f..52fc1b65c7 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/node" "github.com/sourcenetwork/defradb/tests/clients" @@ -78,6 +79,9 @@ type eventState struct { // p2pTopic is the `event.P2PTopicCompletedName` subscription p2pTopic *event.Subscription + + // encKeyRetrieved is the `encryption.KeyRetrieved` subscription + encKeyRetrieved *event.Subscription } // newEventState returns an eventState with all required subscriptions. @@ -98,11 +102,16 @@ func newEventState(bus *event.Bus) (*eventState, error) { if err != nil { return nil, err } + encKeyRetrieved, err := bus.Subscribe(encryption.KeyRetrievedEventName) + if err != nil { + return nil, err + } return &eventState{ - merge: merge, - update: update, - replicator: replicator, - p2pTopic: p2pTopic, + merge: merge, + update: update, + replicator: replicator, + p2pTopic: p2pTopic, + encKeyRetrieved: encKeyRetrieved, }, nil } diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 139b53652b..ee8d685e77 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -342,7 +342,7 @@ func performAction( assertClientIntrospectionResults(s, action) case WaitForSync: - waitForMergeEvents(s) + waitForSync(s, action) case Benchmark: benchmarkAction(s, actionIndex, action) @@ -706,7 +706,7 @@ func restartNodes( nodeOpts := s.nodeConfigs[i] nodeOpts = append(nodeOpts, net.WithListenAddresses(addresses...)) - p, err := net.NewPeer(s.ctx, node.DB.Rootstore(), node.DB.Blockstore(), node.DB.Events(), nodeOpts...) + p, err := net.NewPeer(s.ctx, node.DB.Rootstore(), node.DB.Blockstore(), node.DB.Encstore(), node.DB.Events(), nodeOpts...) require.NoError(s.t, err) if err := p.Start(); err != nil { @@ -780,7 +780,7 @@ func configureNode( nodeOpts := action() nodeOpts = append(nodeOpts, net.WithPrivateKey(privateKey)) - p, err := net.NewPeer(s.ctx, node.DB.Rootstore(), node.DB.Blockstore(), node.DB.Events(), nodeOpts...) + p, err := net.NewPeer(s.ctx, node.DB.Rootstore(), node.DB.Blockstore(), node.DB.Encstore(), node.DB.Events(), nodeOpts...) require.NoError(s.t, err) log.InfoContext(s.ctx, "Starting P2P node", corelog.Any("P2P address", p.PeerInfo())) From 5d189706db29d45cdef8efa6db0090ff2cf4ac96 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 27 Jul 2024 09:09:43 +0200 Subject: [PATCH 04/88] Add protobuf data for key request/response --- net/pb/Makefile | 7 +- net/pb/net.pb.go | 365 ++++++++++++++++++++++----------- net/pb/net.proto | 25 ++- net/pb/net_vtproto.pb.go | 422 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 691 insertions(+), 128 deletions(-) diff --git a/net/pb/Makefile b/net/pb/Makefile index db8b72e810..099a9ccc74 100644 --- a/net/pb/Makefile +++ b/net/pb/Makefile @@ -5,6 +5,8 @@ PROTOC_GEN_GO := $(shell which protoc-gen-go) PROTOC_GEN_GO_GRPC := $(shell which protoc-gen-go-grpc) PROTOC_GEN_GO_VTPROTO := $(shell which protoc-gen-go-vtproto) +LIBP2P_PATH := $(shell go list -f '{{.Dir}}' github.com/libp2p/go-libp2p) + all: $(GO) deps: @@ -14,6 +16,9 @@ deps: %.pb.go: %.proto protoc \ + -I. \ + -I$(LIBP2P_PATH) \ + -I$(GOPATH)/src \ --go_out=. --plugin protoc-gen-go="$(PROTOC_GEN_GO)" \ --go-grpc_out=. --plugin protoc-gen-go-grpc="$(PROTOC_GEN_GO_GRPC)" \ --go-vtproto_out=. --plugin protoc-gen-go-vtproto="$(PROTOC_GEN_GO_VTPROTO)" \ @@ -24,4 +29,4 @@ clean: rm -f *.pb.go rm -f *pb_test.go -.PHONY: clean deps debug \ No newline at end of file +.PHONY: clean deps debug diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 768e7ddb0e..40972130f2 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -7,6 +7,7 @@ package net_pb import ( + pb "github.com/libp2p/go-libp2p/core/crypto/pb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -344,6 +345,67 @@ func (x *PushLogRequest) GetBody() *PushLogRequest_Body { return nil } +// PeerInfo contains information about a specific peer +type PeerInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // id is a libp2p peer identity. + Id []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // addresses is a list of peer's address in multiaddr format + // Learn more here https://github.com/multiformats/multiadd + Addresses []string `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses,omitempty"` +} + +func (x *PeerInfo) Reset() { + *x = PeerInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_net_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PeerInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerInfo) ProtoMessage() {} + +func (x *PeerInfo) ProtoReflect() protoreflect.Message { + mi := &file_net_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerInfo.ProtoReflect.Descriptor instead. +func (*PeerInfo) Descriptor() ([]byte, []int) { + return file_net_proto_rawDescGZIP(), []int{8} +} + +func (x *PeerInfo) GetId() []byte { + if x != nil { + return x.Id + } + return nil +} + +func (x *PeerInfo) GetAddresses() []string { + if x != nil { + return x.Addresses + } + return nil +} + +// FetchEncryptionKeyRequest is a request to receive a doc encryption key +// from a peer that holds it. type FetchEncryptionKeyRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -355,12 +417,18 @@ type FetchEncryptionKeyRequest struct { Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + // publicKey is a public of the requesting peer + PublicKey *pb.PublicKey `protobuf:"bytes,4,opt,name=publicKey,proto3" json:"publicKey,omitempty"` + // peerInfo is peer information is the requesting peer + PeerInfo *PeerInfo `protobuf:"bytes,5,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` + // signature is the requesting peer's signature of the request + Signature []byte `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { *x = FetchEncryptionKeyRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[8] + mi := &file_net_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -373,7 +441,7 @@ func (x *FetchEncryptionKeyRequest) String() string { func (*FetchEncryptionKeyRequest) ProtoMessage() {} func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[8] + mi := &file_net_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -386,7 +454,7 @@ func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{8} + return file_net_proto_rawDescGZIP(), []int{9} } func (x *FetchEncryptionKeyRequest) GetDocID() []byte { @@ -410,6 +478,27 @@ func (x *FetchEncryptionKeyRequest) GetSchemaRoot() []byte { return nil } +func (x *FetchEncryptionKeyRequest) GetPublicKey() *pb.PublicKey { + if x != nil { + return x.PublicKey + } + return nil +} + +func (x *FetchEncryptionKeyRequest) GetPeerInfo() *PeerInfo { + if x != nil { + return x.PeerInfo + } + return nil +} + +func (x *FetchEncryptionKeyRequest) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + type GetHeadLogRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -419,7 +508,7 @@ type GetHeadLogRequest struct { func (x *GetHeadLogRequest) Reset() { *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -432,7 +521,7 @@ func (x *GetHeadLogRequest) String() string { func (*GetHeadLogRequest) ProtoMessage() {} func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -445,7 +534,7 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{9} + return file_net_proto_rawDescGZIP(), []int{10} } type PushLogReply struct { @@ -457,7 +546,7 @@ type PushLogReply struct { func (x *PushLogReply) Reset() { *x = PushLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -470,7 +559,7 @@ func (x *PushLogReply) String() string { func (*PushLogReply) ProtoMessage() {} func (x *PushLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -483,25 +572,29 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. func (*PushLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{10} + return file_net_proto_rawDescGZIP(), []int{11} } +// FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request +// by a peer that holds the requested doc encryption key. type FetchEncryptionKeyReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - EncryptionKey []byte `protobuf:"bytes,1,opt,name=encryptionKey,proto3" json:"encryptionKey,omitempty"` + EncryptedKey []byte `protobuf:"bytes,1,opt,name=encryptedKey,proto3" json:"encryptedKey,omitempty"` // cid is the CID of the composite of the document. Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + // signature is the responding peer's signature of the reply + Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyReply) Reset() { *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -514,7 +607,7 @@ func (x *FetchEncryptionKeyReply) String() string { func (*FetchEncryptionKeyReply) ProtoMessage() {} func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -527,12 +620,12 @@ func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{11} + return file_net_proto_rawDescGZIP(), []int{12} } -func (x *FetchEncryptionKeyReply) GetEncryptionKey() []byte { +func (x *FetchEncryptionKeyReply) GetEncryptedKey() []byte { if x != nil { - return x.EncryptionKey + return x.EncryptedKey } return nil } @@ -551,6 +644,13 @@ func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { return nil } +func (x *FetchEncryptionKeyReply) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + type GetHeadLogReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -560,7 +660,7 @@ type GetHeadLogReply struct { func (x *GetHeadLogReply) Reset() { *x = GetHeadLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -573,7 +673,7 @@ func (x *GetHeadLogReply) String() string { func (*GetHeadLogReply) ProtoMessage() {} func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -586,7 +686,7 @@ func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. func (*GetHeadLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{12} + return file_net_proto_rawDescGZIP(), []int{13} } type PushLogRequest_Body struct { @@ -609,7 +709,7 @@ type PushLogRequest_Body struct { func (x *PushLogRequest_Body) Reset() { *x = PushLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -622,7 +722,7 @@ func (x *PushLogRequest_Body) String() string { func (*PushLogRequest_Body) ProtoMessage() {} func (x *PushLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -677,74 +777,89 @@ var File_net_proto protoreflect.FileDescriptor var file_net_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, - 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, - 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x63, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, - 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, - 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x71, - 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x65, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, + 0x2e, 0x70, 0x62, 0x1a, 0x1b, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, + 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x22, 0x14, 0x0a, + 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x44, + 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x13, + 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x03, 0x6c, 0x6f, + 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, + 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xe3, 0x01, 0x0a, 0x19, + 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, + 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, - 0x74, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, - 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, - 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, - 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, - 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x12, 0x32, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, + 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, + 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, + 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, + 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, + 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, + 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, + 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -759,7 +874,7 @@ func file_net_proto_rawDescGZIP() []byte { return file_net_proto_rawDescData } -var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_net_proto_goTypes = []any{ (*Log)(nil), // 0: net.pb.Log (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest @@ -769,33 +884,37 @@ var file_net_proto_goTypes = []any{ (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest (*GetLogReply)(nil), // 6: net.pb.GetLogReply (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest - (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest - (*GetHeadLogRequest)(nil), // 9: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 10: net.pb.PushLogReply - (*FetchEncryptionKeyReply)(nil), // 11: net.pb.FetchEncryptionKeyReply - (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply - (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body + (*PeerInfo)(nil), // 8: net.pb.PeerInfo + (*FetchEncryptionKeyRequest)(nil), // 9: net.pb.FetchEncryptionKeyRequest + (*GetHeadLogRequest)(nil), // 10: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 11: net.pb.PushLogReply + (*FetchEncryptionKeyReply)(nil), // 12: net.pb.FetchEncryptionKeyReply + (*GetHeadLogReply)(nil), // 13: net.pb.GetHeadLogReply + (*PushLogRequest_Body)(nil), // 14: net.pb.PushLogRequest.Body + (*pb.PublicKey)(nil), // 15: crypto.pb.PublicKey } var file_net_proto_depIdxs = []int32{ - 13, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 0, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log - 1, // 2: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest - 3, // 3: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest - 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest - 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 8, // 6: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 9, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 8: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 9: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 10: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 10, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 11, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply - 12, // 13: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 8, // [8:14] is the sub-list for method output_type - 2, // [2:8] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 14, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body + 15, // 1: net.pb.FetchEncryptionKeyRequest.publicKey:type_name -> crypto.pb.PublicKey + 8, // 2: net.pb.FetchEncryptionKeyRequest.peerInfo:type_name -> net.pb.PeerInfo + 0, // 3: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log + 1, // 4: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest + 3, // 5: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest + 5, // 6: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest + 7, // 7: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest + 9, // 8: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest + 10, // 9: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 10: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 11: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 12: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 11, // 13: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 12, // 14: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 13, // 15: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 10, // [10:16] is the sub-list for method output_type + 4, // [4:10] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_net_proto_init() } @@ -901,7 +1020,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*FetchEncryptionKeyRequest); i { + switch v := v.(*PeerInfo); i { case 0: return &v.state case 1: @@ -913,7 +1032,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogRequest); i { + switch v := v.(*FetchEncryptionKeyRequest); i { case 0: return &v.state case 1: @@ -925,7 +1044,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*PushLogReply); i { + switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state case 1: @@ -937,7 +1056,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*FetchEncryptionKeyReply); i { + switch v := v.(*PushLogReply); i { case 0: return &v.state case 1: @@ -949,7 +1068,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogReply); i { + switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state case 1: @@ -961,6 +1080,18 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { + switch v := v.(*GetHeadLogReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_net_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state @@ -979,7 +1110,7 @@ func file_net_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_net_proto_rawDesc, NumEnums: 0, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, diff --git a/net/pb/net.proto b/net/pb/net.proto index e39ab9fa91..6191efce49 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -1,6 +1,8 @@ syntax = "proto3"; package net.pb; +import "core/crypto/pb/crypto.proto"; + option go_package = "/;net_pb"; // Log represents a thread log. @@ -38,6 +40,17 @@ message PushLogRequest { } } +// PeerInfo contains information about a specific peer +message PeerInfo { + // id is a libp2p peer identity. + bytes id = 1; + // addresses is a list of peer's address in multiaddr format + // Learn more here https://github.com/multiformats/multiadd + repeated string addresses = 2; +} + +// FetchEncryptionKeyRequest is a request to receive a doc encryption key +// from a peer that holds it. message FetchEncryptionKeyRequest { // docID is the ID of the document that is affected by the log. bytes docID = 1; @@ -45,18 +58,28 @@ message FetchEncryptionKeyRequest { bytes cid = 2; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 3; + // publicKey is a public of the requesting peer + crypto.pb.PublicKey publicKey = 4; + // peerInfo is peer information is the requesting peer + PeerInfo peerInfo = 5; + // signature is the requesting peer's signature of the request + bytes signature = 6; } message GetHeadLogRequest {} message PushLogReply {} +// FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request +// by a peer that holds the requested doc encryption key. message FetchEncryptionKeyReply { - bytes encryptionKey = 1; + bytes encryptedKey = 1; // cid is the CID of the composite of the document. bytes cid = 2; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 3; + // signature is the responding peer's signature of the reply + bytes signature = 4; } message GetHeadLogReply {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 4456228e6e..6f6a7de1e3 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -6,7 +6,9 @@ package net_pb import ( fmt "fmt" + pb "github.com/libp2p/go-libp2p/core/crypto/pb" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" protoimpl "google.golang.org/protobuf/runtime/protoimpl" io "io" ) @@ -370,6 +372,55 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *PeerInfo) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PeerInfo) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PeerInfo) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Addresses) > 0 { + for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Addresses[iNdEx]) + copy(dAtA[i:], m.Addresses[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Addresses[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Id) > 0 { + i -= len(m.Id) + copy(dAtA[i:], m.Id) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Id))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *FetchEncryptionKeyRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -400,6 +451,45 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x32 + } + if m.PeerInfo != nil { + size, err := m.PeerInfo.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x2a + } + if m.PublicKey != nil { + if vtmsg, ok := interface{}(m.PublicKey).(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + } else { + encoded, err := proto.Marshal(m.PublicKey) + if err != nil { + return 0, err + } + i -= len(encoded) + copy(dAtA[i:], encoded) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) + } + i-- + dAtA[i] = 0x22 + } if len(m.SchemaRoot) > 0 { i -= len(m.SchemaRoot) copy(dAtA[i:], m.SchemaRoot) @@ -520,6 +610,13 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x22 + } if len(m.SchemaRoot) > 0 { i -= len(m.SchemaRoot) copy(dAtA[i:], m.SchemaRoot) @@ -534,10 +631,10 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro i-- dAtA[i] = 0x12 } - if len(m.EncryptionKey) > 0 { - i -= len(m.EncryptionKey) - copy(dAtA[i:], m.EncryptionKey) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptionKey))) + if len(m.EncryptedKey) > 0 { + i -= len(m.EncryptedKey) + copy(dAtA[i:], m.EncryptedKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKey))) i-- dAtA[i] = 0xa } @@ -695,6 +792,26 @@ func (m *PushLogRequest) SizeVT() (n int) { return n } +func (m *PeerInfo) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Id) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Addresses) > 0 { + for _, s := range m.Addresses { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if m == nil { return 0 @@ -713,6 +830,24 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.PublicKey != nil { + if size, ok := interface{}(m.PublicKey).(interface { + SizeVT() int + }); ok { + l = size.SizeVT() + } else { + l = proto.Size(m.PublicKey) + } + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.PeerInfo != nil { + l = m.PeerInfo.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -743,7 +878,7 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { } var l int _ = l - l = len(m.EncryptionKey) + l = len(m.EncryptedKey) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } @@ -755,6 +890,10 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -1468,6 +1607,123 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *PeerInfo) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PeerInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PeerInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Id = append(m.Id[:0], dAtA[iNdEx:postIndex]...) + if m.Id == nil { + m.Id = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Addresses = append(m.Addresses, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1599,6 +1855,120 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.SchemaRoot = []byte{} } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PublicKey", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PublicKey == nil { + m.PublicKey = &pb.PublicKey{} + } + if unmarshal, ok := interface{}(m.PublicKey).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.PublicKey); err != nil { + return err + } + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PeerInfo == nil { + m.PeerInfo = &PeerInfo{} + } + if err := m.PeerInfo.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -1754,7 +2124,7 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptionKey", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field EncryptedKey", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1781,9 +2151,9 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.EncryptionKey = append(m.EncryptionKey[:0], dAtA[iNdEx:postIndex]...) - if m.EncryptionKey == nil { - m.EncryptionKey = []byte{} + m.EncryptedKey = append(m.EncryptedKey[:0], dAtA[iNdEx:postIndex]...) + if m.EncryptedKey == nil { + m.EncryptedKey = []byte{} } iNdEx = postIndex case 2: @@ -1854,6 +2224,40 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { m.SchemaRoot = []byte{} } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) From 83cb8da7b0aea251d50d19dbae0a4a0d5faa242b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 27 Jul 2024 11:53:03 +0200 Subject: [PATCH 05/88] Encrypt peer-to-peer data exchange --- crypto/ephemeral.go | 47 +++++++++++ net/peer.go | 8 +- net/server.go | 202 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 222 insertions(+), 35 deletions(-) create mode 100644 crypto/ephemeral.go diff --git a/crypto/ephemeral.go b/crypto/ephemeral.go new file mode 100644 index 0000000000..87e35b8fa7 --- /dev/null +++ b/crypto/ephemeral.go @@ -0,0 +1,47 @@ +package crypto + +import ( + "crypto/ecdh" + "crypto/rand" + "crypto/sha256" + "fmt" +) + +const ( + // EphemeralKeyLength is the size of the ECDH ephemeral key in bytes. + EphemeralKeyLength = 65 +) + +// EncryptWithEphemeralKey encrypts a key using a randomly generated ephemeral ECDH key and a provided public key. +// It returns the encrypted key prepended with the ephemeral public key. +func EncryptWithEphemeralKey(plainText, publicKeyBytes []byte) ([]byte, error) { + ephemeralPriv, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) + } + + ephPubKeyBytes := ephemeralPriv.PublicKey().Bytes() + sharedSecret := sha256.Sum256(append(ephPubKeyBytes, publicKeyBytes...)) + + return append(ephPubKeyBytes, xorBytes(plainText, sharedSecret[:])...), nil +} + +func xorBytes(data, xor []byte) []byte { + result := make([]byte, len(data)) + for i := range data { + result[i] = data[i] ^ xor[i%len(xor)] + } + return result +} + +// DecryptWithEphemeralKey decrypts data that was encrypted using EncryptWithEphemeralKey. +// It expects the input to be the ephemeral public key followed by the encrypted data. +func DecryptWithEphemeralKey(encryptedData, publicKeyBytes []byte) ([]byte, error) { + ephPubKeyBytes := encryptedData[:EphemeralKeyLength] + cipherText := make([]byte, len(encryptedData)-EphemeralKeyLength) + copy(cipherText, encryptedData[EphemeralKeyLength:]) + + sharedSecret := sha256.Sum256(append(ephPubKeyBytes, publicKeyBytes...)) + + return xorBytes(cipherText, sharedSecret[:]), nil +} diff --git a/net/peer.go b/net/peer.go index 5ffce49bb0..06666d87dc 100644 --- a/net/peer.go +++ b/net/peer.go @@ -461,13 +461,7 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { } func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeyEvent) error { - req := &pb.FetchEncryptionKeyRequest{ - DocID: []byte(evt.DocID), - Cid: evt.Cid.Bytes(), - SchemaRoot: []byte(evt.SchemaRoot), - } - - if err := p.server.requestEncryptionKey(p.ctx, req); err != nil { + if err := p.server.requestEncryptionKey(p.ctx, evt.DocID, evt.Cid, evt.SchemaRoot); err != nil { return NewErrPublishingToDocIDTopic(err, evt.Cid.String(), evt.DocID) } diff --git a/net/server.go b/net/server.go index 8dd65b6c5e..a3d326d8a2 100644 --- a/net/server.go +++ b/net/server.go @@ -14,6 +14,7 @@ package net import ( "context" + "crypto/sha256" "fmt" "sync" @@ -28,7 +29,9 @@ import ( grpcpeer "google.golang.org/grpc/peer" "google.golang.org/protobuf/proto" + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" coreblock "github.com/sourcenetwork/defradb/internal/core/block" @@ -167,32 +170,91 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return &pb.PushLogReply{}, nil } -func (s *server) TryGenEncryptionKey( - ctx context.Context, - req *pb.FetchEncryptionKeyRequest, -) (*pb.FetchEncryptionKeyReply, error) { +func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { + isValid, err := s.verifyRequestSignature(req) + if err != nil { + return nil, errors.Wrap("invalid signature", err) + } + + if !isValid { + return nil, errors.New("invalid signature") + } + + pubKey, err := libp2pCrypto.PublicKeyFromProto(req.PublicKey) + if err != nil { + return nil, errors.Wrap("failed to unmarshal public key", err) + } + + if err := s.verifyPeerInfo(libpeer.ID(req.PeerInfo.Id), pubKey); err != nil { + return nil, errors.Wrap("invalid peer info", err) + } + docID, err := client.NewDocIDFromString(string(req.DocID)) if err != nil { return nil, err } encKey, err := encryption.GetDocKey(encryption.ContextWithStore(ctx, s.peer.encstore), docID.String()) + if err != nil || len(encKey) == 0 { + return nil, err + } + pubKeyBytes, err := pubKey.Raw() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get raw Ed25519 public key: %w", err) } - if len(encKey) == 0 { - return nil, nil + encryptedKey, err := crypto.EncryptWithEphemeralKey(encKey, pubKeyBytes) + if err != nil { + return nil, errors.Wrap("failed to encrypt key for requester", err) } res := &pb.FetchEncryptionKeyReply{ - EncryptionKey: encKey, + EncryptedKey: encryptedKey, + Cid: req.Cid, + SchemaRoot: req.SchemaRoot, + } + + res.Signature, err = s.signResponse(res) + if err != nil { + return nil, errors.Wrap("failed to sign response", err) } return res, nil } +func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest) (bool, error) { + pubKey, err := libp2pCrypto.PublicKeyFromProto(req.PublicKey) + if err != nil { + return false, err + } + + return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) +} + +func (s *server) verifyPeerInfo(peerID libpeer.ID, pubKey libp2pCrypto.PubKey) error { + derivedID, err := peer.IDFromPublicKey(pubKey) + if err != nil { + return err + } + + if peerID != derivedID { + return errors.New("peer ID does not match public key") + } + + return nil +} + +func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { + hash := sha256.New() + hash.Write(res.EncryptedKey) + hash.Write(res.Cid) + hash.Write(res.SchemaRoot) + + privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) + return privKey.Sign(hash.Sum(nil)) +} + // GetHeadLog receives a get head log request func (s *server) GetHeadLog( ctx context.Context, @@ -332,7 +394,7 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe data, err := req.MarshalVT() if err != nil { - return errors.Wrap("failed marshling pubsub message", err) + return errors.Wrap("failed to marshal pubsub message", err) } if _, err := t.Publish(ctx, data, rpc.WithIgnoreResponse(true)); err != nil { @@ -341,55 +403,139 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe return nil } +func toProtoPeerInfo(peerInfo libpeer.AddrInfo) *pb.PeerInfo { + protoPeerInfo := new(pb.PeerInfo) + protoPeerInfo.Id = []byte(peerInfo.ID) + protoPeerInfo.Addresses = make([]string, len(peerInfo.Addrs)) + for i, addr := range peerInfo.Addrs { + protoPeerInfo.Addresses[i] = addr.String() + } + return protoPeerInfo +} + +func (s *server) prepareFetchEncryptionKeyRequest(docID string, cid cid.Cid, schemaRoot string) (*pb.FetchEncryptionKeyRequest, error) { + publicKey := s.peer.host.Peerstore().PubKey(s.peer.host.ID()) + protoPublicKey, err := libp2pCrypto.PublicKeyToProto(publicKey) + if err != nil { + return nil, errors.Wrap("failed to marshal public key", err) + } + + req := &pb.FetchEncryptionKeyRequest{ + DocID: []byte(docID), + Cid: cid.Bytes(), + SchemaRoot: []byte(schemaRoot), + PublicKey: protoPublicKey, + PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), + } + + req.Signature, err = s.signRequest(req) + if err != nil { + return nil, errors.Wrap("failed to sign request", err) + } + + return req, nil +} + // requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *server) requestEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) error { +func (s *server) requestEncryptionKey(ctx context.Context, docID string, cid cid.Cid, schemaRoot string) error { if s.peer.ps == nil { // skip if we aren't running with a pubsub net return nil } - s.mu.Lock() - t, ok := s.topics[encryptionTopic] - s.mu.Unlock() - if !ok { - err := s.addPubSubTopic(encryptionTopic, false) - if err != nil { - return errors.Wrap(fmt.Sprintf("failed to created single use topic %s", encryptionTopic), err) - } - return s.requestEncryptionKey(ctx, req) + + req, err := s.prepareFetchEncryptionKeyRequest(docID, cid, schemaRoot) + if err != nil { + return err } data, err := req.MarshalVT() if err != nil { - return errors.Wrap("failed marshling pubsub message", err) + return errors.Wrap("failed to marshal pubsub message", err) } + s.mu.Lock() + t := s.topics[encryptionTopic] + s.mu.Unlock() respChan, err := t.Publish(ctx, data) if err != nil { return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) } + go func() { s.handleFetchEncryptionKeyResponse(<-respChan, req) }() + return nil } +func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { + hash := sha256.New() + hash.Write(req.DocID) + hash.Write(req.Cid) + hash.Write(req.SchemaRoot) + hash.Write([]byte(req.PublicKey.Type.String())) + hash.Write(req.PublicKey.Data) + hash.Write([]byte(req.PeerInfo.String())) + return hash.Sum(nil) +} + +func (s *server) signRequest(req *pb.FetchEncryptionKeyRequest) ([]byte, error) { + privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) + return privKey.Sign(hashFetchEncryptionKeyRequest(req)) +} + // handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages -func (s *server) handleFetchEncryptionKeyResponse( - resp rpc.Response, - req *pb.FetchEncryptionKeyRequest, -) { +func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.FetchEncryptionKeyRequest) { var keyResp pb.FetchEncryptionKeyReply - err := proto.Unmarshal(resp.Data, &keyResp) - if err != nil { + if err := proto.Unmarshal(resp.Data, &keyResp); err != nil { log.ErrorContextE(s.peer.ctx, "Failed to unmarshal encryption key response", err) return } + + isValid, err := s.verifyResponseSignature(&keyResp, resp.From) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to verify response signature", err) + return + } + + if !isValid { + log.ErrorContext(s.peer.ctx, "Invalid response signature") + return + } + + privateKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) + + // Use the private key to get the public key bytes + ed25519PubKeyBytes, err := privateKey.GetPublic().Raw() + if err != nil { + log.ErrorContextE(s.peer.ctx, "failed to get raw Ed25519 public key", err) + return + } + + decryptedKey, err := crypto.DecryptWithEphemeralKey(keyResp.EncryptedKey, ed25519PubKeyBytes) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) + return + } + cid, err := cid.Cast(req.Cid) if err != nil { log.ErrorContextE(s.peer.ctx, "Failed to parse CID", err) return } + s.peer.bus.Publish(encryption.NewKeyRetrievedMessage( - string(req.DocID), cid, string(req.SchemaRoot), keyResp.EncryptionKey)) + string(req.DocID), cid, string(req.SchemaRoot), decryptedKey)) +} + +func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { + pubKey := s.peer.host.Peerstore().PubKey(fromPeer) + + hash := sha256.New() + hash.Write(res.EncryptedKey) + hash.Write(res.Cid) + hash.Write(res.SchemaRoot) + + return pubKey.Verify(hash.Sum(nil), res.Signature) } // pubSubMessageHandler handles incoming PushLog messages from the pubsub network. @@ -426,7 +572,7 @@ func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, m }) res, err := s.TryGenEncryptionKey(ctx, req) if err != nil { - return nil, errors.Wrap(fmt.Sprintf("Failed pushing log for doc %s", topic), err) + return nil, errors.Wrap("Failed attempt to get encryption key", err) } return res.MarshalVT() //return proto.Marshal(res) From c2ff5a3295d91cd31fe8740b52a5a7a5f9ad760e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 30 Jul 2024 21:55:39 +0200 Subject: [PATCH 06/88] Doc field key exchange --- internal/core/block/block.go | 51 +++++-- internal/core/block/block_test.go | 48 +++++++ internal/core/block/errors.go | 18 +-- internal/db/merge.go | 19 ++- internal/db/messages.go | 2 +- internal/encryption/encryptor.go | 22 ++- internal/encryption/event.go | 28 +++- internal/merkle/clock/clock.go | 27 ++-- net/errors.go | 5 + net/pb/net.pb.go | 128 ++++++++++-------- net/pb/net.proto | 15 +- net/pb/net_vtproto.pb.go | 59 ++++++-- net/peer.go | 9 +- net/server.go | 36 +++-- .../integration/encryption/peer_share_test.go | 52 +++++++ tests/integration/p2p.go | 2 +- 16 files changed, 389 insertions(+), 132 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index d2caa610f7..531c9159da 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -97,15 +97,28 @@ func NewDAGLink(name string, link cidlink.Link) DAGLink { } } +type EncryptionType int + +const ( + NotEncrypted EncryptionType = iota + DocumentEncrypted + FieldEncrypted +) + // Block is a block that contains a CRDT delta and links to other blocks. type Block struct { // Delta is the CRDT delta that is stored in the block. Delta crdt.CRDT // Links are the links to other blocks in the DAG. Links []DAGLink - // IsEncrypted is a flag that indicates if the block's delta is encrypted. - // It needs to be a pointer so that it can be translated from and to `optional Bool` in the IPLD schema. - IsEncrypted *bool + // EncryptionType indicates if the block's delta is encrypted and on what level encryption is applied. + // It needs to be a pointer so that it can be translated from and to `optional` in the IPLD schema. + EncryptionType *EncryptionType +} + +// IsEncrypted returns true if the block is encrypted. +func (b *Block) IsEncrypted() bool { + return b.EncryptionType != nil && *b.EncryptionType != NotEncrypted } // IPLDSchemaBytes returns the IPLD schema representation for the block. @@ -113,11 +126,18 @@ type Block struct { // This needs to match the [Block] struct or [mustSetSchema] will panic on init. func (b Block) IPLDSchemaBytes() []byte { return []byte(` - type Block struct { - delta CRDT - links [ DAGLink ] - isEncrypted optional Bool - }`) + type Block struct { + delta CRDT + links [DAGLink] + encryptionType optional EncryptionType + } + + type EncryptionType enum { + | NotEncrypted ("0") + | DocumentEncrypted ("1") + | FieldEncrypted ("2") + } representation int + `) } // New creates a new block with the given delta and links. @@ -192,6 +212,9 @@ func (block *Block) Unmarshal(b []byte) error { if err != nil { return NewErrUnmarshallingBlock(err) } + if err := block.Validate(); err != nil { + return err + } return nil } @@ -238,3 +261,15 @@ func GetLinkPrototype() cidlink.LinkPrototype { MhLength: 32, }} } + +func (b *Block) Validate() error { + if b.EncryptionType != nil { + switch *b.EncryptionType { + case NotEncrypted, DocumentEncrypted, FieldEncrypted: + // Valid values + default: + return ErrInvalidBlockEncryptionType + } + } + return nil +} diff --git a/internal/core/block/block_test.go b/internal/core/block/block_test.go index 5b68cf9067..554d624c81 100644 --- a/internal/core/block/block_test.go +++ b/internal/core/block/block_test.go @@ -228,3 +228,51 @@ func TestBlockMarshal_IsEncryptedNotSetWithLinkSystem_ShouldLoadWithNoError(t *t _, err = GetFromNode(nd) require.NoError(t, err) } + +func TestBlock_Validate(t *testing.T) { + tests := []struct { + name string + encryptionType *EncryptionType + expectedError error + }{ + { + name: "NotEncrypted is valid", + encryptionType: ptr(NotEncrypted), + expectedError: nil, + }, + { + name: "DocumentEncrypted is valid", + encryptionType: ptr(DocumentEncrypted), + expectedError: nil, + }, + { + name: "FieldEncrypted is valid", + encryptionType: ptr(FieldEncrypted), + expectedError: nil, + }, + { + name: "Nil EncryptionType is valid", + encryptionType: nil, + expectedError: nil, + }, + { + name: "Invalid EncryptionType", + encryptionType: ptr(EncryptionType(99)), + expectedError: ErrInvalidBlockEncryptionType, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &Block{ + EncryptionType: tt.encryptionType, + } + err := b.Validate() + require.Equal(t, tt.expectedError, err) + }) + } +} + +func ptr(e EncryptionType) *EncryptionType { + return &e +} diff --git a/internal/core/block/errors.go b/internal/core/block/errors.go index 9b6b0e8a95..ec91f1e816 100644 --- a/internal/core/block/errors.go +++ b/internal/core/block/errors.go @@ -17,10 +17,11 @@ import ( ) const ( - errNodeToBlock string = "failed to convert node to block" - errEncodingBlock string = "failed to encode block" - errUnmarshallingBlock string = "failed to unmarshal block" - errGeneratingLink string = "failed to generate link" + errNodeToBlock string = "failed to convert node to block" + errEncodingBlock string = "failed to encode block" + errUnmarshallingBlock string = "failed to unmarshal block" + errGeneratingLink string = "failed to generate link" + errInvalidBlockEncryptionType string = "invalid block encryption type" ) // Errors returnable from this package. @@ -28,10 +29,11 @@ const ( // This list is incomplete and undefined errors may also be returned. // Errors returned from this package may be tested against these errors with errors.Is. var ( - ErrNodeToBlock = errors.New(errNodeToBlock) - ErrEncodingBlock = errors.New(errEncodingBlock) - ErrUnmarshallingBlock = errors.New(errUnmarshallingBlock) - ErrGeneratingLink = errors.New(errGeneratingLink) + ErrNodeToBlock = errors.New(errNodeToBlock) + ErrEncodingBlock = errors.New(errEncodingBlock) + ErrUnmarshallingBlock = errors.New(errUnmarshallingBlock) + ErrGeneratingLink = errors.New(errGeneratingLink) + ErrInvalidBlockEncryptionType = errors.New(errInvalidBlockEncryptionType) ) // NewErrFailedToGetPriority returns an error indicating that the priority could not be retrieved. diff --git a/internal/db/merge.go b/internal/db/merge.go index 2c6116f9ce..0b8bb0830d 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -230,7 +230,7 @@ func (mp *mergeProcessor) loadComposites( if b, ok := mt.heads[blockCid]; ok { // the head is already known, but the block might be encrypted // if this time we try to decrypt it, we load the block - if b.IsEncrypted == nil || !*b.IsEncrypted || !willDecrypt { + if !b.IsEncrypted() || !willDecrypt { // We've already processed this block. return nil } @@ -309,7 +309,7 @@ func (mp *mergeProcessor) processBlock( ) error { block := dagBlock var skipMerge bool - if dagBlock.IsEncrypted != nil && *dagBlock.IsEncrypted { + if dagBlock.IsEncrypted() { plainTextBlock, err := decryptBlock(ctx, dagBlock) if err != nil { return err @@ -320,10 +320,15 @@ func (mp *mergeProcessor) processBlock( skipMerge = true block = dagBlock // if we weren't able to decrypt the block we request the encryption key - if dagBlock.Delta.IsComposite() { + if (dagBlock.Delta.IsComposite() && *dagBlock.EncryptionType == coreblock.DocumentEncrypted) || + *dagBlock.EncryptionType == coreblock.FieldEncrypted { docID := string(dagBlock.Delta.GetDocID()) schemaRoot := mp.col.SchemaRoot() - mp.col.db.events.Publish(encryption.NewRequestKeyMessage(docID, blockLink.Cid, schemaRoot)) + fieldName := immutable.None[string]() + if *dagBlock.EncryptionType == coreblock.FieldEncrypted { + fieldName = immutable.Some(dagBlock.Delta.GetFieldName()) + } + mp.col.db.events.Publish(encryption.NewRequestKeyMessage(docID, blockLink.Cid, fieldName, schemaRoot)) } } } @@ -369,9 +374,13 @@ func (mp *mergeProcessor) processBlock( func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { if block.Delta.IsComposite() { + optFieldName := immutable.None[string]() + if *block.EncryptionType == coreblock.FieldEncrypted { + optFieldName = immutable.Some(block.Delta.GetFieldName()) + } // for composite blocks there is nothing to decrypt // so we just check if we have the encryption key for child blocks - bytes, err := encryption.GetDocKey(ctx, string(block.Delta.GetDocID())) + bytes, err := encryption.GetDocKey(ctx, string(block.Delta.GetDocID()), optFieldName) if err != nil { return nil, err } diff --git a/internal/db/messages.go b/internal/db/messages.go index 976f82d8ca..d200447ada 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -83,7 +83,7 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { case encryption.KeyRetrievedEvent: go func() { ctx = encryption.ContextWithStore(ctx, db.Encstore()) - err := encryption.SaveDocKey(ctx, evt.DocID, evt.Key) + err := encryption.SaveDocKey(ctx, evt.DocID, evt.FieldName, evt.Key) if err != nil { log.ErrorContextE( diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 3cae2d33be..ad658de911 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -176,11 +176,15 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName string) ([]byt return encryptionKey, nil } -func (d *DocEncryptor) GetDocKey(docID string) ([]byte, error) { +func (d *DocEncryptor) GetDocKey(docID string, optFieldName immutable.Option[string]) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - storeKey := core.NewEncStoreDocKey(docID, "") + fieldName := "" + if optFieldName.HasValue() { + fieldName = optFieldName.Value() + } + storeKey := core.NewEncStoreDocKey(docID, fieldName) encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) if err != nil && !errors.Is(err, ds.ErrNotFound) { return nil, err @@ -223,7 +227,14 @@ func ShouldEncryptField(ctx context.Context, fieldName string) bool { return shouldEncryptField(GetContextConfig(ctx), fieldName) } -func SaveDocKey(ctx context.Context, docID string, encryptionKey []byte) error { +// ShouldEncryptIndividualField returns true if the given field should be encrypted individually based on +// the context config. +func ShouldEncryptIndividualField(ctx context.Context, fieldName string) bool { + return shouldEncryptIndividualField(GetContextConfig(ctx), fieldName) +} + +// SaveDocKey saves the given encryption key for the given docID and (optional) fieldName with encryptor in the context. +func SaveDocKey(ctx context.Context, docID string, fieldName immutable.Option[string], encryptionKey []byte) error { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil @@ -231,10 +242,11 @@ func SaveDocKey(ctx context.Context, docID string, encryptionKey []byte) error { return enc.SaveDocKey(docID, encryptionKey) } -func GetDocKey(ctx context.Context, docID string) ([]byte, error) { +// GetDocKey returns the encryption key for the given docID and (optional) fieldName with encryptor in the context. +func GetDocKey(ctx context.Context, docID string, fieldName immutable.Option[string]) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.GetDocKey(docID) + return enc.GetDocKey(docID, fieldName) } diff --git a/internal/encryption/event.go b/internal/encryption/event.go index 7a81209cc2..a806fef9cc 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -13,6 +13,7 @@ package encryption import ( "github.com/ipfs/go-cid" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/immutable" ) const RequestKeyEventName = event.Name("enc-key-request") @@ -23,9 +24,13 @@ const KeyRetrievedEventName = event.Name("enc-key-retrieved") // // It must only contain public elements not protected by ACP. type RequestKeyEvent struct { - // DocID is the unique immutable identifier of the document that was updated. + // DocID is the unique immutable identifier of the document that the key is being requested for. DocID string + // FieldName is the name of the field for which the key is being requested. + // If not given, the key is for the whole document. + FieldName immutable.Option[string] + // Cid is the id of the composite commit that formed this update in the DAG. Cid cid.Cid @@ -38,6 +43,10 @@ type KeyRetrievedEvent struct { // DocID is the unique immutable identifier of the document that was updated. DocID string + // FieldName is the name of the field for which the key was retrieved. + // If not given, the key is for the whole document. + FieldName immutable.Option[string] + // Cid is the id of the composite commit that formed this update in the DAG. Cid cid.Cid @@ -51,18 +60,31 @@ type KeyRetrievedEvent struct { // NewRequestKeyMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field -func NewRequestKeyMessage(docID string, cid cid.Cid, schemaRoot string) event.Message { +func NewRequestKeyMessage( + docID string, + cid cid.Cid, + fieldName immutable.Option[string], + schemaRoot string, +) event.Message { return event.NewMessage(RequestKeyEventName, RequestKeyEvent{ DocID: docID, + FieldName: fieldName, Cid: cid, SchemaRoot: schemaRoot, }) } // NewKeyRetrievedMessage creates a new event message for a key that was retrieved -func NewKeyRetrievedMessage(docID string, cid cid.Cid, schemaRoot string, key []byte) event.Message { +func NewKeyRetrievedMessage( + docID string, + fieldName immutable.Option[string], + cid cid.Cid, + schemaRoot string, + key []byte, +) event.Message { return event.NewMessage(KeyRetrievedEventName, KeyRetrievedEvent{ DocID: docID, + FieldName: fieldName, Cid: cid, SchemaRoot: schemaRoot, Key: key, diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index ce6ba8db2b..131d6dc964 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -86,21 +86,20 @@ func (mc *MerkleClock) AddDelta( delta.SetPriority(height) block := coreblock.New(delta, links, heads...) - isEncrypted, err := mc.checkIfBlockEncryptionEnabled(ctx, block.Delta.GetFieldName(), heads) + encryptionType, err := mc.checkIfBlockEncryptionEnabled(ctx, block.Delta.GetFieldName(), heads) if err != nil { return cidlink.Link{}, nil, err } dagBlock := block - if isEncrypted { + if encryptionType != coreblock.NotEncrypted { if !block.Delta.IsComposite() { dagBlock, err = encryptBlock(ctx, block) if err != nil { return cidlink.Link{}, nil, err } - } else { - dagBlock.IsEncrypted = &isEncrypted } + dagBlock.EncryptionType = &encryptionType } link, err := mc.putBlock(ctx, dagBlock) @@ -126,26 +125,29 @@ func (mc *MerkleClock) checkIfBlockEncryptionEnabled( ctx context.Context, fieldName string, heads []cid.Cid, -) (bool, error) { +) (coreblock.EncryptionType, error) { if encryption.ShouldEncryptField(ctx, fieldName) { - return true, nil + if encryption.ShouldEncryptIndividualField(ctx, fieldName) { + return coreblock.FieldEncrypted, nil + } + return coreblock.DocumentEncrypted, nil } for _, headCid := range heads { bytes, err := mc.blockstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) if err != nil { - return false, NewErrCouldNotFindBlock(headCid, err) + return coreblock.NotEncrypted, NewErrCouldNotFindBlock(headCid, err) } prevBlock, err := coreblock.GetFromBytes(bytes) if err != nil { - return false, err + return coreblock.NotEncrypted, err } - if prevBlock.IsEncrypted != nil && *prevBlock.IsEncrypted { - return true, nil + if prevBlock.EncryptionType != nil { + return *prevBlock.EncryptionType, nil } } - return false, nil + return coreblock.NotEncrypted, nil } func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { @@ -156,8 +158,7 @@ func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block return nil, err } clonedCRDT.SetData(bytes) - isEncrypted := true - return &coreblock.Block{Delta: clonedCRDT, Links: block.Links, IsEncrypted: &isEncrypted}, nil + return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil } // ProcessBlock merges the delta CRDT and updates the state accordingly. diff --git a/net/errors.go b/net/errors.go index 615f1088ef..c957baceb8 100644 --- a/net/errors.go +++ b/net/errors.go @@ -22,6 +22,7 @@ const ( errPublishingToDocIDTopic = "can't publish log %s for docID %s" errPublishingToSchemaTopic = "can't publish log %s for schema %s" errCheckingForExistingBlock = "failed to check for existing block" + errRequestingEncryptionKey = "failed to request encryption key with Cid %s for docID %s" ) var ( @@ -52,3 +53,7 @@ func NewErrPublishingToSchemaTopic(inner error, cid, docID string, kv ...errors. func NewErrCheckingForExistingBlock(inner error, cid string) error { return errors.Wrap(errCheckingForExistingBlock, inner, errors.NewKV("cid", cid)) } + +func NewErrRequestingEncryptionKey(inner error, cid, docID string, kv ...errors.KV) error { + return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKey, cid, docID), inner, kv...) +} diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 40972130f2..5072baca82 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -411,18 +411,21 @@ type FetchEncryptionKeyRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // docID is the ID of the document that is affected by the log. + // docID is the ID of the document that the key is being requested for. DocID []byte `protobuf:"bytes,1,opt,name=docID,proto3" json:"docID,omitempty"` + // fieldName if the name of the document field that the key is being requested for. + // If the fieldName is empty, the key is being requested for the whole document. + FieldName string `protobuf:"bytes,2,opt,name=fieldName,proto3" json:"fieldName,omitempty"` // cid is the CID of the composite of the document. - Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` + Cid []byte `protobuf:"bytes,3,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. - SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + SchemaRoot []byte `protobuf:"bytes,4,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` // publicKey is a public of the requesting peer - PublicKey *pb.PublicKey `protobuf:"bytes,4,opt,name=publicKey,proto3" json:"publicKey,omitempty"` + PublicKey *pb.PublicKey `protobuf:"bytes,5,opt,name=publicKey,proto3" json:"publicKey,omitempty"` // peerInfo is peer information is the requesting peer - PeerInfo *PeerInfo `protobuf:"bytes,5,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` + PeerInfo *PeerInfo `protobuf:"bytes,6,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { @@ -464,6 +467,13 @@ func (x *FetchEncryptionKeyRequest) GetDocID() []byte { return nil } +func (x *FetchEncryptionKeyRequest) GetFieldName() string { + if x != nil { + return x.FieldName + } + return "" +} + func (x *FetchEncryptionKeyRequest) GetCid() []byte { if x != nil { return x.Cid @@ -804,62 +814,64 @@ var file_net_proto_rawDesc = []byte{ 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xe3, 0x01, 0x0a, 0x19, + 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x81, 0x02, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, - 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, - 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, - 0x74, 0x12, 0x32, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, - 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, + 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, + 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, + 0x32, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x50, + 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, + 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, + 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, + 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, + 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, + 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, + 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, + 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, + 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, + 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, - 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, - 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, - 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, - 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, + 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, + 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, + 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/net/pb/net.proto b/net/pb/net.proto index 6191efce49..47cbc4e0b8 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -52,18 +52,21 @@ message PeerInfo { // FetchEncryptionKeyRequest is a request to receive a doc encryption key // from a peer that holds it. message FetchEncryptionKeyRequest { - // docID is the ID of the document that is affected by the log. + // docID is the ID of the document that the key is being requested for. bytes docID = 1; + // fieldName if the name of the document field that the key is being requested for. + // If the fieldName is empty, the key is being requested for the whole document. + string fieldName = 2; // cid is the CID of the composite of the document. - bytes cid = 2; + bytes cid = 3; // schemaRoot is the SchemaRoot of the collection that the document resides in. - bytes schemaRoot = 3; + bytes schemaRoot = 4; // publicKey is a public of the requesting peer - crypto.pb.PublicKey publicKey = 4; + crypto.pb.PublicKey publicKey = 5; // peerInfo is peer information is the requesting peer - PeerInfo peerInfo = 5; + PeerInfo peerInfo = 6; // signature is the requesting peer's signature of the request - bytes signature = 6; + bytes signature = 7; } message GetHeadLogRequest {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 6f6a7de1e3..a016809254 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -456,7 +456,7 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x32 + dAtA[i] = 0x3a } if m.PeerInfo != nil { size, err := m.PeerInfo.MarshalToSizedBufferVT(dAtA[:i]) @@ -466,7 +466,7 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i -= size i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- - dAtA[i] = 0x2a + dAtA[i] = 0x32 } if m.PublicKey != nil { if vtmsg, ok := interface{}(m.PublicKey).(interface { @@ -488,20 +488,27 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) } i-- - dAtA[i] = 0x22 + dAtA[i] = 0x2a } if len(m.SchemaRoot) > 0 { i -= len(m.SchemaRoot) copy(dAtA[i:], m.SchemaRoot) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) i-- - dAtA[i] = 0x1a + dAtA[i] = 0x22 } if len(m.Cid) > 0 { i -= len(m.Cid) copy(dAtA[i:], m.Cid) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) i-- + dAtA[i] = 0x1a + } + if len(m.FieldName) > 0 { + i -= len(m.FieldName) + copy(dAtA[i:], m.FieldName) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FieldName))) + i-- dAtA[i] = 0x12 } if len(m.DocID) > 0 { @@ -822,6 +829,10 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.FieldName) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } l = len(m.Cid) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1788,6 +1799,38 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FieldName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Cid", wireType) } @@ -1821,7 +1864,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.Cid = []byte{} } iNdEx = postIndex - case 3: + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) } @@ -1855,7 +1898,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.SchemaRoot = []byte{} } iNdEx = postIndex - case 4: + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PublicKey", wireType) } @@ -1899,7 +1942,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } } iNdEx = postIndex - case 5: + case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerInfo", wireType) } @@ -1935,7 +1978,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex - case 6: + case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } diff --git a/net/peer.go b/net/peer.go index 06666d87dc..3fcba4a1bd 100644 --- a/net/peer.go +++ b/net/peer.go @@ -45,6 +45,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/multiformats/go-multiaddr" "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/immutable" "google.golang.org/grpc" "github.com/sourcenetwork/defradb/client" @@ -461,8 +462,12 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { } func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeyEvent) error { - if err := p.server.requestEncryptionKey(p.ctx, evt.DocID, evt.Cid, evt.SchemaRoot); err != nil { - return NewErrPublishingToDocIDTopic(err, evt.Cid.String(), evt.DocID) + if err := p.server.requestEncryptionKey(p.ctx, evt); err != nil { + kvs := []errors.KV{} + if evt.FieldName.HasValue() { + kvs = append(kvs, errors.NewKV("FieldName", evt.FieldName)) + } + return NewErrRequestingEncryptionKey(err, evt.Cid.String(), evt.DocID, kvs...) } return nil diff --git a/net/server.go b/net/server.go index a3d326d8a2..ee1f853ea6 100644 --- a/net/server.go +++ b/net/server.go @@ -24,6 +24,7 @@ import ( "github.com/libp2p/go-libp2p/core/peerstore" "github.com/sourcenetwork/corelog" rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" + "github.com/sourcenetwork/immutable" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" grpcpeer "google.golang.org/grpc/peer" @@ -189,12 +190,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.Wrap("invalid peer info", err) } - docID, err := client.NewDocIDFromString(string(req.DocID)) - if err != nil { - return nil, err - } - - encKey, err := encryption.GetDocKey(encryption.ContextWithStore(ctx, s.peer.encstore), docID.String()) + encKey, err := s.getEncryptionKey(ctx, req) if err != nil || len(encKey) == 0 { return nil, err } @@ -413,7 +409,9 @@ func toProtoPeerInfo(peerInfo libpeer.AddrInfo) *pb.PeerInfo { return protoPeerInfo } -func (s *server) prepareFetchEncryptionKeyRequest(docID string, cid cid.Cid, schemaRoot string) (*pb.FetchEncryptionKeyRequest, error) { +func (s *server) prepareFetchEncryptionKeyRequest( + evt encryption.RequestKeyEvent, +) (*pb.FetchEncryptionKeyRequest, error) { publicKey := s.peer.host.Peerstore().PubKey(s.peer.host.ID()) protoPublicKey, err := libp2pCrypto.PublicKeyToProto(publicKey) if err != nil { @@ -421,13 +419,17 @@ func (s *server) prepareFetchEncryptionKeyRequest(docID string, cid cid.Cid, sch } req := &pb.FetchEncryptionKeyRequest{ - DocID: []byte(docID), - Cid: cid.Bytes(), - SchemaRoot: []byte(schemaRoot), + DocID: []byte(evt.DocID), + Cid: evt.Cid.Bytes(), + SchemaRoot: []byte(evt.SchemaRoot), PublicKey: protoPublicKey, PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), } + if evt.FieldName.HasValue() { + req.FieldName = evt.FieldName.Value() + } + req.Signature, err = s.signRequest(req) if err != nil { return nil, errors.Wrap("failed to sign request", err) @@ -437,12 +439,12 @@ func (s *server) prepareFetchEncryptionKeyRequest(docID string, cid cid.Cid, sch } // requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *server) requestEncryptionKey(ctx context.Context, docID string, cid cid.Cid, schemaRoot string) error { +func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeyEvent) error { if s.peer.ps == nil { // skip if we aren't running with a pubsub net return nil } - req, err := s.prepareFetchEncryptionKeyRequest(docID, cid, schemaRoot) + req, err := s.prepareFetchEncryptionKeyRequest(evt) if err != nil { return err } @@ -523,12 +525,18 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } + optFieldName := immutable.None[string]() + if req.FieldName != "" { + optFieldName = immutable.Some(req.FieldName) + } + s.peer.bus.Publish(encryption.NewKeyRetrievedMessage( - string(req.DocID), cid, string(req.SchemaRoot), decryptedKey)) + string(req.DocID), optFieldName, cid, string(req.SchemaRoot), decryptedKey)) } func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { - pubKey := s.peer.host.Peerstore().PubKey(fromPeer) + peerStore := s.peer.host.Peerstore() + pubKey := peerStore.PubKey(fromPeer) hash := sha256.New() hash.Write(res.EncryptedKey) diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 4ae3b339e2..48e01aca94 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -67,3 +67,55 @@ func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + EncryptedFields: []string{"age"}, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeyRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: []map[string]any{ + { + "name": "John", + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 17b374016d..8724b3a2bb 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -139,7 +139,7 @@ type GetAllP2PCollections struct { type WaitForSync struct { // Event is the name of the event to wait for. // If is not provided then the action will wait for MergeComplete event. - Event immutable.Option[event.Name] + Event immutable.Option[event.Name] // NodeIDs are the node IDs (indexes) of the nodes to wait for sync on. // If not provided then the action will wait for sync on all nodes. NodeIDs []int From 1505ace32ef70b5c3da0836b1d2fcba2231b291b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 1 Aug 2024 18:55:05 +0200 Subject: [PATCH 07/88] Decrypt doc-level and field-level block simultaneously --- internal/core/key.go | 9 +- internal/db/merge.go | 73 ++++++---- internal/db/messages.go | 2 +- internal/encryption/encryptor.go | 127 +++++++++++------- internal/merkle/clock/clock.go | 21 ++- net/peer.go | 1 - net/server.go | 13 ++ .../integration/encryption/peer_share_test.go | 104 ++++++++++++++ 8 files changed, 259 insertions(+), 91 deletions(-) diff --git a/internal/core/key.go b/internal/core/key.go index 6450540ce2..64116f4150 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -802,11 +802,12 @@ type EncStoreDocKey struct { var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. -func NewEncStoreDocKey(docID string, fieldName string) EncStoreDocKey { - return EncStoreDocKey{ - DocID: docID, - FieldName: fieldName, +func NewEncStoreDocKey(docID string, fieldName immutable.Option[string]) EncStoreDocKey { + key := EncStoreDocKey{DocID: docID} + if fieldName.HasValue() { + key.FieldName = fieldName.Value() } + return key } func (k EncStoreDocKey) ToString() string { diff --git a/internal/db/merge.go b/internal/db/merge.go index 0b8bb0830d..9e50152184 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -284,14 +284,14 @@ func (mp *mergeProcessor) loadComposites( return nil } -func (mp *mergeProcessor) mergeComposites(ctx context.Context, skipHeads bool) error { +func (mp *mergeProcessor) mergeComposites(ctx context.Context, withDecryption bool) error { for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() if err != nil { return err } - err = mp.processBlock(ctx, block, link, skipHeads) + err = mp.processBlock(ctx, block, link, withDecryption) if err != nil { return err } @@ -299,28 +299,26 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context, skipHeads bool) e return nil } -// processBlock merges the block and its children to the datastore and sets the head accordingly. -// If onlyHeads is true, it will skip merging and update only the heads. -func (mp *mergeProcessor) processBlock( +// processEncryptedBlock decrypts the block if it is encrypted and returns the decrypted block. +// If the block is encrypted and we were not able to decrypt it, it returns true as the second return value +// which indicates that the we should skip merging of the block. +// If we were able to decrypt the block, we return the decrypted block and false as the second return value. +func (mp *mergeProcessor) processEncryptedBlock( ctx context.Context, dagBlock *coreblock.Block, blockLink cidlink.Link, - skipHeads bool, -) error { - block := dagBlock - var skipMerge bool + withDecryption bool, +) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { plainTextBlock, err := decryptBlock(ctx, dagBlock) if err != nil { - return err + return nil, false, err } if plainTextBlock != nil { - block = plainTextBlock + return plainTextBlock, false, nil } else { - skipMerge = true - block = dagBlock - // if we weren't able to decrypt the block we request the encryption key - if (dagBlock.Delta.IsComposite() && *dagBlock.EncryptionType == coreblock.DocumentEncrypted) || + // if we weren't able to decrypt the block we request the encryption key unless it's a decryption pass + if !withDecryption && (dagBlock.Delta.IsComposite() && *dagBlock.EncryptionType == coreblock.DocumentEncrypted) || *dagBlock.EncryptionType == coreblock.FieldEncrypted { docID := string(dagBlock.Delta.GetDocID()) schemaRoot := mp.col.SchemaRoot() @@ -330,8 +328,24 @@ func (mp *mergeProcessor) processBlock( } mp.col.db.events.Publish(encryption.NewRequestKeyMessage(docID, blockLink.Cid, fieldName, schemaRoot)) } + return dagBlock, true, nil } } + return dagBlock, false, nil +} + +// processBlock merges the block and its children to the datastore and sets the head accordingly. +// If onlyHeads is true, it will skip merging and update only the heads. +func (mp *mergeProcessor) processBlock( + ctx context.Context, + dagBlock *coreblock.Block, + blockLink cidlink.Link, + withDecryption bool, +) error { + block, skipMerge, err := mp.processEncryptedBlock(ctx, dagBlock, blockLink, withDecryption) + if err != nil { + return err + } crdt, err := mp.initCRDTForType(dagBlock.Delta.GetFieldName()) if err != nil { @@ -344,7 +358,7 @@ func (mp *mergeProcessor) processBlock( return nil } - err = crdt.Clock().ProcessBlock(ctx, block, blockLink, skipMerge, skipHeads) + err = crdt.Clock().ProcessBlock(ctx, block, blockLink, skipMerge, withDecryption) if err != nil { return err } @@ -364,7 +378,13 @@ func (mp *mergeProcessor) processBlock( return err } - if err := mp.processBlock(ctx, childBlock, link.Link, skipHeads); err != nil { + // if this is a decryption pass we skip processing for the field block as field blocks + // should have a dedicated decryption pass (and not as part of the composite block) + if withDecryption && childBlock.EncryptionType != nil && *childBlock.EncryptionType == coreblock.FieldEncrypted { + continue + } + + if err := mp.processBlock(ctx, childBlock, link.Link, withDecryption); err != nil { return err } } @@ -373,14 +393,15 @@ func (mp *mergeProcessor) processBlock( } func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { + optFieldName := immutable.None[string]() + if *block.EncryptionType == coreblock.FieldEncrypted { + optFieldName = immutable.Some(block.Delta.GetFieldName()) + } + if block.Delta.IsComposite() { - optFieldName := immutable.None[string]() - if *block.EncryptionType == coreblock.FieldEncrypted { - optFieldName = immutable.Some(block.Delta.GetFieldName()) - } // for composite blocks there is nothing to decrypt // so we just check if we have the encryption key for child blocks - bytes, err := encryption.GetDocKey(ctx, string(block.Delta.GetDocID()), optFieldName) + bytes, err := encryption.GetKey(ctx, string(block.Delta.GetDocID()), optFieldName) if err != nil { return nil, err } @@ -389,9 +410,9 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } return block, nil } + clonedCRDT := block.Delta.Clone() - bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), - clonedCRDT.GetFieldName(), clonedCRDT.GetData()) + bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), optFieldName, clonedCRDT.GetData()) if err != nil { return nil, err } @@ -402,9 +423,7 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil } -func (mp *mergeProcessor) initCRDTForType( - field string, -) (merklecrdt.MerkleCRDT, error) { +func (mp *mergeProcessor) initCRDTForType(field string) (merklecrdt.MerkleCRDT, error) { mcrdt, exists := mp.mCRDTs[field] if exists { return mcrdt, nil diff --git a/internal/db/messages.go b/internal/db/messages.go index d200447ada..df832fcfc9 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -83,7 +83,7 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { case encryption.KeyRetrievedEvent: go func() { ctx = encryption.ContextWithStore(ctx, db.Encstore()) - err := encryption.SaveDocKey(ctx, evt.DocID, evt.FieldName, evt.Key) + err := encryption.SaveKey(ctx, evt.DocID, evt.FieldName, evt.Key) if err != nil { log.ErrorContextE( diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index ad658de911..6785c506b8 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -31,7 +31,7 @@ const keyLength = 32 // 32 bytes for AES-256 const testEncryptionKey = "examplekey1234567890examplekey12" // generateEncryptionKey generates a random AES key. -func generateEncryptionKey(_, _ string) ([]byte, error) { +func generateEncryptionKey(_ string, _ immutable.Option[string]) ([]byte, error) { key := make([]byte, keyLength) if _, err := io.ReadFull(rand.Reader, key); err != nil { return nil, err @@ -42,8 +42,8 @@ func generateEncryptionKey(_, _ string) ([]byte, error) { // generateTestEncryptionKey generates a deterministic encryption key for testing. // While testing, we also want to make sure different keys are generated for different docs and fields // and that's why we use the docID and fieldName to generate the key. -func generateTestEncryptionKey(docID, fieldName string) ([]byte, error) { - return []byte(fieldName + docID + testEncryptionKey)[0:keyLength], nil +func generateTestEncryptionKey(docID string, fieldName immutable.Option[string]) ([]byte, error) { + return []byte(fieldName.Value() + docID + testEncryptionKey)[0:keyLength], nil } // DocEncryptor is a document encryptor that encrypts and decrypts individual document fields. @@ -53,10 +53,11 @@ type DocEncryptor struct { conf immutable.Option[DocEncConfig] ctx context.Context store datastore.DSReaderWriter + cache map[core.EncStoreDocKey][]byte } func newDocEncryptor(ctx context.Context) *DocEncryptor { - return &DocEncryptor{ctx: ctx} + return &DocEncryptor{ctx: ctx, cache: make(map[core.EncStoreDocKey][]byte)} } // SetConfig sets the configuration for the document encryptor. @@ -69,30 +70,30 @@ func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { d.store = store } -func shouldEncryptIndividualField(conf immutable.Option[DocEncConfig], fieldName string) bool { - if !conf.HasValue() || fieldName == "" { +func shouldEncryptIndividualField(conf immutable.Option[DocEncConfig], fieldName immutable.Option[string]) bool { + if !conf.HasValue() || !fieldName.HasValue() { return false } for _, field := range conf.Value().EncryptedFields { - if field == fieldName { + if field == fieldName.Value() { return true } } return false } -func shouldEncryptField(conf immutable.Option[DocEncConfig], fieldName string) bool { +func shouldEncryptDocField(conf immutable.Option[DocEncConfig], fieldName immutable.Option[string]) bool { if !conf.HasValue() { return false } if conf.Value().IsDocEncrypted { return true } - if fieldName == "" { + if !fieldName.HasValue() { return false } for _, field := range conf.Value().EncryptedFields { - if field == fieldName { + if field == fieldName.Value() { return true } } @@ -102,7 +103,7 @@ func shouldEncryptField(conf immutable.Option[DocEncConfig], fieldName string) b // Encrypt encrypts the given plainText that is associated with the given docID and fieldName. // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. // Otherwise, it will use document-level encryption key. -func (d *DocEncryptor) Encrypt(docID, fieldName string, plainText []byte) ([]byte, error) { +func (d *DocEncryptor) Encrypt(docID string, fieldName immutable.Option[string], plainText []byte) ([]byte, error) { encryptionKey, err := d.fetchEncryptionKey(docID, fieldName) if err != nil { return nil, err @@ -110,10 +111,10 @@ func (d *DocEncryptor) Encrypt(docID, fieldName string, plainText []byte) ([]byt if len(encryptionKey) == 0 { if !shouldEncryptIndividualField(d.conf, fieldName) { - fieldName = "" + fieldName = immutable.None[string]() } - if !shouldEncryptField(d.conf, fieldName) { + if !shouldEncryptDocField(d.conf, fieldName) { return plainText, nil } @@ -122,8 +123,7 @@ func (d *DocEncryptor) Encrypt(docID, fieldName string, plainText []byte) ([]byt return nil, err } - storeKey := core.NewEncStoreDocKey(docID, fieldName) - err = d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) + err = d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName), encryptionKey) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (d *DocEncryptor) Encrypt(docID, fieldName string, plainText []byte) ([]byt // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. // If the corresponding encryption key is not found, it returns nil. -func (d *DocEncryptor) Decrypt(docID, fieldName string, cipherText []byte) ([]byte, error) { +func (d *DocEncryptor) Decrypt(docID string, fieldName immutable.Option[string], cipherText []byte) ([]byte, error) { encKey, err := d.fetchEncryptionKey(docID, fieldName) if err != nil { return nil, err @@ -144,22 +144,43 @@ func (d *DocEncryptor) Decrypt(docID, fieldName string, cipherText []byte) ([]by return DecryptAES(cipherText, encKey) } +func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, error) { + if encryptionKey, ok := d.cache[storeKey]; ok { + return encryptionKey, nil + } + encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) + isNotFound := errors.Is(err, ds.ErrNotFound) + if err != nil { + if isNotFound { + return nil, nil + } + return nil, err + } + + d.cache[storeKey] = encryptionKey + return encryptionKey, nil +} + +func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encryptionKey []byte) error { + d.cache[storeKey] = encryptionKey + return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) +} + // fetchEncryptionKey fetches the encryption key for the given docID and fieldName. // If the key is not found, it returns an empty key. -func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName string) ([]byte, error) { +func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName immutable.Option[string]) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } // first we try to find field-level key storeKey := core.NewEncStoreDocKey(docID, fieldName) - encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) - isNotFound := errors.Is(err, ds.ErrNotFound) + encryptionKey, err := d.fetchByEncStoreKey(storeKey) if err != nil { - if !isNotFound { - return nil, err - } + return nil, err + } + if len(encryptionKey) == 0 { // if previous fetch was for doc-level, there is nothing else to look for - if fieldName == "" { + if !fieldName.HasValue() { return nil, nil } if shouldEncryptIndividualField(d.conf, fieldName) { @@ -167,44 +188,39 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName string) ([]byt } // try to find doc-level key storeKey.FieldName = "" - encryptionKey, err = d.store.Get(d.ctx, storeKey.ToDS()) - isNotFound = errors.Is(err, ds.ErrNotFound) - if err != nil && !isNotFound { - return nil, err - } + encryptionKey, err = d.fetchByEncStoreKey(storeKey) } - return encryptionKey, nil + return encryptionKey, err } -func (d *DocEncryptor) GetDocKey(docID string, optFieldName immutable.Option[string]) ([]byte, error) { +func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string]) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - fieldName := "" - if optFieldName.HasValue() { - fieldName = optFieldName.Value() - } - storeKey := core.NewEncStoreDocKey(docID, fieldName) - encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) - if err != nil && !errors.Is(err, ds.ErrNotFound) { + encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName)) + if err != nil { return nil, err } return encryptionKey, nil } -func (d *DocEncryptor) SaveDocKey(docID string, encryptionKey []byte) error { +func (d *DocEncryptor) SaveKey(docID string, fieldName immutable.Option[string], encryptionKey []byte) error { if d.store == nil { return ErrNoStorageProvided } - storeKey := core.NewEncStoreDocKey(docID, "") - return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) + return d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName), encryptionKey) } // EncryptDoc encrypts the given plainText that is associated with the given docID and fieldName with // encryptor in the context. // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. // Otherwise, it will use document-level encryption key. -func EncryptDoc(ctx context.Context, docID string, fieldName string, plainText []byte) ([]byte, error) { +func EncryptDoc( + ctx context.Context, + docID string, + fieldName immutable.Option[string], + plainText []byte, +) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil @@ -214,7 +230,14 @@ func EncryptDoc(ctx context.Context, docID string, fieldName string, plainText [ // DecryptDoc decrypts the given cipherText that is associated with the given docID and fieldName with // encryptor in the context. -func DecryptDoc(ctx context.Context, docID string, fieldName string, cipherText []byte) ([]byte, error) { +// If fieldName is not provided, it will try to decrypt with the document-level key. Otherwise, it will try to +// decrypt with the field-level key. +func DecryptDoc( + ctx context.Context, + docID string, + fieldName immutable.Option[string], + cipherText []byte, +) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil @@ -222,31 +245,31 @@ func DecryptDoc(ctx context.Context, docID string, fieldName string, cipherText return enc.Decrypt(docID, fieldName, cipherText) } -// ShouldEncryptField returns true if the given field should be encrypted based on the context config. -func ShouldEncryptField(ctx context.Context, fieldName string) bool { - return shouldEncryptField(GetContextConfig(ctx), fieldName) +// ShouldEncryptDocField returns true if the given field should be encrypted based on the context config. +func ShouldEncryptDocField(ctx context.Context, fieldName immutable.Option[string]) bool { + return shouldEncryptDocField(GetContextConfig(ctx), fieldName) } // ShouldEncryptIndividualField returns true if the given field should be encrypted individually based on // the context config. -func ShouldEncryptIndividualField(ctx context.Context, fieldName string) bool { +func ShouldEncryptIndividualField(ctx context.Context, fieldName immutable.Option[string]) bool { return shouldEncryptIndividualField(GetContextConfig(ctx), fieldName) } -// SaveDocKey saves the given encryption key for the given docID and (optional) fieldName with encryptor in the context. -func SaveDocKey(ctx context.Context, docID string, fieldName immutable.Option[string], encryptionKey []byte) error { +// SaveKey saves the given encryption key for the given docID and (optional) fieldName with encryptor in the context. +func SaveKey(ctx context.Context, docID string, fieldName immutable.Option[string], encryptionKey []byte) error { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil } - return enc.SaveDocKey(docID, encryptionKey) + return enc.SaveKey(docID, fieldName, encryptionKey) } -// GetDocKey returns the encryption key for the given docID and (optional) fieldName with encryptor in the context. -func GetDocKey(ctx context.Context, docID string, fieldName immutable.Option[string]) ([]byte, error) { +// GetKey returns the encryption key for the given docID and (optional) fieldName with encryptor in the context. +func GetKey(ctx context.Context, docID string, fieldName immutable.Option[string]) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.GetDocKey(docID, fieldName) + return enc.GetKey(docID, fieldName) } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 131d6dc964..1fe3a88b56 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -21,6 +21,7 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" @@ -86,7 +87,11 @@ func (mc *MerkleClock) AddDelta( delta.SetPriority(height) block := coreblock.New(delta, links, heads...) - encryptionType, err := mc.checkIfBlockEncryptionEnabled(ctx, block.Delta.GetFieldName(), heads) + fieldName := immutable.None[string]() + if block.Delta.GetFieldName() != "" { + fieldName = immutable.Some(block.Delta.GetFieldName()) + } + encryptionType, err := mc.checkIfBlockEncryptionEnabled(ctx, fieldName, heads) if err != nil { return cidlink.Link{}, nil, err } @@ -94,7 +99,7 @@ func (mc *MerkleClock) AddDelta( dagBlock := block if encryptionType != coreblock.NotEncrypted { if !block.Delta.IsComposite() { - dagBlock, err = encryptBlock(ctx, block) + dagBlock, err = encryptBlock(ctx, block, encryptionType == coreblock.FieldEncrypted) if err != nil { return cidlink.Link{}, nil, err } @@ -123,10 +128,10 @@ func (mc *MerkleClock) AddDelta( func (mc *MerkleClock) checkIfBlockEncryptionEnabled( ctx context.Context, - fieldName string, + fieldName immutable.Option[string], heads []cid.Cid, ) (coreblock.EncryptionType, error) { - if encryption.ShouldEncryptField(ctx, fieldName) { + if encryption.ShouldEncryptDocField(ctx, fieldName) { if encryption.ShouldEncryptIndividualField(ctx, fieldName) { return coreblock.FieldEncrypted, nil } @@ -150,10 +155,14 @@ func (mc *MerkleClock) checkIfBlockEncryptionEnabled( return coreblock.NotEncrypted, nil } -func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { +func encryptBlock(ctx context.Context, block *coreblock.Block, isFieldOnly bool) (*coreblock.Block, error) { clonedCRDT := block.Delta.Clone() + fieldName := immutable.None[string]() + if isFieldOnly { + fieldName = immutable.Some(clonedCRDT.GetFieldName()) + } bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), - clonedCRDT.GetFieldName(), clonedCRDT.GetData()) + fieldName, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/net/peer.go b/net/peer.go index 3fcba4a1bd..ae1fcbeb80 100644 --- a/net/peer.go +++ b/net/peer.go @@ -45,7 +45,6 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/multiformats/go-multiaddr" "github.com/sourcenetwork/corelog" - "github.com/sourcenetwork/immutable" "google.golang.org/grpc" "github.com/sourcenetwork/defradb/client" diff --git a/net/server.go b/net/server.go index ee1f853ea6..d67a99152d 100644 --- a/net/server.go +++ b/net/server.go @@ -171,6 +171,19 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return &pb.PushLogReply{}, nil } +func (s *server) getEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) ([]byte, error) { + docID, err := client.NewDocIDFromString(string(req.DocID)) + if err != nil { + return nil, err + } + + optFieldName := immutable.None[string]() + if req.FieldName != "" { + optFieldName = immutable.Some(req.FieldName) + } + return encryption.GetKey(encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName) +} + func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { isValid, err := s.verifyRequestSignature(req) if err != nil { diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 48e01aca94..f3c4ffe5ed 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -119,3 +119,107 @@ func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt testUtils.ExecuteTestCase(t, test) } +func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeysAndDecrypt(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + EncryptedFields: []string{"age"}, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeyRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: []map[string]any{ + { + "name": "John", + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + EncryptedFields: []string{"name", "age"}, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeyRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: []map[string]any{ + { + "name": "John", + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + From 4782ea56249684ca67e6bb63791f2850f1bd5dd8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 3 Aug 2024 20:51:42 +0200 Subject: [PATCH 08/88] Add encryption with ECDH --- {internal/encryption => crypto}/aes.go | 26 ++---- crypto/ecdh.go | 68 ++++++++++++++++ crypto/nonce.go | 52 ++++++++++++ internal/encryption/encryptor.go | 16 +++- internal/encryption/encryptor_test.go | 38 ++++----- net/pb/net.pb.go | 107 ++++++++++++++++--------- net/pb/net.proto | 14 +++- net/pb/net_vtproto.pb.go | 94 +++++++++++++++++++++- net/server.go | 85 +++++++++++++------- 9 files changed, 386 insertions(+), 114 deletions(-) rename {internal/encryption => crypto}/aes.go (74%) create mode 100644 crypto/ecdh.go create mode 100644 crypto/nonce.go diff --git a/internal/encryption/aes.go b/crypto/aes.go similarity index 74% rename from internal/encryption/aes.go rename to crypto/aes.go index e3a7feb563..278ad90012 100644 --- a/internal/encryption/aes.go +++ b/crypto/aes.go @@ -8,16 +8,16 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package encryption +package crypto import ( "crypto/aes" "crypto/cipher" - "encoding/base64" "fmt" ) // EncryptAES encrypts data using AES-GCM with a provided key. +// The nonce is prepended to the cipherText. func EncryptAES(plainText, key []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -36,32 +36,22 @@ func EncryptAES(plainText, key []byte) ([]byte, error) { cipherText := aesGCM.Seal(nonce, nonce, plainText, nil) - buf := make([]byte, base64.StdEncoding.EncodedLen(len(cipherText))) - base64.StdEncoding.Encode(buf, cipherText) - - return buf, nil + return cipherText, nil } // DecryptAES decrypts AES-GCM encrypted data with a provided key. -func DecryptAES(cipherTextBase64, key []byte) ([]byte, error) { - cipherText := make([]byte, base64.StdEncoding.DecodedLen(len(cipherTextBase64))) - n, err := base64.StdEncoding.Decode(cipherText, []byte(cipherTextBase64)) - - if err != nil { - return nil, err +// The nonce is expected to be prepended to the cipherText. +func DecryptAES(cipherText, key []byte) ([]byte, error) { + if len(cipherText) < nonceLength { + // TODO return typed error + return nil, fmt.Errorf("cipherText too short") } - cipherText = cipherText[:n] - block, err := aes.NewCipher(key) if err != nil { return nil, err } - if len(cipherText) < nonceLength { - return nil, fmt.Errorf("cipherText too short") - } - nonce := cipherText[:nonceLength] cipherText = cipherText[nonceLength:] diff --git a/crypto/ecdh.go b/crypto/ecdh.go new file mode 100644 index 0000000000..27a2072ef9 --- /dev/null +++ b/crypto/ecdh.go @@ -0,0 +1,68 @@ +package crypto + +import ( + "crypto/ecdh" + "crypto/rand" + "crypto/sha256" + "fmt" +) + +func GenerateX25519() (*ecdh.PrivateKey, error) { + return ecdh.X25519().GenerateKey(rand.Reader) +} + +const X25519PublicKeySize = 32 + +func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { + return ecdh.X25519().NewPublicKey(publicKeyBytes) +} + +func EncryptECDH(plaintext []byte, publicKey *ecdh.PublicKey) ([]byte, error) { + ephemeralPrivate, err := GenerateX25519() + if err != nil { + return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) + } + ephemeralPublic := ephemeralPrivate.PublicKey() + + sharedSecret, err := ephemeralPrivate.ECDH(publicKey) + if err != nil { + return nil, fmt.Errorf("ECDH failed: %w", err) + } + + key := sha256.Sum256(sharedSecret) + + cipherText, err := EncryptAES(plaintext, key[:]) + if err != nil { + return nil, fmt.Errorf("failed to encrypt: %w", err) + } + + return append(ephemeralPublic.Bytes(), cipherText...), nil +} + +func DecryptECDH(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error) { + if len(cipherText) < X25519PublicKeySize+nonceLength { + return nil, fmt.Errorf("cipherText too short") + } + + ephemeralPublicBytes := cipherText[:X25519PublicKeySize] + ephemeralPublic, err := ecdh.X25519().NewPublicKey(ephemeralPublicBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse ephemeral public key: %w", err) + } + + sharedSecret, err := privateKey.ECDH(ephemeralPublic) + if err != nil { + return nil, fmt.Errorf("ECDH failed: %w", err) + } + + key := sha256.Sum256(sharedSecret) + + cipherText = cipherText[X25519PublicKeySize:] + plainText, err := DecryptAES(cipherText, key[:]) + + if err != nil { + return nil, fmt.Errorf("failed to decrypt: %w", err) + } + + return plainText, nil +} diff --git a/crypto/nonce.go b/crypto/nonce.go new file mode 100644 index 0000000000..a14a3ef5cf --- /dev/null +++ b/crypto/nonce.go @@ -0,0 +1,52 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "crypto/rand" + "errors" + "io" + "os" + "strings" +) + +const nonceLength = 12 + +var generateNonceFunc = generateNonce + +func generateNonce() ([]byte, error) { + nonce := make([]byte, nonceLength) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + return nonce, nil +} + +// generateTestNonce generates a deterministic nonce for testing. +func generateTestNonce() ([]byte, error) { + nonce := []byte("deterministic nonce for testing") + + if len(nonce) < nonceLength { + return nil, errors.New("nonce length is longer than available deterministic nonce") + } + + return nonce[:nonceLength], nil +} + +func init() { + arg := os.Args[0] + // If the binary is a test binary, use a deterministic nonce. + // TODO: We should try to find a better way to detect this https://github.com/sourcenetwork/defradb/issues/2801 + if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { + generateNonceFunc = generateTestNonce + } +} diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 6785c506b8..d6301fede2 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -15,11 +15,14 @@ import ( "crypto/rand" "errors" "io" + "os" + "strings" ds "github.com/ipfs/go-datastore" "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" ) @@ -128,7 +131,7 @@ func (d *DocEncryptor) Encrypt(docID string, fieldName immutable.Option[string], return nil, err } } - return EncryptAES(plainText, encryptionKey) + return crypto.EncryptAES(plainText, encryptionKey) } // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. @@ -141,7 +144,7 @@ func (d *DocEncryptor) Decrypt(docID string, fieldName immutable.Option[string], if len(encKey) == 0 { return nil, nil } - return DecryptAES(cipherText, encKey) + return crypto.DecryptAES(cipherText, encKey) } func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, error) { @@ -273,3 +276,12 @@ func GetKey(ctx context.Context, docID string, fieldName immutable.Option[string } return enc.GetKey(docID, fieldName) } + +func init() { + arg := os.Args[0] + // If the binary is a test binary, use a deterministic nonce. + // TODO: We should try to find a better way to detect this https://github.com/sourcenetwork/defradb/issues/2801 + if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { + generateEncryptionKeyFunc = generateTestEncryptionKey + } +} diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 76888ed4f1..782f6889f5 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/datastore/mocks" "github.com/sourcenetwork/defradb/internal/core" ) @@ -29,19 +30,20 @@ var testErr = errors.New("test error") const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" -const fieldName = "name" +var fieldName = immutable.Some("name") +var noFieldName = immutable.None[string]() func getPlainText() []byte { return []byte("test") } -func getEncKey(fieldName string) []byte { +func getEncKey(fieldName immutable.Option[string]) []byte { key, _ := generateTestEncryptionKey(docID, fieldName) return key } -func getCipherText(t *testing.T, fieldName string) []byte { - cipherText, err := EncryptAES(getPlainText(), getEncKey(fieldName)) +func getCipherText(t *testing.T, fieldName immutable.Option[string]) []byte { + cipherText, err := crypto.EncryptAES(getPlainText(), getEncKey(fieldName)) assert.NoError(t, err) return cipherText } @@ -82,35 +84,35 @@ func TestEncryptorEncrypt_IfStorageReturnsErrorOnSecondCall_Error(t *testing.T) func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - storeKey := core.NewEncStoreDocKey(docID, "") + storeKey := core.NewEncStoreDocKey(docID, noFieldName) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) - st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey("")).Return(nil) + st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, "", getPlainText()) + cipherText, err := enc.Encrypt(docID, noFieldName, getPlainText()) assert.NoError(t, err) - assert.Equal(t, getCipherText(t, ""), cipherText) + assert.Equal(t, getCipherText(t, noFieldName), cipherText) } func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - docStoreKey := core.NewEncStoreDocKey(docID, "").ToDS() + docStoreKey := core.NewEncStoreDocKey(docID, noFieldName).ToDS() fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName).ToDS() st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) st.EXPECT().Get(mock.Anything, docStoreKey).Return(nil, ds.ErrNotFound) - st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey("")).Return(nil) + st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey(noFieldName)).Return(nil) cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) assert.NoError(t, err) - assert.Equal(t, getCipherText(t, ""), cipherText) + assert.Equal(t, getCipherText(t, noFieldName), cipherText) } func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName}}) + enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) storeKey := core.NewEncStoreDocKey(docID, fieldName) @@ -124,7 +126,7 @@ func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreI } func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { - enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName}}) + enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) storeKey := core.NewEncStoreDocKey(docID, fieldName) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) @@ -138,12 +140,12 @@ func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipher func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(""), nil) + st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - cipherText, err := enc.Encrypt(docID, "", getPlainText()) + cipherText, err := enc.Encrypt(docID, noFieldName, getPlainText()) assert.NoError(t, err) - assert.Equal(t, getCipherText(t, ""), cipherText) + assert.Equal(t, getCipherText(t, noFieldName), cipherText) } func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *testing.T) { @@ -201,9 +203,9 @@ func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *testing.T) { enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(""), nil) + st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - plainText, err := enc.Decrypt(docID, fieldName, getCipherText(t, "")) + plainText, err := enc.Decrypt(docID, fieldName, getCipherText(t, noFieldName)) assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 5072baca82..a4b986191a 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -420,12 +420,14 @@ type FetchEncryptionKeyRequest struct { Cid []byte `protobuf:"bytes,3,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,4,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` - // publicKey is a public of the requesting peer + // publicKey is a public of the requesting peer for signature verification PublicKey *pb.PublicKey `protobuf:"bytes,5,opt,name=publicKey,proto3" json:"publicKey,omitempty"` + // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret + EphemeralPublicKey []byte `protobuf:"bytes,6,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` // peerInfo is peer information is the requesting peer - PeerInfo *PeerInfo `protobuf:"bytes,6,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` + PeerInfo *PeerInfo `protobuf:"bytes,7,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,8,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { @@ -495,6 +497,13 @@ func (x *FetchEncryptionKeyRequest) GetPublicKey() *pb.PublicKey { return nil } +func (x *FetchEncryptionKeyRequest) GetEphemeralPublicKey() []byte { + if x != nil { + return x.EphemeralPublicKey + } + return nil +} + func (x *FetchEncryptionKeyRequest) GetPeerInfo() *PeerInfo { if x != nil { return x.PeerInfo @@ -592,13 +601,17 @@ type FetchEncryptionKeyReply struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // encryptedKey is array or bytes representing encrypted doc encryption key + // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM EncryptedKey []byte `protobuf:"bytes,1,opt,name=encryptedKey,proto3" json:"encryptedKey,omitempty"` // cid is the CID of the composite of the document. Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id + ReqEphemeralPublicKey []byte `protobuf:"bytes,4,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` // signature is the responding peer's signature of the reply - Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyReply) Reset() { @@ -654,6 +667,13 @@ func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { return nil } +func (x *FetchEncryptionKeyReply) GetReqEphemeralPublicKey() []byte { + if x != nil { + return x.ReqEphemeralPublicKey + } + return nil +} + func (x *FetchEncryptionKeyReply) GetSignature() []byte { if x != nil { return x.Signature @@ -814,7 +834,7 @@ var file_net_proto_rawDesc = []byte{ 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x81, 0x02, 0x0a, 0x19, + 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xb1, 0x02, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, @@ -826,52 +846,59 @@ var file_net_proto_rawDesc = []byte{ 0x32, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, + 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x07, + 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x8d, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, - 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, - 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, - 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, - 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, - 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, - 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, - 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, + 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, + 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, + 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, + 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, + 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, + 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, + 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/net/pb/net.proto b/net/pb/net.proto index 47cbc4e0b8..711be17870 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -61,12 +61,14 @@ message FetchEncryptionKeyRequest { bytes cid = 3; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 4; - // publicKey is a public of the requesting peer + // publicKey is a public of the requesting peer for signature verification crypto.pb.PublicKey publicKey = 5; + // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret + bytes ephemeralPublicKey = 6; // peerInfo is peer information is the requesting peer - PeerInfo peerInfo = 6; + PeerInfo peerInfo = 7; // signature is the requesting peer's signature of the request - bytes signature = 7; + bytes signature = 8; } message GetHeadLogRequest {} @@ -76,13 +78,17 @@ message PushLogReply {} // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request // by a peer that holds the requested doc encryption key. message FetchEncryptionKeyReply { + // encryptedKey is array or bytes representing encrypted doc encryption key + // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM bytes encryptedKey = 1; // cid is the CID of the composite of the document. bytes cid = 2; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 3; + // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id + bytes reqEphemeralPublicKey = 4; // signature is the responding peer's signature of the reply - bytes signature = 4; + bytes signature = 5; } message GetHeadLogReply {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index a016809254..57a19fc9e9 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -456,7 +456,7 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x3a + dAtA[i] = 0x42 } if m.PeerInfo != nil { size, err := m.PeerInfo.MarshalToSizedBufferVT(dAtA[:i]) @@ -466,6 +466,13 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i -= size i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- + dAtA[i] = 0x3a + } + if len(m.EphemeralPublicKey) > 0 { + i -= len(m.EphemeralPublicKey) + copy(dAtA[i:], m.EphemeralPublicKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EphemeralPublicKey))) + i-- dAtA[i] = 0x32 } if m.PublicKey != nil { @@ -622,6 +629,13 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- + dAtA[i] = 0x2a + } + if len(m.ReqEphemeralPublicKey) > 0 { + i -= len(m.ReqEphemeralPublicKey) + copy(dAtA[i:], m.ReqEphemeralPublicKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReqEphemeralPublicKey))) + i-- dAtA[i] = 0x22 } if len(m.SchemaRoot) > 0 { @@ -851,6 +865,10 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { } n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.EphemeralPublicKey) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } if m.PeerInfo != nil { l = m.PeerInfo.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -901,6 +919,10 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.ReqEphemeralPublicKey) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } l = len(m.Signature) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1943,6 +1965,40 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EphemeralPublicKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EphemeralPublicKey = append(m.EphemeralPublicKey[:0], dAtA[iNdEx:postIndex]...) + if m.EphemeralPublicKey == nil { + m.EphemeralPublicKey = []byte{} + } + iNdEx = postIndex + case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerInfo", wireType) } @@ -1978,7 +2034,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex - case 7: + case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } @@ -2268,6 +2324,40 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ReqEphemeralPublicKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ReqEphemeralPublicKey = append(m.ReqEphemeralPublicKey[:0], dAtA[iNdEx:postIndex]...) + if m.ReqEphemeralPublicKey == nil { + m.ReqEphemeralPublicKey = []byte{} + } + iNdEx = postIndex + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } diff --git a/net/server.go b/net/server.go index d67a99152d..a2701bca3d 100644 --- a/net/server.go +++ b/net/server.go @@ -14,6 +14,7 @@ package net import ( "context" + "crypto/ecdh" "crypto/sha256" "fmt" "sync" @@ -59,6 +60,13 @@ type server struct { conns map[libpeer.ID]*grpc.ClientConn pb.UnimplementedServiceServer + + sessions []session +} + +type session struct { + id string + privateKey *ecdh.PrivateKey } // pubsubTopic is a wrapper of rpc.Topic to be able to track if the topic has @@ -89,6 +97,15 @@ func newServer(p *Peer, opts ...grpc.DialOption) (*server, error) { return s, nil } +func (s *server) findSession(id string) *session { + for _, sess := range s.sessions { + if sess.id == id { + return &sess + } + } + return nil +} + // GetDocGraph receives a get graph request func (s *server) GetDocGraph( ctx context.Context, @@ -194,6 +211,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.New("invalid signature") } + // TODO: check if pubkey can be fetched from peerstore pubKey, err := libp2pCrypto.PublicKeyFromProto(req.PublicKey) if err != nil { return nil, errors.Wrap("failed to unmarshal public key", err) @@ -208,20 +226,21 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, err } - pubKeyBytes, err := pubKey.Raw() + reqEphPubKey, err := crypto.X25519PublicKeyFromBytes(req.EphemeralPublicKey) if err != nil { - return nil, fmt.Errorf("failed to get raw Ed25519 public key: %w", err) + return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) } - encryptedKey, err := crypto.EncryptWithEphemeralKey(encKey, pubKeyBytes) + encryptedKey, err := crypto.EncryptECDH(encKey, reqEphPubKey) if err != nil { return nil, errors.Wrap("failed to encrypt key for requester", err) } res := &pb.FetchEncryptionKeyReply{ - EncryptedKey: encryptedKey, - Cid: req.Cid, - SchemaRoot: req.SchemaRoot, + EncryptedKey: encryptedKey, + Cid: req.Cid, + SchemaRoot: req.SchemaRoot, + ReqEphemeralPublicKey: req.EphemeralPublicKey, } res.Signature, err = s.signResponse(res) @@ -254,14 +273,18 @@ func (s *server) verifyPeerInfo(peerID libpeer.ID, pubKey libp2pCrypto.PubKey) e return nil } -func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { +func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { hash := sha256.New() hash.Write(res.EncryptedKey) hash.Write(res.Cid) hash.Write(res.SchemaRoot) + hash.Write(res.ReqEphemeralPublicKey) + return hash.Sum(nil) +} +func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - return privKey.Sign(hash.Sum(nil)) + return privKey.Sign(hashFetchEncryptionKeyReply(res)) } // GetHeadLog receives a get head log request @@ -424,6 +447,7 @@ func toProtoPeerInfo(peerInfo libpeer.AddrInfo) *pb.PeerInfo { func (s *server) prepareFetchEncryptionKeyRequest( evt encryption.RequestKeyEvent, + ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { publicKey := s.peer.host.Peerstore().PubKey(s.peer.host.ID()) protoPublicKey, err := libp2pCrypto.PublicKeyToProto(publicKey) @@ -432,11 +456,12 @@ func (s *server) prepareFetchEncryptionKeyRequest( } req := &pb.FetchEncryptionKeyRequest{ - DocID: []byte(evt.DocID), - Cid: evt.Cid.Bytes(), - SchemaRoot: []byte(evt.SchemaRoot), - PublicKey: protoPublicKey, - PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), + DocID: []byte(evt.DocID), + Cid: evt.Cid.Bytes(), + SchemaRoot: []byte(evt.SchemaRoot), + PublicKey: protoPublicKey, + EphemeralPublicKey: ephemeralPublicKey, + PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), } if evt.FieldName.HasValue() { @@ -457,7 +482,13 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques return nil } - req, err := s.prepareFetchEncryptionKeyRequest(evt) + ephPrivKey, err := crypto.GenerateX25519() + if err != nil { + return err + } + + ephPubKeyBytes := ephPrivKey.PublicKey().Bytes() + req, err := s.prepareFetchEncryptionKeyRequest(evt, ephPubKeyBytes) if err != nil { return err } @@ -475,6 +506,8 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) } + s.sessions = append(s.sessions, session{id: string(ephPubKeyBytes), privateKey: ephPrivKey}) + go func() { s.handleFetchEncryptionKeyResponse(<-respChan, req) }() @@ -489,6 +522,7 @@ func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { hash.Write(req.SchemaRoot) hash.Write([]byte(req.PublicKey.Type.String())) hash.Write(req.PublicKey.Data) + hash.Write(req.EphemeralPublicKey) hash.Write([]byte(req.PeerInfo.String())) return hash.Sum(nil) } @@ -517,16 +551,14 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - privateKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - - // Use the private key to get the public key bytes - ed25519PubKeyBytes, err := privateKey.GetPublic().Raw() - if err != nil { - log.ErrorContextE(s.peer.ctx, "failed to get raw Ed25519 public key", err) + // TODO: properly handle sessions. At least remove used or old ones + session := s.findSession(string(keyResp.ReqEphemeralPublicKey)) + if session == nil { + log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") return } - decryptedKey, err := crypto.DecryptWithEphemeralKey(keyResp.EncryptedKey, ed25519PubKeyBytes) + decryptedKey, err := crypto.DecryptECDH(keyResp.EncryptedKey, session.privateKey) if err != nil { log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) return @@ -548,15 +580,8 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet } func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { - peerStore := s.peer.host.Peerstore() - pubKey := peerStore.PubKey(fromPeer) - - hash := sha256.New() - hash.Write(res.EncryptedKey) - hash.Write(res.Cid) - hash.Write(res.SchemaRoot) - - return pubKey.Verify(hash.Sum(nil), res.Signature) + pubKey := s.peer.host.Peerstore().PubKey(fromPeer) + return pubKey.Verify(hashFetchEncryptionKeyReply(res), res.Signature) } // pubSubMessageHandler handles incoming PushLog messages from the pubsub network. From 5d83085cfd46e8251e301609330b4c00cad3fc07 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 3 Aug 2024 21:20:06 +0200 Subject: [PATCH 09/88] Remove unnecessary pub key transit --- net/pb/Makefile | 4 - net/pb/net.pb.go | 233 ++++++++++++++++++--------------------- net/pb/net.proto | 10 +- net/pb/net_vtproto.pb.go | 86 +-------------- net/server.go | 32 ++---- 5 files changed, 123 insertions(+), 242 deletions(-) diff --git a/net/pb/Makefile b/net/pb/Makefile index 099a9ccc74..9b253cbf3a 100644 --- a/net/pb/Makefile +++ b/net/pb/Makefile @@ -5,8 +5,6 @@ PROTOC_GEN_GO := $(shell which protoc-gen-go) PROTOC_GEN_GO_GRPC := $(shell which protoc-gen-go-grpc) PROTOC_GEN_GO_VTPROTO := $(shell which protoc-gen-go-vtproto) -LIBP2P_PATH := $(shell go list -f '{{.Dir}}' github.com/libp2p/go-libp2p) - all: $(GO) deps: @@ -17,8 +15,6 @@ deps: %.pb.go: %.proto protoc \ -I. \ - -I$(LIBP2P_PATH) \ - -I$(GOPATH)/src \ --go_out=. --plugin protoc-gen-go="$(PROTOC_GEN_GO)" \ --go-grpc_out=. --plugin protoc-gen-go-grpc="$(PROTOC_GEN_GO_GRPC)" \ --go-vtproto_out=. --plugin protoc-gen-go-vtproto="$(PROTOC_GEN_GO_VTPROTO)" \ diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index a4b986191a..9c0efae055 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -7,7 +7,6 @@ package net_pb import ( - pb "github.com/libp2p/go-libp2p/core/crypto/pb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -420,14 +419,12 @@ type FetchEncryptionKeyRequest struct { Cid []byte `protobuf:"bytes,3,opt,name=cid,proto3" json:"cid,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,4,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` - // publicKey is a public of the requesting peer for signature verification - PublicKey *pb.PublicKey `protobuf:"bytes,5,opt,name=publicKey,proto3" json:"publicKey,omitempty"` // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - EphemeralPublicKey []byte `protobuf:"bytes,6,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` + EphemeralPublicKey []byte `protobuf:"bytes,5,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` // peerInfo is peer information is the requesting peer - PeerInfo *PeerInfo `protobuf:"bytes,7,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` + PeerInfo *PeerInfo `protobuf:"bytes,6,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,8,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { @@ -490,13 +487,6 @@ func (x *FetchEncryptionKeyRequest) GetSchemaRoot() []byte { return nil } -func (x *FetchEncryptionKeyRequest) GetPublicKey() *pb.PublicKey { - if x != nil { - return x.PublicKey - } - return nil -} - func (x *FetchEncryptionKeyRequest) GetEphemeralPublicKey() []byte { if x != nil { return x.EphemeralPublicKey @@ -807,98 +797,93 @@ var File_net_proto protoreflect.FileDescriptor var file_net_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x1a, 0x1b, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, - 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x22, 0x14, 0x0a, - 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, 0x73, 0x68, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x13, - 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, - 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x03, 0x6c, 0x6f, - 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xb1, 0x02, 0x0a, 0x19, + 0x2e, 0x70, 0x62, 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x15, 0x0a, 0x13, 0x50, 0x75, + 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, + 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x0f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x0e, 0x50, 0x75, 0x73, 0x68, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xfd, + 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, + 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, + 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, + 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, + 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, + 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x65, + 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, + 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, + 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, + 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, + 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, + 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, + 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, + 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, + 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, + 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, + 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, - 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, - 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, - 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, - 0x32, 0x0a, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x70, 0x62, 0x2e, 0x50, - 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, - 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, - 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, - 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, - 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, - 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, - 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, - 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, - 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, - 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, - 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, - 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, - 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, - 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, + 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, + 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -930,30 +915,28 @@ var file_net_proto_goTypes = []any{ (*FetchEncryptionKeyReply)(nil), // 12: net.pb.FetchEncryptionKeyReply (*GetHeadLogReply)(nil), // 13: net.pb.GetHeadLogReply (*PushLogRequest_Body)(nil), // 14: net.pb.PushLogRequest.Body - (*pb.PublicKey)(nil), // 15: crypto.pb.PublicKey } var file_net_proto_depIdxs = []int32{ 14, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 15, // 1: net.pb.FetchEncryptionKeyRequest.publicKey:type_name -> crypto.pb.PublicKey - 8, // 2: net.pb.FetchEncryptionKeyRequest.peerInfo:type_name -> net.pb.PeerInfo - 0, // 3: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log - 1, // 4: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest - 3, // 5: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest - 5, // 6: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest - 7, // 7: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 9, // 8: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 10, // 9: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 10: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 11: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 12: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 11, // 13: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 12, // 14: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply - 13, // 15: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 10, // [10:16] is the sub-list for method output_type - 4, // [4:10] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 8, // 1: net.pb.FetchEncryptionKeyRequest.peerInfo:type_name -> net.pb.PeerInfo + 0, // 2: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log + 1, // 3: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest + 3, // 4: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest + 5, // 5: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest + 7, // 6: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest + 9, // 7: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest + 10, // 8: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 9: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 10: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 11: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 11, // 12: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 12, // 13: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 13, // 14: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 9, // [9:15] is the sub-list for method output_type + 3, // [3:9] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_net_proto_init() } diff --git a/net/pb/net.proto b/net/pb/net.proto index 711be17870..d3574b748d 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -1,8 +1,6 @@ syntax = "proto3"; package net.pb; -import "core/crypto/pb/crypto.proto"; - option go_package = "/;net_pb"; // Log represents a thread log. @@ -61,14 +59,12 @@ message FetchEncryptionKeyRequest { bytes cid = 3; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 4; - // publicKey is a public of the requesting peer for signature verification - crypto.pb.PublicKey publicKey = 5; // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - bytes ephemeralPublicKey = 6; + bytes ephemeralPublicKey = 5; // peerInfo is peer information is the requesting peer - PeerInfo peerInfo = 7; + PeerInfo peerInfo = 6; // signature is the requesting peer's signature of the request - bytes signature = 8; + bytes signature = 7; } message GetHeadLogRequest {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 57a19fc9e9..a80929c3c5 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -6,9 +6,7 @@ package net_pb import ( fmt "fmt" - pb "github.com/libp2p/go-libp2p/core/crypto/pb" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" - proto "google.golang.org/protobuf/proto" protoimpl "google.golang.org/protobuf/runtime/protoimpl" io "io" ) @@ -456,7 +454,7 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x42 + dAtA[i] = 0x3a } if m.PeerInfo != nil { size, err := m.PeerInfo.MarshalToSizedBufferVT(dAtA[:i]) @@ -466,35 +464,13 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i -= size i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- - dAtA[i] = 0x3a + dAtA[i] = 0x32 } if len(m.EphemeralPublicKey) > 0 { i -= len(m.EphemeralPublicKey) copy(dAtA[i:], m.EphemeralPublicKey) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EphemeralPublicKey))) i-- - dAtA[i] = 0x32 - } - if m.PublicKey != nil { - if vtmsg, ok := interface{}(m.PublicKey).(interface { - MarshalToSizedBufferVT([]byte) (int, error) - }); ok { - size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - } else { - encoded, err := proto.Marshal(m.PublicKey) - if err != nil { - return 0, err - } - i -= len(encoded) - copy(dAtA[i:], encoded) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) - } - i-- dAtA[i] = 0x2a } if len(m.SchemaRoot) > 0 { @@ -855,16 +831,6 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.PublicKey != nil { - if size, ok := interface{}(m.PublicKey).(interface { - SizeVT() int - }); ok { - l = size.SizeVT() - } else { - l = proto.Size(m.PublicKey) - } - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } l = len(m.EphemeralPublicKey) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1921,50 +1887,6 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PublicKey", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PublicKey == nil { - m.PublicKey = &pb.PublicKey{} - } - if unmarshal, ok := interface{}(m.PublicKey).(interface { - UnmarshalVT([]byte) error - }); ok { - if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.PublicKey); err != nil { - return err - } - } - iNdEx = postIndex - case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field EphemeralPublicKey", wireType) } @@ -1998,7 +1920,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.EphemeralPublicKey = []byte{} } iNdEx = postIndex - case 7: + case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerInfo", wireType) } @@ -2034,7 +1956,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex - case 8: + case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } diff --git a/net/server.go b/net/server.go index a2701bca3d..9f892b9d20 100644 --- a/net/server.go +++ b/net/server.go @@ -202,7 +202,9 @@ func (s *server) getEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKe } func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { - isValid, err := s.verifyRequestSignature(req) + reqPubKey := s.peer.host.Peerstore().PubKey(libpeer.ID(req.PeerInfo.Id)) + + isValid, err := s.verifyRequestSignature(req, reqPubKey) if err != nil { return nil, errors.Wrap("invalid signature", err) } @@ -211,13 +213,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.New("invalid signature") } - // TODO: check if pubkey can be fetched from peerstore - pubKey, err := libp2pCrypto.PublicKeyFromProto(req.PublicKey) - if err != nil { - return nil, errors.Wrap("failed to unmarshal public key", err) - } - - if err := s.verifyPeerInfo(libpeer.ID(req.PeerInfo.Id), pubKey); err != nil { + if err := s.verifyPeerInfo(libpeer.ID(req.PeerInfo.Id), reqPubKey); err != nil { return nil, errors.Wrap("invalid peer info", err) } @@ -251,12 +247,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return res, nil } -func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest) (bool, error) { - pubKey, err := libp2pCrypto.PublicKeyFromProto(req.PublicKey) - if err != nil { - return false, err - } - +func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKey libp2pCrypto.PubKey) (bool, error) { return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) } @@ -449,17 +440,10 @@ func (s *server) prepareFetchEncryptionKeyRequest( evt encryption.RequestKeyEvent, ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { - publicKey := s.peer.host.Peerstore().PubKey(s.peer.host.ID()) - protoPublicKey, err := libp2pCrypto.PublicKeyToProto(publicKey) - if err != nil { - return nil, errors.Wrap("failed to marshal public key", err) - } - req := &pb.FetchEncryptionKeyRequest{ DocID: []byte(evt.DocID), Cid: evt.Cid.Bytes(), SchemaRoot: []byte(evt.SchemaRoot), - PublicKey: protoPublicKey, EphemeralPublicKey: ephemeralPublicKey, PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), } @@ -468,11 +452,13 @@ func (s *server) prepareFetchEncryptionKeyRequest( req.FieldName = evt.FieldName.Value() } - req.Signature, err = s.signRequest(req) + signature, err := s.signRequest(req) if err != nil { return nil, errors.Wrap("failed to sign request", err) } + req.Signature = signature + return req, nil } @@ -520,8 +506,6 @@ func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { hash.Write(req.DocID) hash.Write(req.Cid) hash.Write(req.SchemaRoot) - hash.Write([]byte(req.PublicKey.Type.String())) - hash.Write(req.PublicKey.Data) hash.Write(req.EphemeralPublicKey) hash.Write([]byte(req.PeerInfo.String())) return hash.Sum(nil) From cf5c55986fe2410d9f74b210b6ddea05ff1ea959 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 3 Aug 2024 22:17:14 +0200 Subject: [PATCH 10/88] Implement ECIES --- crypto/aes.go | 6 +- crypto/ecdh.go | 68 --------------- crypto/ecies.go | 120 ++++++++++++++++++++++++++ crypto/nonce.go | 8 +- net/server.go | 4 +- tests/integration/encryption/utils.go | 4 +- 6 files changed, 131 insertions(+), 79 deletions(-) delete mode 100644 crypto/ecdh.go create mode 100644 crypto/ecies.go diff --git a/crypto/aes.go b/crypto/aes.go index 278ad90012..3caed84fe1 100644 --- a/crypto/aes.go +++ b/crypto/aes.go @@ -42,7 +42,7 @@ func EncryptAES(plainText, key []byte) ([]byte, error) { // DecryptAES decrypts AES-GCM encrypted data with a provided key. // The nonce is expected to be prepended to the cipherText. func DecryptAES(cipherText, key []byte) ([]byte, error) { - if len(cipherText) < nonceLength { + if len(cipherText) < AESNonceSize { // TODO return typed error return nil, fmt.Errorf("cipherText too short") } @@ -52,8 +52,8 @@ func DecryptAES(cipherText, key []byte) ([]byte, error) { return nil, err } - nonce := cipherText[:nonceLength] - cipherText = cipherText[nonceLength:] + nonce := cipherText[:AESNonceSize] + cipherText = cipherText[AESNonceSize:] aesGCM, err := cipher.NewGCM(block) if err != nil { diff --git a/crypto/ecdh.go b/crypto/ecdh.go deleted file mode 100644 index 27a2072ef9..0000000000 --- a/crypto/ecdh.go +++ /dev/null @@ -1,68 +0,0 @@ -package crypto - -import ( - "crypto/ecdh" - "crypto/rand" - "crypto/sha256" - "fmt" -) - -func GenerateX25519() (*ecdh.PrivateKey, error) { - return ecdh.X25519().GenerateKey(rand.Reader) -} - -const X25519PublicKeySize = 32 - -func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { - return ecdh.X25519().NewPublicKey(publicKeyBytes) -} - -func EncryptECDH(plaintext []byte, publicKey *ecdh.PublicKey) ([]byte, error) { - ephemeralPrivate, err := GenerateX25519() - if err != nil { - return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) - } - ephemeralPublic := ephemeralPrivate.PublicKey() - - sharedSecret, err := ephemeralPrivate.ECDH(publicKey) - if err != nil { - return nil, fmt.Errorf("ECDH failed: %w", err) - } - - key := sha256.Sum256(sharedSecret) - - cipherText, err := EncryptAES(plaintext, key[:]) - if err != nil { - return nil, fmt.Errorf("failed to encrypt: %w", err) - } - - return append(ephemeralPublic.Bytes(), cipherText...), nil -} - -func DecryptECDH(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error) { - if len(cipherText) < X25519PublicKeySize+nonceLength { - return nil, fmt.Errorf("cipherText too short") - } - - ephemeralPublicBytes := cipherText[:X25519PublicKeySize] - ephemeralPublic, err := ecdh.X25519().NewPublicKey(ephemeralPublicBytes) - if err != nil { - return nil, fmt.Errorf("failed to parse ephemeral public key: %w", err) - } - - sharedSecret, err := privateKey.ECDH(ephemeralPublic) - if err != nil { - return nil, fmt.Errorf("ECDH failed: %w", err) - } - - key := sha256.Sum256(sharedSecret) - - cipherText = cipherText[X25519PublicKeySize:] - plainText, err := DecryptAES(cipherText, key[:]) - - if err != nil { - return nil, fmt.Errorf("failed to decrypt: %w", err) - } - - return plainText, nil -} diff --git a/crypto/ecies.go b/crypto/ecies.go new file mode 100644 index 0000000000..1bc5fe4a1a --- /dev/null +++ b/crypto/ecies.go @@ -0,0 +1,120 @@ +// Current Implementation Analysis: +// 1. Key Generation: Correctly uses X25519 for key generation. +// 2. ECDH: Properly performs the ECDH operation. +// 3. Key Derivation: Uses SHA-256 on the shared secret, which is simplistic. +// 4. Encryption: Uses AES (implementation not shown). +// 5. MAC: Not implemented. + +// Improvements Needed: +// 1. Use a proper Key Derivation Function (KDF) +// 2. Implement HMAC for message authentication +// 3. Use authenticated encryption (e.g., AES-GCM) instead of AES +// 4. Standardize the output format + +// Here's an improved version of the EncryptECDH and DecryptECDH functions: + +package crypto + +import ( + "crypto/ecdh" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "fmt" + + "golang.org/x/crypto/hkdf" +) + +const X25519PublicKeySize = 32 +const HMACSize = 32 +const AESKeySize = 32 + +func GenerateX25519() (*ecdh.PrivateKey, error) { + return ecdh.X25519().GenerateKey(rand.Reader) +} + +func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { + return ecdh.X25519().NewPublicKey(publicKeyBytes) +} + +func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { + ephemeralPrivate, err := GenerateX25519() + if err != nil { + return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) + } + ephemeralPublic := ephemeralPrivate.PublicKey() + + sharedSecret, err := ephemeralPrivate.ECDH(publicKey) + if err != nil { + return nil, fmt.Errorf("ECDH failed: %w", err) + } + + // Key Derivation + kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) + aesKey := make([]byte, AESKeySize) + hmacKey := make([]byte, HMACSize) + if _, err := kdf.Read(aesKey); err != nil { + return nil, fmt.Errorf("KDF failed for AES key: %w", err) + } + if _, err := kdf.Read(hmacKey); err != nil { + return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) + } + + cipherText, err := EncryptAES(plainText, aesKey) + if err != nil { + return nil, fmt.Errorf("failed to encrypt: %w", err) + } + + mac := hmac.New(sha256.New, hmacKey) + mac.Write(cipherText[AESNonceSize:]) + macSum := mac.Sum(nil) + + result := append(ephemeralPublic.Bytes(), cipherText...) + result = append(result, macSum...) + + return result, nil +} + +func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error) { + if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+16 { // public key + min nonce size + HMAC size + min cipherText size + return nil, fmt.Errorf("ciphertext too short") + } + + ephemeralPublicBytes := cipherText[:X25519PublicKeySize] + ephemeralPublic, err := ecdh.X25519().NewPublicKey(ephemeralPublicBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse ephemeral public key: %w", err) + } + + sharedSecret, err := privateKey.ECDH(ephemeralPublic) + if err != nil { + return nil, fmt.Errorf("ECDH failed: %w", err) + } + + kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) + aesKey := make([]byte, AESKeySize) + hmacKey := make([]byte, HMACSize) + if _, err := kdf.Read(aesKey); err != nil { + return nil, fmt.Errorf("KDF failed for AES key: %w", err) + } + if _, err := kdf.Read(hmacKey); err != nil { + return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) + } + + macSum := cipherText[len(cipherText)-HMACSize:] + cipherText = cipherText[:len(cipherText)-HMACSize] + + mac := hmac.New(sha256.New, hmacKey) + mac.Write(cipherText[X25519PublicKeySize+AESNonceSize:]) + expectedMAC := mac.Sum(nil) + if !hmac.Equal(macSum, expectedMAC) { + return nil, fmt.Errorf("HMAC verification failed") + } + + plainText, err := DecryptAES(cipherText[X25519PublicKeySize:], aesKey) + if err != nil { + return nil, fmt.Errorf("failed to decrypt: %w", err) + } + + return plainText, nil +} diff --git a/crypto/nonce.go b/crypto/nonce.go index a14a3ef5cf..9c8f00b31f 100644 --- a/crypto/nonce.go +++ b/crypto/nonce.go @@ -18,12 +18,12 @@ import ( "strings" ) -const nonceLength = 12 +const AESNonceSize = 12 var generateNonceFunc = generateNonce func generateNonce() ([]byte, error) { - nonce := make([]byte, nonceLength) + nonce := make([]byte, AESNonceSize) if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return nil, err } @@ -35,11 +35,11 @@ func generateNonce() ([]byte, error) { func generateTestNonce() ([]byte, error) { nonce := []byte("deterministic nonce for testing") - if len(nonce) < nonceLength { + if len(nonce) < AESNonceSize { return nil, errors.New("nonce length is longer than available deterministic nonce") } - return nonce[:nonceLength], nil + return nonce[:AESNonceSize], nil } func init() { diff --git a/net/server.go b/net/server.go index 9f892b9d20..9a48f815a2 100644 --- a/net/server.go +++ b/net/server.go @@ -227,7 +227,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) } - encryptedKey, err := crypto.EncryptECDH(encKey, reqEphPubKey) + encryptedKey, err := crypto.EncryptECIES(encKey, reqEphPubKey) if err != nil { return nil, errors.Wrap("failed to encrypt key for requester", err) } @@ -542,7 +542,7 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - decryptedKey, err := crypto.DecryptECDH(keyResp.EncryptedKey, session.privateKey) + decryptedKey, err := crypto.DecryptECIES(keyResp.EncryptedKey, session.privateKey) if err != nil { log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) return diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go index fd9c1d17c0..a03e0e92e7 100644 --- a/tests/integration/encryption/utils.go +++ b/tests/integration/encryption/utils.go @@ -11,7 +11,7 @@ package encryption import ( - "github.com/sourcenetwork/defradb/internal/encryption" + "github.com/sourcenetwork/defradb/crypto" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -50,6 +50,6 @@ func updateUserCollectionSchema() testUtils.SchemaUpdate { func encrypt(plaintext []byte, docID, fieldName string) []byte { const keyLength = 32 const testEncKey = "examplekey1234567890examplekey12" - val, _ := encryption.EncryptAES(plaintext, []byte(fieldName + docID + testEncKey)[0:keyLength]) + val, _ := crypto.EncryptAES(plaintext, []byte(fieldName + docID + testEncKey)[0:keyLength]) return val } From d46b8810f05fe2f698fe9a3ca892ecfc2e9da1bd Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 3 Aug 2024 22:19:14 +0200 Subject: [PATCH 11/88] Adjustments --- go.mod | 2 +- go.sum | 2 ++ tests/integration/encryption/commit_test.go | 14 +++++++------- tests/integration/encryption/peer_test.go | 14 +++++++------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3447a74e72..ade3dcbbf7 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( cosmossdk.io/x/feegrant v0.1.0 // indirect cosmossdk.io/x/tx v0.13.2 // indirect cosmossdk.io/x/upgrade v0.1.1 // indirect - filippo.io/edwards25519 v1.0.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/DataDog/datadog-go v3.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index edaba460a8..bdf710542e 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index ffa5333748..f471403ff5 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -47,7 +47,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( `, Results: []map[string]any{ { - "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", + "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -57,7 +57,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", + "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -67,7 +67,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreidxdhzhwjrv5s4x6cho5drz6xq2tc7oymzupf4p4gfk6eelsnc7ke", + "cid": "bafyreia6fqubivj2ms4xwounxsyjr6coxeb2wkcmzaa7yzk3h5cwsj5g6a", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -76,12 +76,12 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", - "name": "age", + "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", + "name": "name", }, { - "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", - "name": "name", + "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", + "name": "age", }, }, }, diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index cc3ea7b1d0..dc642fc0dd 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -94,7 +94,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { `, Results: []map[string]any{ { - "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", + "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -104,7 +104,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", + "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -114,7 +114,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreidxdhzhwjrv5s4x6cho5drz6xq2tc7oymzupf4p4gfk6eelsnc7ke", + "cid": "bafyreia6fqubivj2ms4xwounxsyjr6coxeb2wkcmzaa7yzk3h5cwsj5g6a", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -123,12 +123,12 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", - "name": "age", + "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", + "name": "name", }, { - "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", - "name": "name", + "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", + "name": "age", }, }, }, From 4eafe57877e206d4fcf92bca9bdbfeb89f2bce9b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 4 Aug 2024 08:44:38 +0200 Subject: [PATCH 12/88] Polish --- crypto/ecies.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index 1bc5fe4a1a..dde50479ed 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -29,6 +29,8 @@ const X25519PublicKeySize = 32 const HMACSize = 32 const AESKeySize = 32 +const minCipherTextSize = 16 + func GenerateX25519() (*ecdh.PrivateKey, error) { return ecdh.X25519().GenerateKey(rand.Reader) } @@ -49,7 +51,6 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { return nil, fmt.Errorf("ECDH failed: %w", err) } - // Key Derivation kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) aesKey := make([]byte, AESKeySize) hmacKey := make([]byte, HMACSize) @@ -76,7 +77,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { } func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error) { - if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+16 { // public key + min nonce size + HMAC size + min cipherText size + if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { return nil, fmt.Errorf("ciphertext too short") } From abb3f99ae9f7302dc4fdc8444a6cd1a1fce82d1f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 4 Aug 2024 08:45:50 +0200 Subject: [PATCH 13/88] Remove unnecessary peerInfo transit --- net/pb/net.pb.go | 303 ++++++++++++++------------------------- net/pb/net.proto | 13 +- net/pb/net_vtproto.pb.go | 236 ------------------------------ net/server.go | 22 +-- 4 files changed, 113 insertions(+), 461 deletions(-) diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 9c0efae055..51b042c5fa 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -344,65 +344,6 @@ func (x *PushLogRequest) GetBody() *PushLogRequest_Body { return nil } -// PeerInfo contains information about a specific peer -type PeerInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // id is a libp2p peer identity. - Id []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // addresses is a list of peer's address in multiaddr format - // Learn more here https://github.com/multiformats/multiadd - Addresses []string `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses,omitempty"` -} - -func (x *PeerInfo) Reset() { - *x = PeerInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PeerInfo) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PeerInfo) ProtoMessage() {} - -func (x *PeerInfo) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PeerInfo.ProtoReflect.Descriptor instead. -func (*PeerInfo) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{8} -} - -func (x *PeerInfo) GetId() []byte { - if x != nil { - return x.Id - } - return nil -} - -func (x *PeerInfo) GetAddresses() []string { - if x != nil { - return x.Addresses - } - return nil -} - // FetchEncryptionKeyRequest is a request to receive a doc encryption key // from a peer that holds it. type FetchEncryptionKeyRequest struct { @@ -421,16 +362,14 @@ type FetchEncryptionKeyRequest struct { SchemaRoot []byte `protobuf:"bytes,4,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret EphemeralPublicKey []byte `protobuf:"bytes,5,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` - // peerInfo is peer information is the requesting peer - PeerInfo *PeerInfo `protobuf:"bytes,6,opt,name=peerInfo,proto3" json:"peerInfo,omitempty"` // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { *x = FetchEncryptionKeyRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -443,7 +382,7 @@ func (x *FetchEncryptionKeyRequest) String() string { func (*FetchEncryptionKeyRequest) ProtoMessage() {} func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -456,7 +395,7 @@ func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{9} + return file_net_proto_rawDescGZIP(), []int{8} } func (x *FetchEncryptionKeyRequest) GetDocID() []byte { @@ -494,13 +433,6 @@ func (x *FetchEncryptionKeyRequest) GetEphemeralPublicKey() []byte { return nil } -func (x *FetchEncryptionKeyRequest) GetPeerInfo() *PeerInfo { - if x != nil { - return x.PeerInfo - } - return nil -} - func (x *FetchEncryptionKeyRequest) GetSignature() []byte { if x != nil { return x.Signature @@ -517,7 +449,7 @@ type GetHeadLogRequest struct { func (x *GetHeadLogRequest) Reset() { *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -530,7 +462,7 @@ func (x *GetHeadLogRequest) String() string { func (*GetHeadLogRequest) ProtoMessage() {} func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -543,7 +475,7 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{10} + return file_net_proto_rawDescGZIP(), []int{9} } type PushLogReply struct { @@ -555,7 +487,7 @@ type PushLogReply struct { func (x *PushLogReply) Reset() { *x = PushLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -568,7 +500,7 @@ func (x *PushLogReply) String() string { func (*PushLogReply) ProtoMessage() {} func (x *PushLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -581,7 +513,7 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. func (*PushLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{11} + return file_net_proto_rawDescGZIP(), []int{10} } // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request @@ -607,7 +539,7 @@ type FetchEncryptionKeyReply struct { func (x *FetchEncryptionKeyReply) Reset() { *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -620,7 +552,7 @@ func (x *FetchEncryptionKeyReply) String() string { func (*FetchEncryptionKeyReply) ProtoMessage() {} func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -633,7 +565,7 @@ func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{12} + return file_net_proto_rawDescGZIP(), []int{11} } func (x *FetchEncryptionKeyReply) GetEncryptedKey() []byte { @@ -680,7 +612,7 @@ type GetHeadLogReply struct { func (x *GetHeadLogReply) Reset() { *x = GetHeadLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -693,7 +625,7 @@ func (x *GetHeadLogReply) String() string { func (*GetHeadLogReply) ProtoMessage() {} func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -706,7 +638,7 @@ func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. func (*GetHeadLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{13} + return file_net_proto_rawDescGZIP(), []int{12} } type PushLogRequest_Body struct { @@ -729,7 +661,7 @@ type PushLogRequest_Body struct { func (x *PushLogRequest_Body) Reset() { *x = PushLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[14] + mi := &file_net_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -742,7 +674,7 @@ func (x *PushLogRequest_Body) String() string { func (*PushLogRequest_Body) ProtoMessage() {} func (x *PushLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[14] + mi := &file_net_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -819,71 +751,64 @@ var file_net_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x38, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0xfd, - 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, - 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, - 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, - 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, - 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, - 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x65, - 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x70, 0x65, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, - 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, - 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, - 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, - 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, - 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, - 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, - 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, - 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0xcf, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, + 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, + 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, - 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, - 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, - 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, - 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, + 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, + 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, + 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, + 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, + 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, + 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, + 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, + 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, + 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, + 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, + 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, + 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, + 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -898,7 +823,7 @@ func file_net_proto_rawDescGZIP() []byte { return file_net_proto_rawDescData } -var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_net_proto_goTypes = []any{ (*Log)(nil), // 0: net.pb.Log (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest @@ -908,35 +833,33 @@ var file_net_proto_goTypes = []any{ (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest (*GetLogReply)(nil), // 6: net.pb.GetLogReply (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest - (*PeerInfo)(nil), // 8: net.pb.PeerInfo - (*FetchEncryptionKeyRequest)(nil), // 9: net.pb.FetchEncryptionKeyRequest - (*GetHeadLogRequest)(nil), // 10: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 11: net.pb.PushLogReply - (*FetchEncryptionKeyReply)(nil), // 12: net.pb.FetchEncryptionKeyReply - (*GetHeadLogReply)(nil), // 13: net.pb.GetHeadLogReply - (*PushLogRequest_Body)(nil), // 14: net.pb.PushLogRequest.Body + (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest + (*GetHeadLogRequest)(nil), // 9: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 10: net.pb.PushLogReply + (*FetchEncryptionKeyReply)(nil), // 11: net.pb.FetchEncryptionKeyReply + (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply + (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body } var file_net_proto_depIdxs = []int32{ - 14, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 8, // 1: net.pb.FetchEncryptionKeyRequest.peerInfo:type_name -> net.pb.PeerInfo - 0, // 2: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log - 1, // 3: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest - 3, // 4: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest - 5, // 5: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest - 7, // 6: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 9, // 7: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 10, // 8: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 9: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 10: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 11: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 11, // 12: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 12, // 13: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply - 13, // 14: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 9, // [9:15] is the sub-list for method output_type - 3, // [3:9] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 13, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body + 0, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log + 1, // 2: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest + 3, // 3: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest + 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest + 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest + 8, // 6: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest + 9, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 8: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 9: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 10: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 10, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 11, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 12, // 13: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 8, // [8:14] is the sub-list for method output_type + 2, // [2:8] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_net_proto_init() } @@ -1042,18 +965,6 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*PeerInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*FetchEncryptionKeyRequest); i { case 0: return &v.state @@ -1065,7 +976,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state @@ -1077,7 +988,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*PushLogReply); i { case 0: return &v.state @@ -1089,7 +1000,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state @@ -1101,7 +1012,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogReply); i { case 0: return &v.state @@ -1113,7 +1024,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[14].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state @@ -1132,7 +1043,7 @@ func file_net_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_net_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/net/pb/net.proto b/net/pb/net.proto index d3574b748d..4390cff258 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -38,15 +38,6 @@ message PushLogRequest { } } -// PeerInfo contains information about a specific peer -message PeerInfo { - // id is a libp2p peer identity. - bytes id = 1; - // addresses is a list of peer's address in multiaddr format - // Learn more here https://github.com/multiformats/multiadd - repeated string addresses = 2; -} - // FetchEncryptionKeyRequest is a request to receive a doc encryption key // from a peer that holds it. message FetchEncryptionKeyRequest { @@ -61,10 +52,8 @@ message FetchEncryptionKeyRequest { bytes schemaRoot = 4; // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret bytes ephemeralPublicKey = 5; - // peerInfo is peer information is the requesting peer - PeerInfo peerInfo = 6; // signature is the requesting peer's signature of the request - bytes signature = 7; + bytes signature = 6; } message GetHeadLogRequest {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index a80929c3c5..0a7daf30ba 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -370,55 +370,6 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *PeerInfo) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil - } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *PeerInfo) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *PeerInfo) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) - } - if len(m.Addresses) > 0 { - for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Addresses[iNdEx]) - copy(dAtA[i:], m.Addresses[iNdEx]) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Addresses[iNdEx]))) - i-- - dAtA[i] = 0x12 - } - } - if len(m.Id) > 0 { - i -= len(m.Id) - copy(dAtA[i:], m.Id) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Id))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *FetchEncryptionKeyRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -454,16 +405,6 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x3a - } - if m.PeerInfo != nil { - size, err := m.PeerInfo.MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - i-- dAtA[i] = 0x32 } if len(m.EphemeralPublicKey) > 0 { @@ -789,26 +730,6 @@ func (m *PushLogRequest) SizeVT() (n int) { return n } -func (m *PeerInfo) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Id) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - if len(m.Addresses) > 0 { - for _, s := range m.Addresses { - l = len(s) - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - } - n += len(m.unknownFields) - return n -} - func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if m == nil { return 0 @@ -835,10 +756,6 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.PeerInfo != nil { - l = m.PeerInfo.SizeVT() - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } l = len(m.Signature) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1606,123 +1523,6 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *PeerInfo) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: PeerInfo: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: PeerInfo: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Id = append(m.Id[:0], dAtA[iNdEx:postIndex]...) - if m.Id == nil { - m.Id = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Addresses = append(m.Addresses, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1921,42 +1721,6 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 6: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerInfo", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PeerInfo == nil { - m.PeerInfo = &PeerInfo{} - } - if err := m.PeerInfo.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } diff --git a/net/server.go b/net/server.go index 9a48f815a2..5a8f8b477d 100644 --- a/net/server.go +++ b/net/server.go @@ -202,7 +202,11 @@ func (s *server) getEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKe } func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { - reqPubKey := s.peer.host.Peerstore().PubKey(libpeer.ID(req.PeerInfo.Id)) + peerID, err := peerIDFromContext(ctx) + if err != nil { + return nil, err + } + reqPubKey := s.peer.host.Peerstore().PubKey(peerID) isValid, err := s.verifyRequestSignature(req, reqPubKey) if err != nil { @@ -213,10 +217,6 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.New("invalid signature") } - if err := s.verifyPeerInfo(libpeer.ID(req.PeerInfo.Id), reqPubKey); err != nil { - return nil, errors.Wrap("invalid peer info", err) - } - encKey, err := s.getEncryptionKey(ctx, req) if err != nil || len(encKey) == 0 { return nil, err @@ -426,16 +426,6 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe return nil } -func toProtoPeerInfo(peerInfo libpeer.AddrInfo) *pb.PeerInfo { - protoPeerInfo := new(pb.PeerInfo) - protoPeerInfo.Id = []byte(peerInfo.ID) - protoPeerInfo.Addresses = make([]string, len(peerInfo.Addrs)) - for i, addr := range peerInfo.Addrs { - protoPeerInfo.Addresses[i] = addr.String() - } - return protoPeerInfo -} - func (s *server) prepareFetchEncryptionKeyRequest( evt encryption.RequestKeyEvent, ephemeralPublicKey []byte, @@ -445,7 +435,6 @@ func (s *server) prepareFetchEncryptionKeyRequest( Cid: evt.Cid.Bytes(), SchemaRoot: []byte(evt.SchemaRoot), EphemeralPublicKey: ephemeralPublicKey, - PeerInfo: toProtoPeerInfo(s.peer.PeerInfo()), } if evt.FieldName.HasValue() { @@ -507,7 +496,6 @@ func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { hash.Write(req.Cid) hash.Write(req.SchemaRoot) hash.Write(req.EphemeralPublicKey) - hash.Write([]byte(req.PeerInfo.String())) return hash.Sum(nil) } From 58bf7cecf0fe75e8b480e06edc4343e3271d3d7d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 4 Aug 2024 09:45:10 +0200 Subject: [PATCH 14/88] Polish --- net/pb/net.pb.go | 196 +++++++++++------------ net/pb/net.proto | 8 +- net/pb/net_vtproto.pb.go | 332 +++++++++++++++++++-------------------- net/server.go | 13 -- 4 files changed, 268 insertions(+), 281 deletions(-) diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 51b042c5fa..15cc032271 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -440,14 +440,28 @@ func (x *FetchEncryptionKeyRequest) GetSignature() []byte { return nil } -type GetHeadLogRequest struct { +// FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request +// by a peer that holds the requested doc encryption key. +type FetchEncryptionKeyReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // encryptedKey is array or bytes representing encrypted doc encryption key + // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM + EncryptedKey []byte `protobuf:"bytes,1,opt,name=encryptedKey,proto3" json:"encryptedKey,omitempty"` + // cid is the CID of the composite of the document. + Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` + // schemaRoot is the SchemaRoot of the collection that the document resides in. + SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id + ReqEphemeralPublicKey []byte `protobuf:"bytes,4,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` + // signature is the responding peer's signature of the reply + Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` } -func (x *GetHeadLogRequest) Reset() { - *x = GetHeadLogRequest{} +func (x *FetchEncryptionKeyReply) Reset() { + *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -455,13 +469,13 @@ func (x *GetHeadLogRequest) Reset() { } } -func (x *GetHeadLogRequest) String() string { +func (x *FetchEncryptionKeyReply) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetHeadLogRequest) ProtoMessage() {} +func (*FetchEncryptionKeyReply) ProtoMessage() {} -func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { +func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -473,19 +487,54 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. -func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. +func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{9} } -type PushLogReply struct { +func (x *FetchEncryptionKeyReply) GetEncryptedKey() []byte { + if x != nil { + return x.EncryptedKey + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetCid() []byte { + if x != nil { + return x.Cid + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { + if x != nil { + return x.SchemaRoot + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetReqEphemeralPublicKey() []byte { + if x != nil { + return x.ReqEphemeralPublicKey + } + return nil +} + +func (x *FetchEncryptionKeyReply) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +type GetHeadLogRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields } -func (x *PushLogReply) Reset() { - *x = PushLogReply{} +func (x *GetHeadLogRequest) Reset() { + *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -493,13 +542,13 @@ func (x *PushLogReply) Reset() { } } -func (x *PushLogReply) String() string { +func (x *GetHeadLogRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PushLogReply) ProtoMessage() {} +func (*GetHeadLogRequest) ProtoMessage() {} -func (x *PushLogReply) ProtoReflect() protoreflect.Message { +func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -511,33 +560,19 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. -func (*PushLogReply) Descriptor() ([]byte, []int) { +// Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. +func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{10} } -// FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request -// by a peer that holds the requested doc encryption key. -type FetchEncryptionKeyReply struct { +type PushLogReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - // encryptedKey is array or bytes representing encrypted doc encryption key - // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM - EncryptedKey []byte `protobuf:"bytes,1,opt,name=encryptedKey,proto3" json:"encryptedKey,omitempty"` - // cid is the CID of the composite of the document. - Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` - // schemaRoot is the SchemaRoot of the collection that the document resides in. - SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` - // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id - ReqEphemeralPublicKey []byte `protobuf:"bytes,4,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` - // signature is the responding peer's signature of the reply - Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` } -func (x *FetchEncryptionKeyReply) Reset() { - *x = FetchEncryptionKeyReply{} +func (x *PushLogReply) Reset() { + *x = PushLogReply{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -545,13 +580,13 @@ func (x *FetchEncryptionKeyReply) Reset() { } } -func (x *FetchEncryptionKeyReply) String() string { +func (x *PushLogReply) String() string { return protoimpl.X.MessageStringOf(x) } -func (*FetchEncryptionKeyReply) ProtoMessage() {} +func (*PushLogReply) ProtoMessage() {} -func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { +func (x *PushLogReply) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -563,46 +598,11 @@ func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. -func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { +// Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. +func (*PushLogReply) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{11} } -func (x *FetchEncryptionKeyReply) GetEncryptedKey() []byte { - if x != nil { - return x.EncryptedKey - } - return nil -} - -func (x *FetchEncryptionKeyReply) GetCid() []byte { - if x != nil { - return x.Cid - } - return nil -} - -func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { - if x != nil { - return x.SchemaRoot - } - return nil -} - -func (x *FetchEncryptionKeyReply) GetReqEphemeralPublicKey() []byte { - if x != nil { - return x.ReqEphemeralPublicKey - } - return nil -} - -func (x *FetchEncryptionKeyReply) GetSignature() []byte { - if x != nil { - return x.Signature - } - return nil -} - type GetHeadLogReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -764,22 +764,22 @@ var file_net_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, - 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xc3, 0x01, 0x0a, 0x17, - 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, - 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, - 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, - 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, - 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, - 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, + 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, @@ -834,9 +834,9 @@ var file_net_proto_goTypes = []any{ (*GetLogReply)(nil), // 6: net.pb.GetLogReply (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest - (*GetHeadLogRequest)(nil), // 9: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 10: net.pb.PushLogReply - (*FetchEncryptionKeyReply)(nil), // 11: net.pb.FetchEncryptionKeyReply + (*FetchEncryptionKeyReply)(nil), // 9: net.pb.FetchEncryptionKeyReply + (*GetHeadLogRequest)(nil), // 10: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 11: net.pb.PushLogReply (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body } @@ -848,12 +848,12 @@ var file_net_proto_depIdxs = []int32{ 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest 8, // 6: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 9, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 10, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest 2, // 8: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply 4, // 9: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply 6, // 10: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 10, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 11, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 11, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 9, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply 12, // 13: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply 8, // [8:14] is the sub-list for method output_type 2, // [2:8] is the sub-list for method input_type @@ -977,7 +977,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogRequest); i { + switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state case 1: @@ -989,7 +989,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*PushLogReply); i { + switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state case 1: @@ -1001,7 +1001,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*FetchEncryptionKeyReply); i { + switch v := v.(*PushLogReply); i { case 0: return &v.state case 1: diff --git a/net/pb/net.proto b/net/pb/net.proto index 4390cff258..7c44efe735 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -56,10 +56,6 @@ message FetchEncryptionKeyRequest { bytes signature = 6; } -message GetHeadLogRequest {} - -message PushLogReply {} - // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request // by a peer that holds the requested doc encryption key. message FetchEncryptionKeyReply { @@ -76,6 +72,10 @@ message FetchEncryptionKeyReply { bytes signature = 5; } +message GetHeadLogRequest {} + +message PushLogReply {} + message GetHeadLogReply {} // Service is the peer-to-peer network API for document sync diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 0a7daf30ba..6bc4141229 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -445,7 +445,7 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er return len(dAtA) - i, nil } -func (m *GetHeadLogRequest) MarshalVT() (dAtA []byte, err error) { +func (m *FetchEncryptionKeyReply) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -458,12 +458,12 @@ func (m *GetHeadLogRequest) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *GetHeadLogRequest) MarshalToVT(dAtA []byte) (int, error) { +func (m *FetchEncryptionKeyReply) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *GetHeadLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -475,10 +475,45 @@ func (m *GetHeadLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x2a + } + if len(m.ReqEphemeralPublicKey) > 0 { + i -= len(m.ReqEphemeralPublicKey) + copy(dAtA[i:], m.ReqEphemeralPublicKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReqEphemeralPublicKey))) + i-- + dAtA[i] = 0x22 + } + if len(m.SchemaRoot) > 0 { + i -= len(m.SchemaRoot) + copy(dAtA[i:], m.SchemaRoot) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) + i-- + dAtA[i] = 0x1a + } + if len(m.Cid) > 0 { + i -= len(m.Cid) + copy(dAtA[i:], m.Cid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + i-- + dAtA[i] = 0x12 + } + if len(m.EncryptedKey) > 0 { + i -= len(m.EncryptedKey) + copy(dAtA[i:], m.EncryptedKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKey))) + i-- + dAtA[i] = 0xa + } return len(dAtA) - i, nil } -func (m *PushLogReply) MarshalVT() (dAtA []byte, err error) { +func (m *GetHeadLogRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -491,12 +526,12 @@ func (m *PushLogReply) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *PushLogReply) MarshalToVT(dAtA []byte) (int, error) { +func (m *GetHeadLogRequest) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *PushLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *GetHeadLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -511,7 +546,7 @@ func (m *PushLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *FetchEncryptionKeyReply) MarshalVT() (dAtA []byte, err error) { +func (m *PushLogReply) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -524,12 +559,12 @@ func (m *FetchEncryptionKeyReply) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *FetchEncryptionKeyReply) MarshalToVT(dAtA []byte) (int, error) { +func (m *PushLogReply) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *PushLogReply) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -541,41 +576,6 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if len(m.Signature) > 0 { - i -= len(m.Signature) - copy(dAtA[i:], m.Signature) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) - i-- - dAtA[i] = 0x2a - } - if len(m.ReqEphemeralPublicKey) > 0 { - i -= len(m.ReqEphemeralPublicKey) - copy(dAtA[i:], m.ReqEphemeralPublicKey) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReqEphemeralPublicKey))) - i-- - dAtA[i] = 0x22 - } - if len(m.SchemaRoot) > 0 { - i -= len(m.SchemaRoot) - copy(dAtA[i:], m.SchemaRoot) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) - i-- - dAtA[i] = 0x1a - } - if len(m.Cid) > 0 { - i -= len(m.Cid) - copy(dAtA[i:], m.Cid) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) - i-- - dAtA[i] = 0x12 - } - if len(m.EncryptedKey) > 0 { - i -= len(m.EncryptedKey) - copy(dAtA[i:], m.EncryptedKey) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKey))) - i-- - dAtA[i] = 0xa - } return len(dAtA) - i, nil } @@ -764,26 +764,6 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { return n } -func (m *GetHeadLogRequest) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - n += len(m.unknownFields) - return n -} - -func (m *PushLogReply) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - n += len(m.unknownFields) - return n -} - func (m *FetchEncryptionKeyReply) SizeVT() (n int) { if m == nil { return 0 @@ -814,6 +794,26 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { return n } +func (m *GetHeadLogRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *PushLogReply) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + func (m *GetHeadLogReply) SizeVT() (n int) { if m == nil { return 0 @@ -1776,108 +1776,6 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *GetHeadLogRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: GetHeadLogRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: GetHeadLogRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *PushLogReply) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: PushLogReply: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: PushLogReply: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2099,6 +1997,108 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *GetHeadLogRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetHeadLogRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetHeadLogRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PushLogReply) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PushLogReply: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PushLogReply: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GetHeadLogReply) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/net/server.go b/net/server.go index 5a8f8b477d..d2bf37c381 100644 --- a/net/server.go +++ b/net/server.go @@ -251,19 +251,6 @@ func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKe return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) } -func (s *server) verifyPeerInfo(peerID libpeer.ID, pubKey libp2pCrypto.PubKey) error { - derivedID, err := peer.IDFromPublicKey(pubKey) - if err != nil { - return err - } - - if peerID != derivedID { - return errors.New("peer ID does not match public key") - } - - return nil -} - func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { hash := sha256.New() hash.Write(res.EncryptedKey) From a24c9b0d962f0ca2cf577a5498f31aa287f7285e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Aug 2024 12:45:53 +0200 Subject: [PATCH 15/88] Make all mission enc keys be batched --- internal/db/db.go | 2 +- internal/db/merge.go | 124 ++++++++----- internal/db/messages.go | 23 ++- internal/encryption/event.go | 71 +++----- net/errors.go | 8 +- net/pb/net.pb.go | 334 +++++++++++++++++++++-------------- net/pb/net.proto | 27 +-- net/pb/net_vtproto.pb.go | 263 +++++++++++++++++++++------ net/peer.go | 13 +- net/server.go | 109 ++++++++---- tests/integration/events.go | 2 +- tests/integration/state.go | 16 +- 12 files changed, 642 insertions(+), 350 deletions(-) diff --git a/internal/db/db.go b/internal/db/db.go index 07bc118750..50c1ecc38c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -131,7 +131,7 @@ func newDB( return nil, err } - sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName, encryption.KeyRetrievedEventName) + sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName, encryption.KeysRetrievedEventName) if err != nil { return nil, err } diff --git a/internal/db/merge.go b/internal/db/merge.go index 9e50152184..f0b1830650 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -64,7 +64,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.loadComposites(ctx, dagMerge.Cid, mt, false) + err = mp.loadBlocks(ctx, dagMerge.Cid, mt, false) if err != nil { return err } @@ -74,6 +74,8 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } + mp.sendPendingEncryptionRequest() + err = syncIndexedDoc(ctx, docID, col) if err != nil { return err @@ -89,7 +91,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return nil } -func (db *db) mergeEncryptedBlock(ctx context.Context, keyEvent encryption.KeyRetrievedEvent) error { +func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyRetrievedEvent) error { ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { return err @@ -104,35 +106,53 @@ func (db *db) mergeEncryptedBlock(ctx context.Context, keyEvent encryption.KeyRe ls := cidlink.DefaultLinkSystem() ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) - docID, err := client.NewDocIDFromString(keyEvent.DocID) - if err != nil { - return err - } - dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(col.Description(), docID.String()) + mergedDocs := make(map[string]struct{}) - mp, err := db.newMergeProcessor(txn, ls, col, dsKey) - if err != nil { - return err - } + for encStoreKey := range keyEvent.Data { + if _, ok := mergedDocs[encStoreKey.DocID]; ok { + continue + } - mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey) - if err != nil { - return err - } + mergedDocs[encStoreKey.DocID] = struct{}{} - err = mp.loadComposites(ctx, keyEvent.Cid, mt, true) - if err != nil { - return err - } + docID, err := client.NewDocIDFromString(encStoreKey.DocID) + if err != nil { + return err + } + dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(col.Description(), docID.String()) - err = mp.mergeComposites(ctx, true) - if err != nil { - return err - } + mp, err := db.newMergeProcessor(txn, ls, col, dsKey) + if err != nil { + return err + } - err = syncIndexedDoc(ctx, docID, col) - if err != nil { - return err + mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey) + if err != nil { + return err + } + + for encStoreKey2, data := range keyEvent.Data { + // load all blocks related to the document + if encStoreKey2.DocID == encStoreKey.DocID { + // TODO: check if it's really necessary to pass cid here and over the wire + // Looks like once a key is retrieved, we should traverse the DAG from the head + // and decrypt all the blocks that are encrypted with this key. CID should not matter. + err = mp.loadBlocks(ctx, data.Cid, mt, true) + if err != nil { + return err + } + } + } + + err = mp.mergeComposites(ctx, true) + if err != nil { + return err + } + + err = syncIndexedDoc(ctx, docID, col) + if err != nil { + return err + } } err = txn.Commit(ctx) @@ -184,12 +204,13 @@ func (m *mergeQueue) done(docID string) { } type mergeProcessor struct { - txn datastore.Txn - lsys linking.LinkSystem - mCRDTs map[string]merklecrdt.MerkleCRDT - col *collection - dsKey core.DataStoreKey - composites *list.List + txn datastore.Txn + lsys linking.LinkSystem + mCRDTs map[string]merklecrdt.MerkleCRDT + col *collection + dsKey core.DataStoreKey + blocks *list.List + pendingEncryptionKeyRequests map[core.EncStoreDocKey]cid.Cid } func (db *db) newMergeProcessor( @@ -199,12 +220,13 @@ func (db *db) newMergeProcessor( dsKey core.DataStoreKey, ) (*mergeProcessor, error) { return &mergeProcessor{ - txn: txn, - lsys: lsys, - mCRDTs: make(map[string]merklecrdt.MerkleCRDT), - col: col, - dsKey: dsKey, - composites: list.New(), + txn: txn, + lsys: lsys, + mCRDTs: make(map[string]merklecrdt.MerkleCRDT), + col: col, + dsKey: dsKey, + blocks: list.New(), + pendingEncryptionKeyRequests: make(map[core.EncStoreDocKey]cid.Cid), }, nil } @@ -219,9 +241,9 @@ func newMergeTarget() mergeTarget { } } -// loadComposites retrieves and stores into the merge processor the composite blocks for the given -// document until it reaches a block that has already been merged or until we reach the genesis block. -func (mp *mergeProcessor) loadComposites( +// loadBlocks retrieves and stores into the merge processor the blocks for the given +// CID until it reaches a block that has already been merged or until we reach the genesis block. +func (mp *mergeProcessor) loadBlocks( ctx context.Context, blockCid cid.Cid, mt mergeTarget, @@ -250,10 +272,10 @@ func (mp *mergeProcessor) loadComposites( // of the composite DAG. However, the new block and its children might have branched off from an older block. // In this case, we also need to walk back the merge target's DAG until we reach a common block. if block.Delta.GetPriority() >= mt.headHeight { - mp.composites.PushFront(block) + mp.blocks.PushFront(block) for _, link := range block.Links { if link.Name == core.HEAD { - err := mp.loadComposites(ctx, link.Cid, mt, willDecrypt) + err := mp.loadBlocks(ctx, link.Cid, mt, willDecrypt) if err != nil { return err } @@ -279,13 +301,13 @@ func (mp *mergeProcessor) loadComposites( } } } - return mp.loadComposites(ctx, blockCid, newMT, willDecrypt) + return mp.loadBlocks(ctx, blockCid, newMT, willDecrypt) } return nil } func (mp *mergeProcessor) mergeComposites(ctx context.Context, withDecryption bool) error { - for e := mp.composites.Front(); e != nil; e = e.Next() { + for e := mp.blocks.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() if err != nil { @@ -321,12 +343,11 @@ func (mp *mergeProcessor) processEncryptedBlock( if !withDecryption && (dagBlock.Delta.IsComposite() && *dagBlock.EncryptionType == coreblock.DocumentEncrypted) || *dagBlock.EncryptionType == coreblock.FieldEncrypted { docID := string(dagBlock.Delta.GetDocID()) - schemaRoot := mp.col.SchemaRoot() fieldName := immutable.None[string]() if *dagBlock.EncryptionType == coreblock.FieldEncrypted { fieldName = immutable.Some(dagBlock.Delta.GetFieldName()) } - mp.col.db.events.Publish(encryption.NewRequestKeyMessage(docID, blockLink.Cid, fieldName, schemaRoot)) + mp.addPendingEncryptionRequest(docID, blockLink.Cid, fieldName) } return dagBlock, true, nil } @@ -334,6 +355,15 @@ func (mp *mergeProcessor) processEncryptedBlock( return dagBlock, false, nil } +func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, blockCid cid.Cid, fieldName immutable.Option[string]) { + mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName)] = blockCid +} + +func (mp *mergeProcessor) sendPendingEncryptionRequest() { + schemaRoot := mp.col.SchemaRoot() + mp.col.db.events.Publish(encryption.NewRequestKeysMessage(schemaRoot, mp.pendingEncryptionKeyRequests)) +} + // processBlock merges the block and its children to the datastore and sets the head accordingly. // If onlyHeads is true, it will skip merging and update only the heads. func (mp *mergeProcessor) processBlock( diff --git a/internal/db/messages.go b/internal/db/messages.go index df832fcfc9..b66939a302 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -15,6 +15,7 @@ import ( "sync" "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" @@ -83,17 +84,23 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { case encryption.KeyRetrievedEvent: go func() { ctx = encryption.ContextWithStore(ctx, db.Encstore()) - err := encryption.SaveKey(ctx, evt.DocID, evt.FieldName, evt.Key) + for encStoreKey, data := range evt.Data { + optFieldName := immutable.None[string]() + if encStoreKey.FieldName != "" { + optFieldName = immutable.Some(encStoreKey.FieldName) + } + err := encryption.SaveKey(ctx, encStoreKey.DocID, optFieldName, data.Key) - if err != nil { - log.ErrorContextE( - ctx, - "Failed to save doc encryption key", - err, - corelog.Any("Event", evt)) + if err != nil { + log.ErrorContextE( + ctx, + "Failed to save doc encryption key", + err, + corelog.Any("Event", evt)) + } } - err = db.mergeEncryptedBlock(ctx, evt) + err := db.mergeEncryptedBlocks(ctx, evt) if err != nil { log.ErrorContextE( diff --git a/internal/encryption/event.go b/internal/encryption/event.go index a806fef9cc..8efbdef2f4 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -13,80 +13,61 @@ package encryption import ( "github.com/ipfs/go-cid" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/internal/core" ) -const RequestKeyEventName = event.Name("enc-key-request") -const KeyRetrievedEventName = event.Name("enc-key-retrieved") +const RequestKeysEventName = event.Name("enc-keys-request") +const KeysRetrievedEventName = event.Name("enc-keys-retrieved") -// RequestKeyEvent represents a request of a node to fetch an encryption key for a specific +// RequestKeysEvent represents a request of a node to fetch an encryption key for a specific // docID/field // // It must only contain public elements not protected by ACP. -type RequestKeyEvent struct { - // DocID is the unique immutable identifier of the document that the key is being requested for. - DocID string +type RequestKeysEvent struct { + // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. + SchemaRoot string - // FieldName is the name of the field for which the key is being requested. - // If not given, the key is for the whole document. - FieldName immutable.Option[string] + // Keys is a map of the keys that are being requested. + Keys map[core.EncStoreDocKey]cid.Cid +} +// RequestedKeyEventData represents the data that was retrieved for a specific key. +type RequestedKeyEventData struct { // Cid is the id of the composite commit that formed this update in the DAG. Cid cid.Cid - // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. - SchemaRoot string + // Key is the encryption key that was retrieved. + Key []byte } // KeyRetrievedEvent represents a key that was retrieved. type KeyRetrievedEvent struct { - // DocID is the unique immutable identifier of the document that was updated. - DocID string - - // FieldName is the name of the field for which the key was retrieved. - // If not given, the key is for the whole document. - FieldName immutable.Option[string] - - // Cid is the id of the composite commit that formed this update in the DAG. - Cid cid.Cid - // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. SchemaRoot string - // TODO: should be encrypted - // Key is the encryption key that was retrieved. - Key []byte + // Data is a map of the requested keys to the data that was retrieved. + Data map[core.EncStoreDocKey]RequestedKeyEventData } -// NewRequestKeyMessage creates a new event message for a request of a node to fetch an encryption key +// NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field -func NewRequestKeyMessage( - docID string, - cid cid.Cid, - fieldName immutable.Option[string], +func NewRequestKeysMessage( schemaRoot string, + keys map[core.EncStoreDocKey]cid.Cid, ) event.Message { - return event.NewMessage(RequestKeyEventName, RequestKeyEvent{ - DocID: docID, - FieldName: fieldName, - Cid: cid, + return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ SchemaRoot: schemaRoot, + Keys: keys, }) } -// NewKeyRetrievedMessage creates a new event message for a key that was retrieved -func NewKeyRetrievedMessage( - docID string, - fieldName immutable.Option[string], - cid cid.Cid, +// NewKeysRetrievedMessage creates a new event message for a key that was retrieved +func NewKeysRetrievedMessage( schemaRoot string, - key []byte, + data map[core.EncStoreDocKey]RequestedKeyEventData, ) event.Message { - return event.NewMessage(KeyRetrievedEventName, KeyRetrievedEvent{ - DocID: docID, - FieldName: fieldName, - Cid: cid, + return event.NewMessage(KeysRetrievedEventName, KeyRetrievedEvent{ SchemaRoot: schemaRoot, - Key: key, + Data: data, }) } diff --git a/net/errors.go b/net/errors.go index c957baceb8..6d2fe36b15 100644 --- a/net/errors.go +++ b/net/errors.go @@ -13,7 +13,9 @@ package net import ( "fmt" + "github.com/ipfs/go-cid" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/internal/core" ) const ( @@ -22,7 +24,7 @@ const ( errPublishingToDocIDTopic = "can't publish log %s for docID %s" errPublishingToSchemaTopic = "can't publish log %s for schema %s" errCheckingForExistingBlock = "failed to check for existing block" - errRequestingEncryptionKey = "failed to request encryption key with Cid %s for docID %s" + errRequestingEncryptionKeys = "failed to request encryption keys with %v" ) var ( @@ -54,6 +56,6 @@ func NewErrCheckingForExistingBlock(inner error, cid string) error { return errors.Wrap(errCheckingForExistingBlock, inner, errors.NewKV("cid", cid)) } -func NewErrRequestingEncryptionKey(inner error, cid, docID string, kv ...errors.KV) error { - return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKey, cid, docID), inner, kv...) +func NewErrRequestingEncryptionKeys(inner error, keys map[core.EncStoreDocKey]cid.Cid) error { + return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKeys, keys), inner) } diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 15cc032271..51efe7bfc3 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -344,9 +344,8 @@ func (x *PushLogRequest) GetBody() *PushLogRequest_Body { return nil } -// FetchEncryptionKeyRequest is a request to receive a doc encryption key -// from a peer that holds it. -type FetchEncryptionKeyRequest struct { +// EncryptionKeyTarget is a struct containing information about required doc encryption key. +type EncryptionKeyTarget struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -358,16 +357,10 @@ type FetchEncryptionKeyRequest struct { FieldName string `protobuf:"bytes,2,opt,name=fieldName,proto3" json:"fieldName,omitempty"` // cid is the CID of the composite of the document. Cid []byte `protobuf:"bytes,3,opt,name=cid,proto3" json:"cid,omitempty"` - // schemaRoot is the SchemaRoot of the collection that the document resides in. - SchemaRoot []byte `protobuf:"bytes,4,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` - // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - EphemeralPublicKey []byte `protobuf:"bytes,5,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` - // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` } -func (x *FetchEncryptionKeyRequest) Reset() { - *x = FetchEncryptionKeyRequest{} +func (x *EncryptionKeyTarget) Reset() { + *x = EncryptionKeyTarget{} if protoimpl.UnsafeEnabled { mi := &file_net_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -375,13 +368,13 @@ func (x *FetchEncryptionKeyRequest) Reset() { } } -func (x *FetchEncryptionKeyRequest) String() string { +func (x *EncryptionKeyTarget) String() string { return protoimpl.X.MessageStringOf(x) } -func (*FetchEncryptionKeyRequest) ProtoMessage() {} +func (*EncryptionKeyTarget) ProtoMessage() {} -func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { +func (x *EncryptionKeyTarget) ProtoReflect() protoreflect.Message { mi := &file_net_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -393,32 +386,88 @@ func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. -func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use EncryptionKeyTarget.ProtoReflect.Descriptor instead. +func (*EncryptionKeyTarget) Descriptor() ([]byte, []int) { return file_net_proto_rawDescGZIP(), []int{8} } -func (x *FetchEncryptionKeyRequest) GetDocID() []byte { +func (x *EncryptionKeyTarget) GetDocID() []byte { if x != nil { return x.DocID } return nil } -func (x *FetchEncryptionKeyRequest) GetFieldName() string { +func (x *EncryptionKeyTarget) GetFieldName() string { if x != nil { return x.FieldName } return "" } -func (x *FetchEncryptionKeyRequest) GetCid() []byte { +func (x *EncryptionKeyTarget) GetCid() []byte { if x != nil { return x.Cid } return nil } +// FetchEncryptionKeyRequest is a request to receive a doc encryption key +// from a peer that holds it. +type FetchEncryptionKeyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // targets is the list of docs/fields for which encryption keys are being requested. + Targets []*EncryptionKeyTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` + // schemaRoot is the SchemaRoot of the collection that the document resides in. + SchemaRoot []byte `protobuf:"bytes,2,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` + // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret + EphemeralPublicKey []byte `protobuf:"bytes,3,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` + // signature is the requesting peer's signature of the request + Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *FetchEncryptionKeyRequest) Reset() { + *x = FetchEncryptionKeyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_net_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FetchEncryptionKeyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchEncryptionKeyRequest) ProtoMessage() {} + +func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { + mi := &file_net_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. +func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { + return file_net_proto_rawDescGZIP(), []int{9} +} + +func (x *FetchEncryptionKeyRequest) GetTargets() []*EncryptionKeyTarget { + if x != nil { + return x.Targets + } + return nil +} + func (x *FetchEncryptionKeyRequest) GetSchemaRoot() []byte { if x != nil { return x.SchemaRoot @@ -447,11 +496,12 @@ type FetchEncryptionKeyReply struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // encryptedKey is array or bytes representing encrypted doc encryption key + // targets is the list of docs/fields for which encryption keys are being requested. + Targets []*EncryptionKeyTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` + // encryptedKeys is an encrypted list of doc encryption keys. // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM - EncryptedKey []byte `protobuf:"bytes,1,opt,name=encryptedKey,proto3" json:"encryptedKey,omitempty"` - // cid is the CID of the composite of the document. - Cid []byte `protobuf:"bytes,2,opt,name=cid,proto3" json:"cid,omitempty"` + // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. + EncryptedKeys []byte `protobuf:"bytes,2,opt,name=encryptedKeys,proto3" json:"encryptedKeys,omitempty"` // schemaRoot is the SchemaRoot of the collection that the document resides in. SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id @@ -463,7 +513,7 @@ type FetchEncryptionKeyReply struct { func (x *FetchEncryptionKeyReply) Reset() { *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -476,7 +526,7 @@ func (x *FetchEncryptionKeyReply) String() string { func (*FetchEncryptionKeyReply) ProtoMessage() {} func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -489,19 +539,19 @@ func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{9} + return file_net_proto_rawDescGZIP(), []int{10} } -func (x *FetchEncryptionKeyReply) GetEncryptedKey() []byte { +func (x *FetchEncryptionKeyReply) GetTargets() []*EncryptionKeyTarget { if x != nil { - return x.EncryptedKey + return x.Targets } return nil } -func (x *FetchEncryptionKeyReply) GetCid() []byte { +func (x *FetchEncryptionKeyReply) GetEncryptedKeys() []byte { if x != nil { - return x.Cid + return x.EncryptedKeys } return nil } @@ -536,7 +586,7 @@ type GetHeadLogRequest struct { func (x *GetHeadLogRequest) Reset() { *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -549,7 +599,7 @@ func (x *GetHeadLogRequest) String() string { func (*GetHeadLogRequest) ProtoMessage() {} func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -562,7 +612,7 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{10} + return file_net_proto_rawDescGZIP(), []int{11} } type PushLogReply struct { @@ -574,7 +624,7 @@ type PushLogReply struct { func (x *PushLogReply) Reset() { *x = PushLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -587,7 +637,7 @@ func (x *PushLogReply) String() string { func (*PushLogReply) ProtoMessage() {} func (x *PushLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -600,7 +650,7 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. func (*PushLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{11} + return file_net_proto_rawDescGZIP(), []int{12} } type GetHeadLogReply struct { @@ -612,7 +662,7 @@ type GetHeadLogReply struct { func (x *GetHeadLogReply) Reset() { *x = GetHeadLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -625,7 +675,7 @@ func (x *GetHeadLogReply) String() string { func (*GetHeadLogReply) ProtoMessage() {} func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -638,7 +688,7 @@ func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. func (*GetHeadLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{12} + return file_net_proto_rawDescGZIP(), []int{13} } type PushLogRequest_Body struct { @@ -661,7 +711,7 @@ type PushLogRequest_Body struct { func (x *PushLogRequest_Body) Reset() { *x = PushLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -674,7 +724,7 @@ func (x *PushLogRequest_Body) String() string { func (*PushLogRequest_Body) ProtoMessage() {} func (x *PushLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -751,64 +801,71 @@ var file_net_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0xcf, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, - 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xc3, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, - 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, - 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, - 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, - 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, - 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, - 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, - 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, - 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, - 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x5b, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, + 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, + 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, + 0x64, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, + 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, + 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x1e, 0x0a, + 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, + 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, + 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, + 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, + 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, + 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, + 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, + 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, + 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, + 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, + 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -823,7 +880,7 @@ func file_net_proto_rawDescGZIP() []byte { return file_net_proto_rawDescData } -var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_net_proto_goTypes = []any{ (*Log)(nil), // 0: net.pb.Log (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest @@ -833,33 +890,36 @@ var file_net_proto_goTypes = []any{ (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest (*GetLogReply)(nil), // 6: net.pb.GetLogReply (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest - (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest - (*FetchEncryptionKeyReply)(nil), // 9: net.pb.FetchEncryptionKeyReply - (*GetHeadLogRequest)(nil), // 10: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 11: net.pb.PushLogReply - (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply - (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body + (*EncryptionKeyTarget)(nil), // 8: net.pb.EncryptionKeyTarget + (*FetchEncryptionKeyRequest)(nil), // 9: net.pb.FetchEncryptionKeyRequest + (*FetchEncryptionKeyReply)(nil), // 10: net.pb.FetchEncryptionKeyReply + (*GetHeadLogRequest)(nil), // 11: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 12: net.pb.PushLogReply + (*GetHeadLogReply)(nil), // 13: net.pb.GetHeadLogReply + (*PushLogRequest_Body)(nil), // 14: net.pb.PushLogRequest.Body } var file_net_proto_depIdxs = []int32{ - 13, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 0, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log - 1, // 2: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest - 3, // 3: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest - 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest - 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 8, // 6: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 10, // 7: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 8: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 9: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 10: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 11, // 11: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 9, // 12: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply - 12, // 13: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 8, // [8:14] is the sub-list for method output_type - 2, // [2:8] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 14, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body + 8, // 1: net.pb.FetchEncryptionKeyRequest.targets:type_name -> net.pb.EncryptionKeyTarget + 8, // 2: net.pb.FetchEncryptionKeyReply.targets:type_name -> net.pb.EncryptionKeyTarget + 0, // 3: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log + 1, // 4: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest + 3, // 5: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest + 5, // 6: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest + 7, // 7: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest + 9, // 8: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest + 11, // 9: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 10: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 11: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 12: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 12, // 13: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 10, // 14: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply + 13, // 15: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 10, // [10:16] is the sub-list for method output_type + 4, // [4:10] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_net_proto_init() } @@ -965,7 +1025,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*FetchEncryptionKeyRequest); i { + switch v := v.(*EncryptionKeyTarget); i { case 0: return &v.state case 1: @@ -977,7 +1037,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { - switch v := v.(*FetchEncryptionKeyReply); i { + switch v := v.(*FetchEncryptionKeyRequest); i { case 0: return &v.state case 1: @@ -989,7 +1049,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogRequest); i { + switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state case 1: @@ -1001,7 +1061,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { - switch v := v.(*PushLogReply); i { + switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state case 1: @@ -1013,7 +1073,7 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { - switch v := v.(*GetHeadLogReply); i { + switch v := v.(*PushLogReply); i { case 0: return &v.state case 1: @@ -1025,6 +1085,18 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { + switch v := v.(*GetHeadLogReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_net_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state @@ -1043,7 +1115,7 @@ func file_net_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_net_proto_rawDesc, NumEnums: 0, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, diff --git a/net/pb/net.proto b/net/pb/net.proto index 7c44efe735..55fad90777 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -38,9 +38,8 @@ message PushLogRequest { } } -// FetchEncryptionKeyRequest is a request to receive a doc encryption key -// from a peer that holds it. -message FetchEncryptionKeyRequest { +// EncryptionKeyTarget is a struct containing information about required doc encryption key. +message EncryptionKeyTarget { // docID is the ID of the document that the key is being requested for. bytes docID = 1; // fieldName if the name of the document field that the key is being requested for. @@ -48,22 +47,30 @@ message FetchEncryptionKeyRequest { string fieldName = 2; // cid is the CID of the composite of the document. bytes cid = 3; +} + +// FetchEncryptionKeyRequest is a request to receive a doc encryption key +// from a peer that holds it. +message FetchEncryptionKeyRequest { + // targets is the list of docs/fields for which encryption keys are being requested. + repeated EncryptionKeyTarget targets = 1; // schemaRoot is the SchemaRoot of the collection that the document resides in. - bytes schemaRoot = 4; + bytes schemaRoot = 2; // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - bytes ephemeralPublicKey = 5; + bytes ephemeralPublicKey = 3; // signature is the requesting peer's signature of the request - bytes signature = 6; + bytes signature = 4; } // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request // by a peer that holds the requested doc encryption key. message FetchEncryptionKeyReply { - // encryptedKey is array or bytes representing encrypted doc encryption key + // targets is the list of docs/fields for which encryption keys are being requested. + repeated EncryptionKeyTarget targets = 1; + // encryptedKeys is an encrypted list of doc encryption keys. // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM - bytes encryptedKey = 1; - // cid is the CID of the composite of the document. - bytes cid = 2; + // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. + bytes encryptedKeys = 2; // schemaRoot is the SchemaRoot of the collection that the document resides in. bytes schemaRoot = 3; // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 6bc4141229..b3ea2465e8 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -370,6 +370,60 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *EncryptionKeyTarget) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *EncryptionKeyTarget) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *EncryptionKeyTarget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Cid) > 0 { + i -= len(m.Cid) + copy(dAtA[i:], m.Cid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + i-- + dAtA[i] = 0x1a + } + if len(m.FieldName) > 0 { + i -= len(m.FieldName) + copy(dAtA[i:], m.FieldName) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FieldName))) + i-- + dAtA[i] = 0x12 + } + if len(m.DocID) > 0 { + i -= len(m.DocID) + copy(dAtA[i:], m.DocID) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *FetchEncryptionKeyRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -405,42 +459,33 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x32 + dAtA[i] = 0x22 } if len(m.EphemeralPublicKey) > 0 { i -= len(m.EphemeralPublicKey) copy(dAtA[i:], m.EphemeralPublicKey) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EphemeralPublicKey))) i-- - dAtA[i] = 0x2a + dAtA[i] = 0x1a } if len(m.SchemaRoot) > 0 { i -= len(m.SchemaRoot) copy(dAtA[i:], m.SchemaRoot) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) i-- - dAtA[i] = 0x22 - } - if len(m.Cid) > 0 { - i -= len(m.Cid) - copy(dAtA[i:], m.Cid) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) - i-- - dAtA[i] = 0x1a - } - if len(m.FieldName) > 0 { - i -= len(m.FieldName) - copy(dAtA[i:], m.FieldName) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FieldName))) - i-- dAtA[i] = 0x12 } - if len(m.DocID) > 0 { - i -= len(m.DocID) - copy(dAtA[i:], m.DocID) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) - i-- - dAtA[i] = 0xa + if len(m.Targets) > 0 { + for iNdEx := len(m.Targets) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Targets[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } } return len(dAtA) - i, nil } @@ -496,19 +541,24 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro i-- dAtA[i] = 0x1a } - if len(m.Cid) > 0 { - i -= len(m.Cid) - copy(dAtA[i:], m.Cid) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + if len(m.EncryptedKeys) > 0 { + i -= len(m.EncryptedKeys) + copy(dAtA[i:], m.EncryptedKeys) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKeys))) i-- dAtA[i] = 0x12 } - if len(m.EncryptedKey) > 0 { - i -= len(m.EncryptedKey) - copy(dAtA[i:], m.EncryptedKey) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKey))) - i-- - dAtA[i] = 0xa + if len(m.Targets) > 0 { + for iNdEx := len(m.Targets) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Targets[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } } return len(dAtA) - i, nil } @@ -730,7 +780,7 @@ func (m *PushLogRequest) SizeVT() (n int) { return n } -func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { +func (m *EncryptionKeyTarget) SizeVT() (n int) { if m == nil { return 0 } @@ -748,6 +798,22 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + n += len(m.unknownFields) + return n +} + +func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Targets) > 0 { + for _, e := range m.Targets { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } l = len(m.SchemaRoot) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -770,11 +836,13 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { } var l int _ = l - l = len(m.EncryptedKey) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + if len(m.Targets) > 0 { + for _, e := range m.Targets { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } } - l = len(m.Cid) + l = len(m.EncryptedKeys) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } @@ -1523,7 +1591,7 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { +func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1546,10 +1614,10 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: FetchEncryptionKeyRequest: wiretype end group for non-group") + return fmt.Errorf("proto: EncryptionKeyTarget: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: FetchEncryptionKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: EncryptionKeyTarget: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1652,7 +1720,92 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.Cid = []byte{} } iNdEx = postIndex - case 4: + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FetchEncryptionKeyRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FetchEncryptionKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Targets", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Targets = append(m.Targets, &EncryptionKeyTarget{}) + if err := m.Targets[len(m.Targets)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) } @@ -1686,7 +1839,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.SchemaRoot = []byte{} } iNdEx = postIndex - case 5: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field EphemeralPublicKey", wireType) } @@ -1720,7 +1873,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.EphemeralPublicKey = []byte{} } iNdEx = postIndex - case 6: + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } @@ -1807,9 +1960,9 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptedKey", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Targets", wireType) } - var byteLen int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -1819,29 +1972,29 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + if msglen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + byteLen + postIndex := iNdEx + msglen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.EncryptedKey = append(m.EncryptedKey[:0], dAtA[iNdEx:postIndex]...) - if m.EncryptedKey == nil { - m.EncryptedKey = []byte{} + m.Targets = append(m.Targets, &EncryptionKeyTarget{}) + if err := m.Targets[len(m.Targets)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err } iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Cid", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field EncryptedKeys", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1868,9 +2021,9 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Cid = append(m.Cid[:0], dAtA[iNdEx:postIndex]...) - if m.Cid == nil { - m.Cid = []byte{} + m.EncryptedKeys = append(m.EncryptedKeys[:0], dAtA[iNdEx:postIndex]...) + if m.EncryptedKeys == nil { + m.EncryptedKeys = []byte{} } iNdEx = postIndex case 3: diff --git a/net/peer.go b/net/peer.go index ae1fcbeb80..94460821d2 100644 --- a/net/peer.go +++ b/net/peer.go @@ -263,7 +263,8 @@ func (p *Peer) Start() error { } if p.ps != nil { - sub, err := p.bus.Subscribe(event.UpdateName, event.P2PTopicName, event.ReplicatorName, encryption.RequestKeyEventName) + sub, err := p.bus.Subscribe(event.UpdateName, event.P2PTopicName, + event.ReplicatorName, encryption.RequestKeysEventName) if err != nil { return err } @@ -362,7 +363,7 @@ func (p *Peer) handleMessageLoop() { case event.Replicator: p.server.updateReplicators(evt) - case encryption.RequestKeyEvent: + case encryption.RequestKeysEvent: err := p.handleEncryptionKeyRequest(evt) if err != nil { log.ErrorContextE(p.ctx, "Error while handling broadcast log", err) @@ -460,13 +461,9 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { return nil } -func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeyEvent) error { +func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeysEvent) error { if err := p.server.requestEncryptionKey(p.ctx, evt); err != nil { - kvs := []errors.KV{} - if evt.FieldName.HasValue() { - kvs = append(kvs, errors.NewKV("FieldName", evt.FieldName)) - } - return NewErrRequestingEncryptionKey(err, evt.Cid.String(), evt.DocID, kvs...) + return NewErrRequestingEncryptionKeys(err, evt.Keys) } return nil diff --git a/net/server.go b/net/server.go index d2bf37c381..ba26a9c335 100644 --- a/net/server.go +++ b/net/server.go @@ -36,7 +36,9 @@ import ( "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" + "github.com/sourcenetwork/defradb/internal/encoding" "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -188,17 +190,36 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return &pb.PushLogReply{}, nil } -func (s *server) getEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) ([]byte, error) { - docID, err := client.NewDocIDFromString(string(req.DocID)) - if err != nil { - return nil, err - } +func (s *server) getEncryptionKeys( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) ([]byte, []*pb.EncryptionKeyTarget, error) { + encryptionKeys := make([]byte, 0) + targets := make([]*pb.EncryptionKeyTarget, 0) + for _, target := range req.Targets { + docID, err := client.NewDocIDFromString(string(target.DocID)) + if err != nil { + return nil, nil, err + } - optFieldName := immutable.None[string]() - if req.FieldName != "" { - optFieldName = immutable.Some(req.FieldName) + optFieldName := immutable.None[string]() + if target.FieldName != "" { + optFieldName = immutable.Some(target.FieldName) + } + encKey, err := encryption.GetKey( + encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName) + if err != nil { + return nil, nil, err + } + // TODO: we should test it somehow. For this this one peer should have some keys and + // another one should have the others + if len(encKey) == 0 { + continue + } + targets = append(targets, target) + encryptionKeys = encoding.EncodeBytesAscending(encryptionKeys, encKey) } - return encryption.GetKey(encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName) + return encryptionKeys, targets, nil } func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { @@ -217,7 +238,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.New("invalid signature") } - encKey, err := s.getEncryptionKey(ctx, req) + encKey, targets, err := s.getEncryptionKeys(ctx, req) if err != nil || len(encKey) == 0 { return nil, err } @@ -233,10 +254,10 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio } res := &pb.FetchEncryptionKeyReply{ - EncryptedKey: encryptedKey, - Cid: req.Cid, SchemaRoot: req.SchemaRoot, ReqEphemeralPublicKey: req.EphemeralPublicKey, + Targets: targets, + EncryptedKeys: encryptedKey, } res.Signature, err = s.signResponse(res) @@ -253,10 +274,14 @@ func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKe func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { hash := sha256.New() - hash.Write(res.EncryptedKey) - hash.Write(res.Cid) + hash.Write(res.EncryptedKeys) hash.Write(res.SchemaRoot) hash.Write(res.ReqEphemeralPublicKey) + for _, target := range res.Targets { + hash.Write(target.DocID) + hash.Write([]byte(target.FieldName)) + hash.Write(target.Cid) + } return hash.Sum(nil) } @@ -414,18 +439,21 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe } func (s *server) prepareFetchEncryptionKeyRequest( - evt encryption.RequestKeyEvent, + evt encryption.RequestKeysEvent, ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { req := &pb.FetchEncryptionKeyRequest{ - DocID: []byte(evt.DocID), - Cid: evt.Cid.Bytes(), SchemaRoot: []byte(evt.SchemaRoot), EphemeralPublicKey: ephemeralPublicKey, } - if evt.FieldName.HasValue() { - req.FieldName = evt.FieldName.Value() + for key, cid := range evt.Keys { + encKey := &pb.EncryptionKeyTarget{ + DocID: []byte(key.DocID), + FieldName: key.FieldName, + Cid: cid.Bytes(), + } + req.Targets = append(req.Targets, encKey) } signature, err := s.signRequest(req) @@ -439,7 +467,7 @@ func (s *server) prepareFetchEncryptionKeyRequest( } // requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeyEvent) error { +func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeysEvent) error { if s.peer.ps == nil { // skip if we aren't running with a pubsub net return nil } @@ -479,10 +507,13 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { hash := sha256.New() - hash.Write(req.DocID) - hash.Write(req.Cid) hash.Write(req.SchemaRoot) hash.Write(req.EphemeralPublicKey) + for _, target := range req.Targets { + hash.Write(target.DocID) + hash.Write([]byte(target.FieldName)) + hash.Write(target.Cid) + } return hash.Sum(nil) } @@ -517,25 +548,37 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - decryptedKey, err := crypto.DecryptECIES(keyResp.EncryptedKey, session.privateKey) + decryptedData, err := crypto.DecryptECIES(keyResp.EncryptedKeys, session.privateKey) if err != nil { log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) return } - cid, err := cid.Cast(req.Cid) - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to parse CID", err) - return - } + eventData := make(map[core.EncStoreDocKey]encryption.RequestedKeyEventData) + for _, target := range keyResp.Targets { + cid, err := cid.Cast(target.Cid) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to parse CID", err) + return + } + + optFieldName := immutable.None[string]() + if target.FieldName != "" { + optFieldName = immutable.Some(target.FieldName) + } + + var encKey []byte + decryptedData, encKey, err = encoding.DecodeBytesAscending(decryptedData) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to decode encrypted key", err) + return + } - optFieldName := immutable.None[string]() - if req.FieldName != "" { - optFieldName = immutable.Some(req.FieldName) + d := encryption.RequestedKeyEventData{Cid: cid, Key: encKey} + eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName)] = d } - s.peer.bus.Publish(encryption.NewKeyRetrievedMessage( - string(req.DocID), optFieldName, cid, string(req.SchemaRoot), decryptedKey)) + s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) } func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { diff --git a/tests/integration/events.go b/tests/integration/events.go index 2ed9a01b4f..0c1779078a 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -372,7 +372,7 @@ func waitForKeyRetrievedEvent(s *state, nodeIDs []int) { func waitForSync(s *state, action WaitForSync) { if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { waitForMergeEvents(s) - } else if action.Event.Value() == encryption.KeyRetrievedEventName { + } else if action.Event.Value() == encryption.KeysRetrievedEventName { waitForKeyRetrievedEvent(s, action.NodeIDs) } else { require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) diff --git a/tests/integration/state.go b/tests/integration/state.go index 52fc1b65c7..8573d6ddad 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -80,8 +80,8 @@ type eventState struct { // p2pTopic is the `event.P2PTopicCompletedName` subscription p2pTopic *event.Subscription - // encKeyRetrieved is the `encryption.KeyRetrieved` subscription - encKeyRetrieved *event.Subscription + // encKeysRetrieved is the `encryption.KeyRetrieved` subscription + encKeysRetrieved *event.Subscription } // newEventState returns an eventState with all required subscriptions. @@ -102,16 +102,16 @@ func newEventState(bus *event.Bus) (*eventState, error) { if err != nil { return nil, err } - encKeyRetrieved, err := bus.Subscribe(encryption.KeyRetrievedEventName) + encKeysRetrieved, err := bus.Subscribe(encryption.KeysRetrievedEventName) if err != nil { return nil, err } return &eventState{ - merge: merge, - update: update, - replicator: replicator, - p2pTopic: p2pTopic, - encKeyRetrieved: encKeyRetrieved, + merge: merge, + update: update, + replicator: replicator, + p2pTopic: p2pTopic, + encKeysRetrieved: encKeysRetrieved, }, nil } From 31e0c37df842b8bcaa7461a9a8cb13fae9a7b749 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 5 Aug 2024 14:12:11 +0200 Subject: [PATCH 16/88] Load latest available encrypted block from Blockstore instead of fetching it by CID --- internal/db/merge.go | 80 ++++++++++++------- .../integration/encryption/peer_share_test.go | 61 +++++++++++++- tests/integration/events.go | 2 +- 3 files changed, 107 insertions(+), 36 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index f0b1830650..3c2fba6b9d 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -59,7 +59,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey) + mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) if err != nil { return err } @@ -69,7 +69,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.mergeComposites(ctx, false) + err = mp.mergeBlocks(ctx, false) if err != nil { return err } @@ -106,16 +106,27 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR ls := cidlink.DefaultLinkSystem() ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) - mergedDocs := make(map[string]struct{}) + type mergeGroup struct { + includeComposite bool + fields []string + } + + mergeGroups := make(map[string]mergeGroup) for encStoreKey := range keyEvent.Data { - if _, ok := mergedDocs[encStoreKey.DocID]; ok { - continue + g := mergeGroups[encStoreKey.DocID] + + if encStoreKey.FieldName == "" { + g.includeComposite = true + } else { + g.fields = append(g.fields, encStoreKey.FieldName) } - mergedDocs[encStoreKey.DocID] = struct{}{} + mergeGroups[encStoreKey.DocID] = g + } - docID, err := client.NewDocIDFromString(encStoreKey.DocID) + for docID, mergeGroup := range mergeGroups { + docID, err := client.NewDocIDFromString(docID) if err != nil { return err } @@ -126,29 +137,45 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR return err } - mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey) - if err != nil { - return err - } + // if we need to process the composite block, we load only it + // as the children will be loaded by the merge processor + // Otherwise, we load the blocks for each field + if mergeGroup.includeComposite { + mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) + if err != nil { + return err + } - for encStoreKey2, data := range keyEvent.Data { - // load all blocks related to the document - if encStoreKey2.DocID == encStoreKey.DocID { - // TODO: check if it's really necessary to pass cid here and over the wire - // Looks like once a key is retrieved, we should traverse the DAG from the head - // and decrypt all the blocks that are encrypted with this key. CID should not matter. - err = mp.loadBlocks(ctx, data.Cid, mt, true) + for _, block := range mt.heads { + mp.blocks.PushFront(block) + break + } + } else { + for _, fieldName := range mergeGroup.fields { + fd, ok := mp.col.Definition().GetFieldByName(fieldName) + if !ok { + return client.NewErrFieldNotExist(fieldName) + } + + fieldDsKey := dsKey.WithFieldId(fd.ID.String()) + mt, err := getHeadsAsMergeTarget(ctx, txn, fieldDsKey) if err != nil { return err } + + for _, block := range mt.heads { + mp.blocks.PushFront(block) + break + } } } - err = mp.mergeComposites(ctx, true) + err = mp.mergeBlocks(ctx, true) if err != nil { return err } + // TODO: test is doc field was indexed after decryption err = syncIndexedDoc(ctx, docID, col) if err != nil { return err @@ -306,7 +333,7 @@ func (mp *mergeProcessor) loadBlocks( return nil } -func (mp *mergeProcessor) mergeComposites(ctx context.Context, withDecryption bool) error { +func (mp *mergeProcessor) mergeBlocks(ctx context.Context, withDecryption bool) error { for e := mp.blocks.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() @@ -365,7 +392,7 @@ func (mp *mergeProcessor) sendPendingEncryptionRequest() { } // processBlock merges the block and its children to the datastore and sets the head accordingly. -// If onlyHeads is true, it will skip merging and update only the heads. +// withDecryption is a flag that indicates this is the decryption pass instead of a normal merge. func (mp *mergeProcessor) processBlock( ctx context.Context, dagBlock *coreblock.Block, @@ -408,12 +435,6 @@ func (mp *mergeProcessor) processBlock( return err } - // if this is a decryption pass we skip processing for the field block as field blocks - // should have a dedicated decryption pass (and not as part of the composite block) - if withDecryption && childBlock.EncryptionType != nil && *childBlock.EncryptionType == coreblock.FieldEncrypted { - continue - } - if err := mp.processBlock(ctx, childBlock, link.Link, withDecryption); err != nil { return err } @@ -518,10 +539,7 @@ func getCollectionFromRootSchema(ctx context.Context, db *db, rootSchema string) // getHeadsAsMergeTarget retrieves the heads of the composite DAG for the given document // and returns them as a merge target. func getHeadsAsMergeTarget(ctx context.Context, txn datastore.Txn, dsKey core.DataStoreKey) (mergeTarget, error) { - headset := clock.NewHeadSet( - txn.Headstore(), - dsKey.WithFieldId(core.COMPOSITE_NAMESPACE).ToHeadStoreKey(), - ) + headset := clock.NewHeadSet(txn.Headstore(), dsKey.ToHeadStoreKey()) cids, _, err := headset.List(ctx) if err != nil { diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 5be3676477..1f116b2e58 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -46,7 +46,7 @@ func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) IsDocEncrypted: true, }, testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeyRetrievedEventName), + Event: immutable.Some(encryption.KeysRetrievedEventName), NodeIDs: []int{1}, }, testUtils.Request{ @@ -97,7 +97,7 @@ func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt EncryptedFields: []string{"age"}, }, testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeyRetrievedEventName), + Event: immutable.Some(encryption.KeysRetrievedEventName), NodeIDs: []int{1}, }, testUtils.Request{ @@ -151,7 +151,7 @@ func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeys EncryptedFields: []string{"age"}, }, testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeyRetrievedEventName), + Event: immutable.Some(encryption.KeysRetrievedEventName), NodeIDs: []int{1}, }, testUtils.Request{ @@ -205,7 +205,60 @@ func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypt EncryptedFields: []string{"name", "age"}, }, testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeyRetrievedEventName), + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "name": "John", + "age": int64(21), + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + EncryptedFields: []string{"name", "age"}, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeysRetrievedEventName), NodeIDs: []int{1}, }, testUtils.Request{ diff --git a/tests/integration/events.go b/tests/integration/events.go index 0c1779078a..0df8bb6f63 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -358,7 +358,7 @@ func waitForKeyRetrievedEvent(s *state, nodeIDs []int) { } select { - case _, ok := <-s.nodeEvents[nodeID].encKeyRetrieved.Message(): + case _, ok := <-s.nodeEvents[nodeID].encKeysRetrieved.Message(): if !ok { require.Fail(s.t, "subscription closed waiting for key retrieved event") } From 809e181a2f9256e5bf947fe10a33308f91d0b8ac Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 13:04:58 +0200 Subject: [PATCH 17/88] Make block store height of where encryption started --- internal/core/block/block.go | 32 +++-- internal/core/block/block_test.go | 47 +++---- internal/core/block/errors.go | 2 + internal/core/key.go | 26 ++-- internal/db/merge.go | 44 ++++--- internal/db/messages.go | 15 +-- internal/encryption/encryptor.go | 76 +++++++---- internal/encryption/encryptor_test.go | 41 +++--- internal/encryption/event.go | 18 ++- internal/merkle/clock/clock.go | 43 ++++--- net/errors.go | 3 +- net/pb/net.pb.go | 133 ++++++++++---------- net/pb/net.proto | 4 +- net/pb/net_vtproto.pb.go | 36 ++---- net/server.go | 32 ++--- tests/integration/encryption/commit_test.go | 14 +-- tests/integration/encryption/peer_test.go | 14 +-- 17 files changed, 315 insertions(+), 265 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 531c9159da..1f1e9c4057 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -105,20 +105,27 @@ const ( FieldEncrypted ) +type Encryption struct { + // Type indicates on what level encryption is applied. + Type EncryptionType + // From specifies the block height from which the encryption is applied. + From uint64 +} + // Block is a block that contains a CRDT delta and links to other blocks. type Block struct { // Delta is the CRDT delta that is stored in the block. Delta crdt.CRDT // Links are the links to other blocks in the DAG. Links []DAGLink - // EncryptionType indicates if the block's delta is encrypted and on what level encryption is applied. + // Encryption contains the encryption information for the block's delta. // It needs to be a pointer so that it can be translated from and to `optional` in the IPLD schema. - EncryptionType *EncryptionType + Encryption *Encryption } // IsEncrypted returns true if the block is encrypted. func (b *Block) IsEncrypted() bool { - return b.EncryptionType != nil && *b.EncryptionType != NotEncrypted + return b.Encryption != nil && (*b.Encryption).Type != NotEncrypted } // IPLDSchemaBytes returns the IPLD schema representation for the block. @@ -127,9 +134,14 @@ func (b *Block) IsEncrypted() bool { func (b Block) IPLDSchemaBytes() []byte { return []byte(` type Block struct { - delta CRDT - links [DAGLink] - encryptionType optional EncryptionType + delta CRDT + links [DAGLink] + encryption optional Encryption + } + + type Encryption struct { + type EncryptionType + from Int } type EncryptionType enum { @@ -263,13 +275,17 @@ func GetLinkPrototype() cidlink.LinkPrototype { } func (b *Block) Validate() error { - if b.EncryptionType != nil { - switch *b.EncryptionType { + if b.Encryption != nil { + switch (*b.Encryption).Type { case NotEncrypted, DocumentEncrypted, FieldEncrypted: // Valid values default: return ErrInvalidBlockEncryptionType } + + if (*b.Encryption).From == 0 { + return ErrInvalidBlockEncryptionFrom + } } return nil } diff --git a/internal/core/block/block_test.go b/internal/core/block/block_test.go index 554d624c81..3c5a374ed9 100644 --- a/internal/core/block/block_test.go +++ b/internal/core/block/block_test.go @@ -231,48 +231,49 @@ func TestBlockMarshal_IsEncryptedNotSetWithLinkSystem_ShouldLoadWithNoError(t *t func TestBlock_Validate(t *testing.T) { tests := []struct { - name string - encryptionType *EncryptionType - expectedError error + name string + encryption *Encryption + expectedError error }{ { - name: "NotEncrypted is valid", - encryptionType: ptr(NotEncrypted), - expectedError: nil, + name: "NotEncrypted type is valid", + encryption: &Encryption{Type: NotEncrypted, From: 1}, + expectedError: nil, }, { - name: "DocumentEncrypted is valid", - encryptionType: ptr(DocumentEncrypted), - expectedError: nil, + name: "DocumentEncrypted type is valid", + encryption: &Encryption{Type: DocumentEncrypted, From: 1}, + expectedError: nil, }, { - name: "FieldEncrypted is valid", - encryptionType: ptr(FieldEncrypted), - expectedError: nil, + name: "FieldEncrypted type is valid", + encryption: &Encryption{Type: FieldEncrypted, From: 1}, + expectedError: nil, }, { - name: "Nil EncryptionType is valid", - encryptionType: nil, - expectedError: nil, + name: "Nil Encryption is valid", + encryption: nil, + expectedError: nil, }, { - name: "Invalid EncryptionType", - encryptionType: ptr(EncryptionType(99)), - expectedError: ErrInvalidBlockEncryptionType, + name: "Invalid encryption type", + encryption: &Encryption{Type: EncryptionType(99), From: 1}, + expectedError: ErrInvalidBlockEncryptionType, + }, + { + name: "Invalid encryption from parameter", + encryption: &Encryption{Type: DocumentEncrypted}, + expectedError: ErrInvalidBlockEncryptionFrom, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &Block{ - EncryptionType: tt.encryptionType, + Encryption: tt.encryption, } err := b.Validate() require.Equal(t, tt.expectedError, err) }) } } - -func ptr(e EncryptionType) *EncryptionType { - return &e -} diff --git a/internal/core/block/errors.go b/internal/core/block/errors.go index ec91f1e816..c45e2b0cca 100644 --- a/internal/core/block/errors.go +++ b/internal/core/block/errors.go @@ -22,6 +22,7 @@ const ( errUnmarshallingBlock string = "failed to unmarshal block" errGeneratingLink string = "failed to generate link" errInvalidBlockEncryptionType string = "invalid block encryption type" + errInvalidBlockEncryptionFrom string = "invalid block encryption from parameter" ) // Errors returnable from this package. @@ -34,6 +35,7 @@ var ( ErrUnmarshallingBlock = errors.New(errUnmarshallingBlock) ErrGeneratingLink = errors.New(errGeneratingLink) ErrInvalidBlockEncryptionType = errors.New(errInvalidBlockEncryptionType) + ErrInvalidBlockEncryptionFrom = errors.New(errInvalidBlockEncryptionFrom) ) // NewErrFailedToGetPriority returns an error indicating that the priority could not be retrieved. diff --git a/internal/core/key.go b/internal/core/key.go index 64116f4150..15c5e14de8 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -795,26 +795,30 @@ func bytesPrefixEnd(b []byte) []byte { // EncStoreDocKey is a key for the encryption store. type EncStoreDocKey struct { - DocID string - FieldName string + // DocID is the ID of the document that the key is for. + DocID string + // FieldName is the name of the field that the key is for. + // If unset, it indicates the key is for the whole document. + FieldName immutable.Option[string] + // BlockHeight is the height of the block that the key is for. + // It is used to differentiate keys that are used in different point in time. + BlockHeight uint64 } var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. -func NewEncStoreDocKey(docID string, fieldName immutable.Option[string]) EncStoreDocKey { - key := EncStoreDocKey{DocID: docID} - if fieldName.HasValue() { - key.FieldName = fieldName.Value() - } - return key +// Unset fieldName indicates the key is for the whole document. +// blockHeight is the height of the block that the key is for. +func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], blockHeight uint64) EncStoreDocKey { + return EncStoreDocKey{DocID: docID, FieldName: fieldName, BlockHeight: blockHeight} } func (k EncStoreDocKey) ToString() string { - if k.FieldName == "" { - return k.DocID + if k.FieldName.HasValue() { + return fmt.Sprintf("%s/%s/%d", k.DocID, k.FieldName.Value(), k.BlockHeight) } - return fmt.Sprintf("%s/%s", k.DocID, k.FieldName) + return fmt.Sprintf("%s/%d", k.DocID, k.BlockHeight) } func (k EncStoreDocKey) Bytes() []byte { diff --git a/internal/db/merge.go b/internal/db/merge.go index 3c2fba6b9d..526de1e867 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -113,13 +113,13 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR mergeGroups := make(map[string]mergeGroup) - for encStoreKey := range keyEvent.Data { + for encStoreKey := range keyEvent.Keys { g := mergeGroups[encStoreKey.DocID] - if encStoreKey.FieldName == "" { - g.includeComposite = true + if encStoreKey.FieldName.HasValue() { + g.fields = append(g.fields, encStoreKey.FieldName.Value()) } else { - g.fields = append(g.fields, encStoreKey.FieldName) + g.includeComposite = true } mergeGroups[encStoreKey.DocID] = g @@ -237,7 +237,7 @@ type mergeProcessor struct { col *collection dsKey core.DataStoreKey blocks *list.List - pendingEncryptionKeyRequests map[core.EncStoreDocKey]cid.Cid + pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} } func (db *db) newMergeProcessor( @@ -253,7 +253,7 @@ func (db *db) newMergeProcessor( col: col, dsKey: dsKey, blocks: list.New(), - pendingEncryptionKeyRequests: make(map[core.EncStoreDocKey]cid.Cid), + pendingEncryptionKeyRequests: make(map[core.EncStoreDocKey]struct{}), }, nil } @@ -355,7 +355,6 @@ func (mp *mergeProcessor) mergeBlocks(ctx context.Context, withDecryption bool) func (mp *mergeProcessor) processEncryptedBlock( ctx context.Context, dagBlock *coreblock.Block, - blockLink cidlink.Link, withDecryption bool, ) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { @@ -366,15 +365,16 @@ func (mp *mergeProcessor) processEncryptedBlock( if plainTextBlock != nil { return plainTextBlock, false, nil } else { - // if we weren't able to decrypt the block we request the encryption key unless it's a decryption pass - if !withDecryption && (dagBlock.Delta.IsComposite() && *dagBlock.EncryptionType == coreblock.DocumentEncrypted) || - *dagBlock.EncryptionType == coreblock.FieldEncrypted { + blockEnc := dagBlock.Encryption + // we weren't able to decrypt the block, so we request the encryption key unless it's a decryption pass + if !withDecryption && (dagBlock.Delta.IsComposite() && blockEnc.Type == coreblock.DocumentEncrypted) || + blockEnc.Type == coreblock.FieldEncrypted { docID := string(dagBlock.Delta.GetDocID()) fieldName := immutable.None[string]() - if *dagBlock.EncryptionType == coreblock.FieldEncrypted { + if blockEnc.Type == coreblock.FieldEncrypted { fieldName = immutable.Some(dagBlock.Delta.GetFieldName()) } - mp.addPendingEncryptionRequest(docID, blockLink.Cid, fieldName) + mp.addPendingEncryptionRequest(docID, fieldName, blockEnc.From) } return dagBlock, true, nil } @@ -382,13 +382,17 @@ func (mp *mergeProcessor) processEncryptedBlock( return dagBlock, false, nil } -func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, blockCid cid.Cid, fieldName immutable.Option[string]) { - mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName)] = blockCid +func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], height uint64) { + mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, height)] = struct{}{} } func (mp *mergeProcessor) sendPendingEncryptionRequest() { schemaRoot := mp.col.SchemaRoot() - mp.col.db.events.Publish(encryption.NewRequestKeysMessage(schemaRoot, mp.pendingEncryptionKeyRequests)) + storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) + for k := range mp.pendingEncryptionKeyRequests { + storeKeys = append(storeKeys, k) + } + mp.col.db.events.Publish(encryption.NewRequestKeysMessage(schemaRoot, storeKeys)) } // processBlock merges the block and its children to the datastore and sets the head accordingly. @@ -399,7 +403,7 @@ func (mp *mergeProcessor) processBlock( blockLink cidlink.Link, withDecryption bool, ) error { - block, skipMerge, err := mp.processEncryptedBlock(ctx, dagBlock, blockLink, withDecryption) + block, skipMerge, err := mp.processEncryptedBlock(ctx, dagBlock, withDecryption) if err != nil { return err } @@ -445,14 +449,15 @@ func (mp *mergeProcessor) processBlock( func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { optFieldName := immutable.None[string]() - if *block.EncryptionType == coreblock.FieldEncrypted { + blockEnc := block.Encryption + if blockEnc.Type == coreblock.FieldEncrypted { optFieldName = immutable.Some(block.Delta.GetFieldName()) } if block.Delta.IsComposite() { // for composite blocks there is nothing to decrypt // so we just check if we have the encryption key for child blocks - bytes, err := encryption.GetKey(ctx, string(block.Delta.GetDocID()), optFieldName) + bytes, err := encryption.GetKey(ctx, string(block.Delta.GetDocID()), optFieldName, blockEnc.From) if err != nil { return nil, err } @@ -463,7 +468,8 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } clonedCRDT := block.Delta.Clone() - bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), optFieldName, clonedCRDT.GetData()) + bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), optFieldName, + blockEnc.From, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/internal/db/messages.go b/internal/db/messages.go index b66939a302..4f4d34f4ea 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -15,7 +15,6 @@ import ( "sync" "github.com/sourcenetwork/corelog" - "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" @@ -84,12 +83,14 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { case encryption.KeyRetrievedEvent: go func() { ctx = encryption.ContextWithStore(ctx, db.Encstore()) - for encStoreKey, data := range evt.Data { - optFieldName := immutable.None[string]() - if encStoreKey.FieldName != "" { - optFieldName = immutable.Some(encStoreKey.FieldName) - } - err := encryption.SaveKey(ctx, encStoreKey.DocID, optFieldName, data.Key) + for encStoreKey, encKey := range evt.Keys { + err := encryption.SaveKey( + ctx, + encStoreKey.DocID, + encStoreKey.FieldName, + encStoreKey.BlockHeight, + encKey, + ) if err != nil { log.ErrorContextE( diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index d6301fede2..4757df4edd 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -103,11 +103,16 @@ func shouldEncryptDocField(conf immutable.Option[DocEncConfig], fieldName immuta return false } -// Encrypt encrypts the given plainText that is associated with the given docID and fieldName. +// Encrypt encrypts the given plainText that is associated with the given docID, fieldName and block height. // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. // Otherwise, it will use document-level encryption key. -func (d *DocEncryptor) Encrypt(docID string, fieldName immutable.Option[string], plainText []byte) ([]byte, error) { - encryptionKey, err := d.fetchEncryptionKey(docID, fieldName) +func (d *DocEncryptor) Encrypt( + docID string, + fieldName immutable.Option[string], + blockHeight uint64, + plainText []byte, +) ([]byte, error) { + encryptionKey, err := d.fetchEncryptionKey(docID, fieldName, blockHeight) if err != nil { return nil, err } @@ -126,7 +131,7 @@ func (d *DocEncryptor) Encrypt(docID string, fieldName immutable.Option[string], return nil, err } - err = d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName), encryptionKey) + err = d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight), encryptionKey) if err != nil { return nil, err } @@ -136,8 +141,13 @@ func (d *DocEncryptor) Encrypt(docID string, fieldName immutable.Option[string], // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. // If the corresponding encryption key is not found, it returns nil. -func (d *DocEncryptor) Decrypt(docID string, fieldName immutable.Option[string], cipherText []byte) ([]byte, error) { - encKey, err := d.fetchEncryptionKey(docID, fieldName) +func (d *DocEncryptor) Decrypt( + docID string, + fieldName immutable.Option[string], + blockHeight uint64, + cipherText []byte, +) ([]byte, error) { + encKey, err := d.fetchEncryptionKey(docID, fieldName, blockHeight) if err != nil { return nil, err } @@ -171,16 +181,21 @@ func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encrypti // fetchEncryptionKey fetches the encryption key for the given docID and fieldName. // If the key is not found, it returns an empty key. -func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName immutable.Option[string]) ([]byte, error) { +func (d *DocEncryptor) fetchEncryptionKey( + docID string, + fieldName immutable.Option[string], + blockHeight uint64, +) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } // first we try to find field-level key - storeKey := core.NewEncStoreDocKey(docID, fieldName) + storeKey := core.NewEncStoreDocKey(docID, fieldName, blockHeight) encryptionKey, err := d.fetchByEncStoreKey(storeKey) if err != nil { return nil, err } + // TODO: check if it's still needed to try to find doc-level key if field-level key is not found if len(encryptionKey) == 0 { // if previous fetch was for doc-level, there is nothing else to look for if !fieldName.HasValue() { @@ -190,31 +205,38 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName immutable.Opti return nil, nil } // try to find doc-level key - storeKey.FieldName = "" + storeKey.FieldName = immutable.None[string]() encryptionKey, err = d.fetchByEncStoreKey(storeKey) } return encryptionKey, err } -func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string]) ([]byte, error) { +// GetKey returns the encryption key for the given docID, (optional) fieldName and block height. +func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string], blockHeight uint64) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName)) + encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) if err != nil { return nil, err } return encryptionKey, nil } -func (d *DocEncryptor) SaveKey(docID string, fieldName immutable.Option[string], encryptionKey []byte) error { +// SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height. +func (d *DocEncryptor) SaveKey( + docID string, + fieldName immutable.Option[string], + blockHeight uint64, + encryptionKey []byte, +) error { if d.store == nil { return ErrNoStorageProvided } - return d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName), encryptionKey) + return d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight), encryptionKey) } -// EncryptDoc encrypts the given plainText that is associated with the given docID and fieldName with +// EncryptDoc encrypts the given plainText that is associated with the given docID, fieldName and block height with // encryptor in the context. // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. // Otherwise, it will use document-level encryption key. @@ -222,13 +244,14 @@ func EncryptDoc( ctx context.Context, docID string, fieldName immutable.Option[string], + blockHeight uint64, plainText []byte, ) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Encrypt(docID, fieldName, plainText) + return enc.Encrypt(docID, fieldName, blockHeight, plainText) } // DecryptDoc decrypts the given cipherText that is associated with the given docID and fieldName with @@ -239,13 +262,14 @@ func DecryptDoc( ctx context.Context, docID string, fieldName immutable.Option[string], + blockHeight uint64, cipherText []byte, ) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Decrypt(docID, fieldName, cipherText) + return enc.Decrypt(docID, fieldName, blockHeight, cipherText) } // ShouldEncryptDocField returns true if the given field should be encrypted based on the context config. @@ -259,22 +283,30 @@ func ShouldEncryptIndividualField(ctx context.Context, fieldName immutable.Optio return shouldEncryptIndividualField(GetContextConfig(ctx), fieldName) } -// SaveKey saves the given encryption key for the given docID and (optional) fieldName with encryptor in the context. -func SaveKey(ctx context.Context, docID string, fieldName immutable.Option[string], encryptionKey []byte) error { +// SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height with +// encryptor in the context. +func SaveKey( + ctx context.Context, + docID string, + fieldName immutable.Option[string], + blockHeight uint64, + encryptionKey []byte, +) error { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil } - return enc.SaveKey(docID, fieldName, encryptionKey) + return enc.SaveKey(docID, fieldName, blockHeight, encryptionKey) } -// GetKey returns the encryption key for the given docID and (optional) fieldName with encryptor in the context. -func GetKey(ctx context.Context, docID string, fieldName immutable.Option[string]) ([]byte, error) { +// GetKey returns the encryption key for the given docID, (optional) fieldName and block height with encryptor +// in the context. +func GetKey(ctx context.Context, docID string, fieldName immutable.Option[string], blockHeight uint64) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.GetKey(docID, fieldName) + return enc.GetKey(docID, fieldName, blockHeight) } func init() { diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 782f6889f5..643a733f05 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -29,6 +29,7 @@ import ( var testErr = errors.New("test error") const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" +const height = 1 var fieldName = immutable.Some("name") var noFieldName = immutable.None[string]() @@ -65,7 +66,7 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Encrypt(docID, fieldName, []byte("test")) + _, err := enc.Encrypt(docID, fieldName, height, []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -76,7 +77,7 @@ func TestEncryptorEncrypt_IfStorageReturnsErrorOnSecondCall_Error(t *testing.T) st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound).Once() st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Encrypt(docID, fieldName, []byte("test")) + _, err := enc.Encrypt(docID, fieldName, height, []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -84,12 +85,12 @@ func TestEncryptorEncrypt_IfStorageReturnsErrorOnSecondCall_Error(t *testing.T) func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - storeKey := core.NewEncStoreDocKey(docID, noFieldName) + storeKey := core.NewEncStoreDocKey(docID, noFieldName, height) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, noFieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, noFieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -98,14 +99,14 @@ func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerate func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - docStoreKey := core.NewEncStoreDocKey(docID, noFieldName).ToDS() - fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName).ToDS() + docStoreKey := core.NewEncStoreDocKey(docID, noFieldName, height).ToDS() + fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName, height).ToDS() st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) st.EXPECT().Get(mock.Anything, docStoreKey).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -114,12 +115,12 @@ func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStor func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) - storeKey := core.NewEncStoreDocKey(docID, fieldName) + storeKey := core.NewEncStoreDocKey(docID, fieldName, height) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(fieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, fieldName), cipherText) @@ -128,10 +129,10 @@ func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreI func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) - storeKey := core.NewEncStoreDocKey(docID, fieldName) + storeKey := core.NewEncStoreDocKey(docID, fieldName, height) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) - cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, fieldName), cipherText) @@ -142,7 +143,7 @@ func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *t st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - cipherText, err := enc.Encrypt(docID, noFieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, noFieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -155,7 +156,7 @@ func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *test st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) - _, err := enc.Encrypt(docID, fieldName, getPlainText()) + _, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.ErrorIs(t, err, testErr) } @@ -166,7 +167,7 @@ func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *t st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.NoError(t, err) assert.Equal(t, getPlainText(), cipherText) @@ -176,7 +177,7 @@ func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Encrypt(docID, fieldName, getPlainText()) + _, err := enc.Encrypt(docID, fieldName, height, getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -185,7 +186,7 @@ func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Decrypt(docID, fieldName, getPlainText()) + _, err := enc.Decrypt(docID, fieldName, height, getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -195,7 +196,7 @@ func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Decrypt(docID, fieldName, []byte("test")) + _, err := enc.Decrypt(docID, fieldName, height, []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -205,20 +206,20 @@ func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *te st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - plainText, err := enc.Decrypt(docID, fieldName, getCipherText(t, noFieldName)) + plainText, err := enc.Decrypt(docID, fieldName, height, getCipherText(t, noFieldName)) assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) } func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := EncryptDoc(context.Background(), docID, fieldName, getPlainText()) + data, err := EncryptDoc(context.Background(), docID, fieldName, height, getPlainText()) assert.Nil(t, data, "data should be nil") assert.NoError(t, err, "error should be nil") } func TestDecryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := DecryptDoc(context.Background(), docID, fieldName, getCipherText(t, fieldName)) + data, err := DecryptDoc(context.Background(), docID, fieldName, height, getCipherText(t, fieldName)) assert.Nil(t, data, "data should be nil") assert.NoError(t, err, "error should be nil") } diff --git a/internal/encryption/event.go b/internal/encryption/event.go index 8efbdef2f4..abe828c2d5 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -11,7 +11,6 @@ package encryption import ( - "github.com/ipfs/go-cid" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" ) @@ -27,15 +26,12 @@ type RequestKeysEvent struct { // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. SchemaRoot string - // Keys is a map of the keys that are being requested. - Keys map[core.EncStoreDocKey]cid.Cid + // Keys is a list of the keys that are being requested. + Keys []core.EncStoreDocKey } // RequestedKeyEventData represents the data that was retrieved for a specific key. type RequestedKeyEventData struct { - // Cid is the id of the composite commit that formed this update in the DAG. - Cid cid.Cid - // Key is the encryption key that was retrieved. Key []byte } @@ -45,15 +41,15 @@ type KeyRetrievedEvent struct { // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. SchemaRoot string - // Data is a map of the requested keys to the data that was retrieved. - Data map[core.EncStoreDocKey]RequestedKeyEventData + // Keys is a map of the requested keys to the corresponding raw encryption keys. + Keys map[core.EncStoreDocKey][]byte } // NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field func NewRequestKeysMessage( schemaRoot string, - keys map[core.EncStoreDocKey]cid.Cid, + keys []core.EncStoreDocKey, ) event.Message { return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ SchemaRoot: schemaRoot, @@ -64,10 +60,10 @@ func NewRequestKeysMessage( // NewKeysRetrievedMessage creates a new event message for a key that was retrieved func NewKeysRetrievedMessage( schemaRoot string, - data map[core.EncStoreDocKey]RequestedKeyEventData, + data map[core.EncStoreDocKey][]byte, ) event.Message { return event.NewMessage(KeysRetrievedEventName, KeyRetrievedEvent{ SchemaRoot: schemaRoot, - Data: data, + Keys: data, }) } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 1fe3a88b56..9ac26ffd5d 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -91,20 +91,21 @@ func (mc *MerkleClock) AddDelta( if block.Delta.GetFieldName() != "" { fieldName = immutable.Some(block.Delta.GetFieldName()) } - encryptionType, err := mc.checkIfBlockEncryptionEnabled(ctx, fieldName, heads) + blockEnc, err := mc.determineBlockEncryptionData(ctx, fieldName, height, heads) if err != nil { return cidlink.Link{}, nil, err } dagBlock := block - if encryptionType != coreblock.NotEncrypted { + if blockEnc != nil && blockEnc.Type != coreblock.NotEncrypted { if !block.Delta.IsComposite() { - dagBlock, err = encryptBlock(ctx, block, encryptionType == coreblock.FieldEncrypted) + dagBlock, err = encryptBlock(ctx, block, blockEnc) if err != nil { return cidlink.Link{}, nil, err } + } else { + dagBlock.Encryption = blockEnc } - dagBlock.EncryptionType = &encryptionType } link, err := mc.putBlock(ctx, dagBlock) @@ -126,48 +127,56 @@ func (mc *MerkleClock) AddDelta( return link, b, err } -func (mc *MerkleClock) checkIfBlockEncryptionEnabled( +func (mc *MerkleClock) determineBlockEncryptionData( ctx context.Context, fieldName immutable.Option[string], + height uint64, heads []cid.Cid, -) (coreblock.EncryptionType, error) { +) (*coreblock.Encryption, error) { + // if new encryption was requested by the user if encryption.ShouldEncryptDocField(ctx, fieldName) { + blockEnc := &coreblock.Encryption{From: height} if encryption.ShouldEncryptIndividualField(ctx, fieldName) { - return coreblock.FieldEncrypted, nil + blockEnc.Type = coreblock.FieldEncrypted + } else { + blockEnc.Type = coreblock.DocumentEncrypted } - return coreblock.DocumentEncrypted, nil + return blockEnc, nil } + // otherwise we use the same encryption as the previous block for _, headCid := range heads { bytes, err := mc.blockstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) if err != nil { - return coreblock.NotEncrypted, NewErrCouldNotFindBlock(headCid, err) + return nil, NewErrCouldNotFindBlock(headCid, err) } prevBlock, err := coreblock.GetFromBytes(bytes) if err != nil { - return coreblock.NotEncrypted, err + return nil, err } - if prevBlock.EncryptionType != nil { - return *prevBlock.EncryptionType, nil + if prevBlock.Encryption != nil { + blockEnc := &coreblock.Encryption{Type: (*prevBlock.Encryption).Type} + blockEnc.From = prevBlock.Encryption.From + return blockEnc, nil } } - return coreblock.NotEncrypted, nil + return nil, nil } -func encryptBlock(ctx context.Context, block *coreblock.Block, isFieldOnly bool) (*coreblock.Block, error) { +func encryptBlock(ctx context.Context, block *coreblock.Block, blockEnc *coreblock.Encryption) (*coreblock.Block, error) { clonedCRDT := block.Delta.Clone() fieldName := immutable.None[string]() - if isFieldOnly { + if blockEnc.Type == coreblock.FieldEncrypted { fieldName = immutable.Some(clonedCRDT.GetFieldName()) } bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), - fieldName, clonedCRDT.GetData()) + fieldName, blockEnc.From, clonedCRDT.GetData()) if err != nil { return nil, err } clonedCRDT.SetData(bytes) - return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil + return &coreblock.Block{Delta: clonedCRDT, Links: block.Links, Encryption: blockEnc}, nil } // ProcessBlock merges the delta CRDT and updates the state accordingly. diff --git a/net/errors.go b/net/errors.go index 6d2fe36b15..34f27121a5 100644 --- a/net/errors.go +++ b/net/errors.go @@ -13,7 +13,6 @@ package net import ( "fmt" - "github.com/ipfs/go-cid" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/internal/core" ) @@ -56,6 +55,6 @@ func NewErrCheckingForExistingBlock(inner error, cid string) error { return errors.Wrap(errCheckingForExistingBlock, inner, errors.NewKV("cid", cid)) } -func NewErrRequestingEncryptionKeys(inner error, keys map[core.EncStoreDocKey]cid.Cid) error { +func NewErrRequestingEncryptionKeys(inner error, keys []core.EncStoreDocKey) error { return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKeys, keys), inner) } diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 51efe7bfc3..ec09f861a4 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -355,8 +355,8 @@ type EncryptionKeyTarget struct { // fieldName if the name of the document field that the key is being requested for. // If the fieldName is empty, the key is being requested for the whole document. FieldName string `protobuf:"bytes,2,opt,name=fieldName,proto3" json:"fieldName,omitempty"` - // cid is the CID of the composite of the document. - Cid []byte `protobuf:"bytes,3,opt,name=cid,proto3" json:"cid,omitempty"` + // height is the height of the block the encryption key is being requested for. + Height uint64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` } func (x *EncryptionKeyTarget) Reset() { @@ -405,11 +405,11 @@ func (x *EncryptionKeyTarget) GetFieldName() string { return "" } -func (x *EncryptionKeyTarget) GetCid() []byte { +func (x *EncryptionKeyTarget) GetHeight() uint64 { if x != nil { - return x.Cid + return x.Height } - return nil + return 0 } // FetchEncryptionKeyRequest is a request to receive a doc encryption key @@ -801,71 +801,72 @@ var file_net_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x5b, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x61, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x10, 0x0a, 0x03, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x63, 0x69, - 0x64, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, - 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, - 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x1e, 0x0a, - 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, - 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, - 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, - 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, - 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, - 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, - 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, - 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, - 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, - 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, - 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, + 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, + 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, + 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, + 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, + 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, + 0x65, 0x79, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, + 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, + 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, + 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, + 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, + 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, + 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, + 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, + 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, + 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, + 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, + 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, + 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/net/pb/net.proto b/net/pb/net.proto index 55fad90777..6632077c7b 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -45,8 +45,8 @@ message EncryptionKeyTarget { // fieldName if the name of the document field that the key is being requested for. // If the fieldName is empty, the key is being requested for the whole document. string fieldName = 2; - // cid is the CID of the composite of the document. - bytes cid = 3; + // height is the height of the block the encryption key is being requested for. + uint64 height = 3; } // FetchEncryptionKeyRequest is a request to receive a doc encryption key diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index b3ea2465e8..f7c53f117b 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -400,12 +400,10 @@ func (m *EncryptionKeyTarget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if len(m.Cid) > 0 { - i -= len(m.Cid) - copy(dAtA[i:], m.Cid) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Cid))) + if m.Height != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Height)) i-- - dAtA[i] = 0x1a + dAtA[i] = 0x18 } if len(m.FieldName) > 0 { i -= len(m.FieldName) @@ -794,9 +792,8 @@ func (m *EncryptionKeyTarget) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - l = len(m.Cid) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + if m.Height != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Height)) } n += len(m.unknownFields) return n @@ -1687,10 +1684,10 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { m.FieldName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Cid", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) } - var byteLen int + m.Height = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -1700,26 +1697,11 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.Height |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Cid = append(m.Cid[:0], dAtA[iNdEx:postIndex]...) - if m.Cid == nil { - m.Cid = []byte{} - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/net/server.go b/net/server.go index ba26a9c335..cd7c2cbea5 100644 --- a/net/server.go +++ b/net/server.go @@ -16,6 +16,7 @@ import ( "context" "crypto/ecdh" "crypto/sha256" + "encoding/binary" "fmt" "sync" @@ -207,7 +208,7 @@ func (s *server) getEncryptionKeys( optFieldName = immutable.Some(target.FieldName) } encKey, err := encryption.GetKey( - encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName) + encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName, target.Height) if err != nil { return nil, nil, err } @@ -280,7 +281,9 @@ func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { for _, target := range res.Targets { hash.Write(target.DocID) hash.Write([]byte(target.FieldName)) - hash.Write(target.Cid) + heightBytes := make([]byte, 8) + binary.BigEndian.PutUint64(heightBytes, target.Height) + hash.Write(heightBytes) } return hash.Sum(nil) } @@ -447,11 +450,13 @@ func (s *server) prepareFetchEncryptionKeyRequest( EphemeralPublicKey: ephemeralPublicKey, } - for key, cid := range evt.Keys { + for _, encStoreKey := range evt.Keys { encKey := &pb.EncryptionKeyTarget{ - DocID: []byte(key.DocID), - FieldName: key.FieldName, - Cid: cid.Bytes(), + DocID: []byte(encStoreKey.DocID), + Height: encStoreKey.BlockHeight, + } + if encStoreKey.FieldName.HasValue() { + encKey.FieldName = encStoreKey.FieldName.Value() } req.Targets = append(req.Targets, encKey) } @@ -512,7 +517,9 @@ func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { for _, target := range req.Targets { hash.Write(target.DocID) hash.Write([]byte(target.FieldName)) - hash.Write(target.Cid) + heightBytes := make([]byte, 8) + binary.BigEndian.PutUint64(heightBytes, target.Height) + hash.Write(heightBytes) } return hash.Sum(nil) } @@ -554,14 +561,8 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - eventData := make(map[core.EncStoreDocKey]encryption.RequestedKeyEventData) + eventData := make(map[core.EncStoreDocKey][]byte) for _, target := range keyResp.Targets { - cid, err := cid.Cast(target.Cid) - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to parse CID", err) - return - } - optFieldName := immutable.None[string]() if target.FieldName != "" { optFieldName = immutable.Some(target.FieldName) @@ -574,8 +575,7 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - d := encryption.RequestedKeyEventData{Cid: cid, Key: encKey} - eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName)] = d + eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, target.Height)] = encKey } s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 444165e036..12a363c8b5 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -48,7 +48,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", + "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -58,7 +58,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", + "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -68,7 +68,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreia6fqubivj2ms4xwounxsyjr6coxeb2wkcmzaa7yzk3h5cwsj5g6a", + "cid": "bafyreibdoxaj6yeybokybtjl3mprzryjzufoxzhiumibuvtvpsylf5hufi", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -77,12 +77,12 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", - "name": "name", + "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "name": "age", }, { - "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", - "name": "age", + "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "name": "name", }, }, }, diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 4f40b8cde5..f02854c885 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -97,7 +97,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", + "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -107,7 +107,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", + "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -117,7 +117,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreia6fqubivj2ms4xwounxsyjr6coxeb2wkcmzaa7yzk3h5cwsj5g6a", + "cid": "bafyreibdoxaj6yeybokybtjl3mprzryjzufoxzhiumibuvtvpsylf5hufi", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -126,12 +126,12 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreic76n53ygh5ljaoa2njmllalgtozfquhzbjonz5d4sdrjza2kmvle", - "name": "name", + "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "name": "age", }, { - "cid": "bafyreicuxrw3lnccwdzvfgmqwlg67eolma6oyspyejfkxagtuukjrymxlu", - "name": "age", + "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "name": "name", }, }, }, From a1272983c3b77cce6edb3465e62cf07a927dc3a9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 14:48:12 +0200 Subject: [PATCH 18/88] Make failed tests show this 'path: commits[2].links[1].cid' instead of this 'doc: 1' --- tests/integration/assert_stack.go | 53 +++++++++++++++++++++++++++++++ tests/integration/utils2.go | 19 ++++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 tests/integration/assert_stack.go diff --git a/tests/integration/assert_stack.go b/tests/integration/assert_stack.go new file mode 100644 index 0000000000..0662db3390 --- /dev/null +++ b/tests/integration/assert_stack.go @@ -0,0 +1,53 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tests + +import ( + "strconv" + "strings" +) + +type assertStack struct { + stack []string + isMap []bool +} + +func (a *assertStack) pushMap(key string) { + a.stack = append(a.stack, key) + a.isMap = append(a.isMap, true) +} + +func (a *assertStack) pushArray(index int) { + a.stack = append(a.stack, strconv.Itoa(index)) + a.isMap = append(a.isMap, false) +} + +func (a *assertStack) pop() { + a.stack = a.stack[:len(a.stack)-1] + a.isMap = a.isMap[:len(a.isMap)-1] +} + +func (a *assertStack) String() string { + var b strings.Builder + for i, key := range a.stack { + if a.isMap[i] { + if i > 0 { + b.WriteString(".") + } + b.WriteString(key) + } else { + b.WriteString("[") + b.WriteString(key) + b.WriteString("]") + } + } + return b.String() +} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 2cc5a2c627..c5e90bb706 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1866,7 +1866,9 @@ func assertRequestResults( keys[key] = struct{}{} } + stack := &assertStack{} for key := range keys { + stack.pushMap(key) expect, ok := expectedResults[key] require.True(s.t, ok, "expected key not found: %s", key) @@ -1881,16 +1883,19 @@ func assertRequestResults( nodeID, expectDocs, actualDocs, - anyOfByField) + anyOfByField, + stack, + ) } else { assertResultsEqual( s.t, s.clientType, expect, actual, - fmt.Sprintf("node: %v, key: %v", nodeID, key), + fmt.Sprintf("node: %v, path: %s", nodeID, stack), ) } + stack.pop() } return false @@ -1902,12 +1907,14 @@ func assertRequestResultDocs( expectedResults []map[string]any, actualResults []map[string]any, anyOfByField map[docFieldKey][]any, + stack *assertStack, ) bool { // compare results require.Equal(s.t, len(expectedResults), len(actualResults), s.testCase.Description+" \n(number of results don't match)") for actualDocIndex, actualDoc := range actualResults { + stack.pushArray(actualDocIndex) expectedDoc := expectedResults[actualDocIndex] require.Equal( @@ -1922,6 +1929,7 @@ func assertRequestResultDocs( ) for field, actualValue := range actualDoc { + stack.pushMap(field) switch expectedValue := expectedDoc[field].(type) { case AnyOf: assertResultsAnyOf(s.t, s.clientType, expectedValue, actualValue) @@ -1937,7 +1945,7 @@ func assertRequestResultDocs( s.clientType, expectedDocID, actualValue, - fmt.Sprintf("node: %v, doc: %v", nodeID, actualDocIndex), + fmt.Sprintf("node: %v, path: %s", nodeID, stack), ) case []map[string]any: actualValueMap := ConvertToArrayOfMaps(s.t, actualValue) @@ -1948,6 +1956,7 @@ func assertRequestResultDocs( expectedValue, actualValueMap, anyOfByField, + stack, ) default: @@ -1956,10 +1965,12 @@ func assertRequestResultDocs( s.clientType, expectedValue, actualValue, - fmt.Sprintf("node: %v, doc: %v", nodeID, actualDocIndex), + fmt.Sprintf("node: %v, path: %s", nodeID, stack), ) } + stack.pop() } + stack.pop() } return false From f62156c2507abf8867e216ff16c6adca5562e9c6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 16:11:49 +0200 Subject: [PATCH 19/88] Merge blocks starting from the first encrypted --- internal/core/block/block.go | 11 ++ internal/db/merge.go | 118 ++++++++++++----- .../integration/encryption/peer_share_test.go | 123 ++++++++++++++++++ 3 files changed, 222 insertions(+), 30 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 1f1e9c4057..74ff74fff8 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -25,6 +25,7 @@ import ( "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/core/crdt" + "github.com/sourcenetwork/immutable" ) // Schema is the IPLD schema type that represents a `Block`. @@ -128,6 +129,16 @@ func (b *Block) IsEncrypted() bool { return b.Encryption != nil && (*b.Encryption).Type != NotEncrypted } +// GetPrevBlockCid returns the CID of the previous block. +func (b *Block) GetPrevBlockCid() immutable.Option[cid.Cid] { + for _, link := range b.Links { + if link.Name == core.HEAD { + return immutable.Some(link.Cid) + } + } + return immutable.None[cid.Cid]() +} + // IPLDSchemaBytes returns the IPLD schema representation for the block. // // This needs to match the [Block] struct or [mustSetSchema] will panic on init. diff --git a/internal/db/merge.go b/internal/db/merge.go index 526de1e867..f1feedeb14 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -107,8 +107,8 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) type mergeGroup struct { - includeComposite bool - fields []string + compositeKey core.EncStoreDocKey + fieldsKeys []core.EncStoreDocKey } mergeGroups := make(map[string]mergeGroup) @@ -117,9 +117,9 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR g := mergeGroups[encStoreKey.DocID] if encStoreKey.FieldName.HasValue() { - g.fields = append(g.fields, encStoreKey.FieldName.Value()) + g.fieldsKeys = append(g.fieldsKeys, encStoreKey) } else { - g.includeComposite = true + g.compositeKey = encStoreKey } mergeGroups[encStoreKey.DocID] = g @@ -137,39 +137,46 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR return err } - // if we need to process the composite block, we load only it - // as the children will be loaded by the merge processor + var blocks []*coreblock.Block + // if merge ground includes composite, we process only the composite block and skip the field + // blocks, as they will be processed as part of the composite block anyway. // Otherwise, we load the blocks for each field - if mergeGroup.includeComposite { - mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) + if mergeGroup.compositeKey.DocID != "" { + cids, err := getHeads(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) if err != nil { return err } - for _, block := range mt.heads { - mp.blocks.PushFront(block) - break + blocks, err = loadBlocksChainFromBlockstoreTillHeight(ctx, txn, cids, mergeGroup.compositeKey.BlockHeight) + if err != nil { + return err } } else { - for _, fieldName := range mergeGroup.fields { - fd, ok := mp.col.Definition().GetFieldByName(fieldName) + for _, fieldStoreKey := range mergeGroup.fieldsKeys { + fd, ok := mp.col.Definition().GetFieldByName(fieldStoreKey.FieldName.Value()) if !ok { - return client.NewErrFieldNotExist(fieldName) + return client.NewErrFieldNotExist(fieldStoreKey.FieldName.Value()) } fieldDsKey := dsKey.WithFieldId(fd.ID.String()) - mt, err := getHeadsAsMergeTarget(ctx, txn, fieldDsKey) + + cids, err := getHeads(ctx, txn, fieldDsKey) if err != nil { return err } - for _, block := range mt.heads { - mp.blocks.PushFront(block) - break + fieldBlocks, err := loadBlocksChainFromBlockstoreTillHeight(ctx, txn, cids, fieldStoreKey.BlockHeight) + if err != nil { + return err } + + blocks = append(blocks, fieldBlocks...) } } + for _, block := range blocks { + mp.blocks.PushFront(block) + } err = mp.mergeBlocks(ctx, true) if err != nil { return err @@ -300,12 +307,11 @@ func (mp *mergeProcessor) loadBlocks( // In this case, we also need to walk back the merge target's DAG until we reach a common block. if block.Delta.GetPriority() >= mt.headHeight { mp.blocks.PushFront(block) - for _, link := range block.Links { - if link.Name == core.HEAD { - err := mp.loadBlocks(ctx, link.Cid, mt, willDecrypt) + prevCid := block.GetPrevBlockCid() + if prevCid.HasValue() { + err := mp.loadBlocks(ctx, prevCid.Value(), mt, willDecrypt) if err != nil { return err - } } } } else { @@ -545,21 +551,15 @@ func getCollectionFromRootSchema(ctx context.Context, db *db, rootSchema string) // getHeadsAsMergeTarget retrieves the heads of the composite DAG for the given document // and returns them as a merge target. func getHeadsAsMergeTarget(ctx context.Context, txn datastore.Txn, dsKey core.DataStoreKey) (mergeTarget, error) { - headset := clock.NewHeadSet(txn.Headstore(), dsKey.ToHeadStoreKey()) + cids, err := getHeads(ctx, txn, dsKey) - cids, _, err := headset.List(ctx) if err != nil { return mergeTarget{}, err } mt := newMergeTarget() for _, cid := range cids { - b, err := txn.Blockstore().Get(ctx, cid) - if err != nil { - return mergeTarget{}, err - } - - block, err := coreblock.GetFromBytes(b.RawData()) + block, err := loadBlockFromBlockStore(ctx, txn, cid) if err != nil { return mergeTarget{}, err } @@ -571,6 +571,64 @@ func getHeadsAsMergeTarget(ctx context.Context, txn datastore.Txn, dsKey core.Da return mt, nil } +// getHeads retrieves the heads associated with the given datastore key. +func getHeads(ctx context.Context, txn datastore.Txn, dsKey core.DataStoreKey) ([]cid.Cid, error) { + headset := clock.NewHeadSet(txn.Headstore(), dsKey.ToHeadStoreKey()) + + cids, _, err := headset.List(ctx) + if err != nil { + return nil, err + } + + return cids, nil +} + +// loadBlockFromBlockStore loads a block from the blockstore. +func loadBlockFromBlockStore(ctx context.Context, txn datastore.Txn, cid cid.Cid) (*coreblock.Block, error) { + b, err := txn.Blockstore().Get(ctx, cid) + if err != nil { + return nil, err + } + + block, err := coreblock.GetFromBytes(b.RawData()) + if err != nil { + return nil, err + } + + return block, nil +} + +// loadBlocksChainFromBlockstoreTillHeight loads the blocks from the blockstore starting from the given CIDs +// until it reaches a block with a height equal to the given height (including that block). +// The returned blocks are ordered from the highest height to the lowest. +func loadBlocksChainFromBlockstoreTillHeight( + ctx context.Context, + txn datastore.Txn, + cids []cid.Cid, + height uint64, +) ([]*coreblock.Block, error) { + var blocks []*coreblock.Block + for len(cids) > 0 { + cid := cids[0] + block, err := loadBlockFromBlockStore(ctx, txn, cid) + if err != nil { + return nil, err + } + + if block.Delta.GetPriority() >= height { + blocks = append(blocks, block) + if block.Delta.GetPriority() != height { + prevCid := block.GetPrevBlockCid() + if prevCid.HasValue() { + cids = append(cids, prevCid.Value()) + } + } + } + cids = cids[1:] + } + return blocks, nil +} + func syncIndexedDoc( ctx context.Context, docID client.DocID, diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 1f116b2e58..8be09b4877 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -283,3 +283,126 @@ func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_Should testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecryptAndCorrectlyMerge(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int @crdt(type: "pcounter") + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + EncryptedFields: []string{"age"}, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"age": 3}`, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"age": 2}`, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "name": "John", + "age": int64(26), + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_ShouldDecryptAndCorrectlyMerge(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int @crdt(type: "pcounter") + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"age": 3}`, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"age": 2}`, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, + }, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + age + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "name": "John", + "age": int64(26), + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + From cf4ac1c67887ee31ed478df016669b56f68bcec7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 17:29:56 +0200 Subject: [PATCH 20/88] Add more options to AES encryption/decryption --- crypto/aes.go | 34 ++++++++++++++++----------- crypto/ecies.go | 4 ++-- internal/encryption/encryptor.go | 5 ++-- tests/integration/encryption/utils.go | 2 +- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/crypto/aes.go b/crypto/aes.go index 3caed84fe1..0f1d299204 100644 --- a/crypto/aes.go +++ b/crypto/aes.go @@ -18,33 +18,42 @@ import ( // EncryptAES encrypts data using AES-GCM with a provided key. // The nonce is prepended to the cipherText. -func EncryptAES(plainText, key []byte) ([]byte, error) { +func EncryptAES(plainText, key, additionalData []byte, prependNonce bool) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { - return nil, err + return nil, nil, err } nonce, err := generateNonceFunc() if err != nil { - return nil, err + return nil, nil, err } aesGCM, err := cipher.NewGCM(block) if err != nil { - return nil, err + return nil, nil, err } - cipherText := aesGCM.Seal(nonce, nonce, plainText, nil) + var cipherText []byte + if prependNonce { + cipherText = aesGCM.Seal(nonce, nonce, plainText, additionalData) + } else { + cipherText = aesGCM.Seal(nil, nonce, plainText, additionalData) + } - return cipherText, nil + return cipherText, nonce, nil } // DecryptAES decrypts AES-GCM encrypted data with a provided key. // The nonce is expected to be prepended to the cipherText. -func DecryptAES(cipherText, key []byte) ([]byte, error) { - if len(cipherText) < AESNonceSize { - // TODO return typed error - return nil, fmt.Errorf("cipherText too short") +func DecryptAES(nonce, cipherText, key, additionalData []byte) ([]byte, error) { + if len(nonce) == 0 { + if len(cipherText) < AESNonceSize { + // TODO return typed error + return nil, fmt.Errorf("cipherText too short") + } + nonce = cipherText[:AESNonceSize] + cipherText = cipherText[AESNonceSize:] } block, err := aes.NewCipher(key) @@ -52,15 +61,12 @@ func DecryptAES(cipherText, key []byte) ([]byte, error) { return nil, err } - nonce := cipherText[:AESNonceSize] - cipherText = cipherText[AESNonceSize:] - aesGCM, err := cipher.NewGCM(block) if err != nil { return nil, err } - plainText, err := aesGCM.Open(nil, nonce, cipherText, nil) + plainText, err := aesGCM.Open(nil, nonce, cipherText, additionalData) if err != nil { return nil, err } diff --git a/crypto/ecies.go b/crypto/ecies.go index dde50479ed..490d8251d7 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -61,7 +61,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) } - cipherText, err := EncryptAES(plainText, aesKey) + cipherText, _, err := EncryptAES(plainText, aesKey, nil, true) if err != nil { return nil, fmt.Errorf("failed to encrypt: %w", err) } @@ -112,7 +112,7 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error return nil, fmt.Errorf("HMAC verification failed") } - plainText, err := DecryptAES(cipherText[X25519PublicKeySize:], aesKey) + plainText, err := DecryptAES(nil, cipherText[X25519PublicKeySize:], aesKey, nil) if err != nil { return nil, fmt.Errorf("failed to decrypt: %w", err) } diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 4757df4edd..fe7585821b 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -136,7 +136,8 @@ func (d *DocEncryptor) Encrypt( return nil, err } } - return crypto.EncryptAES(plainText, encryptionKey) + cipherText, _, err := crypto.EncryptAES(plainText, encryptionKey, nil, true) + return cipherText, err } // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. @@ -154,7 +155,7 @@ func (d *DocEncryptor) Decrypt( if len(encKey) == 0 { return nil, nil } - return crypto.DecryptAES(cipherText, encKey) + return crypto.DecryptAES(nil, cipherText, encKey, nil) } func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, error) { diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go index a03e0e92e7..685a2567fb 100644 --- a/tests/integration/encryption/utils.go +++ b/tests/integration/encryption/utils.go @@ -50,6 +50,6 @@ func updateUserCollectionSchema() testUtils.SchemaUpdate { func encrypt(plaintext []byte, docID, fieldName string) []byte { const keyLength = 32 const testEncKey = "examplekey1234567890examplekey12" - val, _ := crypto.EncryptAES(plaintext, []byte(fieldName + docID + testEncKey)[0:keyLength]) + val, _, _ := crypto.EncryptAES(plaintext, []byte(fieldName + docID + testEncKey)[0:keyLength], nil, true) return val } From e5e5fc2061a0579924fa65dc1475eb0eb08419b3 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 19:30:20 +0200 Subject: [PATCH 21/88] Add associated data to ECIES --- crypto/ecies.go | 57 +++++++++++++++++++++++++++++++++++++++++++------ net/server.go | 39 ++++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index 490d8251d7..d57008e6eb 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -31,15 +31,37 @@ const AESKeySize = 32 const minCipherTextSize = 16 +// GenerateX25519 generates a new X25519 private key. func GenerateX25519() (*ecdh.PrivateKey, error) { return ecdh.X25519().GenerateKey(rand.Reader) } +// X25519PublicKeyFromBytes creates a new X25519 public key from the given bytes. func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { return ecdh.X25519().NewPublicKey(publicKeyBytes) } -func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { +// EncryptECIES encrypts plaintext using a custom Elliptic Curve Integrated Encryption Scheme (ECIES) +// with X25519 for key agreement, HKDF for key derivation, AES for encryption, and HMAC for authentication. +// +// The function: +// - Generates an ephemeral X25519 key pair +// - Performs ECDH with the provided public key +// - Derives encryption and HMAC keys using HKDF +// - Encrypts the plaintext using a custom AES encryption function +// - Computes an HMAC over the ciphertext +// +// The output format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// +// Parameters: +// - plainText: The message to encrypt +// - publicKey: The recipient's X25519 public key +// - associatedData: Optional associated data for additional authentication +// +// Returns: +// - Byte slice containing the encrypted message and necessary metadata for decryption +// - Error if any step of the encryption process fails +func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData []byte) ([]byte, error) { ephemeralPrivate, err := GenerateX25519() if err != nil { return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) @@ -61,13 +83,14 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) } - cipherText, _, err := EncryptAES(plainText, aesKey, nil, true) + fullAssociatedData := append(ephemeralPublic.Bytes(), associatedData...) + cipherText, _, err := EncryptAES(plainText, aesKey, fullAssociatedData, true) if err != nil { return nil, fmt.Errorf("failed to encrypt: %w", err) } mac := hmac.New(sha256.New, hmacKey) - mac.Write(cipherText[AESNonceSize:]) + mac.Write(cipherText) macSum := mac.Sum(nil) result := append(ephemeralPublic.Bytes(), cipherText...) @@ -76,7 +99,26 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey) ([]byte, error) { return result, nil } -func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error) { +// DecryptECIES decrypts ciphertext encrypted with EncryptECIES using the provided private key. +// +// The function: +// - Extracts the ephemeral public key from the ciphertext +// - Performs ECDH with the provided private key +// - Derives decryption and HMAC keys using HKDF +// - Verifies the HMAC +// - Decrypts the message using a custom AES decryption function +// +// The expected input format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// +// Parameters: +// - cipherText: The encrypted message, including all necessary metadata +// - privateKey: The recipient's X25519 private key +// - associatedData: Optional associated data used during encryption for additional authentication +// +// Returns: +// - Byte slice containing the decrypted plaintext +// - Error if any step of the decryption process fails, including authentication failure +func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData []byte) ([]byte, error) { if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { return nil, fmt.Errorf("ciphertext too short") } @@ -103,16 +145,17 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey) ([]byte, error } macSum := cipherText[len(cipherText)-HMACSize:] - cipherText = cipherText[:len(cipherText)-HMACSize] + cipherTextWithNonce := cipherText[X25519PublicKeySize : len(cipherText)-HMACSize] mac := hmac.New(sha256.New, hmacKey) - mac.Write(cipherText[X25519PublicKeySize+AESNonceSize:]) + mac.Write(cipherTextWithNonce) expectedMAC := mac.Sum(nil) if !hmac.Equal(macSum, expectedMAC) { return nil, fmt.Errorf("HMAC verification failed") } - plainText, err := DecryptAES(nil, cipherText[X25519PublicKeySize:], aesKey, nil) + fullAssociatedData := append(ephemeralPublicBytes, associatedData...) + plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, fullAssociatedData) if err != nil { return nil, fmt.Errorf("failed to decrypt: %w", err) } diff --git a/net/server.go b/net/server.go index cd7c2cbea5..31774d2eda 100644 --- a/net/server.go +++ b/net/server.go @@ -13,6 +13,7 @@ package net import ( + "bytes" "context" "crypto/ecdh" "crypto/sha256" @@ -39,7 +40,6 @@ import ( "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" - "github.com/sourcenetwork/defradb/internal/encoding" "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -218,7 +218,7 @@ func (s *server) getEncryptionKeys( continue } targets = append(targets, target) - encryptionKeys = encoding.EncodeBytesAscending(encryptionKeys, encKey) + encryptionKeys = append(encryptionKeys, encKey...) } return encryptionKeys, targets, nil } @@ -239,8 +239,8 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.New("invalid signature") } - encKey, targets, err := s.getEncryptionKeys(ctx, req) - if err != nil || len(encKey) == 0 { + encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) + if err != nil || len(encryptionKeys) == 0 { return nil, err } @@ -249,7 +249,7 @@ func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptio return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) } - encryptedKey, err := crypto.EncryptECIES(encKey, reqEphPubKey) + encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peer.PeerID())) if err != nil { return nil, errors.Wrap("failed to encrypt key for requester", err) } @@ -555,12 +555,22 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - decryptedData, err := crypto.DecryptECIES(keyResp.EncryptedKeys, session.privateKey) + decryptedData, err := crypto.DecryptECIES( + keyResp.EncryptedKeys, + session.privateKey, + makeAssociatedData(req, resp.From), + ) + if err != nil { log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) return } + if len(decryptedData) != crypto.AESKeySize*len(keyResp.Targets) { + log.ErrorContext(s.peer.ctx, "Invalid decrypted data length") + return + } + eventData := make(map[core.EncStoreDocKey][]byte) for _, target := range keyResp.Targets { optFieldName := immutable.None[string]() @@ -568,12 +578,8 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet optFieldName = immutable.Some(target.FieldName) } - var encKey []byte - decryptedData, encKey, err = encoding.DecodeBytesAscending(decryptedData) - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to decode encrypted key", err) - return - } + encKey := decryptedData[:crypto.AESKeySize] + decryptedData = decryptedData[crypto.AESKeySize:] eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, target.Height)] = encKey } @@ -581,6 +587,15 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) } +// makeAssociatedData creates the associated data for the encryption key request +func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) []byte { + return bytes.Join([][]byte{ + []byte(req.SchemaRoot), + []byte(req.EphemeralPublicKey), + []byte(peerID), + }, []byte{}) +} + func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { pubKey := s.peer.host.Peerstore().PubKey(fromPeer) return pubKey.Verify(hashFetchEncryptionKeyReply(res), res.Signature) From 8bd388096f1bc108f1ecf8a07df2f434dcf04b86 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 19:49:00 +0200 Subject: [PATCH 22/88] Improve session handling --- net/server.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/net/server.go b/net/server.go index 31774d2eda..d19113627a 100644 --- a/net/server.go +++ b/net/server.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "fmt" "sync" + "time" cid "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" @@ -67,9 +68,16 @@ type server struct { sessions []session } +const sessionTimeout = 5 * time.Second + type session struct { id string privateKey *ecdh.PrivateKey + t time.Time +} + +func newSession(id string, privateKey *ecdh.PrivateKey) session { + return session{id: id, privateKey: privateKey, t: time.Now()} } // pubsubTopic is a wrapper of rpc.Topic to be able to track if the topic has @@ -100,13 +108,21 @@ func newServer(p *Peer, opts ...grpc.DialOption) (*server, error) { return s, nil } -func (s *server) findSession(id string) *session { - for _, sess := range s.sessions { - if sess.id == id { - return &sess +func (s *server) extractSessionAndRemoveOldOnes(id string) *session { + var result *session + swapLast := func(i int) { + s.sessions[i] = s.sessions[len(s.sessions)-1] + s.sessions = s.sessions[:len(s.sessions)-1] + } + for i, session := range s.sessions { + if session.id == id { + result = &session + swapLast(i) + } else if time.Since(session.t) > sessionTimeout { + swapLast(i) } } - return nil + return result } // GetDocGraph receives a get graph request @@ -501,7 +517,7 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) } - s.sessions = append(s.sessions, session{id: string(ephPubKeyBytes), privateKey: ephPrivKey}) + s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), ephPrivKey)) go func() { s.handleFetchEncryptionKeyResponse(<-respChan, req) @@ -548,8 +564,7 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - // TODO: properly handle sessions. At least remove used or old ones - session := s.findSession(string(keyResp.ReqEphemeralPublicKey)) + session := s.extractSessionAndRemoveOldOnes(string(keyResp.ReqEphemeralPublicKey)) if session == nil { log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") return From 5e72e9cb9929e8a2f1804c6d929133268542710a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 22:57:52 +0200 Subject: [PATCH 23/88] Split server into 2 files --- net/server.go | 314 -------------------------------- net/server_encryption_key.go | 342 +++++++++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+), 314 deletions(-) create mode 100644 net/server_encryption_key.go diff --git a/net/server.go b/net/server.go index d19113627a..1a22aae29b 100644 --- a/net/server.go +++ b/net/server.go @@ -13,11 +13,8 @@ package net import ( - "bytes" "context" "crypto/ecdh" - "crypto/sha256" - "encoding/binary" "fmt" "sync" "time" @@ -28,25 +25,18 @@ import ( "github.com/libp2p/go-libp2p/core/peerstore" "github.com/sourcenetwork/corelog" rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" - "github.com/sourcenetwork/immutable" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" grpcpeer "google.golang.org/grpc/peer" "google.golang.org/protobuf/proto" - libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" - "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) -const encryptionTopic = "encryption" - // Server is the request/response instance for all P2P RPC communication. // Implements gRPC server. See net/pb/net.proto for corresponding service definitions. // @@ -207,108 +197,6 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL return &pb.PushLogReply{}, nil } -func (s *server) getEncryptionKeys( - ctx context.Context, - req *pb.FetchEncryptionKeyRequest, -) ([]byte, []*pb.EncryptionKeyTarget, error) { - encryptionKeys := make([]byte, 0) - targets := make([]*pb.EncryptionKeyTarget, 0) - for _, target := range req.Targets { - docID, err := client.NewDocIDFromString(string(target.DocID)) - if err != nil { - return nil, nil, err - } - - optFieldName := immutable.None[string]() - if target.FieldName != "" { - optFieldName = immutable.Some(target.FieldName) - } - encKey, err := encryption.GetKey( - encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName, target.Height) - if err != nil { - return nil, nil, err - } - // TODO: we should test it somehow. For this this one peer should have some keys and - // another one should have the others - if len(encKey) == 0 { - continue - } - targets = append(targets, target) - encryptionKeys = append(encryptionKeys, encKey...) - } - return encryptionKeys, targets, nil -} - -func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { - peerID, err := peerIDFromContext(ctx) - if err != nil { - return nil, err - } - reqPubKey := s.peer.host.Peerstore().PubKey(peerID) - - isValid, err := s.verifyRequestSignature(req, reqPubKey) - if err != nil { - return nil, errors.Wrap("invalid signature", err) - } - - if !isValid { - return nil, errors.New("invalid signature") - } - - encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) - if err != nil || len(encryptionKeys) == 0 { - return nil, err - } - - reqEphPubKey, err := crypto.X25519PublicKeyFromBytes(req.EphemeralPublicKey) - if err != nil { - return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) - } - - encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peer.PeerID())) - if err != nil { - return nil, errors.Wrap("failed to encrypt key for requester", err) - } - - res := &pb.FetchEncryptionKeyReply{ - SchemaRoot: req.SchemaRoot, - ReqEphemeralPublicKey: req.EphemeralPublicKey, - Targets: targets, - EncryptedKeys: encryptedKey, - } - - res.Signature, err = s.signResponse(res) - if err != nil { - return nil, errors.Wrap("failed to sign response", err) - } - - return res, nil -} - -func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKey libp2pCrypto.PubKey) (bool, error) { - return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) -} - -func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { - hash := sha256.New() - hash.Write(res.EncryptedKeys) - hash.Write(res.SchemaRoot) - hash.Write(res.ReqEphemeralPublicKey) - for _, target := range res.Targets { - hash.Write(target.DocID) - hash.Write([]byte(target.FieldName)) - heightBytes := make([]byte, 8) - binary.BigEndian.PutUint64(heightBytes, target.Height) - hash.Write(heightBytes) - } - return hash.Sum(nil) -} - -func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { - privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - return privKey.Sign(hashFetchEncryptionKeyReply(res)) -} - // GetHeadLog receives a get head log request func (s *server) GetHeadLog( ctx context.Context, @@ -355,30 +243,6 @@ func (s *server) addPubSubTopic(topic string, subscribe bool) error { return nil } -// addPubSubEncryptionTopic subscribes to a topic on the pubsub network -func (s *server) addPubSubEncryptionTopic() error { - if s.peer.ps == nil { - return nil - } - - t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), encryptionTopic, true) - if err != nil { - return err - } - - t.SetEventHandler(s.pubSubEventHandler) - t.SetMessageHandler(s.pubSubEncryptionMessageHandler) - - s.mu.Lock() - defer s.mu.Unlock() - - s.topics[encryptionTopic] = pubsubTopic{ - Topic: t, - subscribed: true, - } - return nil -} - // hasPubSubTopic checks if we are subscribed to a topic. func (s *server) hasPubSubTopic(topic string) bool { s.mu.Lock() @@ -457,165 +321,6 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe return nil } -func (s *server) prepareFetchEncryptionKeyRequest( - evt encryption.RequestKeysEvent, - ephemeralPublicKey []byte, -) (*pb.FetchEncryptionKeyRequest, error) { - req := &pb.FetchEncryptionKeyRequest{ - SchemaRoot: []byte(evt.SchemaRoot), - EphemeralPublicKey: ephemeralPublicKey, - } - - for _, encStoreKey := range evt.Keys { - encKey := &pb.EncryptionKeyTarget{ - DocID: []byte(encStoreKey.DocID), - Height: encStoreKey.BlockHeight, - } - if encStoreKey.FieldName.HasValue() { - encKey.FieldName = encStoreKey.FieldName.Value() - } - req.Targets = append(req.Targets, encKey) - } - - signature, err := s.signRequest(req) - if err != nil { - return nil, errors.Wrap("failed to sign request", err) - } - - req.Signature = signature - - return req, nil -} - -// requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeysEvent) error { - if s.peer.ps == nil { // skip if we aren't running with a pubsub net - return nil - } - - ephPrivKey, err := crypto.GenerateX25519() - if err != nil { - return err - } - - ephPubKeyBytes := ephPrivKey.PublicKey().Bytes() - req, err := s.prepareFetchEncryptionKeyRequest(evt, ephPubKeyBytes) - if err != nil { - return err - } - - data, err := req.MarshalVT() - if err != nil { - return errors.Wrap("failed to marshal pubsub message", err) - } - - s.mu.Lock() - t := s.topics[encryptionTopic] - s.mu.Unlock() - respChan, err := t.Publish(ctx, data) - if err != nil { - return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) - } - - s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), ephPrivKey)) - - go func() { - s.handleFetchEncryptionKeyResponse(<-respChan, req) - }() - - return nil -} - -func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { - hash := sha256.New() - hash.Write(req.SchemaRoot) - hash.Write(req.EphemeralPublicKey) - for _, target := range req.Targets { - hash.Write(target.DocID) - hash.Write([]byte(target.FieldName)) - heightBytes := make([]byte, 8) - binary.BigEndian.PutUint64(heightBytes, target.Height) - hash.Write(heightBytes) - } - return hash.Sum(nil) -} - -func (s *server) signRequest(req *pb.FetchEncryptionKeyRequest) ([]byte, error) { - privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - return privKey.Sign(hashFetchEncryptionKeyRequest(req)) -} - -// handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages -func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.FetchEncryptionKeyRequest) { - var keyResp pb.FetchEncryptionKeyReply - if err := proto.Unmarshal(resp.Data, &keyResp); err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to unmarshal encryption key response", err) - return - } - - isValid, err := s.verifyResponseSignature(&keyResp, resp.From) - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to verify response signature", err) - return - } - - if !isValid { - log.ErrorContext(s.peer.ctx, "Invalid response signature") - return - } - - session := s.extractSessionAndRemoveOldOnes(string(keyResp.ReqEphemeralPublicKey)) - if session == nil { - log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") - return - } - - decryptedData, err := crypto.DecryptECIES( - keyResp.EncryptedKeys, - session.privateKey, - makeAssociatedData(req, resp.From), - ) - - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) - return - } - - if len(decryptedData) != crypto.AESKeySize*len(keyResp.Targets) { - log.ErrorContext(s.peer.ctx, "Invalid decrypted data length") - return - } - - eventData := make(map[core.EncStoreDocKey][]byte) - for _, target := range keyResp.Targets { - optFieldName := immutable.None[string]() - if target.FieldName != "" { - optFieldName = immutable.Some(target.FieldName) - } - - encKey := decryptedData[:crypto.AESKeySize] - decryptedData = decryptedData[crypto.AESKeySize:] - - eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, target.Height)] = encKey - } - - s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) -} - -// makeAssociatedData creates the associated data for the encryption key request -func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) []byte { - return bytes.Join([][]byte{ - []byte(req.SchemaRoot), - []byte(req.EphemeralPublicKey), - []byte(peerID), - }, []byte{}) -} - -func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { - pubKey := s.peer.host.Peerstore().PubKey(fromPeer) - return pubKey.Verify(hashFetchEncryptionKeyReply(res), res.Signature) -} - // pubSubMessageHandler handles incoming PushLog messages from the pubsub network. func (s *server) pubSubMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { log.InfoContext(s.peer.ctx, "Received new pubsub event", @@ -637,25 +342,6 @@ func (s *server) pubSubMessageHandler(from libpeer.ID, topic string, msg []byte) return nil, nil } -// pubSubEncryptionMessageHandler handles incoming FetchEncryptionKeyRequest messages from the pubsub network. -func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { - req := new(pb.FetchEncryptionKeyRequest) - if err := proto.Unmarshal(msg, req); err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to unmarshal pubsub message %s", err) - return nil, err - } - - ctx := grpcpeer.NewContext(s.peer.ctx, &grpcpeer.Peer{ - Addr: addr{from}, - }) - res, err := s.TryGenEncryptionKey(ctx, req) - if err != nil { - return nil, errors.Wrap("Failed attempt to get encryption key", err) - } - return res.MarshalVT() - //return proto.Marshal(res) -} - // pubSubEventHandler logs events from the subscribed DocID topics. func (s *server) pubSubEventHandler(from libpeer.ID, topic string, msg []byte) { log.InfoContext(s.peer.ctx, "Received new pubsub event", diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go new file mode 100644 index 0000000000..6a527711b8 --- /dev/null +++ b/net/server_encryption_key.go @@ -0,0 +1,342 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package net + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" + libpeer "github.com/libp2p/go-libp2p/core/peer" + rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" + "github.com/sourcenetwork/immutable" + grpcpeer "google.golang.org/grpc/peer" + "google.golang.org/protobuf/proto" + + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/crypto" + "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/internal/core" + "github.com/sourcenetwork/defradb/internal/encryption" + pb "github.com/sourcenetwork/defradb/net/pb" +) + +const encryptionTopic = "encryption" + +// Server is the request/response instance for all P2P RPC communication. +// Implements gRPC server. See net/pb/net.proto for corresponding service definitions. +func (s *server) getEncryptionKeys( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) ([]byte, []*pb.EncryptionKeyTarget, error) { + encryptionKeys := make([]byte, 0) + targets := make([]*pb.EncryptionKeyTarget, 0) + for _, target := range req.Targets { + docID, err := client.NewDocIDFromString(string(target.DocID)) + if err != nil { + return nil, nil, err + } + + optFieldName := immutable.None[string]() + if target.FieldName != "" { + optFieldName = immutable.Some(target.FieldName) + } + encKey, err := encryption.GetKey( + encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName, target.Height) + if err != nil { + return nil, nil, err + } + // TODO: we should test it somehow. For this this one peer should have some keys and + // another one should have the others + if len(encKey) == 0 { + continue + } + targets = append(targets, target) + encryptionKeys = append(encryptionKeys, encKey...) + } + return encryptionKeys, targets, nil +} + +func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { + peerID, err := peerIDFromContext(ctx) + if err != nil { + return nil, err + } + reqPubKey := s.peer.host.Peerstore().PubKey(peerID) + + isValid, err := s.verifyRequestSignature(req, reqPubKey) + if err != nil { + return nil, errors.Wrap("invalid signature", err) + } + + if !isValid { + return nil, errors.New("invalid signature") + } + + encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) + if err != nil || len(encryptionKeys) == 0 { + return nil, err + } + + reqEphPubKey, err := crypto.X25519PublicKeyFromBytes(req.EphemeralPublicKey) + if err != nil { + return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) + } + + encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peer.PeerID())) + if err != nil { + return nil, errors.Wrap("failed to encrypt key for requester", err) + } + + res := &pb.FetchEncryptionKeyReply{ + SchemaRoot: req.SchemaRoot, + ReqEphemeralPublicKey: req.EphemeralPublicKey, + Targets: targets, + EncryptedKeys: encryptedKey, + } + + res.Signature, err = s.signResponse(res) + if err != nil { + return nil, errors.Wrap("failed to sign response", err) + } + + return res, nil +} + +func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKey libp2pCrypto.PubKey) (bool, error) { + return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) +} + +func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { + hash := sha256.New() + hash.Write(res.EncryptedKeys) + hash.Write(res.SchemaRoot) + hash.Write(res.ReqEphemeralPublicKey) + for _, target := range res.Targets { + hash.Write(target.DocID) + hash.Write([]byte(target.FieldName)) + heightBytes := make([]byte, 8) + binary.BigEndian.PutUint64(heightBytes, target.Height) + hash.Write(heightBytes) + } + return hash.Sum(nil) +} + +func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { + privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) + return privKey.Sign(hashFetchEncryptionKeyReply(res)) +} + +// addPubSubEncryptionTopic subscribes to a topic on the pubsub network +func (s *server) addPubSubEncryptionTopic() error { + if s.peer.ps == nil { + return nil + } + + t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), encryptionTopic, true) + if err != nil { + return err + } + + t.SetEventHandler(s.pubSubEventHandler) + t.SetMessageHandler(s.pubSubEncryptionMessageHandler) + + s.mu.Lock() + defer s.mu.Unlock() + + s.topics[encryptionTopic] = pubsubTopic{ + Topic: t, + subscribed: true, + } + return nil +} + +// pubSubEncryptionMessageHandler handles incoming FetchEncryptionKeyRequest messages from the pubsub network. +func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { + req := new(pb.FetchEncryptionKeyRequest) + if err := proto.Unmarshal(msg, req); err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to unmarshal pubsub message %s", err) + return nil, err + } + + ctx := grpcpeer.NewContext(s.peer.ctx, &grpcpeer.Peer{ + Addr: addr{from}, + }) + res, err := s.TryGenEncryptionKey(ctx, req) + if err != nil { + return nil, errors.Wrap("Failed attempt to get encryption key", err) + } + return res.MarshalVT() + //return proto.Marshal(res) +} + +func (s *server) prepareFetchEncryptionKeyRequest( + evt encryption.RequestKeysEvent, + ephemeralPublicKey []byte, +) (*pb.FetchEncryptionKeyRequest, error) { + req := &pb.FetchEncryptionKeyRequest{ + SchemaRoot: []byte(evt.SchemaRoot), + EphemeralPublicKey: ephemeralPublicKey, + } + + for _, encStoreKey := range evt.Keys { + encKey := &pb.EncryptionKeyTarget{ + DocID: []byte(encStoreKey.DocID), + Height: encStoreKey.BlockHeight, + } + if encStoreKey.FieldName.HasValue() { + encKey.FieldName = encStoreKey.FieldName.Value() + } + req.Targets = append(req.Targets, encKey) + } + + signature, err := s.signRequest(req) + if err != nil { + return nil, errors.Wrap("failed to sign request", err) + } + + req.Signature = signature + + return req, nil +} + +// requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network +func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeysEvent) error { + if s.peer.ps == nil { // skip if we aren't running with a pubsub net + return nil + } + + ephPrivKey, err := crypto.GenerateX25519() + if err != nil { + return err + } + + ephPubKeyBytes := ephPrivKey.PublicKey().Bytes() + req, err := s.prepareFetchEncryptionKeyRequest(evt, ephPubKeyBytes) + if err != nil { + return err + } + + data, err := req.MarshalVT() + if err != nil { + return errors.Wrap("failed to marshal pubsub message", err) + } + + s.mu.Lock() + t := s.topics[encryptionTopic] + s.mu.Unlock() + respChan, err := t.Publish(ctx, data) + if err != nil { + return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) + } + + s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), ephPrivKey)) + + go func() { + s.handleFetchEncryptionKeyResponse(<-respChan, req) + }() + + return nil +} + +func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { + hash := sha256.New() + hash.Write(req.SchemaRoot) + hash.Write(req.EphemeralPublicKey) + for _, target := range req.Targets { + hash.Write(target.DocID) + hash.Write([]byte(target.FieldName)) + heightBytes := make([]byte, 8) + binary.BigEndian.PutUint64(heightBytes, target.Height) + hash.Write(heightBytes) + } + return hash.Sum(nil) +} + +func (s *server) signRequest(req *pb.FetchEncryptionKeyRequest) ([]byte, error) { + privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) + return privKey.Sign(hashFetchEncryptionKeyRequest(req)) +} + +// handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages +func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.FetchEncryptionKeyRequest) { + var keyResp pb.FetchEncryptionKeyReply + if err := proto.Unmarshal(resp.Data, &keyResp); err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to unmarshal encryption key response", err) + return + } + + isValid, err := s.verifyResponseSignature(&keyResp, resp.From) + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to verify response signature", err) + return + } + + if !isValid { + log.ErrorContext(s.peer.ctx, "Invalid response signature") + return + } + + session := s.extractSessionAndRemoveOldOnes(string(keyResp.ReqEphemeralPublicKey)) + if session == nil { + log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") + return + } + + decryptedData, err := crypto.DecryptECIES( + keyResp.EncryptedKeys, + session.privateKey, + makeAssociatedData(req, resp.From), + ) + + if err != nil { + log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) + return + } + + if len(decryptedData) != crypto.AESKeySize*len(keyResp.Targets) { + log.ErrorContext(s.peer.ctx, "Invalid decrypted data length") + return + } + + eventData := make(map[core.EncStoreDocKey][]byte) + for _, target := range keyResp.Targets { + optFieldName := immutable.None[string]() + if target.FieldName != "" { + optFieldName = immutable.Some(target.FieldName) + } + + encKey := decryptedData[:crypto.AESKeySize] + decryptedData = decryptedData[crypto.AESKeySize:] + + eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, target.Height)] = encKey + } + + s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) +} + +// makeAssociatedData creates the associated data for the encryption key request +func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) []byte { + return bytes.Join([][]byte{ + []byte(req.SchemaRoot), + []byte(req.EphemeralPublicKey), + []byte(peerID), + }, []byte{}) +} + +func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { + pubKey := s.peer.host.Peerstore().PubKey(fromPeer) + return pubKey.Verify(hashFetchEncryptionKeyReply(res), res.Signature) +} From 7357e9af281785c7c716c6503906e5122c15f409 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 22:58:01 +0200 Subject: [PATCH 24/88] Polish --- internal/encryption/encryptor_test.go | 2 +- internal/encryption/nonce.go | 53 --------------------------- 2 files changed, 1 insertion(+), 54 deletions(-) delete mode 100644 internal/encryption/nonce.go diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 643a733f05..7738adfd9e 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -44,7 +44,7 @@ func getEncKey(fieldName immutable.Option[string]) []byte { } func getCipherText(t *testing.T, fieldName immutable.Option[string]) []byte { - cipherText, err := crypto.EncryptAES(getPlainText(), getEncKey(fieldName)) + cipherText, _, err := crypto.EncryptAES(getPlainText(), getEncKey(fieldName), nil, true) assert.NoError(t, err) return cipherText } diff --git a/internal/encryption/nonce.go b/internal/encryption/nonce.go deleted file mode 100644 index 67a5467a4e..0000000000 --- a/internal/encryption/nonce.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package encryption - -import ( - "crypto/rand" - "errors" - "io" - "os" - "strings" -) - -const nonceLength = 12 - -var generateNonceFunc = generateNonce - -func generateNonce() ([]byte, error) { - nonce := make([]byte, nonceLength) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return nil, err - } - - return nonce, nil -} - -// generateTestNonce generates a deterministic nonce for testing. -func generateTestNonce() ([]byte, error) { - nonce := []byte("deterministic nonce for testing") - - if len(nonce) < nonceLength { - return nil, errors.New("nonce length is longer than available deterministic nonce") - } - - return nonce[:nonceLength], nil -} - -func init() { - arg := os.Args[0] - // If the binary is a test binary, use a deterministic nonce. - // TODO: We should try to find a better way to detect this https://github.com/sourcenetwork/defradb/issues/2801 - if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { - generateNonceFunc = generateTestNonce - generateEncryptionKeyFunc = generateTestEncryptionKey - } -} From 60bbb91ff9200758e339836de9f7367f3339619b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 23:25:07 +0200 Subject: [PATCH 25/88] Improve documentation --- crypto/aes.go | 30 +++++++++++++++++++++++++----- crypto/ecies.go | 23 +++++++++-------------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/crypto/aes.go b/crypto/aes.go index 0f1d299204..6fc27c54c6 100644 --- a/crypto/aes.go +++ b/crypto/aes.go @@ -16,8 +16,19 @@ import ( "fmt" ) -// EncryptAES encrypts data using AES-GCM with a provided key. -// The nonce is prepended to the cipherText. +// EncryptAES encrypts data using AES-GCM with a provided key and additional data. +// It generates a nonce internally and optionally prepends it to the cipherText. +// +// Parameters: +// - plainText: The data to be encrypted +// - key: The AES encryption key +// - additionalData: Additional authenticated data (AAD) to be used in the encryption +// - prependNonce: If true, the nonce is prepended to the returned cipherText +// +// Returns: +// - cipherText: The encrypted data, with the nonce prepended if prependNonce is true +// - nonce: The generated nonce +// - error: Any error encountered during the encryption process func EncryptAES(plainText, key, additionalData []byte, prependNonce bool) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -44,12 +55,21 @@ func EncryptAES(plainText, key, additionalData []byte, prependNonce bool) ([]byt return cipherText, nonce, nil } -// DecryptAES decrypts AES-GCM encrypted data with a provided key. -// The nonce is expected to be prepended to the cipherText. +// DecryptAES decrypts AES-GCM encrypted data with a provided key and additional data. +// If no separate nonce is provided, it assumes the nonce is prepended to the cipherText. +// +// Parameters: +// - nonce: The nonce used for decryption. If empty, it's assumed to be prepended to cipherText +// - cipherText: The data to be decrypted +// - key: The AES decryption key +// - additionalData: Additional authenticated data (AAD) used during encryption +// +// Returns: +// - plainText: The decrypted data +// - error: Any error encountered during the decryption process, including authentication failures func DecryptAES(nonce, cipherText, key, additionalData []byte) ([]byte, error) { if len(nonce) == 0 { if len(cipherText) < AESNonceSize { - // TODO return typed error return nil, fmt.Errorf("cipherText too short") } nonce = cipherText[:AESNonceSize] diff --git a/crypto/ecies.go b/crypto/ecies.go index d57008e6eb..97af499279 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -1,17 +1,12 @@ -// Current Implementation Analysis: -// 1. Key Generation: Correctly uses X25519 for key generation. -// 2. ECDH: Properly performs the ECDH operation. -// 3. Key Derivation: Uses SHA-256 on the shared secret, which is simplistic. -// 4. Encryption: Uses AES (implementation not shown). -// 5. MAC: Not implemented. - -// Improvements Needed: -// 1. Use a proper Key Derivation Function (KDF) -// 2. Implement HMAC for message authentication -// 3. Use authenticated encryption (e.g., AES-GCM) instead of AES -// 4. Standardize the output format - -// Here's an improved version of the EncryptECDH and DecryptECDH functions: +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. package crypto From cdf797c9d94b4301e475833a6ed0804a6b7fb883 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 7 Aug 2024 23:46:05 +0200 Subject: [PATCH 26/88] Polish --- client/mocks/db.go | 47 +++++++++++++++++++ crypto/ecies.go | 14 +++--- crypto/ephemeral.go | 47 ------------------- go.mod | 2 +- go.sum | 2 - internal/core/block/block.go | 3 +- internal/db/merge.go | 6 +-- internal/merkle/clock/clock.go | 6 ++- net/server.go | 3 +- net/server_encryption_key.go | 10 ++-- .../integration/encryption/peer_share_test.go | 1 - tests/integration/p2p.go | 3 +- 12 files changed, 76 insertions(+), 68 deletions(-) delete mode 100644 crypto/ephemeral.go diff --git a/client/mocks/db.go b/client/mocks/db.go index 089e41c159..5f4c7ee78e 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -479,6 +479,53 @@ func (_c *DB_DeleteReplicator_Call) RunAndReturn(run func(context.Context, clien return _c } +// Encstore provides a mock function with given fields: +func (_m *DB) Encstore() datastore.DSReaderWriter { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Encstore") + } + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// DB_Encstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Encstore' +type DB_Encstore_Call struct { + *mock.Call +} + +// Encstore is a helper method to define mock.On call +func (_e *DB_Expecter) Encstore() *DB_Encstore_Call { + return &DB_Encstore_Call{Call: _e.mock.On("Encstore")} +} + +func (_c *DB_Encstore_Call) Run(run func()) *DB_Encstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *DB_Encstore_Call) Return(_a0 datastore.DSReaderWriter) *DB_Encstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DB_Encstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *DB_Encstore_Call { + _c.Call.Return(run) + return _c +} + // Events provides a mock function with given fields: func (_m *DB) Events() *event.Bus { ret := _m.Called() diff --git a/crypto/ecies.go b/crypto/ecies.go index 97af499279..e9d579120d 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -65,17 +65,17 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] sharedSecret, err := ephemeralPrivate.ECDH(publicKey) if err != nil { - return nil, fmt.Errorf("ECDH failed: %w", err) + return nil, fmt.Errorf("failed ECDH operation: %w", err) } kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) aesKey := make([]byte, AESKeySize) hmacKey := make([]byte, HMACSize) if _, err := kdf.Read(aesKey); err != nil { - return nil, fmt.Errorf("KDF failed for AES key: %w", err) + return nil, fmt.Errorf("failed KDF operation for AES key: %w", err) } if _, err := kdf.Read(hmacKey); err != nil { - return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) + return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) } fullAssociatedData := append(ephemeralPublic.Bytes(), associatedData...) @@ -126,17 +126,17 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData sharedSecret, err := privateKey.ECDH(ephemeralPublic) if err != nil { - return nil, fmt.Errorf("ECDH failed: %w", err) + return nil, fmt.Errorf("failed ECDH operation: %w", err) } kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) aesKey := make([]byte, AESKeySize) hmacKey := make([]byte, HMACSize) if _, err := kdf.Read(aesKey); err != nil { - return nil, fmt.Errorf("KDF failed for AES key: %w", err) + return nil, fmt.Errorf("failed KDF operation for AES key: %w", err) } if _, err := kdf.Read(hmacKey); err != nil { - return nil, fmt.Errorf("KDF failed for HMAC key: %w", err) + return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) } macSum := cipherText[len(cipherText)-HMACSize:] @@ -146,7 +146,7 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData mac.Write(cipherTextWithNonce) expectedMAC := mac.Sum(nil) if !hmac.Equal(macSum, expectedMAC) { - return nil, fmt.Errorf("HMAC verification failed") + return nil, fmt.Errorf("verification with HMAC failed") } fullAssociatedData := append(ephemeralPublicBytes, associatedData...) diff --git a/crypto/ephemeral.go b/crypto/ephemeral.go deleted file mode 100644 index 87e35b8fa7..0000000000 --- a/crypto/ephemeral.go +++ /dev/null @@ -1,47 +0,0 @@ -package crypto - -import ( - "crypto/ecdh" - "crypto/rand" - "crypto/sha256" - "fmt" -) - -const ( - // EphemeralKeyLength is the size of the ECDH ephemeral key in bytes. - EphemeralKeyLength = 65 -) - -// EncryptWithEphemeralKey encrypts a key using a randomly generated ephemeral ECDH key and a provided public key. -// It returns the encrypted key prepended with the ephemeral public key. -func EncryptWithEphemeralKey(plainText, publicKeyBytes []byte) ([]byte, error) { - ephemeralPriv, err := ecdh.P256().GenerateKey(rand.Reader) - if err != nil { - return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) - } - - ephPubKeyBytes := ephemeralPriv.PublicKey().Bytes() - sharedSecret := sha256.Sum256(append(ephPubKeyBytes, publicKeyBytes...)) - - return append(ephPubKeyBytes, xorBytes(plainText, sharedSecret[:])...), nil -} - -func xorBytes(data, xor []byte) []byte { - result := make([]byte, len(data)) - for i := range data { - result[i] = data[i] ^ xor[i%len(xor)] - } - return result -} - -// DecryptWithEphemeralKey decrypts data that was encrypted using EncryptWithEphemeralKey. -// It expects the input to be the ephemeral public key followed by the encrypted data. -func DecryptWithEphemeralKey(encryptedData, publicKeyBytes []byte) ([]byte, error) { - ephPubKeyBytes := encryptedData[:EphemeralKeyLength] - cipherText := make([]byte, len(encryptedData)-EphemeralKeyLength) - copy(cipherText, encryptedData[EphemeralKeyLength:]) - - sharedSecret := sha256.Sum256(append(ephPubKeyBytes, publicKeyBytes...)) - - return xorBytes(cipherText, sharedSecret[:]), nil -} diff --git a/go.mod b/go.mod index 2dbd9f0324..fbf5a20b28 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.25.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/term v0.23.0 google.golang.org/grpc v1.65.0 @@ -356,7 +357,6 @@ require ( go.uber.org/fx v1.22.0 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.25.0 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect diff --git a/go.sum b/go.sum index bad0d6308c..acfe01ac1d 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= -filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 74ff74fff8..091830b5c6 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -23,9 +23,10 @@ import ( "github.com/ipld/go-ipld-prime/schema" "github.com/multiformats/go-multicodec" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/core/crdt" - "github.com/sourcenetwork/immutable" ) // Schema is the IPLD schema type that represents a `Block`. diff --git a/internal/db/merge.go b/internal/db/merge.go index f1feedeb14..bf4b9aa500 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -175,7 +175,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR } for _, block := range blocks { - mp.blocks.PushFront(block) + mp.blocks.PushFront(block) } err = mp.mergeBlocks(ctx, true) if err != nil { @@ -310,8 +310,8 @@ func (mp *mergeProcessor) loadBlocks( prevCid := block.GetPrevBlockCid() if prevCid.HasValue() { err := mp.loadBlocks(ctx, prevCid.Value(), mt, willDecrypt) - if err != nil { - return err + if err != nil { + return err } } } else { diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 9ac26ffd5d..45cef24ef2 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -164,7 +164,11 @@ func (mc *MerkleClock) determineBlockEncryptionData( return nil, nil } -func encryptBlock(ctx context.Context, block *coreblock.Block, blockEnc *coreblock.Encryption) (*coreblock.Block, error) { +func encryptBlock( + ctx context.Context, + block *coreblock.Block, + blockEnc *coreblock.Encryption, +) (*coreblock.Block, error) { clonedCRDT := block.Delta.Clone() fieldName := immutable.None[string]() if blockEnc.Type == coreblock.FieldEncrypted { diff --git a/net/server.go b/net/server.go index 1a22aae29b..a5a6472145 100644 --- a/net/server.go +++ b/net/server.go @@ -106,7 +106,8 @@ func (s *server) extractSessionAndRemoveOldOnes(id string) *session { } for i, session := range s.sessions { if session.id == id { - result = &session + tmpSession := session + result = &tmpSession swapLast(i) } else if time.Since(session.t) > sessionTimeout { swapLast(i) diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 6a527711b8..0c1a27eb9f 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -25,6 +25,7 @@ import ( "google.golang.org/protobuf/proto" libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/errors" @@ -69,7 +70,10 @@ func (s *server) getEncryptionKeys( return encryptionKeys, targets, nil } -func (s *server) TryGenEncryptionKey(ctx context.Context, req *pb.FetchEncryptionKeyRequest) (*pb.FetchEncryptionKeyReply, error) { +func (s *server) TryGenEncryptionKey( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) (*pb.FetchEncryptionKeyReply, error) { peerID, err := peerIDFromContext(ctx) if err != nil { return nil, err @@ -176,10 +180,10 @@ func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, m }) res, err := s.TryGenEncryptionKey(ctx, req) if err != nil { - return nil, errors.Wrap("Failed attempt to get encryption key", err) + log.ErrorContextE(s.peer.ctx, "failed attempt to get encryption key", err) + return nil, errors.Wrap("failed attempt to get encryption key", err) } return res.MarshalVT() - //return proto.Marshal(res) } func (s *server) prepareFetchEncryptionKeyRequest( diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 8be09b4877..cb111286a3 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -405,4 +405,3 @@ func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_Should testUtils.ExecuteTestCase(t, test) } - diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 8724b3a2bb..fe4012634a 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -13,10 +13,11 @@ package tests import ( "time" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" - "github.com/sourcenetwork/immutable" "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/corelog" From bf9d6857bbb109ee5d6ed76bea21238dfbf668ff Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 00:01:10 +0200 Subject: [PATCH 27/88] Minor improvements --- internal/core/block/block.go | 2 ++ internal/core/key.go | 4 ++-- internal/db/merge.go | 44 ++++++++++++++++++++---------------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 091830b5c6..be410a76cd 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -99,6 +99,7 @@ func NewDAGLink(name string, link cidlink.Link) DAGLink { } } +// EncryptionType represents the type (or level) of encryption applied to the block. type EncryptionType int const ( @@ -107,6 +108,7 @@ const ( FieldEncrypted ) +// Encryption contains the encryption information for the block's delta. type Encryption struct { // Type indicates on what level encryption is applied. Type EncryptionType diff --git a/internal/core/key.go b/internal/core/key.go index 15c5e14de8..bdddfb2478 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -798,7 +798,7 @@ type EncStoreDocKey struct { // DocID is the ID of the document that the key is for. DocID string // FieldName is the name of the field that the key is for. - // If unset, it indicates the key is for the whole document. + // If unset, it indicates that the key is for the whole document. FieldName immutable.Option[string] // BlockHeight is the height of the block that the key is for. // It is used to differentiate keys that are used in different point in time. @@ -808,7 +808,7 @@ type EncStoreDocKey struct { var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. -// Unset fieldName indicates the key is for the whole document. +// Unset fieldName indicates that the key is for the whole document. // blockHeight is the height of the block that the key is for. func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], blockHeight uint64) EncStoreDocKey { return EncStoreDocKey{DocID: docID, FieldName: fieldName, BlockHeight: blockHeight} diff --git a/internal/db/merge.go b/internal/db/merge.go index bf4b9aa500..0749f6d1d4 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -91,6 +91,29 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return nil } +type encryptionMergeGroup struct { + compositeKey core.EncStoreDocKey + fieldsKeys []core.EncStoreDocKey +} + +func createMergeGroups(keyEvent encryption.KeyRetrievedEvent) map[string]encryptionMergeGroup { + mergeGroups := make(map[string]encryptionMergeGroup) + + for encStoreKey := range keyEvent.Keys { + g := mergeGroups[encStoreKey.DocID] + + if encStoreKey.FieldName.HasValue() { + g.fieldsKeys = append(g.fieldsKeys, encStoreKey) + } else { + g.compositeKey = encStoreKey + } + + mergeGroups[encStoreKey.DocID] = g + } + + return mergeGroups +} + func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyRetrievedEvent) error { ctx, txn, err := ensureContextTxn(ctx, db, false) if err != nil { @@ -106,26 +129,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR ls := cidlink.DefaultLinkSystem() ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) - type mergeGroup struct { - compositeKey core.EncStoreDocKey - fieldsKeys []core.EncStoreDocKey - } - - mergeGroups := make(map[string]mergeGroup) - - for encStoreKey := range keyEvent.Keys { - g := mergeGroups[encStoreKey.DocID] - - if encStoreKey.FieldName.HasValue() { - g.fieldsKeys = append(g.fieldsKeys, encStoreKey) - } else { - g.compositeKey = encStoreKey - } - - mergeGroups[encStoreKey.DocID] = g - } - - for docID, mergeGroup := range mergeGroups { + for docID, mergeGroup := range createMergeGroups(keyEvent) { docID, err := client.NewDocIDFromString(docID) if err != nil { return err From 42648bc91c2283db4a9d35ad591f8baa670518ae Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 07:40:39 +0200 Subject: [PATCH 28/88] Remove unnecessary method --- internal/core/block/block.go | 1 + internal/encryption/encryptor.go | 36 ++------------------------------ 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index be410a76cd..c7527f7170 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -288,6 +288,7 @@ func GetLinkPrototype() cidlink.LinkPrototype { }} } +// Validate checks if the block is valid. func (b *Block) Validate() error { if b.Encryption != nil { switch (*b.Encryption).Type { diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index fe7585821b..108d0d883c 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -112,7 +112,7 @@ func (d *DocEncryptor) Encrypt( blockHeight uint64, plainText []byte, ) ([]byte, error) { - encryptionKey, err := d.fetchEncryptionKey(docID, fieldName, blockHeight) + encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) if err != nil { return nil, err } @@ -148,7 +148,7 @@ func (d *DocEncryptor) Decrypt( blockHeight uint64, cipherText []byte, ) ([]byte, error) { - encKey, err := d.fetchEncryptionKey(docID, fieldName, blockHeight) + encKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) if err != nil { return nil, err } @@ -180,38 +180,6 @@ func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encrypti return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) } -// fetchEncryptionKey fetches the encryption key for the given docID and fieldName. -// If the key is not found, it returns an empty key. -func (d *DocEncryptor) fetchEncryptionKey( - docID string, - fieldName immutable.Option[string], - blockHeight uint64, -) ([]byte, error) { - if d.store == nil { - return nil, ErrNoStorageProvided - } - // first we try to find field-level key - storeKey := core.NewEncStoreDocKey(docID, fieldName, blockHeight) - encryptionKey, err := d.fetchByEncStoreKey(storeKey) - if err != nil { - return nil, err - } - // TODO: check if it's still needed to try to find doc-level key if field-level key is not found - if len(encryptionKey) == 0 { - // if previous fetch was for doc-level, there is nothing else to look for - if !fieldName.HasValue() { - return nil, nil - } - if shouldEncryptIndividualField(d.conf, fieldName) { - return nil, nil - } - // try to find doc-level key - storeKey.FieldName = immutable.None[string]() - encryptionKey, err = d.fetchByEncStoreKey(storeKey) - } - return encryptionKey, err -} - // GetKey returns the encryption key for the given docID, (optional) fieldName and block height. func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string], blockHeight uint64) ([]byte, error) { if d.store == nil { From 1aafe656d6013240d8ac0c8191e1470b6ba69333 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 07:59:46 +0200 Subject: [PATCH 29/88] Fixed encryptor tests --- internal/encryption/encryptor.go | 6 ++++++ internal/encryption/encryptor_test.go | 12 ------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 108d0d883c..6f822daa93 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -112,6 +112,9 @@ func (d *DocEncryptor) Encrypt( blockHeight uint64, plainText []byte, ) ([]byte, error) { + if d.store == nil { + return nil, ErrNoStorageProvided + } encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) if err != nil { return nil, err @@ -148,6 +151,9 @@ func (d *DocEncryptor) Decrypt( blockHeight uint64, cipherText []byte, ) ([]byte, error) { + if d.store == nil { + return nil, ErrNoStorageProvided + } encKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) if err != nil { return nil, err diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 7738adfd9e..75e0c1c0eb 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -71,17 +71,6 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { assert.ErrorIs(t, err, testErr) } -func TestEncryptorEncrypt_IfStorageReturnsErrorOnSecondCall_Error(t *testing.T) { - enc, st := newDefaultEncryptor(t) - - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound).Once() - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - - _, err := enc.Encrypt(docID, fieldName, height, []byte("test")) - - assert.ErrorIs(t, err, testErr) -} - func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) @@ -103,7 +92,6 @@ func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStor fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName, height).ToDS() st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) - st.EXPECT().Get(mock.Anything, docStoreKey).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey(noFieldName)).Return(nil) cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) From 5cd897714ef05c8d82b6ed6cce52f73349f9c4c5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 08:24:32 +0200 Subject: [PATCH 30/88] Polish --- internal/encryption/event.go | 4 +-- net/pb/Makefile | 2 +- net/server_encryption_key.go | 6 ++-- tests/integration/encryption/peer_test.go | 37 ----------------------- 4 files changed, 6 insertions(+), 43 deletions(-) diff --git a/internal/encryption/event.go b/internal/encryption/event.go index abe828c2d5..0760e3f583 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -60,10 +60,10 @@ func NewRequestKeysMessage( // NewKeysRetrievedMessage creates a new event message for a key that was retrieved func NewKeysRetrievedMessage( schemaRoot string, - data map[core.EncStoreDocKey][]byte, + keys map[core.EncStoreDocKey][]byte, ) event.Message { return event.NewMessage(KeysRetrievedEventName, KeyRetrievedEvent{ SchemaRoot: schemaRoot, - Keys: data, + Keys: keys, }) } diff --git a/net/pb/Makefile b/net/pb/Makefile index 9b253cbf3a..eb2c950466 100644 --- a/net/pb/Makefile +++ b/net/pb/Makefile @@ -25,4 +25,4 @@ clean: rm -f *.pb.go rm -f *pb_test.go -.PHONY: clean deps debug +.PHONY: clean deps diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 0c1a27eb9f..9ad719d37b 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -36,14 +36,14 @@ import ( const encryptionTopic = "encryption" -// Server is the request/response instance for all P2P RPC communication. -// Implements gRPC server. See net/pb/net.proto for corresponding service definitions. +// getEncryptionKeys retrieves the encryption keys for the given targets. +// It returns the encryption keys and the targets for which the keys were found. func (s *server) getEncryptionKeys( ctx context.Context, req *pb.FetchEncryptionKeyRequest, ) ([]byte, []*pb.EncryptionKeyTarget, error) { encryptionKeys := make([]byte, 0) - targets := make([]*pb.EncryptionKeyTarget, 0) + targets := make([]*pb.EncryptionKeyTarget, 0, len(req.Targets)) for _, target := range req.Targets { docID, err := client.NewDocIDFromString(string(target.DocID)) if err != nil { diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index f02854c885..e88765addf 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -18,43 +18,6 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -/*func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { - test := testUtils.TestCase{ - Actions: []any{ - testUtils.RandomNetworkingConfig(), - testUtils.RandomNetworkingConfig(), - updateUserCollectionSchema(), - testUtils.ConnectPeers{ - SourceNodeID: 1, - TargetNodeID: 0, - }, - testUtils.SubscribeToCollection{ - NodeID: 1, - CollectionIDs: []int{0}, - }, - testUtils.CreateDoc{ - NodeID: immutable.Some(0), - Doc: john21Doc, - IsDocEncrypted: true, - }, - testUtils.WaitForSync{}, - testUtils.Request{ - NodeID: immutable.Some(1), - Request: `query { - Users { - age - } - }`, - Results: map[string]any{ - "Users": []map[string]any{}, - }, - }, - }, - } - - testUtils.ExecuteTestCase(t, test) -}*/ - func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { test := testUtils.TestCase{ Actions: []any{ From f79bf4756b076ddc34a6750da121004353ea28d1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 08:49:16 +0200 Subject: [PATCH 31/88] Patch for change detector --- docs/data_format_changes/i2891-no-change-tests-updated.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/data_format_changes/i2891-no-change-tests-updated.md diff --git a/docs/data_format_changes/i2891-no-change-tests-updated.md b/docs/data_format_changes/i2891-no-change-tests-updated.md new file mode 100644 index 0000000000..8d22b94c15 --- /dev/null +++ b/docs/data_format_changes/i2891-no-change-tests-updated.md @@ -0,0 +1,3 @@ +# Doc encryption key exchange + +For the key exchange mechanism we changed slightly the structure of DAG block to hold an additional information. From f0c3cde7f084bcd70f7e7f80d890248151316466 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 8 Aug 2024 23:54:07 +0200 Subject: [PATCH 32/88] Adjust encryption to work with sec. indexes --- internal/db/merge.go | 12 +- .../encryption/peer_sec_index_test.go | 150 ++++++++++++++++++ tests/integration/events.go | 18 ++- tests/integration/p2p.go | 2 + 4 files changed, 173 insertions(+), 9 deletions(-) create mode 100644 tests/integration/encryption/peer_sec_index_test.go diff --git a/internal/db/merge.go b/internal/db/merge.go index 0749f6d1d4..55c0a96ac5 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -76,9 +76,11 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { mp.sendPendingEncryptionRequest() - err = syncIndexedDoc(ctx, docID, col) - if err != nil { - return err + if !mp.hasPendingCompositeBlock { + err = syncIndexedDoc(ctx, docID, col) + if err != nil { + return err + } } err = txn.Commit(ctx) @@ -249,6 +251,7 @@ type mergeProcessor struct { dsKey core.DataStoreKey blocks *list.List pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} + hasPendingCompositeBlock bool } func (db *db) newMergeProcessor( @@ -394,6 +397,9 @@ func (mp *mergeProcessor) processEncryptedBlock( func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], height uint64) { mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, height)] = struct{}{} + if !fieldName.HasValue() { + mp.hasPendingCompositeBlock = true + } } func (mp *mergeProcessor) sendPendingEncryptionRequest() { diff --git a/tests/integration/encryption/peer_sec_index_test.go b/tests/integration/encryption/peer_sec_index_test.go new file mode 100644 index 0000000000..0c45f56321 --- /dev/null +++ b/tests/integration/encryption/peer_sec_index_test.go @@ -0,0 +1,150 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + "github.com/sourcenetwork/immutable" + + "github.com/sourcenetwork/defradb/internal/encryption" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecryption(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int @index + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "Shahzad", + "age": 25 + }`, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: islam33Doc, + IsDocEncrypted: true, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "Andy", + "age": 21 + }`, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, + Count: 2, + }, + testUtils.Request{ + Request: ` + query @explain(type: execute) { + User(filter: {age: {_eq: 21}}) { + age + } + }`, + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(2), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionPeer_IfDocDocHasEncryptedIndexedField_ShouldIndexAfterDecryption(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int @index + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "Shahzad", + "age": 25 + }`, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: islam33Doc, + EncryptedFields: []string{"age"}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "Andy", + "age": 21 + }`, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + EncryptedFields: []string{"age"}, + }, + testUtils.WaitForSync{ + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, + Count: 2, + }, + testUtils.Request{ + Request: ` + query @explain(type: execute) { + User(filter: {age: {_eq: 21}}) { + age + } + }`, + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(2), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/events.go b/tests/integration/events.go index 0df8bb6f63..3ec8c577c6 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -370,12 +370,18 @@ func waitForKeyRetrievedEvent(s *state, nodeIDs []int) { } func waitForSync(s *state, action WaitForSync) { - if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { - waitForMergeEvents(s) - } else if action.Event.Value() == encryption.KeysRetrievedEventName { - waitForKeyRetrievedEvent(s, action.NodeIDs) - } else { - require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) + count := int(action.Count) + if count == 0 { + count = 1 + } + for i := 0; i < count; i++ { + if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { + waitForMergeEvents(s) + } else if action.Event.Value() == encryption.KeysRetrievedEventName { + waitForKeyRetrievedEvent(s, action.NodeIDs) + } else { + require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) + } } } diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index fe4012634a..b81504f1dd 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -144,6 +144,8 @@ type WaitForSync struct { // NodeIDs are the node IDs (indexes) of the nodes to wait for sync on. // If not provided then the action will wait for sync on all nodes. NodeIDs []int + // Count is the number of times the event should be received. + Count uint } // connectPeers connects two existing, started, nodes as peers. It returns a channel From 20f574c5140da54672fde46ab270c0c7507a8a5a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 12 Aug 2024 18:42:49 +0200 Subject: [PATCH 33/88] Pass EncStoreDocKey to encryptor --- internal/db/merge.go | 7 +-- internal/db/messages.go | 8 +--- internal/encryption/encryptor.go | 61 +++++++++------------------ internal/encryption/encryptor_test.go | 32 ++++++++------ internal/merkle/clock/clock.go | 7 ++- net/server_encryption_key.go | 4 +- 6 files changed, 52 insertions(+), 67 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 55c0a96ac5..27def3d213 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -470,10 +470,12 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block optFieldName = immutable.Some(block.Delta.GetFieldName()) } + encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), optFieldName, blockEnc.From) + if block.Delta.IsComposite() { // for composite blocks there is nothing to decrypt // so we just check if we have the encryption key for child blocks - bytes, err := encryption.GetKey(ctx, string(block.Delta.GetDocID()), optFieldName, blockEnc.From) + bytes, err := encryption.GetKey(ctx, encStoreKey) if err != nil { return nil, err } @@ -484,8 +486,7 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } clonedCRDT := block.Delta.Clone() - bytes, err := encryption.DecryptDoc(ctx, string(clonedCRDT.GetDocID()), optFieldName, - blockEnc.From, clonedCRDT.GetData()) + bytes, err := encryption.DecryptDoc(ctx, encStoreKey, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/internal/db/messages.go b/internal/db/messages.go index 4f4d34f4ea..ef9631dbcd 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -84,13 +84,7 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { go func() { ctx = encryption.ContextWithStore(ctx, db.Encstore()) for encStoreKey, encKey := range evt.Keys { - err := encryption.SaveKey( - ctx, - encStoreKey.DocID, - encStoreKey.FieldName, - encStoreKey.BlockHeight, - encKey, - ) + err := encryption.SaveKey(ctx, encStoreKey, encKey) if err != nil { log.ErrorContextE( diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 6f822daa93..72f44dec01 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -107,34 +107,32 @@ func shouldEncryptDocField(conf immutable.Option[DocEncConfig], fieldName immuta // If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. // Otherwise, it will use document-level encryption key. func (d *DocEncryptor) Encrypt( - docID string, - fieldName immutable.Option[string], - blockHeight uint64, + encStoreKey core.EncStoreDocKey, plainText []byte, ) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) + encryptionKey, err := d.fetchByEncStoreKey(encStoreKey) if err != nil { return nil, err } if len(encryptionKey) == 0 { - if !shouldEncryptIndividualField(d.conf, fieldName) { - fieldName = immutable.None[string]() + if !shouldEncryptIndividualField(d.conf, encStoreKey.FieldName) { + encStoreKey.FieldName = immutable.None[string]() } - if !shouldEncryptDocField(d.conf, fieldName) { + if !shouldEncryptDocField(d.conf, encStoreKey.FieldName) { return plainText, nil } - encryptionKey, err = generateEncryptionKeyFunc(docID, fieldName) + encryptionKey, err = generateEncryptionKeyFunc(encStoreKey.DocID, encStoreKey.FieldName) if err != nil { return nil, err } - err = d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight), encryptionKey) + err = d.storeByEncStoreKey(encStoreKey, encryptionKey) if err != nil { return nil, err } @@ -146,15 +144,13 @@ func (d *DocEncryptor) Encrypt( // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. // If the corresponding encryption key is not found, it returns nil. func (d *DocEncryptor) Decrypt( - docID string, - fieldName immutable.Option[string], - blockHeight uint64, + encStoreKey core.EncStoreDocKey, cipherText []byte, ) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - encKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) + encKey, err := d.fetchByEncStoreKey(encStoreKey) if err != nil { return nil, err } @@ -187,11 +183,11 @@ func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encrypti } // GetKey returns the encryption key for the given docID, (optional) fieldName and block height. -func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string], blockHeight uint64) ([]byte, error) { +func (d *DocEncryptor) GetKey(encStoreKey core.EncStoreDocKey) ([]byte, error) { if d.store == nil { return nil, ErrNoStorageProvided } - encryptionKey, err := d.fetchByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight)) + encryptionKey, err := d.fetchByEncStoreKey(encStoreKey) if err != nil { return nil, err } @@ -199,16 +195,11 @@ func (d *DocEncryptor) GetKey(docID string, fieldName immutable.Option[string], } // SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height. -func (d *DocEncryptor) SaveKey( - docID string, - fieldName immutable.Option[string], - blockHeight uint64, - encryptionKey []byte, -) error { +func (d *DocEncryptor) SaveKey(encStoreKey core.EncStoreDocKey, encryptionKey []byte) error { if d.store == nil { return ErrNoStorageProvided } - return d.storeByEncStoreKey(core.NewEncStoreDocKey(docID, fieldName, blockHeight), encryptionKey) + return d.storeByEncStoreKey(encStoreKey, encryptionKey) } // EncryptDoc encrypts the given plainText that is associated with the given docID, fieldName and block height with @@ -217,16 +208,14 @@ func (d *DocEncryptor) SaveKey( // Otherwise, it will use document-level encryption key. func EncryptDoc( ctx context.Context, - docID string, - fieldName immutable.Option[string], - blockHeight uint64, + encStoreKey core.EncStoreDocKey, plainText []byte, ) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Encrypt(docID, fieldName, blockHeight, plainText) + return enc.Encrypt(encStoreKey, plainText) } // DecryptDoc decrypts the given cipherText that is associated with the given docID and fieldName with @@ -235,16 +224,14 @@ func EncryptDoc( // decrypt with the field-level key. func DecryptDoc( ctx context.Context, - docID string, - fieldName immutable.Option[string], - blockHeight uint64, + encStoreKey core.EncStoreDocKey, cipherText []byte, ) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Decrypt(docID, fieldName, blockHeight, cipherText) + return enc.Decrypt(encStoreKey, cipherText) } // ShouldEncryptDocField returns true if the given field should be encrypted based on the context config. @@ -260,28 +247,22 @@ func ShouldEncryptIndividualField(ctx context.Context, fieldName immutable.Optio // SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height with // encryptor in the context. -func SaveKey( - ctx context.Context, - docID string, - fieldName immutable.Option[string], - blockHeight uint64, - encryptionKey []byte, -) error { +func SaveKey(ctx context.Context, encStoreKey core.EncStoreDocKey, encryptionKey []byte) error { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil } - return enc.SaveKey(docID, fieldName, blockHeight, encryptionKey) + return enc.SaveKey(encStoreKey, encryptionKey) } // GetKey returns the encryption key for the given docID, (optional) fieldName and block height with encryptor // in the context. -func GetKey(ctx context.Context, docID string, fieldName immutable.Option[string], blockHeight uint64) ([]byte, error) { +func GetKey(ctx context.Context, encStoreKey core.EncStoreDocKey) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.GetKey(docID, fieldName, blockHeight) + return enc.GetKey(encStoreKey) } func init() { diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 75e0c1c0eb..e89a573c92 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -66,7 +66,7 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Encrypt(docID, fieldName, height, []byte("test")) + _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -79,7 +79,7 @@ func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerate st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, noFieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, noFieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -94,7 +94,7 @@ func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStor st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -108,7 +108,7 @@ func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreI st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(fieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, fieldName), cipherText) @@ -120,7 +120,7 @@ func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipher storeKey := core.NewEncStoreDocKey(docID, fieldName, height) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) - cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, fieldName), cipherText) @@ -131,7 +131,7 @@ func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *t st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - cipherText, err := enc.Encrypt(docID, noFieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, noFieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -144,7 +144,7 @@ func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *test st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) - _, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.ErrorIs(t, err, testErr) } @@ -155,7 +155,7 @@ func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *t st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - cipherText, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.NoError(t, err) assert.Equal(t, getPlainText(), cipherText) @@ -165,7 +165,7 @@ func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Encrypt(docID, fieldName, height, getPlainText()) + _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -174,7 +174,7 @@ func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Decrypt(docID, fieldName, height, getPlainText()) + _, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -184,7 +184,7 @@ func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Decrypt(docID, fieldName, height, []byte("test")) + _, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -194,20 +194,24 @@ func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *te st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - plainText, err := enc.Decrypt(docID, fieldName, height, getCipherText(t, noFieldName)) + plainText, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), getCipherText(t, noFieldName)) assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) } func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := EncryptDoc(context.Background(), docID, fieldName, height, getPlainText()) + data, err := EncryptDoc(context.Background(), core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) assert.Nil(t, data, "data should be nil") assert.NoError(t, err, "error should be nil") } func TestDecryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := DecryptDoc(context.Background(), docID, fieldName, height, getCipherText(t, fieldName)) + data, err := DecryptDoc( + context.Background(), + core.NewEncStoreDocKey(docID, fieldName, height), + getCipherText(t, fieldName), + ) assert.Nil(t, data, "data should be nil") assert.NoError(t, err, "error should be nil") } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 45cef24ef2..2dec134d14 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -174,8 +174,11 @@ func encryptBlock( if blockEnc.Type == coreblock.FieldEncrypted { fieldName = immutable.Some(clonedCRDT.GetFieldName()) } - bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), - fieldName, blockEnc.From, clonedCRDT.GetData()) + bytes, err := encryption.EncryptDoc( + ctx, + core.NewEncStoreDocKey(string(clonedCRDT.GetDocID()), fieldName, blockEnc.From), + clonedCRDT.GetData(), + ) if err != nil { return nil, err } diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 9ad719d37b..e96d3b9688 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -55,7 +55,9 @@ func (s *server) getEncryptionKeys( optFieldName = immutable.Some(target.FieldName) } encKey, err := encryption.GetKey( - encryption.ContextWithStore(ctx, s.peer.encstore), docID.String(), optFieldName, target.Height) + encryption.ContextWithStore(ctx, s.peer.encstore), + core.NewEncStoreDocKey(docID.String(), optFieldName, target.Height), + ) if err != nil { return nil, nil, err } From dcf36e71ca6e3597f21bb64c660005f275c30ac8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 10:56:00 +0200 Subject: [PATCH 34/88] Store enc key CID in a block instead of height --- crypto/cid.go | 26 ++++ internal/core/block/block.go | 12 +- internal/core/block/block_test.go | 14 +- internal/core/block/errors.go | 24 ++-- internal/core/key.go | 13 +- internal/db/merge.go | 33 +++-- internal/encryption/encryptor.go | 118 +++++++++++++---- internal/encryption/encryptor_test.go | 80 ++++-------- internal/merkle/clock/clock.go | 48 ++++--- internal/merkle/clock/errors.go | 2 + net/pb/net.pb.go | 134 ++++++++++---------- net/pb/net.proto | 4 +- net/pb/net_vtproto.pb.go | 36 ++++-- net/server_encryption_key.go | 31 +++-- tests/integration/encryption/commit_test.go | 10 +- tests/integration/encryption/peer_test.go | 10 +- 16 files changed, 345 insertions(+), 250 deletions(-) create mode 100644 crypto/cid.go diff --git a/crypto/cid.go b/crypto/cid.go new file mode 100644 index 0000000000..a7cc4a952d --- /dev/null +++ b/crypto/cid.go @@ -0,0 +1,26 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +// GenerateCid generates a CID from the given data. +func GenerateCid(data []byte) (cid.Cid, error) { + mh, err := multihash.Sum(data, multihash.SHA2_256, -1) + if err != nil { + return cid.Cid{}, err + } + + return cid.NewCidV1(cid.Raw, mh), nil +} diff --git a/internal/core/block/block.go b/internal/core/block/block.go index c7527f7170..de6ddcdebb 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -112,8 +112,8 @@ const ( type Encryption struct { // Type indicates on what level encryption is applied. Type EncryptionType - // From specifies the block height from which the encryption is applied. - From uint64 + // CID of the key used for encryption. + KeyID []byte } // Block is a block that contains a CRDT delta and links to other blocks. @@ -129,7 +129,7 @@ type Block struct { // IsEncrypted returns true if the block is encrypted. func (b *Block) IsEncrypted() bool { - return b.Encryption != nil && (*b.Encryption).Type != NotEncrypted + return b.Encryption != nil && b.Encryption.Type != NotEncrypted } // GetPrevBlockCid returns the CID of the previous block. @@ -155,7 +155,7 @@ func (b Block) IPLDSchemaBytes() []byte { type Encryption struct { type EncryptionType - from Int + keyID Bytes } type EncryptionType enum { @@ -298,8 +298,8 @@ func (b *Block) Validate() error { return ErrInvalidBlockEncryptionType } - if (*b.Encryption).From == 0 { - return ErrInvalidBlockEncryptionFrom + if len(b.Encryption.KeyID) == 0 { + return ErrInvalidBlockEncryptionKeyID } } return nil diff --git a/internal/core/block/block_test.go b/internal/core/block/block_test.go index 3c5a374ed9..bfd949077a 100644 --- a/internal/core/block/block_test.go +++ b/internal/core/block/block_test.go @@ -237,17 +237,17 @@ func TestBlock_Validate(t *testing.T) { }{ { name: "NotEncrypted type is valid", - encryption: &Encryption{Type: NotEncrypted, From: 1}, + encryption: &Encryption{Type: NotEncrypted, KeyID: []byte{1}}, expectedError: nil, }, { name: "DocumentEncrypted type is valid", - encryption: &Encryption{Type: DocumentEncrypted, From: 1}, + encryption: &Encryption{Type: DocumentEncrypted, KeyID: []byte{1}}, expectedError: nil, }, { name: "FieldEncrypted type is valid", - encryption: &Encryption{Type: FieldEncrypted, From: 1}, + encryption: &Encryption{Type: FieldEncrypted, KeyID: []byte{1}}, expectedError: nil, }, { @@ -257,13 +257,13 @@ func TestBlock_Validate(t *testing.T) { }, { name: "Invalid encryption type", - encryption: &Encryption{Type: EncryptionType(99), From: 1}, + encryption: &Encryption{Type: EncryptionType(99), KeyID: []byte{1}}, expectedError: ErrInvalidBlockEncryptionType, }, { - name: "Invalid encryption from parameter", - encryption: &Encryption{Type: DocumentEncrypted}, - expectedError: ErrInvalidBlockEncryptionFrom, + name: "Invalid encryption key id parameter", + encryption: &Encryption{Type: DocumentEncrypted, KeyID: []byte{}}, + expectedError: ErrInvalidBlockEncryptionKeyID, }, } diff --git a/internal/core/block/errors.go b/internal/core/block/errors.go index c45e2b0cca..ced4c4d6a1 100644 --- a/internal/core/block/errors.go +++ b/internal/core/block/errors.go @@ -17,12 +17,12 @@ import ( ) const ( - errNodeToBlock string = "failed to convert node to block" - errEncodingBlock string = "failed to encode block" - errUnmarshallingBlock string = "failed to unmarshal block" - errGeneratingLink string = "failed to generate link" - errInvalidBlockEncryptionType string = "invalid block encryption type" - errInvalidBlockEncryptionFrom string = "invalid block encryption from parameter" + errNodeToBlock string = "failed to convert node to block" + errEncodingBlock string = "failed to encode block" + errUnmarshallingBlock string = "failed to unmarshal block" + errGeneratingLink string = "failed to generate link" + errInvalidBlockEncryptionType string = "invalid block encryption type" + errInvalidBlockEncryptionKeyID string = "invalid block encryption key id" ) // Errors returnable from this package. @@ -30,12 +30,12 @@ const ( // This list is incomplete and undefined errors may also be returned. // Errors returned from this package may be tested against these errors with errors.Is. var ( - ErrNodeToBlock = errors.New(errNodeToBlock) - ErrEncodingBlock = errors.New(errEncodingBlock) - ErrUnmarshallingBlock = errors.New(errUnmarshallingBlock) - ErrGeneratingLink = errors.New(errGeneratingLink) - ErrInvalidBlockEncryptionType = errors.New(errInvalidBlockEncryptionType) - ErrInvalidBlockEncryptionFrom = errors.New(errInvalidBlockEncryptionFrom) + ErrNodeToBlock = errors.New(errNodeToBlock) + ErrEncodingBlock = errors.New(errEncodingBlock) + ErrUnmarshallingBlock = errors.New(errUnmarshallingBlock) + ErrGeneratingLink = errors.New(errGeneratingLink) + ErrInvalidBlockEncryptionType = errors.New(errInvalidBlockEncryptionType) + ErrInvalidBlockEncryptionKeyID = errors.New(errInvalidBlockEncryptionKeyID) ) // NewErrFailedToGetPriority returns an error indicating that the priority could not be retrieved. diff --git a/internal/core/key.go b/internal/core/key.go index bdddfb2478..a61e6ac854 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -800,9 +800,8 @@ type EncStoreDocKey struct { // FieldName is the name of the field that the key is for. // If unset, it indicates that the key is for the whole document. FieldName immutable.Option[string] - // BlockHeight is the height of the block that the key is for. - // It is used to differentiate keys that are used in different point in time. - BlockHeight uint64 + // KeyID is a hash (Cid) of the of the encryption key. + KeyID string } var _ Key = (*EncStoreDocKey)(nil) @@ -810,15 +809,15 @@ var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. // Unset fieldName indicates that the key is for the whole document. // blockHeight is the height of the block that the key is for. -func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], blockHeight uint64) EncStoreDocKey { - return EncStoreDocKey{DocID: docID, FieldName: fieldName, BlockHeight: blockHeight} +func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], keyID string) EncStoreDocKey { + return EncStoreDocKey{DocID: docID, FieldName: fieldName, KeyID: keyID} } func (k EncStoreDocKey) ToString() string { if k.FieldName.HasValue() { - return fmt.Sprintf("%s/%s/%d", k.DocID, k.FieldName.Value(), k.BlockHeight) + return fmt.Sprintf("%s/%s/%s", k.DocID, k.FieldName.Value(), k.KeyID) } - return fmt.Sprintf("%s/%d", k.DocID, k.BlockHeight) + return fmt.Sprintf("%s/%s", k.DocID, k.KeyID) } func (k EncStoreDocKey) Bytes() []byte { diff --git a/internal/db/merge.go b/internal/db/merge.go index 27def3d213..7052f616d0 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -11,6 +11,7 @@ package db import ( + "bytes" "container/list" "context" "sync" @@ -153,7 +154,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR return err } - blocks, err = loadBlocksChainFromBlockstoreTillHeight(ctx, txn, cids, mergeGroup.compositeKey.BlockHeight) + blocks, err = loadBlocksWithKeyIDFromBlockstore(ctx, txn, cids, mergeGroup.compositeKey.KeyID) if err != nil { return err } @@ -171,7 +172,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR return err } - fieldBlocks, err := loadBlocksChainFromBlockstoreTillHeight(ctx, txn, cids, fieldStoreKey.BlockHeight) + fieldBlocks, err := loadBlocksWithKeyIDFromBlockstore(ctx, txn, cids, fieldStoreKey.KeyID) if err != nil { return err } @@ -387,7 +388,7 @@ func (mp *mergeProcessor) processEncryptedBlock( if blockEnc.Type == coreblock.FieldEncrypted { fieldName = immutable.Some(dagBlock.Delta.GetFieldName()) } - mp.addPendingEncryptionRequest(docID, fieldName, blockEnc.From) + mp.addPendingEncryptionRequest(docID, fieldName, string(blockEnc.KeyID)) } return dagBlock, true, nil } @@ -395,8 +396,8 @@ func (mp *mergeProcessor) processEncryptedBlock( return dagBlock, false, nil } -func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], height uint64) { - mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, height)] = struct{}{} +func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], keyID string) { + mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, keyID)] = struct{}{} if !fieldName.HasValue() { mp.hasPendingCompositeBlock = true } @@ -470,7 +471,7 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block optFieldName = immutable.Some(block.Delta.GetFieldName()) } - encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), optFieldName, blockEnc.From) + encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), optFieldName, string(blockEnc.KeyID)) if block.Delta.IsComposite() { // for composite blocks there is nothing to decrypt @@ -609,14 +610,14 @@ func loadBlockFromBlockStore(ctx context.Context, txn datastore.Txn, cid cid.Cid return block, nil } -// loadBlocksChainFromBlockstoreTillHeight loads the blocks from the blockstore starting from the given CIDs -// until it reaches a block with a height equal to the given height (including that block). -// The returned blocks are ordered from the highest height to the lowest. -func loadBlocksChainFromBlockstoreTillHeight( +// loadBlocksWithKeyIDFromBlockstore loads the blocks from the blockstore that have given encryption +// keyID until it reaches a block with a different keyID or without any. +// The returned blocks are ordered from the newest to the oldest. +func loadBlocksWithKeyIDFromBlockstore( ctx context.Context, txn datastore.Txn, cids []cid.Cid, - height uint64, + keyID string, ) ([]*coreblock.Block, error) { var blocks []*coreblock.Block for len(cids) > 0 { @@ -626,13 +627,11 @@ func loadBlocksChainFromBlockstoreTillHeight( return nil, err } - if block.Delta.GetPriority() >= height { + if block.Encryption != nil && bytes.Equal(block.Encryption.KeyID, []byte(keyID)) { blocks = append(blocks, block) - if block.Delta.GetPriority() != height { - prevCid := block.GetPrevBlockCid() - if prevCid.HasValue() { - cids = append(cids, prevCid.Value()) - } + prevCid := block.GetPrevBlockCid() + if prevCid.HasValue() { + cids = append(cids, prevCid.Value()) } } cids = cids[1:] diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 72f44dec01..ebab69262c 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -53,10 +53,11 @@ func generateTestEncryptionKey(docID string, fieldName immutable.Option[string]) // It acts based on the configuration [DocEncConfig] provided and data stored in the provided store. // It uses [core.EncStoreDocKey] to store and retrieve encryption keys. type DocEncryptor struct { - conf immutable.Option[DocEncConfig] - ctx context.Context - store datastore.DSReaderWriter - cache map[core.EncStoreDocKey][]byte + conf immutable.Option[DocEncConfig] + ctx context.Context + store datastore.DSReaderWriter + cache map[core.EncStoreDocKey][]byte + generatedKeys []core.EncStoreDocKey } func newDocEncryptor(ctx context.Context) *DocEncryptor { @@ -103,9 +104,8 @@ func shouldEncryptDocField(conf immutable.Option[DocEncConfig], fieldName immuta return false } -// Encrypt encrypts the given plainText that is associated with the given docID, fieldName and block height. -// If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. -// Otherwise, it will use document-level encryption key. +// Encrypt encrypts the given plainText with the encryption key that is associated with the given docID, +// fieldName and key id. func (d *DocEncryptor) Encrypt( encStoreKey core.EncStoreDocKey, plainText []byte, @@ -113,31 +113,17 @@ func (d *DocEncryptor) Encrypt( if d.store == nil { return nil, ErrNoStorageProvided } + encryptionKey, err := d.fetchByEncStoreKey(encStoreKey) if err != nil { return nil, err } - if len(encryptionKey) == 0 { - if !shouldEncryptIndividualField(d.conf, encStoreKey.FieldName) { - encStoreKey.FieldName = immutable.None[string]() - } - - if !shouldEncryptDocField(d.conf, encStoreKey.FieldName) { - return plainText, nil - } - - encryptionKey, err = generateEncryptionKeyFunc(encStoreKey.DocID, encStoreKey.FieldName) - if err != nil { - return nil, err - } - - err = d.storeByEncStoreKey(encStoreKey, encryptionKey) - if err != nil { - return nil, err - } + var cipherText []byte + if len(plainText) > 0 { + cipherText, _, err = crypto.EncryptAES(plainText, encryptionKey, nil, true) } - cipherText, _, err := crypto.EncryptAES(plainText, encryptionKey, nil, true) + return cipherText, err } @@ -194,6 +180,68 @@ func (d *DocEncryptor) GetKey(encStoreKey core.EncStoreDocKey) ([]byte, error) { return encryptionKey, nil } +// getGeneratedKeyFor returns the generated key for the given docID and fieldName. +func (d *DocEncryptor) getGeneratedKeyFor( + docID string, + fieldName immutable.Option[string], +) (immutable.Option[core.EncStoreDocKey], []byte) { + for _, key := range d.generatedKeys { + if key.DocID == docID && key.FieldName == fieldName { + return immutable.Some(key), d.cache[key] + } + } + return immutable.None[core.EncStoreDocKey](), nil +} + +// GetOrGenerateEncryptionKey returns the generated encryption key for the given docID, (optional) fieldName. +// If the key is not generated before, it generates a new key and stores it. +func (d *DocEncryptor) GetOrGenerateEncryptionKey( + docID string, + fieldName immutable.Option[string], +) (immutable.Option[core.EncStoreDocKey], []byte, error) { + encStoreKey, encryptionKey := d.getGeneratedKeyFor(docID, fieldName) + if encStoreKey.HasValue() { + return encStoreKey, encryptionKey, nil + } + + return d.generateEncryptionKey(docID, fieldName) +} + +// generateEncryptionKey generates a new encryption key for the given docID and fieldName. +func (d *DocEncryptor) generateEncryptionKey( + docID string, + fieldName immutable.Option[string], +) (immutable.Option[core.EncStoreDocKey], []byte, error) { + encStoreKey := core.NewEncStoreDocKey(docID, fieldName, "") + if !shouldEncryptIndividualField(d.conf, fieldName) { + encStoreKey.FieldName = immutable.None[string]() + } + + if !shouldEncryptDocField(d.conf, encStoreKey.FieldName) { + return immutable.None[core.EncStoreDocKey](), nil, nil + } + + encryptionKey, err := generateEncryptionKeyFunc(encStoreKey.DocID, encStoreKey.FieldName) + if err != nil { + return immutable.None[core.EncStoreDocKey](), nil, err + } + + keyID, err := crypto.GenerateCid(encryptionKey) + if err != nil { + return immutable.None[core.EncStoreDocKey](), nil, err + } + encStoreKey.KeyID = keyID.String() + + err = d.storeByEncStoreKey(encStoreKey, encryptionKey) + if err != nil { + return immutable.None[core.EncStoreDocKey](), nil, err + } + + d.generatedKeys = append(d.generatedKeys, encStoreKey) + + return immutable.Some(encStoreKey), encryptionKey, nil +} + // SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height. func (d *DocEncryptor) SaveKey(encStoreKey core.EncStoreDocKey, encryptionKey []byte) error { if d.store == nil { @@ -265,11 +313,27 @@ func GetKey(ctx context.Context, encStoreKey core.EncStoreDocKey) ([]byte, error return enc.GetKey(encStoreKey) } +// GetOrGenerateEncryptionKey returns the generated encryption key for the given docID, (optional) fieldName. +// If the key is not generated before, it generates a new key and stores it. +func GetOrGenerateEncryptionKey( + ctx context.Context, + docID string, + fieldName immutable.Option[string], +) (immutable.Option[core.EncStoreDocKey], []byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return immutable.None[core.EncStoreDocKey](), nil, nil + } + return enc.GetOrGenerateEncryptionKey(docID, fieldName) +} + func init() { arg := os.Args[0] // If the binary is a test binary, use a deterministic nonce. // TODO: We should try to find a better way to detect this https://github.com/sourcenetwork/defradb/issues/2801 - if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { + if strings.HasSuffix(arg, ".test") || + strings.Contains(arg, "/defradb/tests/") || + strings.Contains(arg, "/__debug_bin") { generateEncryptionKeyFunc = generateTestEncryptionKey } } diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index e89a573c92..2b979d1dec 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -15,7 +15,6 @@ import ( "errors" "testing" - ds "github.com/ipfs/go-datastore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -29,7 +28,6 @@ import ( var testErr = errors.New("test error") const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" -const height = 1 var fieldName = immutable.Some("name") var noFieldName = immutable.None[string]() @@ -43,6 +41,16 @@ func getEncKey(fieldName immutable.Option[string]) []byte { return key } +func getKeyID(fieldName immutable.Option[string]) string { + key := getEncKey(fieldName) + cid, _ := crypto.GenerateCid(key) + return cid.String() +} + +func makeStoreKey(docID string, fieldName immutable.Option[string]) core.EncStoreDocKey { + return core.NewEncStoreDocKey(docID, fieldName, getKeyID(fieldName)) +} + func getCipherText(t *testing.T, fieldName immutable.Option[string]) []byte { cipherText, _, err := crypto.EncryptAES(getPlainText(), getEncKey(fieldName), nil, true) assert.NoError(t, err) @@ -66,7 +74,7 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), []byte("test")) + _, err := enc.Encrypt(makeStoreKey(docID, fieldName), []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -74,53 +82,26 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { enc, st := newDefaultEncryptor(t) - storeKey := core.NewEncStoreDocKey(docID, noFieldName, height) + storeKey := makeStoreKey(docID, noFieldName) - st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(noFieldName)).Return(nil) - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, noFieldName, height), getPlainText()) - + _, _, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) assert.NoError(t, err) - assert.Equal(t, getCipherText(t, noFieldName), cipherText) -} -func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc, st := newDefaultEncryptor(t) - - docStoreKey := core.NewEncStoreDocKey(docID, noFieldName, height).ToDS() - fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName, height).ToDS() - - st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) - st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey(noFieldName)).Return(nil) - - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) + cipherText, err := enc.Encrypt(makeStoreKey(docID, noFieldName), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) } -func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) - - storeKey := core.NewEncStoreDocKey(docID, fieldName, height) - - st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) - st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(fieldName)).Return(nil) - - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) - - assert.NoError(t, err) - assert.Equal(t, getCipherText(t, fieldName), cipherText) -} - func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) - storeKey := core.NewEncStoreDocKey(docID, fieldName, height) + storeKey := makeStoreKey(docID, fieldName) st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) + cipherText, err := enc.Encrypt(makeStoreKey(docID, fieldName), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, fieldName), cipherText) @@ -131,7 +112,7 @@ func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *t st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, noFieldName, height), getPlainText()) + cipherText, err := enc.Encrypt(makeStoreKey(docID, noFieldName), getPlainText()) assert.NoError(t, err) assert.Equal(t, getCipherText(t, noFieldName), cipherText) @@ -140,32 +121,27 @@ func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *t func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *testing.T) { enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) - _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) - + _, _, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) assert.ErrorIs(t, err, testErr) } func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *testing.T) { - enc, st := newDefaultEncryptor(t) + enc, _ := newDefaultEncryptor(t) enc.SetConfig(immutable.None[DocEncConfig]()) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - - cipherText, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) - + encStoreKey, encryptionKey, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) assert.NoError(t, err) - assert.Equal(t, getPlainText(), cipherText) + assert.Len(t, encryptionKey, 0) + assert.Equal(t, encStoreKey, immutable.None[core.EncStoreDocKey]()) } func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Encrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) + _, err := enc.Encrypt(makeStoreKey(docID, fieldName), getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -174,7 +150,7 @@ func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { enc, _ := newDefaultEncryptor(t) enc.SetStore(nil) - _, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) + _, err := enc.Decrypt(makeStoreKey(docID, fieldName), getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } @@ -184,7 +160,7 @@ func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), []byte("test")) + _, err := enc.Decrypt(makeStoreKey(docID, fieldName), []byte("test")) assert.ErrorIs(t, err, testErr) } @@ -194,14 +170,14 @@ func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *te st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) - plainText, err := enc.Decrypt(core.NewEncStoreDocKey(docID, fieldName, height), getCipherText(t, noFieldName)) + plainText, err := enc.Decrypt(makeStoreKey(docID, fieldName), getCipherText(t, noFieldName)) assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) } func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := EncryptDoc(context.Background(), core.NewEncStoreDocKey(docID, fieldName, height), getPlainText()) + data, err := EncryptDoc(context.Background(), makeStoreKey(docID, fieldName), getPlainText()) assert.Nil(t, data, "data should be nil") assert.NoError(t, err, "error should be nil") } @@ -209,7 +185,7 @@ func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { func TestDecryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { data, err := DecryptDoc( context.Background(), - core.NewEncStoreDocKey(docID, fieldName, height), + makeStoreKey(docID, fieldName), getCipherText(t, fieldName), ) assert.Nil(t, data, "data should be nil") diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 2dec134d14..3d24138ca7 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -91,20 +91,16 @@ func (mc *MerkleClock) AddDelta( if block.Delta.GetFieldName() != "" { fieldName = immutable.Some(block.Delta.GetFieldName()) } - blockEnc, err := mc.determineBlockEncryptionData(ctx, fieldName, height, heads) + blockEnc, err := mc.determineBlockEncryptionData(ctx, string(block.Delta.GetDocID()), fieldName, heads) if err != nil { return cidlink.Link{}, nil, err } dagBlock := block if blockEnc != nil && blockEnc.Type != coreblock.NotEncrypted { - if !block.Delta.IsComposite() { - dagBlock, err = encryptBlock(ctx, block, blockEnc) - if err != nil { - return cidlink.Link{}, nil, err - } - } else { - dagBlock.Encryption = blockEnc + dagBlock, err = encryptBlock(ctx, block, blockEnc) + if err != nil { + return cidlink.Link{}, nil, err } } @@ -129,18 +125,25 @@ func (mc *MerkleClock) AddDelta( func (mc *MerkleClock) determineBlockEncryptionData( ctx context.Context, + docID string, fieldName immutable.Option[string], - height uint64, heads []cid.Cid, ) (*coreblock.Encryption, error) { // if new encryption was requested by the user if encryption.ShouldEncryptDocField(ctx, fieldName) { - blockEnc := &coreblock.Encryption{From: height} + blockEnc := &coreblock.Encryption{} if encryption.ShouldEncryptIndividualField(ctx, fieldName) { blockEnc.Type = coreblock.FieldEncrypted } else { blockEnc.Type = coreblock.DocumentEncrypted } + encStoreKey, _, err := encryption.GetOrGenerateEncryptionKey(ctx, docID, fieldName) + if err != nil { + return nil, err + } + if encStoreKey.HasValue() { + blockEnc.KeyID = []byte(encStoreKey.Value().KeyID) + } return blockEnc, nil } @@ -155,9 +158,10 @@ func (mc *MerkleClock) determineBlockEncryptionData( return nil, err } if prevBlock.Encryption != nil { - blockEnc := &coreblock.Encryption{Type: (*prevBlock.Encryption).Type} - blockEnc.From = prevBlock.Encryption.From - return blockEnc, nil + return &coreblock.Encryption{ + Type: prevBlock.Encryption.Type, + KeyID: prevBlock.Encryption.KeyID, + }, nil } } @@ -169,16 +173,20 @@ func encryptBlock( block *coreblock.Block, blockEnc *coreblock.Encryption, ) (*coreblock.Block, error) { - clonedCRDT := block.Delta.Clone() fieldName := immutable.None[string]() if blockEnc.Type == coreblock.FieldEncrypted { - fieldName = immutable.Some(clonedCRDT.GetFieldName()) + fieldName = immutable.Some(block.Delta.GetFieldName()) } - bytes, err := encryption.EncryptDoc( - ctx, - core.NewEncStoreDocKey(string(clonedCRDT.GetDocID()), fieldName, blockEnc.From), - clonedCRDT.GetData(), - ) + + encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), fieldName, string(blockEnc.KeyID)) + blockEnc.KeyID = []byte(encStoreKey.KeyID) + if block.Delta.IsComposite() { + block.Encryption = blockEnc + return block, nil + } + + clonedCRDT := block.Delta.Clone() + bytes, err := encryption.EncryptDoc(ctx, encStoreKey, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/internal/merkle/clock/errors.go b/internal/merkle/clock/errors.go index 9903f777a9..a20ce30731 100644 --- a/internal/merkle/clock/errors.go +++ b/internal/merkle/clock/errors.go @@ -26,6 +26,7 @@ const ( errReplacingHead = "error replacing head" errCouldNotFindBlock = "error checking for known block " errFailedToGetNextQResult = "failed to get next query result" + errCouldNotGetEncKey = "could not get encryption key" ) var ( @@ -39,6 +40,7 @@ var ( ErrCouldNotFindBlock = errors.New(errCouldNotFindBlock) ErrFailedToGetNextQResult = errors.New(errFailedToGetNextQResult) ErrDecodingHeight = errors.New("error decoding height") + ErrCouldNotGetEncKey = errors.New(errCouldNotGetEncKey) ) func NewErrCreatingBlock(inner error) error { diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index ec09f861a4..125e58f286 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -355,8 +355,8 @@ type EncryptionKeyTarget struct { // fieldName if the name of the document field that the key is being requested for. // If the fieldName is empty, the key is being requested for the whole document. FieldName string `protobuf:"bytes,2,opt,name=fieldName,proto3" json:"fieldName,omitempty"` - // height is the height of the block the encryption key is being requested for. - Height uint64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` + // keyID is the hash (Cid) of the key that is being requested. + KeyID []byte `protobuf:"bytes,3,opt,name=keyID,proto3" json:"keyID,omitempty"` } func (x *EncryptionKeyTarget) Reset() { @@ -405,11 +405,11 @@ func (x *EncryptionKeyTarget) GetFieldName() string { return "" } -func (x *EncryptionKeyTarget) GetHeight() uint64 { +func (x *EncryptionKeyTarget) GetKeyID() []byte { if x != nil { - return x.Height + return x.KeyID } - return 0 + return nil } // FetchEncryptionKeyRequest is a request to receive a doc encryption key @@ -801,72 +801,72 @@ var file_net_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x61, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x5f, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, - 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, - 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, - 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, - 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, - 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, - 0x65, 0x79, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x6f, 0x6f, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, - 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, - 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, - 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, - 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, - 0xae, 0x03, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, - 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, - 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, - 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, - 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, - 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x5b, 0x0a, 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, - 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, - 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, - 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, - 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, - 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, - 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x42, 0x0a, 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x14, 0x0a, 0x05, 0x6b, 0x65, 0x79, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, + 0x6b, 0x65, 0x79, 0x49, 0x44, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, + 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, + 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x65, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, + 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, + 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, + 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, + 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, + 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, + 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, + 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, + 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, + 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, + 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, + 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, + 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, + 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, + 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/net/pb/net.proto b/net/pb/net.proto index 6632077c7b..4bde3a4b88 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -45,8 +45,8 @@ message EncryptionKeyTarget { // fieldName if the name of the document field that the key is being requested for. // If the fieldName is empty, the key is being requested for the whole document. string fieldName = 2; - // height is the height of the block the encryption key is being requested for. - uint64 height = 3; + // keyID is the hash (Cid) of the key that is being requested. + bytes keyID = 3; } // FetchEncryptionKeyRequest is a request to receive a doc encryption key diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index f7c53f117b..4212fb1e3a 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -400,10 +400,12 @@ func (m *EncryptionKeyTarget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if m.Height != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Height)) + if len(m.KeyID) > 0 { + i -= len(m.KeyID) + copy(dAtA[i:], m.KeyID) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.KeyID))) i-- - dAtA[i] = 0x18 + dAtA[i] = 0x1a } if len(m.FieldName) > 0 { i -= len(m.FieldName) @@ -792,8 +794,9 @@ func (m *EncryptionKeyTarget) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Height != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Height)) + l = len(m.KeyID) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n @@ -1684,10 +1687,10 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { m.FieldName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KeyID", wireType) } - m.Height = 0 + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -1697,11 +1700,26 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Height |= uint64(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KeyID = append(m.KeyID[:0], dAtA[iNdEx:postIndex]...) + if m.KeyID == nil { + m.KeyID = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index e96d3b9688..dfea15d18e 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -14,9 +14,10 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/binary" "fmt" + "encoding/base64" + "github.com/libp2p/go-libp2p/core/peer" libpeer "github.com/libp2p/go-libp2p/core/peer" rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" @@ -42,7 +43,7 @@ func (s *server) getEncryptionKeys( ctx context.Context, req *pb.FetchEncryptionKeyRequest, ) ([]byte, []*pb.EncryptionKeyTarget, error) { - encryptionKeys := make([]byte, 0) + encryptionKeys := make([]byte, 0, len(req.Targets)) targets := make([]*pb.EncryptionKeyTarget, 0, len(req.Targets)) for _, target := range req.Targets { docID, err := client.NewDocIDFromString(string(target.DocID)) @@ -56,7 +57,7 @@ func (s *server) getEncryptionKeys( } encKey, err := encryption.GetKey( encryption.ContextWithStore(ctx, s.peer.encstore), - core.NewEncStoreDocKey(docID.String(), optFieldName, target.Height), + core.NewEncStoreDocKey(docID.String(), optFieldName, string(target.KeyID)), ) if err != nil { return nil, nil, err @@ -133,9 +134,7 @@ func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { for _, target := range res.Targets { hash.Write(target.DocID) hash.Write([]byte(target.FieldName)) - heightBytes := make([]byte, 8) - binary.BigEndian.PutUint64(heightBytes, target.Height) - hash.Write(heightBytes) + hash.Write([]byte(target.KeyID)) } return hash.Sum(nil) } @@ -199,8 +198,8 @@ func (s *server) prepareFetchEncryptionKeyRequest( for _, encStoreKey := range evt.Keys { encKey := &pb.EncryptionKeyTarget{ - DocID: []byte(encStoreKey.DocID), - Height: encStoreKey.BlockHeight, + DocID: []byte(encStoreKey.DocID), + KeyID: []byte(encStoreKey.KeyID), } if encStoreKey.FieldName.HasValue() { encKey.FieldName = encStoreKey.FieldName.Value() @@ -264,9 +263,7 @@ func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { for _, target := range req.Targets { hash.Write(target.DocID) hash.Write([]byte(target.FieldName)) - heightBytes := make([]byte, 8) - binary.BigEndian.PutUint64(heightBytes, target.Height) - hash.Write(heightBytes) + hash.Write([]byte(target.KeyID)) } return hash.Sum(nil) } @@ -327,19 +324,25 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet encKey := decryptedData[:crypto.AESKeySize] decryptedData = decryptedData[crypto.AESKeySize:] - eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, target.Height)] = encKey + eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, string(target.KeyID))] = encKey } s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) } +func encodeToBase64(data []byte) []byte { + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(data))) + base64.StdEncoding.Encode(encoded, data) + return encoded +} + // makeAssociatedData creates the associated data for the encryption key request func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) []byte { - return bytes.Join([][]byte{ + return encodeToBase64(bytes.Join([][]byte{ []byte(req.SchemaRoot), []byte(req.EphemeralPublicKey), []byte(peerID), - }, []byte{}) + }, []byte{})) } func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 12a363c8b5..4d1efbc2fa 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -48,7 +48,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -58,7 +58,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -68,7 +68,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreibdoxaj6yeybokybtjl3mprzryjzufoxzhiumibuvtvpsylf5hufi", + "cid": "bafyreiey25ljav736mpbs6ghkmvwstxil4lt4jrte33p6jnixg4nvzz264", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -77,11 +77,11 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", "name": "age", }, { - "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", "name": "name", }, }, diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index e88765addf..7b33998e80 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -60,7 +60,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -70,7 +70,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -80,7 +80,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreibdoxaj6yeybokybtjl3mprzryjzufoxzhiumibuvtvpsylf5hufi", + "cid": "bafyreiey25ljav736mpbs6ghkmvwstxil4lt4jrte33p6jnixg4nvzz264", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -89,11 +89,11 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreiapgmaxzdyvhmpoh3oibsem5xsl2hpl7wtfalzfv37ikcfjsmt5d4", + "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", "name": "age", }, { - "cid": "bafyreid7f742ai5fyhvofwn5ortiqcx3jtwahkxcvfb5kspufndfq6iqeu", + "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", "name": "name", }, }, From 21c98e7ce228435f4e520aca49b299e4fa73e46c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 12:11:10 +0200 Subject: [PATCH 35/88] Add minor PR changes --- internal/db/errors.go | 1 + internal/db/messages.go | 37 +++++++++++++------------------ net/server.go | 4 +++- net/server_encryption_key.go | 2 +- tests/integration/assert_stack.go | 4 ++++ tests/integration/events.go | 8 +++++-- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/internal/db/errors.go b/internal/db/errors.go index e8a835f3f2..38878616ac 100644 --- a/internal/db/errors.go +++ b/internal/db/errors.go @@ -101,6 +101,7 @@ const ( errReplicatorCollections string = "failed to get collections for replicator" errReplicatorNotFound string = "replicator not found" errCanNotEncryptBuiltinField string = "can not encrypt build-in field" + errFailedToHandleEncKeysReceivedEvent string = "failed to handle encryption-keys-received event" ) var ( diff --git a/internal/db/messages.go b/internal/db/messages.go index ef9631dbcd..c8891b6dd3 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -82,30 +82,25 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { case encryption.KeyRetrievedEvent: go func() { - ctx = encryption.ContextWithStore(ctx, db.Encstore()) - for encStoreKey, encKey := range evt.Keys { - err := encryption.SaveKey(ctx, encStoreKey, encKey) - - if err != nil { - log.ErrorContextE( - ctx, - "Failed to save doc encryption key", - err, - corelog.Any("Event", evt)) - } - } - - err := db.mergeEncryptedBlocks(ctx, evt) - - if err != nil { - log.ErrorContextE( - ctx, - "Failed to merge encrypted block", - err, - corelog.Any("Event", evt)) + if err := db.handleEncryptionKeysRetrievedEvent(ctx, evt); err != nil { + log.ErrorContextE(ctx, errFailedToHandleEncKeysReceivedEvent, err, corelog.Any("Event", evt)) } }() } } } } + +// handleEncryptionKeysRetrievedEvent handles the event when requested encryption keys are retrieved from other peers. +func (db *db) handleEncryptionKeysRetrievedEvent(ctx context.Context, evt encryption.KeyRetrievedEvent) error { + ctx = encryption.ContextWithStore(ctx, db.Encstore()) + for encStoreKey, encKey := range evt.Keys { + err := encryption.SaveKey(ctx, encStoreKey, encKey) + + if err != nil { + return err + } + } + + return db.mergeEncryptedBlocks(ctx, evt) +} diff --git a/net/server.go b/net/server.go index a5a6472145..5bf878c2d8 100644 --- a/net/server.go +++ b/net/server.go @@ -98,7 +98,9 @@ func newServer(p *Peer, opts ...grpc.DialOption) (*server, error) { return s, nil } -func (s *server) extractSessionAndRemoveOldOnes(id string) *session { +// extractSessionAndRemoveOld extracts a session with the given id from the server's session list +// and removes any old sessions by comparing their timestamps. +func (s *server) extractSessionAndRemoveOld(id string) *session { var result *session swapLast := func(i int) { s.sessions[i] = s.sessions[len(s.sessions)-1] diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index dfea15d18e..498e866b04 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -292,7 +292,7 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet return } - session := s.extractSessionAndRemoveOldOnes(string(keyResp.ReqEphemeralPublicKey)) + session := s.extractSessionAndRemoveOld(string(keyResp.ReqEphemeralPublicKey)) if session == nil { log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") return diff --git a/tests/integration/assert_stack.go b/tests/integration/assert_stack.go index 0662db3390..a341c96a31 100644 --- a/tests/integration/assert_stack.go +++ b/tests/integration/assert_stack.go @@ -15,6 +15,10 @@ import ( "strings" ) +// assertStack keeps track of the current assertion path. +// GraphQL response can be traversed by a key of a map and/or an index of an array. +// So whenever we have a mismatch in a large response, we can use this stack to find the exact path. +// Example output: "commits[2].links[1].cid" type assertStack struct { stack []string isMap []bool diff --git a/tests/integration/events.go b/tests/integration/events.go index 3ec8c577c6..20a74a16b8 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -351,9 +351,13 @@ func getEventsForCreateDoc(s *state, action CreateDoc) map[string]struct{} { return expect } -func waitForKeyRetrievedEvent(s *state, nodeIDs []int) { +// waitForKeyRetrievedEvent waits for nodes to receive a key retrieved event. +// If targetNodeIDs is empty, all nodes will be waiting on the event sync. +// Otherwise, only the nodes with the specified IDs will be checked. +func waitForKeyRetrievedEvent(s *state, targetNodeIDs []int) { for nodeID := 0; nodeID < len(s.nodes); nodeID++ { - if len(nodeIDs) > 0 && !slices.Contains(nodeIDs, nodeID) { + // if we have target nodes and the current node is not in the list, skip it + if len(targetNodeIDs) > 0 && !slices.Contains(targetNodeIDs, nodeID) { continue } From 09fb85513ed0ed8760b774be4771dc5949413530 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 12:33:12 +0200 Subject: [PATCH 36/88] Wait for sync on goroutine if count > 1 --- tests/integration/events.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/integration/events.go b/tests/integration/events.go index 20a74a16b8..1a127ed497 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -13,6 +13,7 @@ package tests import ( "encoding/json" "slices" + "sync" "time" "github.com/sourcenetwork/immutable" @@ -378,7 +379,8 @@ func waitForSync(s *state, action WaitForSync) { if count == 0 { count = 1 } - for i := 0; i < count; i++ { + + syncFunc := func(s *state, action WaitForSync) { if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { waitForMergeEvents(s) } else if action.Event.Value() == encryption.KeysRetrievedEventName { @@ -387,6 +389,20 @@ func waitForSync(s *state, action WaitForSync) { require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) } } + + if count == 1 { + syncFunc(s, action) + } else { + var wg sync.WaitGroup + wg.Add(count) + for i := 0; i < count; i++ { + go func(s *state, action WaitForSync) { + defer wg.Done() + syncFunc(s, action) + }(s, action) + } + wg.Wait() + } } // getEventsForUpdateWithFilter returns a map of docIDs that should be From f85e9a2e661cd4fefeef971790c2a9362c8344fc Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 12:33:19 +0200 Subject: [PATCH 37/88] Add docs --- tests/integration/p2p.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index b81504f1dd..9d1e50e2e6 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -144,7 +144,8 @@ type WaitForSync struct { // NodeIDs are the node IDs (indexes) of the nodes to wait for sync on. // If not provided then the action will wait for sync on all nodes. NodeIDs []int - // Count is the number of times the event should be received. + // Count is the number of times the event should be received by each node. + // Default value (zero) is treated as 1, e.g. the event should be received once. Count uint } From a8415bf07eb8b2603043007b4ea862f72ceb3ba2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 13:14:48 +0200 Subject: [PATCH 38/88] Add tests for ECIES --- crypto/ecies_test.go | 100 +++++++++++++++++++++++++++++++++++++ tests/integration/utils.go | 1 + 2 files changed, 101 insertions(+) create mode 100644 crypto/ecies_test.go diff --git a/crypto/ecies_test.go b/crypto/ecies_test.go new file mode 100644 index 0000000000..d973535f6a --- /dev/null +++ b/crypto/ecies_test.go @@ -0,0 +1,100 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "crypto/ecdh" + "strings" + "testing" +) + +func TestEncryptECIES_Errors(t *testing.T) { + validAssociatedData := []byte("associated data") + + tests := []struct { + name string + plainText []byte + publicKey *ecdh.PublicKey + associatedData []byte + expectError string + }{ + { + name: "Invalid public key", + plainText: []byte("test data"), + publicKey: &ecdh.PublicKey{}, + associatedData: validAssociatedData, + expectError: "failed ECDH operation", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := EncryptECIES(tt.plainText, tt.publicKey, tt.associatedData) + if err == nil { + t.Errorf("Expected an error, but got nil") + } else if !strings.Contains(err.Error(), tt.expectError) { + t.Errorf("Expected error containing '%s', got '%v'", tt.expectError, err) + } + }) + } +} + +func TestDecryptECIES_Errors(t *testing.T) { + // Setup + validPrivateKey, _ := GenerateX25519() + validCipherText, _ := EncryptECIES([]byte("test data"), validPrivateKey.PublicKey(), []byte("associated data")) + + tests := []struct { + name string + cipherText []byte + privateKey *ecdh.PrivateKey + associatedData []byte + expectError string + }{ + { + name: "Ciphertext too short", + cipherText: []byte("short"), + privateKey: validPrivateKey, + associatedData: []byte("associated data"), + expectError: "ciphertext too short", + }, + { + name: "Invalid private key", + cipherText: validCipherText, + privateKey: &ecdh.PrivateKey{}, + associatedData: []byte("associated data"), + expectError: "failed ECDH operation", + }, + { + name: "Tampered ciphertext", + cipherText: append(validCipherText, byte(0)), + privateKey: validPrivateKey, + associatedData: []byte("associated data"), + expectError: "verification with HMAC failed", + }, + { + name: "Wrong associated data", + cipherText: validCipherText, + privateKey: validPrivateKey, + associatedData: []byte("wrong data"), + expectError: "failed to decrypt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DecryptECIES(tt.cipherText, tt.privateKey, tt.associatedData) + if err == nil || !strings.Contains(err.Error(), tt.expectError) { + t.Errorf("Expected error containing '%s', got %v", tt.expectError, err) + } + }) + } +} diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 5e4600ef7c..49921d8971 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -106,6 +106,7 @@ func init() { if value, ok := os.LookupEnv(skipNetworkTestsEnvName); ok { skipNetworkTests, _ = strconv.ParseBool(value) } + mutationType = GQLRequestMutationType } // AssertPanic asserts that the code inside the specified PanicTestFunc panics. From c8369684e4a985c6d05b111e93cf29ad7a1386c9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 13:20:27 +0200 Subject: [PATCH 39/88] Add AES tests --- crypto/aes_test.go | 175 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 crypto/aes_test.go diff --git a/crypto/aes_test.go b/crypto/aes_test.go new file mode 100644 index 0000000000..25009304ff --- /dev/null +++ b/crypto/aes_test.go @@ -0,0 +1,175 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncryptAES(t *testing.T) { + validKey := make([]byte, 32) // AES-256 + _, err := rand.Read(validKey) + require.NoError(t, err) + validPlaintext := []byte("Hello, World!") + validAAD := []byte("Additional Authenticated Data") + + tests := []struct { + name string + plainText []byte + key []byte + additionalData []byte + prependNonce bool + expectError bool + errorContains string + }{ + { + name: "Valid encryption with prepended nonce", + plainText: validPlaintext, + key: validKey, + additionalData: validAAD, + prependNonce: true, + expectError: false, + }, + { + name: "Valid encryption without prepended nonce", + plainText: validPlaintext, + key: validKey, + additionalData: validAAD, + prependNonce: false, + expectError: false, + }, + { + name: "Invalid key size", + plainText: validPlaintext, + key: make([]byte, 31), // Invalid key size + additionalData: validAAD, + prependNonce: true, + expectError: true, + errorContains: "invalid key size", + }, + { + name: "Nil plaintext", + plainText: nil, + key: validKey, + additionalData: validAAD, + prependNonce: true, + expectError: false, // AES-GCM can encrypt nil/empty plaintext + }, + { + name: "Nil additional data", + plainText: validPlaintext, + key: validKey, + additionalData: nil, + prependNonce: true, + expectError: false, // Nil AAD is valid + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cipherText, nonce, err := EncryptAES(tt.plainText, tt.key, tt.additionalData, tt.prependNonce) + + if tt.expectError { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errorContains) + } else { + require.NoError(t, err) + if tt.prependNonce { + require.Greater(t, len(cipherText), len(nonce), "Ciphertext length not greater than nonce length") + } else { + require.Equal(t, AESNonceSize, len(nonce), "Nonce length != AESNonceSize") + } + } + }) + } +} + +func TestDecryptAES(t *testing.T) { + validKey := make([]byte, 32) // AES-256 + _, err := rand.Read(validKey) + require.NoError(t, err) + validPlaintext := []byte("Hello, World!") + validAAD := []byte("Additional Authenticated Data") + validCiphertext, validNonce, _ := EncryptAES(validPlaintext, validKey, validAAD, true) + + tests := []struct { + name string + nonce []byte + cipherText []byte + key []byte + additionalData []byte + expectError bool + errorContains string + }{ + { + name: "Valid decryption", + nonce: nil, // Should be extracted from cipherText + cipherText: validCiphertext, + key: validKey, + additionalData: validAAD, + expectError: false, + }, + { + name: "Invalid key size", + nonce: validNonce, + cipherText: validCiphertext[AESNonceSize:], + key: make([]byte, 31), // Invalid key size + additionalData: validAAD, + expectError: true, + errorContains: "invalid key size", + }, + { + name: "Ciphertext too short", + nonce: nil, + cipherText: make([]byte, AESNonceSize-1), // Too short to contain nonce + key: validKey, + additionalData: validAAD, + expectError: true, + errorContains: "cipherText too short", + }, + { + name: "Invalid additional data", + nonce: validNonce, + cipherText: validCiphertext[AESNonceSize:], + key: validKey, + additionalData: []byte("Wrong AAD"), + expectError: true, + errorContains: "message authentication failed", + }, + { + name: "Tampered ciphertext", + nonce: validNonce, + cipherText: append([]byte{0}, validCiphertext[AESNonceSize+1:]...), + key: validKey, + additionalData: validAAD, + expectError: true, + errorContains: "message authentication failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plainText, err := DecryptAES(tt.nonce, tt.cipherText, tt.key, tt.additionalData) + + if tt.expectError { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errorContains) + } else { + require.NoError(t, err) + require.True(t, bytes.Equal(plainText, validPlaintext), "Decrypted plaintext does not match original") + } + }) + } +} From fdfa6d456f628544c4c74ca61bd9796d6a66332a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 13:40:18 +0200 Subject: [PATCH 40/88] Polish --- net/errors.go | 4 ---- net/server_encryption_key.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/net/errors.go b/net/errors.go index 34f27121a5..fe8f286ab7 100644 --- a/net/errors.go +++ b/net/errors.go @@ -51,10 +51,6 @@ func NewErrPublishingToSchemaTopic(inner error, cid, docID string, kv ...errors. return errors.Wrap(fmt.Sprintf(errPublishingToSchemaTopic, cid, docID), inner, kv...) } -func NewErrCheckingForExistingBlock(inner error, cid string) error { - return errors.Wrap(errCheckingForExistingBlock, inner, errors.NewKV("cid", cid)) -} - func NewErrRequestingEncryptionKeys(inner error, keys []core.EncStoreDocKey) error { return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKeys, keys), inner) } diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 498e866b04..8fece8bf47 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -63,7 +63,7 @@ func (s *server) getEncryptionKeys( return nil, nil, err } // TODO: we should test it somehow. For this this one peer should have some keys and - // another one should have the others + // another one should have the others. https://github.com/sourcenetwork/defradb/issues/2895 if len(encKey) == 0 { continue } From 25d4373d36250342a1fe0323ce2c1a63a9302986 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 13:40:51 +0200 Subject: [PATCH 41/88] Add unmarshal tests for block --- internal/core/block/block_test.go | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/internal/core/block/block_test.go b/internal/core/block/block_test.go index bfd949077a..f30c06451c 100644 --- a/internal/core/block/block_test.go +++ b/internal/core/block/block_test.go @@ -229,6 +229,91 @@ func TestBlockMarshal_IsEncryptedNotSetWithLinkSystem_ShouldLoadWithNoError(t *t require.NoError(t, err) } +func TestBlockUnmarshal_ValidInput_Succeed(t *testing.T) { + validBlock := Block{ + Delta: crdt.CRDT{ + LWWRegDelta: &crdt.LWWRegDelta{ + DocID: []byte("docID"), + FieldName: "name", + Priority: 1, + SchemaVersionID: "schemaVersionID", + Data: []byte("John"), + }, + }, + } + + marshaledData, err := validBlock.Marshal() + require.NoError(t, err) + + var unmarshaledBlock Block + err = unmarshaledBlock.Unmarshal(marshaledData) + require.NoError(t, err) + + require.Equal(t, validBlock, unmarshaledBlock) +} + +func TestBlockUnmarshal_InvalidCBOR_Error(t *testing.T) { + invalidData := []byte("invalid CBOR data") + var block Block + err := block.Unmarshal(invalidData) + require.Error(t, err) +} + +func TestBlockUnmarshal_WithValidEncryption_Succeed(t *testing.T) { + encryptedBlock := Block{ + Delta: crdt.CRDT{ + LWWRegDelta: &crdt.LWWRegDelta{ + DocID: []byte("docID"), + FieldName: "name", + Priority: 1, + SchemaVersionID: "schemaVersionID", + Data: []byte("John"), + }, + }, + Encryption: &Encryption{ + Type: FieldEncrypted, + KeyID: []byte("keyID"), + }, + } + + marshaledData, err := encryptedBlock.Marshal() + require.NoError(t, err) + + var unmarshaledBlock Block + err = unmarshaledBlock.Unmarshal(marshaledData) + require.NoError(t, err) + + require.Equal(t, encryptedBlock, unmarshaledBlock) + require.NotNil(t, unmarshaledBlock.Encryption) + require.Equal(t, FieldEncrypted, unmarshaledBlock.Encryption.Type) + require.Equal(t, []byte("keyID"), unmarshaledBlock.Encryption.KeyID) +} + +func TestBlockUnmarshal_WithInvalidEncryption_Error(t *testing.T) { + encryptedBlock := Block{ + Delta: crdt.CRDT{ + LWWRegDelta: &crdt.LWWRegDelta{ + DocID: []byte("docID"), + FieldName: "name", + Priority: 1, + SchemaVersionID: "schemaVersionID", + Data: []byte("John"), + }, + }, + Encryption: &Encryption{ + Type: FieldEncrypted, + KeyID: []byte{}, + }, + } + + marshaledData, err := encryptedBlock.Marshal() + require.NoError(t, err) + + var unmarshaledBlock Block + err = unmarshaledBlock.Unmarshal(marshaledData) + require.Error(t, err) +} + func TestBlock_Validate(t *testing.T) { tests := []struct { name string From 658a092a2ee4e1316de5588d27e07ef42c45e4aa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 13:53:13 +0200 Subject: [PATCH 42/88] Add tests for Cid --- crypto/cid_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 crypto/cid_test.go diff --git a/crypto/cid_test.go b/crypto/cid_test.go new file mode 100644 index 0000000000..11be7f04b8 --- /dev/null +++ b/crypto/cid_test.go @@ -0,0 +1,46 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestGenerateCid_HappyPath(t *testing.T) { + testData := []byte("Hello, world!") + + generatedCid, err := GenerateCid(testData) + + require.NoError(t, err) + require.NotEmpty(t, generatedCid) +} + +func TestGenerateCid_ErrorPath(t *testing.T) { + // Define a custom GenerateCid function that uses a failing hash function + generateCidWithError := func(data []byte) (cid.Cid, error) { + _, err := multihash.Sum(data, 0xffff, -1) // Use an invalid hash function code + if err != nil { + return cid.Cid{}, err + } + return cid.Cid{}, nil // This line should never be reached + } + + testData := []byte("This data doesn't matter as we're forcing an error") + generatedCid, err := generateCidWithError(testData) + + require.Error(t, err) + require.Contains(t, err.Error(), "unknown multihash code") + require.Equal(t, cid.Cid{}, generatedCid) // Should be an empty CID +} From 168f943e43a66f31fafde4783dfa5b93186828ac Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 16:03:56 +0200 Subject: [PATCH 43/88] Skip even attempt to index if doc is encrypted --- internal/db/merge.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 7052f616d0..d7d224df3e 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -659,10 +659,15 @@ func syncIndexedDoc( return err } - if isDeletedDoc { - return col.deleteIndexedDoc(ctx, oldDoc) - } else if isNewDoc { + // this can happen we received an encrypted document that we haven't decrypted yet + if isNewDoc && isDeletedDoc { + return nil + } + + if isNewDoc { return col.indexNewDoc(ctx, doc) + } else if isDeletedDoc { + return col.deleteIndexedDoc(ctx, oldDoc) } else { return col.updateDocIndex(ctx, oldDoc, doc) } From ff277b6f3de2298333db62e7006785974da0d4a8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 16:14:40 +0200 Subject: [PATCH 44/88] Fix tests --- tests/integration/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 49921d8971..5e4600ef7c 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -106,7 +106,6 @@ func init() { if value, ok := os.LookupEnv(skipNetworkTestsEnvName); ok { skipNetworkTests, _ = strconv.ParseBool(value) } - mutationType = GQLRequestMutationType } // AssertPanic asserts that the code inside the specified PanicTestFunc panics. From 20a1c797bf28631246b6f382e4d19e408a28c6b7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 21:15:30 +0200 Subject: [PATCH 45/88] Don't request enc keys if not pending --- internal/db/merge.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index d7d224df3e..defac9cc3b 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -404,8 +404,12 @@ func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName im } func (mp *mergeProcessor) sendPendingEncryptionRequest() { + n := len(mp.pendingEncryptionKeyRequests) + if n == 0 { + return + } schemaRoot := mp.col.SchemaRoot() - storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) + storeKeys := make([]core.EncStoreDocKey, 0, n) for k := range mp.pendingEncryptionKeyRequests { storeKeys = append(storeKeys, k) } From 3a8f09b6a2953da13079fc5f4050f6b7b4e534d5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 23:17:37 +0200 Subject: [PATCH 46/88] Handle AnyOf for doc (not only fields) --- tests/integration/utils.go | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 5e4600ef7c..ad027c9fe0 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -150,7 +150,7 @@ func ExecuteTestCase( clients = append(clients, HTTPClientType) } if goClient { - clients = append(clients, GoClientType) + clients = append(clients, GoClientType) } if cliClient { clients = append(clients, CLIClientType) @@ -161,7 +161,7 @@ func ExecuteTestCase( databases = append(databases, badgerIMType) } if badgerFile { - databases = append(databases, badgerFileType) + databases = append(databases, badgerFileType) } if inMemoryStore { databases = append(databases, defraIMType) @@ -1709,7 +1709,6 @@ func executeRequest( result := node.ExecRequest(ctx, action.Request) - anyOfByFieldKey := map[docFieldKey][]any{} expectedErrorRaised = assertRequestResults( s, &result.GQL, @@ -1717,7 +1716,6 @@ func executeRequest( action.ExpectedError, action.Asserter, nodeID, - anyOfByFieldKey, ) } @@ -1767,9 +1765,7 @@ func executeSubscriptionRequest( r, action.ExpectedError, nil, - // anyof is not yet supported by subscription requests 0, - map[docFieldKey][]any{}, ) assertExpectedErrorRaised(s.t, s.testCase.Description, action.ExpectedError, expectedErrorRaised) @@ -1826,12 +1822,6 @@ func AssertErrors( return false } -// docFieldKey is an internal key type that wraps docIndex and fieldName -type docFieldKey struct { - docIndex int - fieldName string -} - func assertRequestResults( s *state, result *client.GQLResult, @@ -1839,7 +1829,6 @@ func assertRequestResults( expectedError string, asserter ResultAsserter, nodeID int, - anyOfByField map[docFieldKey][]any, ) bool { // we skip assertion benchmark because you don't specify expected result for benchmark. if AssertErrors(s.t, s.testCase.Description, result.Errors, expectedError) || s.isBench { @@ -1877,18 +1866,19 @@ func assertRequestResults( actual, ok := resultantData[key] require.True(s.t, ok, "result key not found: %s", key) - expectDocs, ok := expect.([]map[string]any) - if ok { + switch exp := expect.(type) { + case []map[string]any: actualDocs := ConvertToArrayOfMaps(s.t, actual) assertRequestResultDocs( s, nodeID, - expectDocs, + exp, actualDocs, - anyOfByField, stack, ) - } else { + case AnyOf: + assertResultsAnyOf(s.t, s.clientType, exp, actual) + default: assertResultsEqual( s.t, s.clientType, @@ -1908,7 +1898,6 @@ func assertRequestResultDocs( nodeID int, expectedResults []map[string]any, actualResults []map[string]any, - anyOfByField map[docFieldKey][]any, stack *assertStack, ) bool { // compare results @@ -1935,11 +1924,6 @@ func assertRequestResultDocs( switch expectedValue := expectedDoc[field].(type) { case AnyOf: assertResultsAnyOf(s.t, s.clientType, expectedValue, actualValue) - - dfk := docFieldKey{actualDocIndex, field} - valueSet := anyOfByField[dfk] - valueSet = append(valueSet, actualValue) - anyOfByField[dfk] = valueSet case DocIndex: expectedDocID := s.docIDs[expectedValue.CollectionIndex][expectedValue.Index].String() assertResultsEqual( @@ -1957,7 +1941,6 @@ func assertRequestResultDocs( nodeID, expectedValue, actualValueMap, - anyOfByField, stack, ) From 53a3e9578a34b9488bf6f1d2ec022d885d59f007 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 23:18:32 +0200 Subject: [PATCH 47/88] Add a test --- tests/integration/encryption/peer_test.go | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 7b33998e80..959263cd83 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -106,3 +106,52 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryptionPeer_IfPeerDidNotReceiveKey_ShouldNotFetch(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + updateUserCollectionSchema(), + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.WaitForSync{}, + // Do not wait for the key sync and request the document as soon as the dag has synced + // The document will be returned if the key-sync has taken place already, if not, the set will + // be empty. + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + Users { + age + } + }`, + Results: map[string]any{ + "Users": testUtils.AnyOf{ + // The key-sync has not yet completed + []map[string]any{}, + // The key-sync has completed + []map[string]any{ + { + "age": int64(21), + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From 89ff5f07f87b5073948f574e39440201d1e27c51 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 13 Aug 2024 23:33:29 +0200 Subject: [PATCH 48/88] Fix lint --- tests/integration/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/utils.go b/tests/integration/utils.go index ad027c9fe0..76105342ec 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -150,7 +150,7 @@ func ExecuteTestCase( clients = append(clients, HTTPClientType) } if goClient { - clients = append(clients, GoClientType) + clients = append(clients, GoClientType) } if cliClient { clients = append(clients, CLIClientType) @@ -161,7 +161,7 @@ func ExecuteTestCase( databases = append(databases, badgerIMType) } if badgerFile { - databases = append(databases, badgerFileType) + databases = append(databases, badgerFileType) } if inMemoryStore { databases = append(databases, defraIMType) From 16b175d078554528c5c809ed87b4af2389642950 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Aug 2024 13:50:28 +0200 Subject: [PATCH 49/88] Fix an issue with overwriting AAD --- crypto/ecies.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index e9d579120d..ea4e785c80 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -78,8 +78,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) } - fullAssociatedData := append(ephemeralPublic.Bytes(), associatedData...) - cipherText, _, err := EncryptAES(plainText, aesKey, fullAssociatedData, true) + cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ephemeralPrivate.Bytes(), associatedData), true) if err != nil { return nil, fmt.Errorf("failed to encrypt: %w", err) } @@ -149,11 +148,19 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData return nil, fmt.Errorf("verification with HMAC failed") } - fullAssociatedData := append(ephemeralPublicBytes, associatedData...) - plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, fullAssociatedData) + plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(ephemeralPublicBytes, associatedData)) if err != nil { return nil, fmt.Errorf("failed to decrypt: %w", err) } return plainText, nil } + +// makeAAD concatenates the ephemeral public key and associated data for use as additional authenticated data. +func makeAAD(ephemeralPublicBytes, associatedData []byte) []byte { + l := len(ephemeralPublicBytes) + len(associatedData) + aad := make([]byte, l) + copy(aad, ephemeralPublicBytes) + copy(aad[len(ephemeralPublicBytes):], associatedData) + return aad +} From f115a14e8ba80b9d9cb2268df9b78566a906dce3 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Aug 2024 13:51:40 +0200 Subject: [PATCH 50/88] Remove cache from encryptor --- internal/encryption/encryptor.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index ebab69262c..faf52de0bc 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -56,12 +56,11 @@ type DocEncryptor struct { conf immutable.Option[DocEncConfig] ctx context.Context store datastore.DSReaderWriter - cache map[core.EncStoreDocKey][]byte generatedKeys []core.EncStoreDocKey } func newDocEncryptor(ctx context.Context) *DocEncryptor { - return &DocEncryptor{ctx: ctx, cache: make(map[core.EncStoreDocKey][]byte)} + return &DocEncryptor{ctx: ctx} } // SetConfig sets the configuration for the document encryptor. @@ -147,9 +146,6 @@ func (d *DocEncryptor) Decrypt( } func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, error) { - if encryptionKey, ok := d.cache[storeKey]; ok { - return encryptionKey, nil - } encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) isNotFound := errors.Is(err, ds.ErrNotFound) if err != nil { @@ -159,12 +155,10 @@ func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, return nil, err } - d.cache[storeKey] = encryptionKey return encryptionKey, nil } func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encryptionKey []byte) error { - d.cache[storeKey] = encryptionKey return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) } @@ -187,7 +181,11 @@ func (d *DocEncryptor) getGeneratedKeyFor( ) (immutable.Option[core.EncStoreDocKey], []byte) { for _, key := range d.generatedKeys { if key.DocID == docID && key.FieldName == fieldName { - return immutable.Some(key), d.cache[key] + fetchByEncStoreKey, err := d.fetchByEncStoreKey(key) + if err != nil { + return immutable.None[core.EncStoreDocKey](), nil + } + return immutable.Some(key), fetchByEncStoreKey } } return immutable.None[core.EncStoreDocKey](), nil From 0831b01b83308f501bdbeea4fadf4c99ee08a960 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Aug 2024 13:52:38 +0200 Subject: [PATCH 51/88] Make block.GetPrevBlockCids return all heads --- internal/core/block/block.go | 11 +++++------ internal/db/merge.go | 10 +++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index de6ddcdebb..bae086bce1 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -23,8 +23,6 @@ import ( "github.com/ipld/go-ipld-prime/schema" "github.com/multiformats/go-multicodec" - "github.com/sourcenetwork/immutable" - "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/core/crdt" ) @@ -132,14 +130,15 @@ func (b *Block) IsEncrypted() bool { return b.Encryption != nil && b.Encryption.Type != NotEncrypted } -// GetPrevBlockCid returns the CID of the previous block. -func (b *Block) GetPrevBlockCid() immutable.Option[cid.Cid] { +// GetPrevBlockCids returns the CIDs of the previous blocks. It can be more than 1 with multiple heads. +func (b *Block) GetPrevBlockCids() []cid.Cid { + var heads []cid.Cid for _, link := range b.Links { if link.Name == core.HEAD { - return immutable.Some(link.Cid) + heads = append(heads, link.Cid) } } - return immutable.None[cid.Cid]() + return heads } // IPLDSchemaBytes returns the IPLD schema representation for the block. diff --git a/internal/db/merge.go b/internal/db/merge.go index defac9cc3b..7d7ceee4bc 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -315,9 +315,8 @@ func (mp *mergeProcessor) loadBlocks( // In this case, we also need to walk back the merge target's DAG until we reach a common block. if block.Delta.GetPriority() >= mt.headHeight { mp.blocks.PushFront(block) - prevCid := block.GetPrevBlockCid() - if prevCid.HasValue() { - err := mp.loadBlocks(ctx, prevCid.Value(), mt, willDecrypt) + for _, prevCid := range block.GetPrevBlockCids() { + err := mp.loadBlocks(ctx, prevCid, mt, willDecrypt) if err != nil { return err } @@ -633,10 +632,7 @@ func loadBlocksWithKeyIDFromBlockstore( if block.Encryption != nil && bytes.Equal(block.Encryption.KeyID, []byte(keyID)) { blocks = append(blocks, block) - prevCid := block.GetPrevBlockCid() - if prevCid.HasValue() { - cids = append(cids, prevCid.Value()) - } + cids = append(cids, block.GetPrevBlockCids()...) } cids = cids[1:] } From af9009ba396c87ed7670530261214a7dd083d866 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 14 Aug 2024 13:53:07 +0200 Subject: [PATCH 52/88] Moved schemaRoot to session --- net/pb/net.pb.go | 46 +++++------------ net/pb/net.proto | 12 ++--- net/pb/net_vtproto.pb.go | 98 ++---------------------------------- net/server.go | 5 +- net/server_encryption_key.go | 11 ++-- 5 files changed, 26 insertions(+), 146 deletions(-) diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index 125e58f286..ed2ce4375b 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -421,12 +421,10 @@ type FetchEncryptionKeyRequest struct { // targets is the list of docs/fields for which encryption keys are being requested. Targets []*EncryptionKeyTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` - // schemaRoot is the SchemaRoot of the collection that the document resides in. - SchemaRoot []byte `protobuf:"bytes,2,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - EphemeralPublicKey []byte `protobuf:"bytes,3,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` + EphemeralPublicKey []byte `protobuf:"bytes,2,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,3,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { @@ -468,13 +466,6 @@ func (x *FetchEncryptionKeyRequest) GetTargets() []*EncryptionKeyTarget { return nil } -func (x *FetchEncryptionKeyRequest) GetSchemaRoot() []byte { - if x != nil { - return x.SchemaRoot - } - return nil -} - func (x *FetchEncryptionKeyRequest) GetEphemeralPublicKey() []byte { if x != nil { return x.EphemeralPublicKey @@ -502,12 +493,10 @@ type FetchEncryptionKeyReply struct { // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. EncryptedKeys []byte `protobuf:"bytes,2,opt,name=encryptedKeys,proto3" json:"encryptedKeys,omitempty"` - // schemaRoot is the SchemaRoot of the collection that the document resides in. - SchemaRoot []byte `protobuf:"bytes,3,opt,name=schemaRoot,proto3" json:"schemaRoot,omitempty"` // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id - ReqEphemeralPublicKey []byte `protobuf:"bytes,4,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` + ReqEphemeralPublicKey []byte `protobuf:"bytes,3,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` // signature is the responding peer's signature of the reply - Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` + Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyReply) Reset() { @@ -556,13 +545,6 @@ func (x *FetchEncryptionKeyReply) GetEncryptedKeys() []byte { return nil } -func (x *FetchEncryptionKeyReply) GetSchemaRoot() []byte { - if x != nil { - return x.SchemaRoot - } - return nil -} - func (x *FetchEncryptionKeyReply) GetReqEphemeralPublicKey() []byte { if x != nil { return x.ReqEphemeralPublicKey @@ -807,19 +789,17 @@ var file_net_proto_rawDesc = []byte{ 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6b, 0x65, 0x79, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x6b, 0x65, 0x79, 0x49, 0x44, 0x22, 0xc0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6b, 0x65, 0x79, 0x49, 0x44, 0x22, 0xa0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, + 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xca, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, @@ -827,13 +807,11 @@ var file_net_proto_rawDesc = []byte{ 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, - 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x6f, 0x6f, - 0x74, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, - 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x73, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, + 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, diff --git a/net/pb/net.proto b/net/pb/net.proto index 4bde3a4b88..82a0799f03 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -54,12 +54,10 @@ message EncryptionKeyTarget { message FetchEncryptionKeyRequest { // targets is the list of docs/fields for which encryption keys are being requested. repeated EncryptionKeyTarget targets = 1; - // schemaRoot is the SchemaRoot of the collection that the document resides in. - bytes schemaRoot = 2; // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret - bytes ephemeralPublicKey = 3; + bytes ephemeralPublicKey = 2; // signature is the requesting peer's signature of the request - bytes signature = 4; + bytes signature = 3; } // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request @@ -71,12 +69,10 @@ message FetchEncryptionKeyReply { // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. bytes encryptedKeys = 2; - // schemaRoot is the SchemaRoot of the collection that the document resides in. - bytes schemaRoot = 3; // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id - bytes reqEphemeralPublicKey = 4; + bytes reqEphemeralPublicKey = 3; // signature is the responding peer's signature of the reply - bytes signature = 5; + bytes signature = 4; } message GetHeadLogRequest {} diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 4212fb1e3a..00b77f4f19 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -459,20 +459,13 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x22 + dAtA[i] = 0x1a } if len(m.EphemeralPublicKey) > 0 { i -= len(m.EphemeralPublicKey) copy(dAtA[i:], m.EphemeralPublicKey) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EphemeralPublicKey))) i-- - dAtA[i] = 0x1a - } - if len(m.SchemaRoot) > 0 { - i -= len(m.SchemaRoot) - copy(dAtA[i:], m.SchemaRoot) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) - i-- dAtA[i] = 0x12 } if len(m.Targets) > 0 { @@ -525,20 +518,13 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro copy(dAtA[i:], m.Signature) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) i-- - dAtA[i] = 0x2a + dAtA[i] = 0x22 } if len(m.ReqEphemeralPublicKey) > 0 { i -= len(m.ReqEphemeralPublicKey) copy(dAtA[i:], m.ReqEphemeralPublicKey) i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReqEphemeralPublicKey))) i-- - dAtA[i] = 0x22 - } - if len(m.SchemaRoot) > 0 { - i -= len(m.SchemaRoot) - copy(dAtA[i:], m.SchemaRoot) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SchemaRoot))) - i-- dAtA[i] = 0x1a } if len(m.EncryptedKeys) > 0 { @@ -814,10 +800,6 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } - l = len(m.SchemaRoot) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } l = len(m.EphemeralPublicKey) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -846,10 +828,6 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - l = len(m.SchemaRoot) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } l = len(m.ReqEphemeralPublicKey) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1806,40 +1784,6 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.SchemaRoot = append(m.SchemaRoot[:0], dAtA[iNdEx:postIndex]...) - if m.SchemaRoot == nil { - m.SchemaRoot = []byte{} - } - iNdEx = postIndex - case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field EphemeralPublicKey", wireType) } @@ -1873,7 +1817,7 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.EphemeralPublicKey = []byte{} } iNdEx = postIndex - case 4: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } @@ -2027,40 +1971,6 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { } iNdEx = postIndex case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SchemaRoot", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.SchemaRoot = append(m.SchemaRoot[:0], dAtA[iNdEx:postIndex]...) - if m.SchemaRoot == nil { - m.SchemaRoot = []byte{} - } - iNdEx = postIndex - case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ReqEphemeralPublicKey", wireType) } @@ -2094,7 +2004,7 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { m.ReqEphemeralPublicKey = []byte{} } iNdEx = postIndex - case 5: + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) } diff --git a/net/server.go b/net/server.go index 5bf878c2d8..6a47f03193 100644 --- a/net/server.go +++ b/net/server.go @@ -63,11 +63,12 @@ const sessionTimeout = 5 * time.Second type session struct { id string privateKey *ecdh.PrivateKey + schemaRoot string t time.Time } -func newSession(id string, privateKey *ecdh.PrivateKey) session { - return session{id: id, privateKey: privateKey, t: time.Now()} +func newSession(id string, schemaRoot string, privateKey *ecdh.PrivateKey) session { + return session{id: id, schemaRoot: schemaRoot, privateKey: privateKey, t: time.Now()} } // pubsubTopic is a wrapper of rpc.Topic to be able to track if the topic has diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 8fece8bf47..0cbdbbf19a 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -108,7 +108,6 @@ func (s *server) TryGenEncryptionKey( } res := &pb.FetchEncryptionKeyReply{ - SchemaRoot: req.SchemaRoot, ReqEphemeralPublicKey: req.EphemeralPublicKey, Targets: targets, EncryptedKeys: encryptedKey, @@ -129,7 +128,6 @@ func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKe func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { hash := sha256.New() hash.Write(res.EncryptedKeys) - hash.Write(res.SchemaRoot) hash.Write(res.ReqEphemeralPublicKey) for _, target := range res.Targets { hash.Write(target.DocID) @@ -192,7 +190,6 @@ func (s *server) prepareFetchEncryptionKeyRequest( ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { req := &pb.FetchEncryptionKeyRequest{ - SchemaRoot: []byte(evt.SchemaRoot), EphemeralPublicKey: ephemeralPublicKey, } @@ -247,7 +244,7 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) } - s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), ephPrivKey)) + s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), evt.SchemaRoot, ephPrivKey)) go func() { s.handleFetchEncryptionKeyResponse(<-respChan, req) @@ -258,7 +255,6 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { hash := sha256.New() - hash.Write(req.SchemaRoot) hash.Write(req.EphemeralPublicKey) for _, target := range req.Targets { hash.Write(target.DocID) @@ -327,7 +323,7 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, string(target.KeyID))] = encKey } - s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(string(req.SchemaRoot), eventData)) + s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(session.schemaRoot, eventData)) } func encodeToBase64(data []byte) []byte { @@ -339,8 +335,7 @@ func encodeToBase64(data []byte) []byte { // makeAssociatedData creates the associated data for the encryption key request func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) []byte { return encodeToBase64(bytes.Join([][]byte{ - []byte(req.SchemaRoot), - []byte(req.EphemeralPublicKey), + req.EphemeralPublicKey, []byte(peerID), }, []byte{})) } From c8d2d444fff4be77f92921e41a1bb1f969523225 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 16 Aug 2024 16:25:04 +0200 Subject: [PATCH 53/88] Rename Id to ID --- internal/core/crdt/composite.go | 4 ++-- internal/core/key.go | 4 ++-- internal/db/base/collection_keys.go | 4 ++-- internal/db/collection.go | 2 +- internal/db/collection_delete.go | 2 +- internal/db/indexed_docs_test.go | 2 +- internal/db/merge.go | 10 +++++----- internal/db/p2p_replicator.go | 2 +- internal/lens/fetcher.go | 4 ++-- internal/planner/commit.go | 12 ++++++------ 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/internal/core/crdt/composite.go b/internal/core/crdt/composite.go index 58372cfb49..71a95b9aac 100644 --- a/internal/core/crdt/composite.go +++ b/internal/core/crdt/composite.go @@ -106,13 +106,13 @@ func (c CompositeDAG) Merge(ctx context.Context, delta core.Delta) error { if err != nil { return err } - return c.deleteWithPrefix(ctx, c.key.WithValueFlag().WithFieldId("")) + return c.deleteWithPrefix(ctx, c.key.WithValueFlag().WithFieldID("")) } // We cannot rely on the dagDelta.Status here as it may have been deleted locally, this is not // reflected in `dagDelta.Status` if sourced via P2P. Updates synced via P2P should not undelete // the local reperesentation of the document. - versionKey := c.key.WithValueFlag().WithFieldId(core.DATASTORE_DOC_VERSION_FIELD_ID) + versionKey := c.key.WithValueFlag().WithFieldID(core.DATASTORE_DOC_VERSION_FIELD_ID) objectMarker, err := c.store.Get(ctx, c.key.ToPrimaryDataStoreKey().ToDS()) hasObjectMarker := !errors.Is(err, ds.ErrNotFound) if err != nil && hasObjectMarker { diff --git a/internal/core/key.go b/internal/core/key.go index a61e6ac854..29f515babf 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -408,9 +408,9 @@ func (k DataStoreKey) WithInstanceInfo(key DataStoreKey) DataStoreKey { return newKey } -func (k DataStoreKey) WithFieldId(fieldId string) DataStoreKey { +func (k DataStoreKey) WithFieldID(fieldID string) DataStoreKey { newKey := k - newKey.FieldID = fieldId + newKey.FieldID = fieldID return newKey } diff --git a/internal/db/base/collection_keys.go b/internal/db/base/collection_keys.go index e23707285c..8878d50b13 100644 --- a/internal/db/base/collection_keys.go +++ b/internal/db/base/collection_keys.go @@ -45,7 +45,7 @@ func MakePrimaryIndexKeyForCRDT( case client.COMPOSITE: return MakeDataStoreKeyWithCollectionDescription(c.Description). WithInstanceInfo(key). - WithFieldId(core.COMPOSITE_NAMESPACE), + WithFieldID(core.COMPOSITE_NAMESPACE), nil case client.LWW_REGISTER, client.PN_COUNTER, client.P_COUNTER: field, ok := c.GetFieldByName(fieldName) @@ -55,7 +55,7 @@ func MakePrimaryIndexKeyForCRDT( return MakeDataStoreKeyWithCollectionDescription(c.Description). WithInstanceInfo(key). - WithFieldId(fmt.Sprint(field.ID)), + WithFieldID(fmt.Sprint(field.ID)), nil } return core.DataStoreKey{}, ErrInvalidCrdtType diff --git a/internal/db/collection.go b/internal/db/collection.go index 088e5075fa..8fe055daae 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -900,7 +900,7 @@ func (c *collection) saveCompositeToMerkleCRDT( status client.DocumentStatus, ) (cidlink.Link, []byte, error) { txn := mustGetContextTxn(ctx) - dsKey = dsKey.WithFieldId(core.COMPOSITE_NAMESPACE) + dsKey = dsKey.WithFieldID(core.COMPOSITE_NAMESPACE) merkleCRDT := merklecrdt.NewMerkleCompositeDAG( txn, core.NewCollectionSchemaVersionKey(c.Schema().VersionID, c.ID()), diff --git a/internal/db/collection_delete.go b/internal/db/collection_delete.go index 082a53caf2..9ccca92ed5 100644 --- a/internal/db/collection_delete.go +++ b/internal/db/collection_delete.go @@ -144,7 +144,7 @@ func (c *collection) applyDelete( dsKey := primaryKey.ToDataStoreKey() headset := clock.NewHeadSet( txn.Headstore(), - dsKey.WithFieldId(core.COMPOSITE_NAMESPACE).ToHeadStoreKey(), + dsKey.WithFieldID(core.COMPOSITE_NAMESPACE).ToHeadStoreKey(), ) cids, _, err := headset.List(ctx) if err != nil { diff --git a/internal/db/indexed_docs_test.go b/internal/db/indexed_docs_test.go index fad45aa11f..4c0a43e08e 100644 --- a/internal/db/indexed_docs_test.go +++ b/internal/db/indexed_docs_test.go @@ -698,7 +698,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) fieldKeyString := core.DataStoreKey{ CollectionRootID: f.users.Description().RootID, }.WithDocID(doc.ID().String()). - WithFieldId("1"). + WithFieldID("1"). WithValueFlag(). ToString() diff --git a/internal/db/merge.go b/internal/db/merge.go index 7d7ceee4bc..ae59f5164c 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -60,7 +60,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) + mt, err := getHeadsAsMergeTarget(ctx, txn, dsKey.WithFieldID(core.COMPOSITE_NAMESPACE)) if err != nil { return err } @@ -149,7 +149,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR // blocks, as they will be processed as part of the composite block anyway. // Otherwise, we load the blocks for each field if mergeGroup.compositeKey.DocID != "" { - cids, err := getHeads(ctx, txn, dsKey.WithFieldId(core.COMPOSITE_NAMESPACE)) + cids, err := getHeads(ctx, txn, dsKey.WithFieldID(core.COMPOSITE_NAMESPACE)) if err != nil { return err } @@ -165,7 +165,7 @@ func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyR return client.NewErrFieldNotExist(fieldStoreKey.FieldName.Value()) } - fieldDsKey := dsKey.WithFieldId(fd.ID.String()) + fieldDsKey := dsKey.WithFieldID(fd.ID.String()) cids, err := getHeads(ctx, txn, fieldDsKey) if err != nil { @@ -516,7 +516,7 @@ func (mp *mergeProcessor) initCRDTForType(field string) (merklecrdt.MerkleCRDT, mcrdt = merklecrdt.NewMerkleCompositeDAG( mp.txn, schemaVersionKey, - mp.dsKey.WithFieldId(core.COMPOSITE_NAMESPACE), + mp.dsKey.WithFieldID(core.COMPOSITE_NAMESPACE), "", ) mp.mCRDTs[field] = mcrdt @@ -534,7 +534,7 @@ func (mp *mergeProcessor) initCRDTForType(field string) (merklecrdt.MerkleCRDT, schemaVersionKey, fd.Typ, fd.Kind, - mp.dsKey.WithFieldId(fd.ID.String()), + mp.dsKey.WithFieldID(fd.ID.String()), field, ) if err != nil { diff --git a/internal/db/p2p_replicator.go b/internal/db/p2p_replicator.go index 409419ea3f..b66ab4f2cf 100644 --- a/internal/db/p2p_replicator.go +++ b/internal/db/p2p_replicator.go @@ -161,7 +161,7 @@ func (db *db) getDocsHeads( docID := core.DataStoreKeyFromDocID(docIDResult.ID) headset := clock.NewHeadSet( txn.Headstore(), - docID.WithFieldId(core.COMPOSITE_NAMESPACE).ToHeadStoreKey(), + docID.WithFieldID(core.COMPOSITE_NAMESPACE).ToHeadStoreKey(), ) cids, _, err := headset.List(ctx) if err != nil { diff --git a/internal/lens/fetcher.go b/internal/lens/fetcher.go index 357bbe9677..bbe0c45a0d 100644 --- a/internal/lens/fetcher.go +++ b/internal/lens/fetcher.go @@ -307,7 +307,7 @@ func (f *lensedFetcher) updateDataStore(ctx context.Context, original map[string // in which case we have to skip them for now. continue } - fieldKey := datastoreKeyBase.WithFieldId(fieldDesc.ID.String()) + fieldKey := datastoreKeyBase.WithFieldID(fieldDesc.ID.String()) bytes, err := cbor.Marshal(value) if err != nil { @@ -320,7 +320,7 @@ func (f *lensedFetcher) updateDataStore(ctx context.Context, original map[string } } - versionKey := datastoreKeyBase.WithFieldId(core.DATASTORE_DOC_VERSION_FIELD_ID) + versionKey := datastoreKeyBase.WithFieldID(core.DATASTORE_DOC_VERSION_FIELD_ID) err := f.txn.Datastore().Put(ctx, versionKey.ToDS(), []byte(f.targetVersionID)) if err != nil { return err diff --git a/internal/planner/commit.go b/internal/planner/commit.go index bbb5fdc09c..001b4d44d5 100644 --- a/internal/planner/commit.go +++ b/internal/planner/commit.go @@ -73,7 +73,7 @@ func (n *dagScanNode) Init() error { if n.commitSelect.FieldID.HasValue() { field := n.commitSelect.FieldID.Value() - dsKey = dsKey.WithFieldId(field) + dsKey = dsKey.WithFieldID(field) } n.spans = core.NewSpans(core.NewSpan(dsKey, dsKey.PrefixEnd())) @@ -104,16 +104,16 @@ func (n *dagScanNode) Spans(spans core.Spans) { } copy(headSetSpans.Value, spans.Value) - var fieldId string + var fieldID string if n.commitSelect.FieldID.HasValue() { - fieldId = n.commitSelect.FieldID.Value() + fieldID = n.commitSelect.FieldID.Value() } else { - fieldId = core.COMPOSITE_NAMESPACE + fieldID = core.COMPOSITE_NAMESPACE } for i, span := range headSetSpans.Value { - if span.Start().FieldID != fieldId { - headSetSpans.Value[i] = core.NewSpan(span.Start().WithFieldId(fieldId), core.DataStoreKey{}) + if span.Start().FieldID != fieldID { + headSetSpans.Value[i] = core.NewSpan(span.Start().WithFieldID(fieldID), core.DataStoreKey{}) } } From bfd89c2639c80c602ff1df4bee4b63463e760c1a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 16 Aug 2024 16:56:48 +0200 Subject: [PATCH 54/88] PR polish --- internal/core/block/block.go | 7 ++----- internal/db/merge.go | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/core/block/block.go b/internal/core/block/block.go index bae086bce1..425bf61b59 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -290,13 +290,10 @@ func GetLinkPrototype() cidlink.LinkPrototype { // Validate checks if the block is valid. func (b *Block) Validate() error { if b.Encryption != nil { - switch (*b.Encryption).Type { - case NotEncrypted, DocumentEncrypted, FieldEncrypted: - // Valid values - default: + eType := b.Encryption.Type + if eType != NotEncrypted && eType != DocumentEncrypted && eType != FieldEncrypted { return ErrInvalidBlockEncryptionType } - if len(b.Encryption.KeyID) == 0 { return ErrInvalidBlockEncryptionKeyID } diff --git a/internal/db/merge.go b/internal/db/merge.go index ae59f5164c..a0953c2fdb 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -245,14 +245,18 @@ func (m *mergeQueue) done(docID string) { } type mergeProcessor struct { - txn datastore.Txn - lsys linking.LinkSystem - mCRDTs map[string]merklecrdt.MerkleCRDT - col *collection - dsKey core.DataStoreKey - blocks *list.List + txn datastore.Txn + lsys linking.LinkSystem + mCRDTs map[string]merklecrdt.MerkleCRDT + col *collection + dsKey core.DataStoreKey + // blocks is a list of blocks that need to be merged. + blocks *list.List + // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge + // and doesn't have locally, so they need to be requested from the network. pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} - hasPendingCompositeBlock bool + // hasPendingCompositeBlock is a flag that indicates if there are any composite blocks that need encryption keys. + hasPendingCompositeBlock bool } func (db *db) newMergeProcessor( @@ -659,7 +663,7 @@ func syncIndexedDoc( return err } - // this can happen we received an encrypted document that we haven't decrypted yet + // this can happen if we received an encrypted document that we haven't decrypted yet if isNewDoc && isDeletedDoc { return nil } From c02e5f7ae4bb6515e6c8ebc023681a3cf9246517 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 16 Aug 2024 23:30:32 +0200 Subject: [PATCH 55/88] Adjust phony --- net/pb/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/pb/Makefile b/net/pb/Makefile index eb2c950466..30b0e92dfa 100644 --- a/net/pb/Makefile +++ b/net/pb/Makefile @@ -7,6 +7,7 @@ PROTOC_GEN_GO_VTPROTO := $(shell which protoc-gen-go-vtproto) all: $(GO) +.PHONY: deps deps: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest @@ -21,8 +22,7 @@ deps: --go-vtproto_opt=features=marshal+unmarshal+size \ $< # This line specifies the input file +.PHONY: clean clean: rm -f *.pb.go rm -f *pb_test.go - -.PHONY: clean deps From a01faa184200c3deaf25f3905ec5b74b4e1bb0a9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 18 Aug 2024 23:28:12 +0200 Subject: [PATCH 56/88] Fix mistake in AAD --- crypto/ecies.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index ea4e785c80..e263a7ca9b 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -78,7 +78,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) } - cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ephemeralPrivate.Bytes(), associatedData), true) + cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ephemeralPublic.Bytes(), associatedData), true) if err != nil { return nil, fmt.Errorf("failed to encrypt: %w", err) } From e19cd2b41e5c696661f4832624ce88b2a34c827e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Aug 2024 09:08:23 +0200 Subject: [PATCH 57/88] Remove unnecessary encryptor test --- internal/encryption/encryptor_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 2b979d1dec..2881abc26f 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -79,22 +79,6 @@ func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { assert.ErrorIs(t, err, testErr) } -func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc, st := newDefaultEncryptor(t) - - storeKey := makeStoreKey(docID, noFieldName) - - st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(noFieldName)).Return(nil) - - _, _, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) - assert.NoError(t, err) - - cipherText, err := enc.Encrypt(makeStoreKey(docID, noFieldName), getPlainText()) - - assert.NoError(t, err) - assert.Equal(t, getCipherText(t, noFieldName), cipherText) -} - func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) From ecd508259fb637a2acc56423769c340acbca8d31 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Aug 2024 09:08:50 +0200 Subject: [PATCH 58/88] Remove unused exch field of Peer struct --- net/peer.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/net/peer.go b/net/peer.go index 94460821d2..52358acd8a 100644 --- a/net/peer.go +++ b/net/peer.go @@ -22,7 +22,6 @@ import ( "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockservice" - exchange "github.com/ipfs/boxo/exchange" "github.com/ipfs/boxo/ipns" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -74,7 +73,6 @@ type Peer struct { p2pRPC *grpc.Server // rpc server over the P2P network // peer DAG service - exch exchange.Interface bserv blockservice.BlockService ctx context.Context @@ -516,7 +514,6 @@ func (p *Peer) setupBlockService() { bswapnet := network.NewFromIpfsHost(p.host, p.dht) bswap := bitswap.New(p.ctx, bswapnet, p.blockstore) p.bserv = blockservice.New(p.blockstore, bswap) - p.exch = bswap } func stopGRPCServer(ctx context.Context, server *grpc.Server) { From 0f561a7bbe854b03ad11cf7123c8eaa23f32c839 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Aug 2024 12:56:20 +0200 Subject: [PATCH 59/88] Remove global functions from encryption package --- cli/collection_create.go | 2 +- internal/db/context.go | 2 +- internal/db/merge.go | 9 +++- internal/db/messages.go | 9 ++-- internal/encryption/context.go | 20 ++++---- internal/encryption/encryptor.go | 66 --------------------------- internal/encryption/encryptor_test.go | 16 ------- internal/encryption/errors.go | 6 ++- internal/merkle/clock/clock.go | 12 ++++- net/server_encryption_key.go | 7 ++- 10 files changed, 44 insertions(+), 105 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index 0bdb5014b7..57c65d6226 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -122,7 +122,7 @@ func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptF } ctx := cmd.Context() if txn != nil { - ctx = encryption.ContextWithStore(ctx, txn.Encstore()) + ctx, _ = encryption.ContextWithStore(ctx, txn.Encstore()) } ctx = encryption.SetContextConfigFromParams(ctx, shouldEncryptDoc, encryptFields) cmd.SetContext(ctx) diff --git a/internal/db/context.go b/internal/db/context.go index 3149bd6205..6e3e9e6e2b 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -63,7 +63,7 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if err != nil { return nil, txn, err } - ctx = encryption.ContextWithStore(ctx, txn.Encstore()) + ctx, _ = encryption.ContextWithStore(ctx, txn.Encstore()) return SetContextTxn(ctx, txn), txn, nil } diff --git a/internal/db/merge.go b/internal/db/merge.go index a0953c2fdb..049a8772fc 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -480,10 +480,15 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), optFieldName, string(blockEnc.KeyID)) + encryptor := encryption.GetEncryptorFromContext(ctx) + if encryptor == nil { + return nil, encryption.ErrContextHasNoEncryptor + } + if block.Delta.IsComposite() { // for composite blocks there is nothing to decrypt // so we just check if we have the encryption key for child blocks - bytes, err := encryption.GetKey(ctx, encStoreKey) + bytes, err := encryptor.GetKey(encStoreKey) if err != nil { return nil, err } @@ -494,7 +499,7 @@ func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } clonedCRDT := block.Delta.Clone() - bytes, err := encryption.DecryptDoc(ctx, encStoreKey, clonedCRDT.GetData()) + bytes, err := encryptor.Decrypt(encStoreKey, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/internal/db/messages.go b/internal/db/messages.go index c8891b6dd3..17d93a5004 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -93,10 +93,13 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { // handleEncryptionKeysRetrievedEvent handles the event when requested encryption keys are retrieved from other peers. func (db *db) handleEncryptionKeysRetrievedEvent(ctx context.Context, evt encryption.KeyRetrievedEvent) error { - ctx = encryption.ContextWithStore(ctx, db.Encstore()) + var encryptor *encryption.DocEncryptor + ctx, encryptor = encryption.ContextWithStore(ctx, db.Encstore()) + if encryptor == nil { + return encryption.ErrContextHasNoEncryptor + } for encStoreKey, encKey := range evt.Keys { - err := encryption.SaveKey(ctx, encStoreKey, encKey) - + err := encryptor.SaveKey(encStoreKey, encKey) if err != nil { return err } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index e1ccb033b0..cc3f9153bb 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -24,14 +24,14 @@ type docEncContextKey struct{} // configContextKey is the key type for encryption context values. type configContextKey struct{} -// TryGetContextDocEnc returns a document encryption and a bool indicating if -// it was retrieved from the given context. -func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { +// GetEncryptorFromContext returns a document encryptor from the given context. +// It returns nil if no encryptor exists in the context. +func GetEncryptorFromContext(ctx context.Context) *DocEncryptor { enc, ok := ctx.Value(docEncContextKey{}).(*DocEncryptor) if ok { setConfig(ctx, enc) } - return enc, ok + return enc } func setConfig(ctx context.Context, enc *DocEncryptor) { @@ -40,20 +40,20 @@ func setConfig(ctx context.Context, enc *DocEncryptor) { } func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { + enc := GetEncryptorFromContext(ctx) + if enc == nil { enc = newDocEncryptor(ctx) ctx = context.WithValue(ctx, docEncContextKey{}, enc) } return ctx, enc } -// ContextWithStore sets the store on the doc encryptor in the context. -// If the doc encryptor is not present, it will be created. -func ContextWithStore(ctx context.Context, encstore datastore.DSReaderWriter) context.Context { +// ContextWithStore sets the store on the doc encryptor in the context and returns the updated +// context and doc encryptor. If the doc encryptor is not present, it will be created. +func ContextWithStore(ctx context.Context, encstore datastore.DSReaderWriter) (context.Context, *DocEncryptor) { ctx, encryptor := ensureContextWithDocEnc(ctx) encryptor.SetStore(encstore) - return ctx + return ctx, encryptor } // GetContextConfig returns the doc encryption config from the given context. diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index faf52de0bc..9c42324c59 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -248,38 +248,6 @@ func (d *DocEncryptor) SaveKey(encStoreKey core.EncStoreDocKey, encryptionKey [] return d.storeByEncStoreKey(encStoreKey, encryptionKey) } -// EncryptDoc encrypts the given plainText that is associated with the given docID, fieldName and block height with -// encryptor in the context. -// If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. -// Otherwise, it will use document-level encryption key. -func EncryptDoc( - ctx context.Context, - encStoreKey core.EncStoreDocKey, - plainText []byte, -) ([]byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return nil, nil - } - return enc.Encrypt(encStoreKey, plainText) -} - -// DecryptDoc decrypts the given cipherText that is associated with the given docID and fieldName with -// encryptor in the context. -// If fieldName is not provided, it will try to decrypt with the document-level key. Otherwise, it will try to -// decrypt with the field-level key. -func DecryptDoc( - ctx context.Context, - encStoreKey core.EncStoreDocKey, - cipherText []byte, -) ([]byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return nil, nil - } - return enc.Decrypt(encStoreKey, cipherText) -} - // ShouldEncryptDocField returns true if the given field should be encrypted based on the context config. func ShouldEncryptDocField(ctx context.Context, fieldName immutable.Option[string]) bool { return shouldEncryptDocField(GetContextConfig(ctx), fieldName) @@ -291,40 +259,6 @@ func ShouldEncryptIndividualField(ctx context.Context, fieldName immutable.Optio return shouldEncryptIndividualField(GetContextConfig(ctx), fieldName) } -// SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height with -// encryptor in the context. -func SaveKey(ctx context.Context, encStoreKey core.EncStoreDocKey, encryptionKey []byte) error { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return nil - } - return enc.SaveKey(encStoreKey, encryptionKey) -} - -// GetKey returns the encryption key for the given docID, (optional) fieldName and block height with encryptor -// in the context. -func GetKey(ctx context.Context, encStoreKey core.EncStoreDocKey) ([]byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return nil, nil - } - return enc.GetKey(encStoreKey) -} - -// GetOrGenerateEncryptionKey returns the generated encryption key for the given docID, (optional) fieldName. -// If the key is not generated before, it generates a new key and stores it. -func GetOrGenerateEncryptionKey( - ctx context.Context, - docID string, - fieldName immutable.Option[string], -) (immutable.Option[core.EncStoreDocKey], []byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return immutable.None[core.EncStoreDocKey](), nil, nil - } - return enc.GetOrGenerateEncryptionKey(docID, fieldName) -} - func init() { arg := os.Args[0] // If the binary is a test binary, use a deterministic nonce. diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 2881abc26f..8611c42046 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -159,19 +159,3 @@ func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *te assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) } - -func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := EncryptDoc(context.Background(), makeStoreKey(docID, fieldName), getPlainText()) - assert.Nil(t, data, "data should be nil") - assert.NoError(t, err, "error should be nil") -} - -func TestDecryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { - data, err := DecryptDoc( - context.Background(), - makeStoreKey(docID, fieldName), - getCipherText(t, fieldName), - ) - assert.Nil(t, data, "data should be nil") - assert.NoError(t, err, "error should be nil") -} diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go index 6a443ad834..a068c20fae 100644 --- a/internal/encryption/errors.go +++ b/internal/encryption/errors.go @@ -15,9 +15,11 @@ import ( ) const ( - errNoStorageProvided string = "no storage provided" + errNoStorageProvided string = "no storage provided" + errContextHasNoEncryptor string = "context has no encryptor" ) var ( - ErrNoStorageProvided = errors.New(errNoStorageProvided) + ErrNoStorageProvided = errors.New(errNoStorageProvided) + ErrContextHasNoEncryptor = errors.New(errContextHasNoEncryptor) ) diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 3d24138ca7..fed5be0112 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -137,7 +137,11 @@ func (mc *MerkleClock) determineBlockEncryptionData( } else { blockEnc.Type = coreblock.DocumentEncrypted } - encStoreKey, _, err := encryption.GetOrGenerateEncryptionKey(ctx, docID, fieldName) + encryptor := encryption.GetEncryptorFromContext(ctx) + if encryptor == nil { + return nil, encryption.ErrContextHasNoEncryptor + } + encStoreKey, _, err := encryptor.GetOrGenerateEncryptionKey(docID, fieldName) if err != nil { return nil, err } @@ -186,7 +190,11 @@ func encryptBlock( } clonedCRDT := block.Delta.Clone() - bytes, err := encryption.EncryptDoc(ctx, encStoreKey, clonedCRDT.GetData()) + encryptor := encryption.GetEncryptorFromContext(ctx) + if encryptor == nil { + return nil, encryption.ErrContextHasNoEncryptor + } + bytes, err := encryptor.Encrypt(encStoreKey, clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/net/server_encryption_key.go b/net/server_encryption_key.go index 0cbdbbf19a..8b0fd14492 100644 --- a/net/server_encryption_key.go +++ b/net/server_encryption_key.go @@ -55,8 +55,11 @@ func (s *server) getEncryptionKeys( if target.FieldName != "" { optFieldName = immutable.Some(target.FieldName) } - encKey, err := encryption.GetKey( - encryption.ContextWithStore(ctx, s.peer.encstore), + _, encryptor := encryption.ContextWithStore(ctx, s.peer.encstore) + if encryptor == nil { + return nil, nil, encryption.ErrContextHasNoEncryptor + } + encKey, err := encryptor.GetKey( core.NewEncStoreDocKey(docID.String(), optFieldName, string(target.KeyID)), ) if err != nil { From ae716ebc6b19091d26af5d883565972ed52ab10d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 19 Aug 2024 13:03:20 +0200 Subject: [PATCH 60/88] Add more docs --- internal/encryption/encryptor.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 9c42324c59..801eb7dbff 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -51,7 +51,9 @@ func generateTestEncryptionKey(docID string, fieldName immutable.Option[string]) // DocEncryptor is a document encryptor that encrypts and decrypts individual document fields. // It acts based on the configuration [DocEncConfig] provided and data stored in the provided store. -// It uses [core.EncStoreDocKey] to store and retrieve encryption keys. +// DocEncryptor is a session-bound, i.e. once a user requests to create (or update) a document or a node +// receives an UpdateEvent on a document (or any other event) a new DocEncryptor is created and stored +// in the context, so that the same DocEncryptor can be used by other object down the call chain. type DocEncryptor struct { conf immutable.Option[DocEncConfig] ctx context.Context From 714fb3cfdfcdbfb3a1f02945b3cee787cce78090 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 8 Sep 2024 21:59:24 +0200 Subject: [PATCH 61/88] Initial KMS implementation --- internal/db/config.go | 10 +- internal/db/config_test.go | 4 +- internal/db/context.go | 5 +- internal/db/db.go | 7 +- internal/db/merge.go | 38 +- internal/encryption/event.go | 33 +- internal/kms/errors.go | 27 ++ .../kms/p2p.go | 405 +++++++++--------- internal/kms/service.go | 33 ++ net/errors.go | 10 + net/peer.go | 60 +-- net/server.go | 83 ++-- node/node.go | 40 +- tests/integration/acp.go | 7 + tests/integration/db.go | 21 +- .../integration/encryption/peer_share_test.go | 6 +- tests/integration/state.go | 4 + tests/integration/test_case.go | 2 + tests/integration/utils.go | 43 +- 19 files changed, 504 insertions(+), 334 deletions(-) create mode 100644 internal/kms/errors.go rename net/server_encryption_key.go => internal/kms/p2p.go (51%) create mode 100644 internal/kms/service.go diff --git a/internal/db/config.go b/internal/db/config.go index 8ce725ebd0..3d69e833c4 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -19,12 +19,16 @@ const ( updateEventBufferSize = 100 ) +type dbOptions struct { + maxTxnRetries immutable.Option[int] +} + // Option is a funtion that sets a config value on the db. -type Option func(*db) +type Option func(*dbOptions) // WithMaxRetries sets the maximum number of retries per transaction. func WithMaxRetries(num int) Option { - return func(db *db) { - db.maxTxnRetries = immutable.Some(num) + return func(opts *dbOptions) { + opts.maxTxnRetries = immutable.Some(num) } } diff --git a/internal/db/config_test.go b/internal/db/config_test.go index 405e192598..a52d494a21 100644 --- a/internal/db/config_test.go +++ b/internal/db/config_test.go @@ -17,8 +17,8 @@ import ( ) func TestWithMaxRetries(t *testing.T) { - d := &db{} - WithMaxRetries(10)(d) + d := dbOptions{} + WithMaxRetries(10)(&d) assert.True(t, d.maxTxnRetries.HasValue()) assert.Equal(t, 10, d.maxTxnRetries.Value()) } diff --git a/internal/db/context.go b/internal/db/context.go index 6e3e9e6e2b..47f5759af8 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -58,7 +58,10 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if ok { return SetContextTxn(ctx, &explicitTxn{txn}), &explicitTxn{txn}, nil } - // implicit transaction + return EnforceNewContextTxn(ctx, db, readOnly) +} + +func EnforceNewContextTxn(ctx context.Context, db transactionDB, readOnly bool) (context.Context, datastore.Txn, error) { txn, err := db.NewTxn(ctx, readOnly) if err != nil { return nil, txn, err diff --git a/internal/db/db.go b/internal/db/db.go index 50c1ecc38c..5177d99edb 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -118,8 +118,13 @@ func newDB( } // apply options + var opts dbOptions for _, opt := range options { - opt(db) + opt(&opts) + } + + if opts.maxTxnRetries.HasValue() { + db.maxTxnRetries = opts.maxTxnRetries } if lens != nil { diff --git a/internal/db/merge.go b/internal/db/merge.go index 049a8772fc..73e4fd8e3d 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -14,6 +14,7 @@ import ( "bytes" "container/list" "context" + "fmt" "sync" "github.com/ipfs/go-cid" @@ -75,7 +76,29 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - mp.sendPendingEncryptionRequest() + if len(mp.pendingEncryptionKeyRequests) != 0 { + entResults := mp.sendPendingEncryptionRequest() + newCtx, newTxn, err := EnforceNewContextTxn(context.TODO(), db, false) + go func(ccc context.Context, ttt datastore.Txn) { + res := <-entResults.Get() + if res.Error != nil { + fmt.Printf("Error fetching keys: %s\n", res.Error) + return + } + fmt.Printf("Received %d keys\n", len(res.Items)) + for i, key := range res.Items { + fmt.Printf("Key %d: %s\n", i, key.StoreKey.ToString()) + } + if err != nil { + fmt.Printf("Error creating new context txn: %s\n", err) + return + } + //err = db.executeMerge(ccc, dagMerge) + //if err != nil { + //fmt.Printf("Error executing merge: %s\n", err) + //} + }(newCtx, newTxn) + } if !mp.hasPendingCompositeBlock { err = syncIndexedDoc(ctx, docID, col) @@ -406,17 +429,14 @@ func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName im } } -func (mp *mergeProcessor) sendPendingEncryptionRequest() { - n := len(mp.pendingEncryptionKeyRequests) - if n == 0 { - return - } - schemaRoot := mp.col.SchemaRoot() - storeKeys := make([]core.EncStoreDocKey, 0, n) +func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { + storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) for k := range mp.pendingEncryptionKeyRequests { storeKeys = append(storeKeys, k) } - mp.col.db.events.Publish(encryption.NewRequestKeysMessage(schemaRoot, storeKeys)) + msg, results := encryption.NewRequestKeysMessage(mp.col.SchemaRoot(), storeKeys) + mp.col.db.events.Publish(msg) + return results } // processBlock merges the block and its children to the datastore and sets the head accordingly. diff --git a/internal/encryption/event.go b/internal/encryption/event.go index 0760e3f583..c767fc6f8a 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -28,6 +28,8 @@ type RequestKeysEvent struct { // Keys is a list of the keys that are being requested. Keys []core.EncStoreDocKey + + Resp chan<- Result } // RequestedKeyEventData represents the data that was retrieved for a specific key. @@ -45,16 +47,43 @@ type KeyRetrievedEvent struct { Keys map[core.EncStoreDocKey][]byte } +type Item struct { + StoreKey core.EncStoreDocKey + EncryptionKey []byte +} + +type Result struct { + Items []Item + Error error +} + +type Results struct { + output chan Result +} + +func (r *Results) Get() <-chan Result { + return r.output +} + +func NewResults() (*Results, chan<- Result) { + ch := make(chan Result, 1) + return &Results{ + output: ch, + }, ch +} + // NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field func NewRequestKeysMessage( schemaRoot string, keys []core.EncStoreDocKey, -) event.Message { +) (event.Message, *Results) { + res, ch := NewResults() return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ SchemaRoot: schemaRoot, Keys: keys, - }) + Resp: ch, + }), res } // NewKeysRetrievedMessage creates a new event message for a key that was retrieved diff --git a/internal/kms/errors.go b/internal/kms/errors.go new file mode 100644 index 0000000000..603d3c3232 --- /dev/null +++ b/internal/kms/errors.go @@ -0,0 +1,27 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package kms + +import ( + "github.com/sourcenetwork/defradb/errors" +) + +const ( + errUnknownKMSType string = "unknown KMS type" +) + +var ( + ErrUnknownKMSType = errors.New(errUnknownKMSType) +) + +func NewErrUnknownKMSType(t ServiceType) error { + return errors.New(errUnknownKMSType, errors.NewKV("Type", t)) +} diff --git a/net/server_encryption_key.go b/internal/kms/p2p.go similarity index 51% rename from net/server_encryption_key.go rename to internal/kms/p2p.go index 8b0fd14492..32faea0e5b 100644 --- a/net/server_encryption_key.go +++ b/internal/kms/p2p.go @@ -8,195 +8,155 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package net +package kms import ( "bytes" "context" - "crypto/sha256" - "fmt" - + "crypto/ecdh" "encoding/base64" - "github.com/libp2p/go-libp2p/core/peer" libpeer "github.com/libp2p/go-libp2p/core/peer" - rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" - "github.com/sourcenetwork/immutable" - grpcpeer "google.golang.org/grpc/peer" - "google.golang.org/protobuf/proto" - - libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" - "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" + "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/encryption" + "github.com/sourcenetwork/defradb/net" pb "github.com/sourcenetwork/defradb/net/pb" + rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" + "github.com/sourcenetwork/immutable" + grpcpeer "google.golang.org/grpc/peer" + "google.golang.org/protobuf/proto" ) -const encryptionTopic = "encryption" +const requestType = "encryption" -// getEncryptionKeys retrieves the encryption keys for the given targets. -// It returns the encryption keys and the targets for which the keys were found. -func (s *server) getEncryptionKeys( - ctx context.Context, - req *pb.FetchEncryptionKeyRequest, -) ([]byte, []*pb.EncryptionKeyTarget, error) { - encryptionKeys := make([]byte, 0, len(req.Targets)) - targets := make([]*pb.EncryptionKeyTarget, 0, len(req.Targets)) - for _, target := range req.Targets { - docID, err := client.NewDocIDFromString(string(target.DocID)) - if err != nil { - return nil, nil, err - } - - optFieldName := immutable.None[string]() - if target.FieldName != "" { - optFieldName = immutable.Some(target.FieldName) - } - _, encryptor := encryption.ContextWithStore(ctx, s.peer.encstore) - if encryptor == nil { - return nil, nil, encryption.ErrContextHasNoEncryptor - } - encKey, err := encryptor.GetKey( - core.NewEncStoreDocKey(docID.String(), optFieldName, string(target.KeyID)), - ) - if err != nil { - return nil, nil, err - } - // TODO: we should test it somehow. For this this one peer should have some keys and - // another one should have the others. https://github.com/sourcenetwork/defradb/issues/2895 - if len(encKey) == 0 { - continue - } - targets = append(targets, target) - encryptionKeys = append(encryptionKeys, encKey...) - } - return encryptionKeys, targets, nil +type PubSubServer interface { + AddPubSubTopic(string, rpc.MessageHandler) error + SendPubSubMessage(context.Context, string, []byte) (<-chan rpc.Response, error) } -func (s *server) TryGenEncryptionKey( - ctx context.Context, - req *pb.FetchEncryptionKeyRequest, -) (*pb.FetchEncryptionKeyReply, error) { - peerID, err := peerIDFromContext(ctx) - if err != nil { - return nil, err - } - reqPubKey := s.peer.host.Peerstore().PubKey(peerID) - - isValid, err := s.verifyRequestSignature(req, reqPubKey) - if err != nil { - return nil, errors.Wrap("invalid signature", err) - } - - if !isValid { - return nil, errors.New("invalid signature") - } - - encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) - if err != nil || len(encryptionKeys) == 0 { - return nil, err - } - - reqEphPubKey, err := crypto.X25519PublicKeyFromBytes(req.EphemeralPublicKey) - if err != nil { - return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) - } +type p2pService struct { + ctx context.Context + peerID libpeer.ID + pubsub PubSubServer + keyRequestedSub *event.Subscription + encstore datastore.DSReaderWriter + eventBus *event.Bus +} - encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peer.PeerID())) - if err != nil { - return nil, errors.Wrap("failed to encrypt key for requester", err) - } +var _ Service = (*p2pService)(nil) - res := &pb.FetchEncryptionKeyReply{ - ReqEphemeralPublicKey: req.EphemeralPublicKey, - Targets: targets, - EncryptedKeys: encryptedKey, - } +func (s *p2pService) GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) { + res, ch := encryption.NewResults() - res.Signature, err = s.signResponse(res) + err := s.requestEncryptionKey(ctx, keys, ch) if err != nil { - return nil, errors.Wrap("failed to sign response", err) + return nil, err } return res, nil } -func (s *server) verifyRequestSignature(req *pb.FetchEncryptionKeyRequest, pubKey libp2pCrypto.PubKey) (bool, error) { - return pubKey.Verify(hashFetchEncryptionKeyRequest(req), req.Signature) -} - -func hashFetchEncryptionKeyReply(res *pb.FetchEncryptionKeyReply) []byte { - hash := sha256.New() - hash.Write(res.EncryptedKeys) - hash.Write(res.ReqEphemeralPublicKey) - for _, target := range res.Targets { - hash.Write(target.DocID) - hash.Write([]byte(target.FieldName)) - hash.Write([]byte(target.KeyID)) +func NewP2PService( + ctx context.Context, + peerID libpeer.ID, + pubsub PubSubServer, + eventBus *event.Bus, + encstore datastore.DSReaderWriter, +) (*p2pService, error) { + s := &p2pService{ + ctx: ctx, + peerID: peerID, + pubsub: pubsub, + encstore: encstore, + eventBus: eventBus, } - return hash.Sum(nil) -} - -func (s *server) signResponse(res *pb.FetchEncryptionKeyReply) ([]byte, error) { - privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - return privKey.Sign(hashFetchEncryptionKeyReply(res)) -} - -// addPubSubEncryptionTopic subscribes to a topic on the pubsub network -func (s *server) addPubSubEncryptionTopic() error { - if s.peer.ps == nil { - return nil + err := pubsub.AddPubSubTopic(requestType, s.handleRequestFromPeer) + if err != nil { + return nil, err } - - t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), encryptionTopic, true) + s.keyRequestedSub, err = eventBus.Subscribe(encryption.RequestKeysEventName) if err != nil { - return err + return nil, err } + go s.handleKeyRequestedEvent() + return s, nil +} - t.SetEventHandler(s.pubSubEventHandler) - t.SetMessageHandler(s.pubSubEncryptionMessageHandler) - - s.mu.Lock() - defer s.mu.Unlock() +func (s *p2pService) handleKeyRequestedEvent() { + for { + msg, isOpen := <-s.keyRequestedSub.Message() + if !isOpen { + return + } - s.topics[encryptionTopic] = pubsubTopic{ - Topic: t, - subscribed: true, + if keyReqEvent, ok := msg.Data.(encryption.RequestKeysEvent); ok { + go func() { + results, err := s.GetKeys(s.ctx, keyReqEvent.Keys...) + if err != nil { + log.ErrorContextE(s.ctx, "Failed to get encryption keys", err) + } + + //_, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) + + encResult := <-results.Get() + /*for _, encItem := range encResult.Items { + err := encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) + if err != nil { + log.ErrorContextE(s.ctx, "Failed to save encryption key", err) + return + } + }*/ + + m := make(map[core.EncStoreDocKey][]byte) + for _, item := range encResult.Items { + m[item.StoreKey] = item.EncryptionKey + } + s.eventBus.Publish(encryption.NewKeysRetrievedMessage(keyReqEvent.SchemaRoot, m)) + keyReqEvent.Resp <- encResult + close(keyReqEvent.Resp) + }() + } else { + log.ErrorContext(s.ctx, "Failed to cast event data to RequestKeysEvent") + } } - return nil } -// pubSubEncryptionMessageHandler handles incoming FetchEncryptionKeyRequest messages from the pubsub network. -func (s *server) pubSubEncryptionMessageHandler(from libpeer.ID, topic string, msg []byte) ([]byte, error) { +// handleEncryptionMessage handles incoming FetchEncryptionKeyRequest messages from the pubsub network. +func (s *p2pService) handleRequestFromPeer(peerID libpeer.ID, topic string, msg []byte) ([]byte, error) { + // TODO: check how it makes sense and how much effort to separate net package so that it has + // client-related and server-related code independently. Conceptually, they should no depend on each other. + // Any common functionality (like hosting peer) can be shared. + // This way we could make kms package depend only on client. The server would depend on kms. req := new(pb.FetchEncryptionKeyRequest) if err := proto.Unmarshal(msg, req); err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to unmarshal pubsub message %s", err) + log.ErrorContextE(s.ctx, "Failed to unmarshal pubsub message %s", err) return nil, err } - ctx := grpcpeer.NewContext(s.peer.ctx, &grpcpeer.Peer{ - Addr: addr{from}, - }) + // TODO: check if this NewGRPCPeer can be abstracted away or copied in this package. + ctx := grpcpeer.NewContext(s.ctx, net.NewGRPCPeer(peerID)) res, err := s.TryGenEncryptionKey(ctx, req) if err != nil { - log.ErrorContextE(s.peer.ctx, "failed attempt to get encryption key", err) + log.ErrorContextE(s.ctx, "failed attempt to get encryption key", err) return nil, errors.Wrap("failed attempt to get encryption key", err) } return res.MarshalVT() } -func (s *server) prepareFetchEncryptionKeyRequest( - evt encryption.RequestKeysEvent, +func (s *p2pService) prepareFetchEncryptionKeyRequest( + encStoreKeys []core.EncStoreDocKey, ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { req := &pb.FetchEncryptionKeyRequest{ EphemeralPublicKey: ephemeralPublicKey, } - for _, encStoreKey := range evt.Keys { + for _, encStoreKey := range encStoreKeys { encKey := &pb.EncryptionKeyTarget{ DocID: []byte(encStoreKey.DocID), KeyID: []byte(encStoreKey.KeyID), @@ -207,29 +167,22 @@ func (s *server) prepareFetchEncryptionKeyRequest( req.Targets = append(req.Targets, encKey) } - signature, err := s.signRequest(req) - if err != nil { - return nil, errors.Wrap("failed to sign request", err) - } - - req.Signature = signature - return req, nil } // requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.RequestKeysEvent) error { - if s.peer.ps == nil { // skip if we aren't running with a pubsub net - return nil - } - +func (s *p2pService) requestEncryptionKey( + ctx context.Context, + encStoreKeys []core.EncStoreDocKey, + result chan<- encryption.Result, +) error { ephPrivKey, err := crypto.GenerateX25519() if err != nil { return err } ephPubKeyBytes := ephPrivKey.PublicKey().Bytes() - req, err := s.prepareFetchEncryptionKeyRequest(evt, ephPubKeyBytes) + req, err := s.prepareFetchEncryptionKeyRequest(encStoreKeys, ephPubKeyBytes) if err != nil { return err } @@ -239,81 +192,56 @@ func (s *server) requestEncryptionKey(ctx context.Context, evt encryption.Reques return errors.Wrap("failed to marshal pubsub message", err) } - s.mu.Lock() - t := s.topics[encryptionTopic] - s.mu.Unlock() - respChan, err := t.Publish(ctx, data) + respChan, err := s.pubsub.SendPubSubMessage(ctx, requestType, data) + //respChan, err := s.topic.Publish(ctx, data) if err != nil { - return errors.Wrap(fmt.Sprintf("failed publishing to thread %s", encryptionTopic), err) + return errors.Wrap("failed publishing to encryption thread", err) } - s.sessions = append(s.sessions, newSession(string(ephPubKeyBytes), evt.SchemaRoot, ephPrivKey)) + //s.server.RequestEncryptionKey(ctx, req, ephPrivKey) go func() { - s.handleFetchEncryptionKeyResponse(<-respChan, req) + s.handleFetchEncryptionKeyResponse(<-respChan, req, ephPrivKey, result) }() return nil } -func hashFetchEncryptionKeyRequest(req *pb.FetchEncryptionKeyRequest) []byte { - hash := sha256.New() - hash.Write(req.EphemeralPublicKey) - for _, target := range req.Targets { - hash.Write(target.DocID) - hash.Write([]byte(target.FieldName)) - hash.Write([]byte(target.KeyID)) - } - return hash.Sum(nil) -} - -func (s *server) signRequest(req *pb.FetchEncryptionKeyRequest) ([]byte, error) { - privKey := s.peer.host.Peerstore().PrivKey(s.peer.host.ID()) - return privKey.Sign(hashFetchEncryptionKeyRequest(req)) -} - // handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages -func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.FetchEncryptionKeyRequest) { +func (s *p2pService) handleFetchEncryptionKeyResponse( + resp rpc.Response, + req *pb.FetchEncryptionKeyRequest, + privateKey *ecdh.PrivateKey, + result chan<- encryption.Result, +) { + defer close(result) + var keyResp pb.FetchEncryptionKeyReply if err := proto.Unmarshal(resp.Data, &keyResp); err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to unmarshal encryption key response", err) - return - } - - isValid, err := s.verifyResponseSignature(&keyResp, resp.From) - if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to verify response signature", err) - return - } - - if !isValid { - log.ErrorContext(s.peer.ctx, "Invalid response signature") - return - } - - session := s.extractSessionAndRemoveOld(string(keyResp.ReqEphemeralPublicKey)) - if session == nil { - log.ErrorContext(s.peer.ctx, "Failed to find session for ephemeral public key") + log.ErrorContextE(s.ctx, "Failed to unmarshal encryption key response", err) + result <- encryption.Result{Error: err} return } decryptedData, err := crypto.DecryptECIES( keyResp.EncryptedKeys, - session.privateKey, + privateKey, makeAssociatedData(req, resp.From), ) if err != nil { - log.ErrorContextE(s.peer.ctx, "Failed to decrypt encryption key", err) + log.ErrorContextE(s.ctx, "Failed to decrypt encryption key", err) + result <- encryption.Result{Error: err} return } if len(decryptedData) != crypto.AESKeySize*len(keyResp.Targets) { - log.ErrorContext(s.peer.ctx, "Invalid decrypted data length") + log.ErrorContext(s.ctx, "Invalid decrypted data length") + result <- encryption.Result{Error: errors.New("invalid decrypted data length")} return } - eventData := make(map[core.EncStoreDocKey][]byte) + resultEncItems := make([]encryption.Item, 0, len(keyResp.Targets)) for _, target := range keyResp.Targets { optFieldName := immutable.None[string]() if target.FieldName != "" { @@ -323,16 +251,15 @@ func (s *server) handleFetchEncryptionKeyResponse(resp rpc.Response, req *pb.Fet encKey := decryptedData[:crypto.AESKeySize] decryptedData = decryptedData[crypto.AESKeySize:] - eventData[core.NewEncStoreDocKey(string(target.DocID), optFieldName, string(target.KeyID))] = encKey + resultEncItems = append(resultEncItems, encryption.Item{ + StoreKey: core.NewEncStoreDocKey(string(target.DocID), optFieldName, string(target.KeyID)), + EncryptionKey: encKey, + }) } - s.peer.bus.Publish(encryption.NewKeysRetrievedMessage(session.schemaRoot, eventData)) -} - -func encodeToBase64(data []byte) []byte { - encoded := make([]byte, base64.StdEncoding.EncodedLen(len(data))) - base64.StdEncoding.Encode(encoded, data) - return encoded + result <- encryption.Result{ + Items: resultEncItems, + } } // makeAssociatedData creates the associated data for the encryption key request @@ -343,7 +270,75 @@ func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) [] }, []byte{})) } -func (s *server) verifyResponseSignature(res *pb.FetchEncryptionKeyReply, fromPeer peer.ID) (bool, error) { - pubKey := s.peer.host.Peerstore().PubKey(fromPeer) - return pubKey.Verify(hashFetchEncryptionKeyReply(res), res.Signature) +func (s *p2pService) TryGenEncryptionKey( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) (*pb.FetchEncryptionKeyReply, error) { + encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) + if err != nil || len(encryptionKeys) == 0 { + return nil, err + } + + reqEphPubKey, err := crypto.X25519PublicKeyFromBytes(req.EphemeralPublicKey) + if err != nil { + return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) + } + + encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peerID)) + if err != nil { + return nil, errors.Wrap("failed to encrypt key for requester", err) + } + + res := &pb.FetchEncryptionKeyReply{ + ReqEphemeralPublicKey: req.EphemeralPublicKey, + Targets: targets, + EncryptedKeys: encryptedKey, + } + + return res, nil +} + +// getEncryptionKeys retrieves the encryption keys for the given targets. +// It returns the encryption keys and the targets for which the keys were found. +func (s *p2pService) getEncryptionKeys( + ctx context.Context, + req *pb.FetchEncryptionKeyRequest, +) ([]byte, []*pb.EncryptionKeyTarget, error) { + encryptionKeys := make([]byte, 0, len(req.Targets)) + targets := make([]*pb.EncryptionKeyTarget, 0, len(req.Targets)) + for _, target := range req.Targets { + docID, err := client.NewDocIDFromString(string(target.DocID)) + if err != nil { + return nil, nil, err + } + + optFieldName := immutable.None[string]() + if target.FieldName != "" { + optFieldName = immutable.Some(target.FieldName) + } + _, encryptor := encryption.ContextWithStore(ctx, s.encstore) + if encryptor == nil { + return nil, nil, encryption.ErrContextHasNoEncryptor + } + encKey, err := encryptor.GetKey( + core.NewEncStoreDocKey(docID.String(), optFieldName, string(target.KeyID)), + ) + if err != nil { + return nil, nil, err + } + // TODO: we should test it somehow. For this this one peer should have some keys and + // another one should have the others. https://github.com/sourcenetwork/defradb/issues/2895 + if len(encKey) == 0 { + continue + } + targets = append(targets, target) + encryptionKeys = append(encryptionKeys, encKey...) + } + return encryptionKeys, targets, nil +} + +func encodeToBase64(data []byte) []byte { + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(data))) + base64.StdEncoding.Encode(encoded, data) + return encoded } diff --git a/internal/kms/service.go b/internal/kms/service.go new file mode 100644 index 0000000000..9ad3a365c8 --- /dev/null +++ b/internal/kms/service.go @@ -0,0 +1,33 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package kms + +import ( + "context" + + "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/defradb/internal/core" + "github.com/sourcenetwork/defradb/internal/encryption" +) + +var ( + log = corelog.NewLogger("kms") +) + +type ServiceType string + +const ( + P2PServiceType = "p2p" +) + +type Service interface { + GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) +} diff --git a/net/errors.go b/net/errors.go index fe8f286ab7..1a8c65a48f 100644 --- a/net/errors.go +++ b/net/errors.go @@ -24,6 +24,8 @@ const ( errPublishingToSchemaTopic = "can't publish log %s for schema %s" errCheckingForExistingBlock = "failed to check for existing block" errRequestingEncryptionKeys = "failed to request encryption keys with %v" + errTopicAlreadyExist = "topic with name \"%s\" already exists" + errTopicDoesNotExist = "topic with name \"%s\" does not exists" ) var ( @@ -54,3 +56,11 @@ func NewErrPublishingToSchemaTopic(inner error, cid, docID string, kv ...errors. func NewErrRequestingEncryptionKeys(inner error, keys []core.EncStoreDocKey) error { return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKeys, keys), inner) } + +func NewErrTopicAlreadyExist(topic string) error { + return errors.New(fmt.Sprintf(errTopicAlreadyExist, topic)) +} + +func NewErrTopicDoesNotExist(topic string) error { + return errors.New(fmt.Sprintf(errTopicDoesNotExist, topic)) +} \ No newline at end of file diff --git a/net/peer.go b/net/peer.go index 75a46e603c..0d1559a80b 100644 --- a/net/peer.go +++ b/net/peer.go @@ -27,18 +27,19 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" + libpeer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" "github.com/sourcenetwork/corelog" "google.golang.org/grpc" + grpcpeer "google.golang.org/grpc/peer" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" corenet "github.com/sourcenetwork/defradb/internal/core/net" - "github.com/sourcenetwork/defradb/internal/encryption" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -139,8 +140,7 @@ func NewPeer( if err != nil { return nil, err } - p.updateSub, err = p.bus.Subscribe(event.UpdateName, event.P2PTopicName, event.ReplicatorName, - encryption.RequestKeysEventName) + p.updateSub, err = p.bus.Subscribe(event.UpdateName, event.P2PTopicName, event.ReplicatorName) if err != nil { return nil, err } @@ -153,7 +153,7 @@ func NewPeer( return nil, err } - p2plistener, err := gostream.Listen(h, corenet.Protocol) + p2pListener, err := gostream.Listen(h, corenet.Protocol) if err != nil { return nil, err } @@ -166,17 +166,12 @@ func NewPeer( // register the P2P gRPC server go func() { pb.RegisterServiceServer(p.p2pRPC, p.server) - if err := p.p2pRPC.Serve(p2plistener); err != nil && + if err := p.p2pRPC.Serve(p2pListener); err != nil && !errors.Is(err, grpc.ErrServerStopped) { log.ErrorE("Fatal P2P RPC server error", err) } }() - err = p.server.addPubSubEncryptionTopic() - if err != nil { - return nil, err - } - bus.Publish(event.NewMessage(event.PeerInfoName, event.PeerInfo{Info: p.PeerInfo()})) return p, nil @@ -234,6 +229,12 @@ func (p *Peer) Close() { } } +func NewGRPCPeer(peerID libpeer.ID) *grpcpeer.Peer { + return &grpcpeer.Peer{ + Addr: addr{peerID}, + } +} + // handleMessage loop manages the transition of messages // from the internal broadcaster to the external pubsub network func (p *Peer) handleMessageLoop() { @@ -261,11 +262,6 @@ func (p *Peer) handleMessageLoop() { case event.Replicator: p.server.updateReplicators(evt) - case encryption.RequestKeysEvent: - err := p.handleEncryptionKeyRequest(evt) - if err != nil { - log.ErrorContextE(p.ctx, "Error while handling broadcast log", err) - } default: // ignore other events continue @@ -359,14 +355,6 @@ func (p *Peer) handleDocUpdateLog(evt event.Update) error { return nil } -func (p *Peer) handleEncryptionKeyRequest(evt encryption.RequestKeysEvent) error { - if err := p.server.requestEncryptionKey(p.ctx, evt); err != nil { - return NewErrRequestingEncryptionKeys(err, evt.Keys) - } - - return nil -} - func (p *Peer) pushLogToReplicators(lg event.Update) { // let the exchange know we have this block // this should speed up the dag sync process @@ -409,28 +397,6 @@ func (p *Peer) pushLogToReplicators(lg event.Update) { } } -func (p *Peer) setupBlockService() { - bswapnet := network.NewFromIpfsHost(p.host, p.dht) - bswap := bitswap.New(p.ctx, bswapnet, p.blockstore) - p.bserv = blockservice.New(p.blockstore, bswap) -} - -func stopGRPCServer(ctx context.Context, server *grpc.Server) { - stopped := make(chan struct{}) - go func() { - server.GracefulStop() - close(stopped) - }() - timer := time.NewTimer(10 * time.Second) - select { - case <-timer.C: - server.Stop() - log.InfoContext(ctx, "Peer gRPC server was shutdown ungracefully") - case <-stopped: - timer.Stop() - } -} - // Connect initiates a connection to the peer with the given address. func (p *Peer) Connect(ctx context.Context, addr peer.AddrInfo) error { return p.host.Connect(ctx, addr) @@ -450,3 +416,7 @@ func (p *Peer) PeerInfo() peer.AddrInfo { Addrs: p.host.Network().ListenAddresses(), } } + +func (p *Peer) Server() *server { + return p.server +} diff --git a/net/server.go b/net/server.go index d7bedfb9fc..e8f433ca79 100644 --- a/net/server.go +++ b/net/server.go @@ -14,10 +14,8 @@ package net import ( "context" - "crypto/ecdh" "fmt" "sync" - "time" cid "github.com/ipfs/go-cid" libpeer "github.com/libp2p/go-libp2p/core/peer" @@ -36,7 +34,7 @@ import ( pb "github.com/sourcenetwork/defradb/net/pb" ) -// Server is the request/response instance for all P2P RPC communication. +// server is the request/response instance for all P2P RPC communication. // Implements gRPC server. See net/pb/net.proto for corresponding service definitions. // // Specifically, server handles the push/get request/response aspects of the RPC service @@ -53,21 +51,6 @@ type server struct { conns map[libpeer.ID]*grpc.ClientConn pb.UnimplementedServiceServer - - sessions []session -} - -const sessionTimeout = 5 * time.Second - -type session struct { - id string - privateKey *ecdh.PrivateKey - schemaRoot string - t time.Time -} - -func newSession(id string, schemaRoot string, privateKey *ecdh.PrivateKey) session { - return session{id: id, schemaRoot: schemaRoot, privateKey: privateKey, t: time.Now()} } // pubsubTopic is a wrapper of rpc.Topic to be able to track if the topic has @@ -98,26 +81,6 @@ func newServer(p *Peer, opts ...grpc.DialOption) (*server, error) { return s, nil } -// extractSessionAndRemoveOld extracts a session with the given id from the server's session list -// and removes any old sessions by comparing their timestamps. -func (s *server) extractSessionAndRemoveOld(id string) *session { - var result *session - swapLast := func(i int) { - s.sessions[i] = s.sessions[len(s.sessions)-1] - s.sessions = s.sessions[:len(s.sessions)-1] - } - for i, session := range s.sessions { - if session.id == id { - tmpSession := session - result = &tmpSession - swapLast(i) - } else if time.Since(session.t) > sessionTimeout { - swapLast(i) - } - } - return result -} - // GetDocGraph receives a get graph request func (s *server) GetDocGraph( ctx context.Context, @@ -446,3 +409,47 @@ func (s *server) updateReplicators(evt event.Replicator) { } s.peer.bus.Publish(event.NewMessage(event.ReplicatorCompletedName, nil)) } + +func (s *server) AddPubSubTopic(topicName string, handler rpc.MessageHandler) error { + if s.peer.ps == nil { + return nil + } + + s.mu.Lock() + _, ok := s.topics[topicName] + s.mu.Unlock() + if ok { + return NewErrTopicAlreadyExist(topicName) + } + + t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), topicName, true) + if err != nil { + return err + } + + t.SetEventHandler(s.pubSubEventHandler) + t.SetMessageHandler(handler) + + s.mu.Lock() + defer s.mu.Unlock() + + s.topics[topicName] = pubsubTopic{ + Topic: t, + subscribed: true, + } + return nil +} + +func (s *server) SendPubSubMessage( + ctx context.Context, + topic string, + data []byte, +) (<-chan rpc.Response, error) { + s.mu.Lock() + t, ok := s.topics[topic] + s.mu.Unlock() + if !ok { + return nil, NewErrTopicDoesNotExist(topic) + } + return t.Publish(ctx, data) +} diff --git a/node/node.go b/node/node.go index 621699dbb2..a5b9e16d8d 100644 --- a/node/node.go +++ b/node/node.go @@ -17,10 +17,12 @@ import ( gohttp "net/http" "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/internal/db" + "github.com/sourcenetwork/defradb/internal/kms" "github.com/sourcenetwork/defradb/net" ) @@ -41,6 +43,7 @@ type Option any type Options struct { disableP2P bool disableAPI bool + kmsType immutable.Option[kms.ServiceType] } // DefaultOptions returns options with default settings. @@ -65,11 +68,18 @@ func WithDisableAPI(disable bool) NodeOpt { } } +func WithKMS(kms kms.ServiceType) NodeOpt { + return func(o *Options) { + o.kmsType = immutable.Some(kms) + } +} + // Node is a DefraDB instance with optional sub-systems. type Node struct { - DB client.DB - Peer *net.Peer - Server *http.Server + DB client.DB + Peer *net.Peer + Server *http.Server + kmsService kms.Service } // NewNode returns a new node instance configured with the given options. @@ -130,12 +140,29 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { } var peer *net.Peer + var kmsService kms.Service if !options.disableP2P { // setup net node peer, err = net.NewPeer(ctx, db.Blockstore(), db.Encstore(), db.Events(), netOpts...) if err != nil { return nil, err } + + if options.kmsType.HasValue() { + switch options.kmsType.Value() { + case kms.P2PServiceType: + kmsService, err = kms.NewP2PService( + ctx, + peer.PeerID(), + peer.Server(), + db.Events(), + db.Encstore(), + ) + } + if err != nil { + return nil, err + } + } } var server *http.Server @@ -152,9 +179,10 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { } return &Node{ - DB: db, - Peer: peer, - Server: server, + DB: db, + Peer: peer, + Server: server, + kmsService: kmsService, }, nil } diff --git a/tests/integration/acp.go b/tests/integration/acp.go index 9242a266fc..ebba25527f 100644 --- a/tests/integration/acp.go +++ b/tests/integration/acp.go @@ -57,6 +57,13 @@ var ( acpType ACPType ) +type KMSType string + +const ( + NoneKMSType = "none" + P2PKMSType = "p2p" +) + func init() { acpType = ACPType(os.Getenv(acpTypeEnvName)) if acpType == "" { diff --git a/tests/integration/db.go b/tests/integration/db.go index 54175784c0..dbc0b4efa3 100644 --- a/tests/integration/db.go +++ b/tests/integration/db.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" + "github.com/sourcenetwork/defradb/internal/kms" "github.com/sourcenetwork/defradb/node" changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" ) @@ -99,10 +100,7 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, error) { return node.DB, err } -// setupNode returns the database implementation for the current -// testing state. The database type on the test state is used to -// select the datastore implementation to use. -func setupNode(s *state) (*node.Node, string, error) { +func getDefaultNodeOpts() []node.Option { opts := []node.Option{ node.WithLensPoolSize(lensPoolSize), // The test framework sets this up elsewhere when required so that it may be wrapped @@ -117,7 +115,7 @@ func setupNode(s *state) (*node.Node, string, error) { if badgerEncryption && encryptionKey == nil { key, err := crypto.GenerateAES256() if err != nil { - return nil, "", err + return nil } encryptionKey = key } @@ -126,6 +124,15 @@ func setupNode(s *state) (*node.Node, string, error) { opts = append(opts, node.WithBadgerEncryptionKey(encryptionKey)) } + return opts +} + +// setupNode returns the database implementation for the current +// testing state. The database type on the test state is used to +// select the datastore implementation to use. +func setupNode(s *state, opts ...node.Option) (*node.Node, string, error) { + opts = append(getDefaultNodeOpts(), opts...) + switch acpType { case LocalACPType: opts = append(opts, node.WithACPType(node.LocalACPType)) @@ -175,6 +182,10 @@ func setupNode(s *state) (*node.Node, string, error) { return nil, "", fmt.Errorf("invalid database type: %v", s.dbt) } + if s.kms == P2PKMSType { + opts = append(opts, node.WithKMS(kms.P2PServiceType)) + } + node, err := node.NewNode(s.ctx, opts...) if err != nil { return nil, "", err diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index cb111286a3..197a0c0f5c 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -21,6 +21,7 @@ import ( func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -45,10 +46,7 @@ func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) Doc: john21Doc, IsDocEncrypted: true, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { diff --git a/tests/integration/state.go b/tests/integration/state.go index 8573d6ddad..2263dd2a97 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -125,6 +125,8 @@ type state struct { // The TestCase currently being executed. testCase TestCase + kms KMSType + // The type of database currently being tested. dbt DatabaseType @@ -195,6 +197,7 @@ func newState( ctx context.Context, t testing.TB, testCase TestCase, + kms KMSType, dbt DatabaseType, clientType ClientType, collectionNames []string, @@ -203,6 +206,7 @@ func newState( ctx: ctx, t: t, testCase: testCase, + kms: kms, dbt: dbt, clientType: clientType, txns: []datastore.Txn{}, diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 689b8cb815..48ef36a155 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -52,6 +52,8 @@ type TestCase struct { // This is to only be used in the very rare cases where we really do want behavioural // differences between acp types, or we need to temporarily document a bug. SupportedACPTypes immutable.Option[[]ACPType] + + TargetKMSTypes []KMSType } // SetupComplete is a flag to explicitly notify the change detector at which point diff --git a/tests/integration/utils.go b/tests/integration/utils.go index ea0944c4e8..fbe273f532 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -14,6 +14,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "reflect" "strconv" @@ -37,6 +38,7 @@ import ( "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/request/graphql" "github.com/sourcenetwork/defradb/net" + "github.com/sourcenetwork/defradb/node" changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" "github.com/sourcenetwork/defradb/tests/clients" "github.com/sourcenetwork/defradb/tests/gen" @@ -167,6 +169,13 @@ func ExecuteTestCase( databases = append(databases, defraIMType) } + var kmsList []KMSType + if len(testCase.TargetKMSTypes) > 0 { + kmsList = testCase.TargetKMSTypes + } else { + kmsList = []KMSType{NoneKMSType} + } + // Assert that these are not empty to protect against accidental mis-configurations, // otherwise an empty set would silently pass all the tests. require.NotEmpty(t, databases) @@ -177,7 +186,9 @@ func ExecuteTestCase( ctx := context.Background() for _, ct := range clients { for _, dbt := range databases { - executeTestCase(ctx, t, collectionNames, testCase, dbt, ct) + for _, kms := range kmsList { + executeTestCase(ctx, t, collectionNames, testCase, kms, dbt, ct) + } } } } @@ -187,12 +198,11 @@ func executeTestCase( t testing.TB, collectionNames []string, testCase TestCase, + kms KMSType, dbt DatabaseType, clientType ClientType, ) { - log.InfoContext( - ctx, - testCase.Description, + logAttrs := []slog.Attr{ corelog.Any("database", dbt), corelog.Any("client", clientType), corelog.Any("mutationType", mutationType), @@ -204,11 +214,17 @@ func executeTestCase( corelog.String("changeDetector.SourceBranch", changeDetector.SourceBranch), corelog.String("changeDetector.TargetBranch", changeDetector.TargetBranch), corelog.String("changeDetector.Repository", changeDetector.Repository), - ) + } + + if kms != NoneKMSType { + logAttrs = append(logAttrs, corelog.Any("kms", kms)) + } + + log.InfoContext(ctx, testCase.Description, logAttrs...) startActionIndex, endActionIndex := getActionRange(t, testCase) - s := newState(ctx, t, testCase, dbt, clientType, collectionNames) + s := newState(ctx, t, testCase, kms, dbt, clientType, collectionNames) setStartingNodes(s) // It is very important that the databases are always closed, otherwise resources will leak @@ -769,20 +785,21 @@ func configureNode( return } - node, path, err := setupNode(s) //disable change dector, or allow it? - require.NoError(s.t, err) - privateKey, err := crypto.GenerateEd25519() require.NoError(s.t, err) - nodeOpts := action() - nodeOpts = append(nodeOpts, net.WithPrivateKey(privateKey)) + netNodeOpts := action() + netNodeOpts = append(netNodeOpts, net.WithPrivateKey(privateKey)) - node.Peer, err = net.NewPeer(s.ctx, node.DB.Blockstore(), node.DB.Encstore(), node.DB.Events(), nodeOpts...) + nodeOpts := []node.Option{node.WithDisableP2P(false)} + for _, opt := range netNodeOpts { + nodeOpts = append(nodeOpts, opt) + } + node, path, err := setupNode(s, nodeOpts...) //disable change dector, or allow it? require.NoError(s.t, err) s.nodeAddresses = append(s.nodeAddresses, node.Peer.PeerInfo()) - s.nodeConfigs = append(s.nodeConfigs, nodeOpts) + s.nodeConfigs = append(s.nodeConfigs, netNodeOpts) c, err := setupClient(s, node) require.NoError(s.t, err) From 7ff6019c11bad26176a97c1b54fa9aeb6579de3c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 9 Sep 2024 22:05:11 +0200 Subject: [PATCH 62/88] Keep only 1 executeMerge (WIP) --- internal/db/fetcher/versioned.go | 2 +- internal/db/merge.go | 151 +++--------------- internal/db/messages.go | 18 +-- internal/encryption/event.go | 8 + internal/kms/p2p.go | 11 +- internal/merkle/clock/clock.go | 17 +- internal/merkle/crdt/merklecrdt.go | 6 +- .../encryption/peer_sec_index_test.go | 2 + .../integration/encryption/peer_share_test.go | 10 +- 9 files changed, 54 insertions(+), 171 deletions(-) diff --git a/internal/db/fetcher/versioned.go b/internal/db/fetcher/versioned.go index a81e105714..80b71cdd88 100644 --- a/internal/db/fetcher/versioned.go +++ b/internal/db/fetcher/versioned.go @@ -415,7 +415,7 @@ func (vf *VersionedFetcher) processBlock( vf.mCRDTs[crdtIndex] = mcrdt } - return mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink, false, false) + return mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink) } func (vf *VersionedFetcher) getDAGBlock(c cid.Cid) (*coreblock.Block, error) { diff --git a/internal/db/merge.go b/internal/db/merge.go index 73e4fd8e3d..0e94a10680 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -71,13 +71,13 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.mergeBlocks(ctx, false) + err = mp.mergeBlocks(ctx) if err != nil { return err } if len(mp.pendingEncryptionKeyRequests) != 0 { - entResults := mp.sendPendingEncryptionRequest() + entResults := mp.sendPendingEncryptionRequest(dagMerge) newCtx, newTxn, err := EnforceNewContextTxn(context.TODO(), db, false) go func(ccc context.Context, ttt datastore.Txn) { res := <-entResults.Get() @@ -93,11 +93,12 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { fmt.Printf("Error creating new context txn: %s\n", err) return } - //err = db.executeMerge(ccc, dagMerge) - //if err != nil { - //fmt.Printf("Error executing merge: %s\n", err) - //} + err = db.executeMerge(ccc, dagMerge) + if err != nil { + fmt.Printf("Error executing merge: %s\n", err) + } }(newCtx, newTxn) + return nil } if !mp.hasPendingCompositeBlock { @@ -117,116 +118,6 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return nil } -type encryptionMergeGroup struct { - compositeKey core.EncStoreDocKey - fieldsKeys []core.EncStoreDocKey -} - -func createMergeGroups(keyEvent encryption.KeyRetrievedEvent) map[string]encryptionMergeGroup { - mergeGroups := make(map[string]encryptionMergeGroup) - - for encStoreKey := range keyEvent.Keys { - g := mergeGroups[encStoreKey.DocID] - - if encStoreKey.FieldName.HasValue() { - g.fieldsKeys = append(g.fieldsKeys, encStoreKey) - } else { - g.compositeKey = encStoreKey - } - - mergeGroups[encStoreKey.DocID] = g - } - - return mergeGroups -} - -func (db *db) mergeEncryptedBlocks(ctx context.Context, keyEvent encryption.KeyRetrievedEvent) error { - ctx, txn, err := ensureContextTxn(ctx, db, false) - if err != nil { - return err - } - defer txn.Discard(ctx) - - col, err := getCollectionFromRootSchema(ctx, db, keyEvent.SchemaRoot) - if err != nil { - return err - } - - ls := cidlink.DefaultLinkSystem() - ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) - - for docID, mergeGroup := range createMergeGroups(keyEvent) { - docID, err := client.NewDocIDFromString(docID) - if err != nil { - return err - } - dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(col.Description(), docID.String()) - - mp, err := db.newMergeProcessor(txn, ls, col, dsKey) - if err != nil { - return err - } - - var blocks []*coreblock.Block - // if merge ground includes composite, we process only the composite block and skip the field - // blocks, as they will be processed as part of the composite block anyway. - // Otherwise, we load the blocks for each field - if mergeGroup.compositeKey.DocID != "" { - cids, err := getHeads(ctx, txn, dsKey.WithFieldID(core.COMPOSITE_NAMESPACE)) - if err != nil { - return err - } - - blocks, err = loadBlocksWithKeyIDFromBlockstore(ctx, txn, cids, mergeGroup.compositeKey.KeyID) - if err != nil { - return err - } - } else { - for _, fieldStoreKey := range mergeGroup.fieldsKeys { - fd, ok := mp.col.Definition().GetFieldByName(fieldStoreKey.FieldName.Value()) - if !ok { - return client.NewErrFieldNotExist(fieldStoreKey.FieldName.Value()) - } - - fieldDsKey := dsKey.WithFieldID(fd.ID.String()) - - cids, err := getHeads(ctx, txn, fieldDsKey) - if err != nil { - return err - } - - fieldBlocks, err := loadBlocksWithKeyIDFromBlockstore(ctx, txn, cids, fieldStoreKey.KeyID) - if err != nil { - return err - } - - blocks = append(blocks, fieldBlocks...) - } - } - - for _, block := range blocks { - mp.blocks.PushFront(block) - } - err = mp.mergeBlocks(ctx, true) - if err != nil { - return err - } - - // TODO: test is doc field was indexed after decryption - err = syncIndexedDoc(ctx, docID, col) - if err != nil { - return err - } - } - - err = txn.Commit(ctx) - if err != nil { - return err - } - - return nil -} - // mergeQueue is synchronization source to ensure that concurrent // document merges do not cause transaction conflicts. type mergeQueue struct { @@ -373,14 +264,14 @@ func (mp *mergeProcessor) loadBlocks( return nil } -func (mp *mergeProcessor) mergeBlocks(ctx context.Context, withDecryption bool) error { +func (mp *mergeProcessor) mergeBlocks(ctx context.Context) error { for e := mp.blocks.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() if err != nil { return err } - err = mp.processBlock(ctx, block, link, withDecryption) + err = mp.processBlock(ctx, block, link) if err != nil { return err } @@ -395,19 +286,18 @@ func (mp *mergeProcessor) mergeBlocks(ctx context.Context, withDecryption bool) func (mp *mergeProcessor) processEncryptedBlock( ctx context.Context, dagBlock *coreblock.Block, - withDecryption bool, -) (*coreblock.Block, bool, error) { +) (*coreblock.Block, error) { if dagBlock.IsEncrypted() { plainTextBlock, err := decryptBlock(ctx, dagBlock) if err != nil { - return nil, false, err + return nil, err } if plainTextBlock != nil { - return plainTextBlock, false, nil + return plainTextBlock, nil } else { blockEnc := dagBlock.Encryption // we weren't able to decrypt the block, so we request the encryption key unless it's a decryption pass - if !withDecryption && (dagBlock.Delta.IsComposite() && blockEnc.Type == coreblock.DocumentEncrypted) || + if (dagBlock.Delta.IsComposite() && blockEnc.Type == coreblock.DocumentEncrypted) || blockEnc.Type == coreblock.FieldEncrypted { docID := string(dagBlock.Delta.GetDocID()) fieldName := immutable.None[string]() @@ -416,10 +306,10 @@ func (mp *mergeProcessor) processEncryptedBlock( } mp.addPendingEncryptionRequest(docID, fieldName, string(blockEnc.KeyID)) } - return dagBlock, true, nil + return dagBlock, nil } } - return dagBlock, false, nil + return dagBlock, nil } func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], keyID string) { @@ -429,12 +319,12 @@ func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName im } } -func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { +func (mp *mergeProcessor) sendPendingEncryptionRequest(mergeEvent event.Merge) *encryption.Results { storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) for k := range mp.pendingEncryptionKeyRequests { storeKeys = append(storeKeys, k) } - msg, results := encryption.NewRequestKeysMessage(mp.col.SchemaRoot(), storeKeys) + msg, results := encryption.NewRequestKeysMessage(mp.col.SchemaRoot(), storeKeys, mergeEvent) mp.col.db.events.Publish(msg) return results } @@ -445,9 +335,8 @@ func (mp *mergeProcessor) processBlock( ctx context.Context, dagBlock *coreblock.Block, blockLink cidlink.Link, - withDecryption bool, ) error { - block, skipMerge, err := mp.processEncryptedBlock(ctx, dagBlock, withDecryption) + block, err := mp.processEncryptedBlock(ctx, dagBlock) if err != nil { return err } @@ -463,7 +352,7 @@ func (mp *mergeProcessor) processBlock( return nil } - err = crdt.Clock().ProcessBlock(ctx, block, blockLink, skipMerge, withDecryption) + err = crdt.Clock().ProcessBlock(ctx, block, blockLink) if err != nil { return err } @@ -483,7 +372,7 @@ func (mp *mergeProcessor) processBlock( return err } - if err := mp.processBlock(ctx, childBlock, link.Link, withDecryption); err != nil { + if err := mp.processBlock(ctx, childBlock, link.Link); err != nil { return err } } diff --git a/internal/db/messages.go b/internal/db/messages.go index 17d93a5004..c49c66b341 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -81,11 +81,11 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { }) case encryption.KeyRetrievedEvent: - go func() { + /*go func() { if err := db.handleEncryptionKeysRetrievedEvent(ctx, evt); err != nil { log.ErrorContextE(ctx, errFailedToHandleEncKeysReceivedEvent, err, corelog.Any("Event", evt)) } - }() + }()*/ } } } @@ -93,17 +93,5 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { // handleEncryptionKeysRetrievedEvent handles the event when requested encryption keys are retrieved from other peers. func (db *db) handleEncryptionKeysRetrievedEvent(ctx context.Context, evt encryption.KeyRetrievedEvent) error { - var encryptor *encryption.DocEncryptor - ctx, encryptor = encryption.ContextWithStore(ctx, db.Encstore()) - if encryptor == nil { - return encryption.ErrContextHasNoEncryptor - } - for encStoreKey, encKey := range evt.Keys { - err := encryptor.SaveKey(encStoreKey, encKey) - if err != nil { - return err - } - } - - return db.mergeEncryptedBlocks(ctx, evt) + return db.executeMerge(ctx, evt.MergeEvent) } diff --git a/internal/encryption/event.go b/internal/encryption/event.go index c767fc6f8a..c617bc72bf 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -30,6 +30,8 @@ type RequestKeysEvent struct { Keys []core.EncStoreDocKey Resp chan<- Result + + MergeEvent event.Merge } // RequestedKeyEventData represents the data that was retrieved for a specific key. @@ -45,6 +47,8 @@ type KeyRetrievedEvent struct { // Keys is a map of the requested keys to the corresponding raw encryption keys. Keys map[core.EncStoreDocKey][]byte + + MergeEvent event.Merge } type Item struct { @@ -77,12 +81,14 @@ func NewResults() (*Results, chan<- Result) { func NewRequestKeysMessage( schemaRoot string, keys []core.EncStoreDocKey, + mergeEvent event.Merge, ) (event.Message, *Results) { res, ch := NewResults() return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ SchemaRoot: schemaRoot, Keys: keys, Resp: ch, + MergeEvent: mergeEvent, }), res } @@ -90,9 +96,11 @@ func NewRequestKeysMessage( func NewKeysRetrievedMessage( schemaRoot string, keys map[core.EncStoreDocKey][]byte, + mergeEvent event.Merge, ) event.Message { return event.NewMessage(KeysRetrievedEventName, KeyRetrievedEvent{ SchemaRoot: schemaRoot, Keys: keys, + MergeEvent: mergeEvent, }) } diff --git a/internal/kms/p2p.go b/internal/kms/p2p.go index 32faea0e5b..27d705681f 100644 --- a/internal/kms/p2p.go +++ b/internal/kms/p2p.go @@ -101,22 +101,23 @@ func (s *p2pService) handleKeyRequestedEvent() { log.ErrorContextE(s.ctx, "Failed to get encryption keys", err) } - //_, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) - encResult := <-results.Get() - /*for _, encItem := range encResult.Items { + + _, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) + + for _, encItem := range encResult.Items { err := encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) if err != nil { log.ErrorContextE(s.ctx, "Failed to save encryption key", err) return } - }*/ + } m := make(map[core.EncStoreDocKey][]byte) for _, item := range encResult.Items { m[item.StoreKey] = item.EncryptionKey } - s.eventBus.Publish(encryption.NewKeysRetrievedMessage(keyReqEvent.SchemaRoot, m)) + s.eventBus.Publish(encryption.NewKeysRetrievedMessage(keyReqEvent.SchemaRoot, m, keyReqEvent.MergeEvent)) keyReqEvent.Resp <- encResult close(keyReqEvent.Resp) }() diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index fed5be0112..eda03c0bbf 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -110,7 +110,7 @@ func (mc *MerkleClock) AddDelta( } // merge the delta and update the state - err = mc.ProcessBlock(ctx, block, link, false, false) + err = mc.ProcessBlock(ctx, block, link) if err != nil { return cidlink.Link{}, nil, err } @@ -209,20 +209,13 @@ func (mc *MerkleClock) ProcessBlock( ctx context.Context, block *coreblock.Block, blockLink cidlink.Link, - skipMerge bool, - skipHeads bool, ) error { - if !skipMerge { - err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) - if err != nil { - return NewErrMergingDelta(blockLink.Cid, err) - } + err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) + if err != nil { + return NewErrMergingDelta(blockLink.Cid, err) } - if !skipHeads { - return mc.updateHeads(ctx, block, blockLink) - } - return nil + return mc.updateHeads(ctx, block, blockLink) } func (mc *MerkleClock) updateHeads( diff --git a/internal/merkle/crdt/merklecrdt.go b/internal/merkle/crdt/merklecrdt.go index 2c99c5f23d..27f92f21ca 100644 --- a/internal/merkle/crdt/merklecrdt.go +++ b/internal/merkle/crdt/merklecrdt.go @@ -48,11 +48,7 @@ type MerkleClock interface { links ...coreblock.DAGLink, ) (cidlink.Link, []byte, error) // ProcessBlock processes a block and updates the CRDT state. - // skipMerge argument indicates whether merging of block's delta should be skipped. It is needed in case - // the block is encrypted and there is not encryption key to decrypt it. - // skipHeads argument indicates whether updating of heads should be skipped. It is needed in case - // the block's head was already updated when the encrypted block was processed. - ProcessBlock(ctx context.Context, block *coreblock.Block, cid cidlink.Link, skipMerge, skipHeads bool) error + ProcessBlock(ctx context.Context, block *coreblock.Block, cid cidlink.Link) error } // baseMerkleCRDT handles the MerkleCRDT overhead functions that aren't CRDT specific like the mutations and state diff --git a/tests/integration/encryption/peer_sec_index_test.go b/tests/integration/encryption/peer_sec_index_test.go index 0c45f56321..3b2ea3dc34 100644 --- a/tests/integration/encryption/peer_sec_index_test.go +++ b/tests/integration/encryption/peer_sec_index_test.go @@ -21,6 +21,7 @@ import ( func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecryption(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -86,6 +87,7 @@ func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecrypt func TestDocEncryptionPeer_IfDocDocHasEncryptedIndexedField_ShouldIndexAfterDecryption(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 197a0c0f5c..2e7a808a7e 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -70,6 +70,7 @@ func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -123,6 +124,7 @@ func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -177,6 +179,7 @@ func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeys func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -231,6 +234,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypt func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -284,6 +288,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_Should func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecryptAndCorrectlyMerge(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -317,8 +322,8 @@ func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecry Doc: `{"age": 2}`, }, testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, + //jEvent: immutable.Some(encryption.KeysRetrievedEventName), + //NodeIDs: []int{1}, }, testUtils.Request{ NodeID: immutable.Some(1), @@ -345,6 +350,7 @@ func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecry func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_ShouldDecryptAndCorrectlyMerge(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), From 55965e63cb2f45f471807d46a24a78e9c2fafa4c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 11:31:21 +0200 Subject: [PATCH 63/88] Make it work with 2 events --- internal/db/merge.go | 22 +--------------------- internal/db/messages.go | 4 ++-- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 0e94a10680..0f936b7952 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -77,27 +77,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { } if len(mp.pendingEncryptionKeyRequests) != 0 { - entResults := mp.sendPendingEncryptionRequest(dagMerge) - newCtx, newTxn, err := EnforceNewContextTxn(context.TODO(), db, false) - go func(ccc context.Context, ttt datastore.Txn) { - res := <-entResults.Get() - if res.Error != nil { - fmt.Printf("Error fetching keys: %s\n", res.Error) - return - } - fmt.Printf("Received %d keys\n", len(res.Items)) - for i, key := range res.Items { - fmt.Printf("Key %d: %s\n", i, key.StoreKey.ToString()) - } - if err != nil { - fmt.Printf("Error creating new context txn: %s\n", err) - return - } - err = db.executeMerge(ccc, dagMerge) - if err != nil { - fmt.Printf("Error executing merge: %s\n", err) - } - }(newCtx, newTxn) + mp.sendPendingEncryptionRequest(dagMerge) return nil } diff --git a/internal/db/messages.go b/internal/db/messages.go index c49c66b341..5e0ff36748 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -81,11 +81,11 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { }) case encryption.KeyRetrievedEvent: - /*go func() { + go func() { if err := db.handleEncryptionKeysRetrievedEvent(ctx, evt); err != nil { log.ErrorContextE(ctx, errFailedToHandleEncKeysReceivedEvent, err, corelog.Any("Event", evt)) } - }()*/ + }() } } } From 807c2529ffb85e6d6fb58b0043c0b6293f7b87be Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 14:07:36 +0200 Subject: [PATCH 64/88] FIx indexing after decryption --- internal/db/merge.go | 51 +++++++++---------- .../encryption/peer_sec_index_test.go | 18 +++++++ .../integration/encryption/peer_share_test.go | 4 +- tests/integration/encryption/peer_test.go | 2 + 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 0f936b7952..38ff74d14f 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -81,11 +81,9 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return nil } - if !mp.hasPendingCompositeBlock { - err = syncIndexedDoc(ctx, docID, col) - if err != nil { - return err - } + err = syncIndexedDoc(ctx, docID, col) + if err != nil { + return err } err = txn.Commit(ctx) @@ -149,8 +147,6 @@ type mergeProcessor struct { // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge // and doesn't have locally, so they need to be requested from the network. pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} - // hasPendingCompositeBlock is a flag that indicates if there are any composite blocks that need encryption keys. - hasPendingCompositeBlock bool } func (db *db) newMergeProcessor( @@ -266,14 +262,14 @@ func (mp *mergeProcessor) mergeBlocks(ctx context.Context) error { func (mp *mergeProcessor) processEncryptedBlock( ctx context.Context, dagBlock *coreblock.Block, -) (*coreblock.Block, error) { +) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { plainTextBlock, err := decryptBlock(ctx, dagBlock) if err != nil { - return nil, err + return nil, false, err } if plainTextBlock != nil { - return plainTextBlock, nil + return plainTextBlock, true, nil } else { blockEnc := dagBlock.Encryption // we weren't able to decrypt the block, so we request the encryption key unless it's a decryption pass @@ -286,17 +282,14 @@ func (mp *mergeProcessor) processEncryptedBlock( } mp.addPendingEncryptionRequest(docID, fieldName, string(blockEnc.KeyID)) } - return dagBlock, nil + return dagBlock, false, nil } } - return dagBlock, nil + return dagBlock, true, nil } func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], keyID string) { mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, keyID)] = struct{}{} - if !fieldName.HasValue() { - mp.hasPendingCompositeBlock = true - } } func (mp *mergeProcessor) sendPendingEncryptionRequest(mergeEvent event.Merge) *encryption.Results { @@ -316,25 +309,27 @@ func (mp *mergeProcessor) processBlock( dagBlock *coreblock.Block, blockLink cidlink.Link, ) error { - block, err := mp.processEncryptedBlock(ctx, dagBlock) + block, canRead, err := mp.processEncryptedBlock(ctx, dagBlock) if err != nil { return err } - crdt, err := mp.initCRDTForType(dagBlock.Delta.GetFieldName()) - if err != nil { - return err - } + if canRead { + crdt, err := mp.initCRDTForType(dagBlock.Delta.GetFieldName()) + if err != nil { + return err + } - // If the CRDT is nil, it means the field is not part - // of the schema and we can safely ignore it. - if crdt == nil { - return nil - } + // If the CRDT is nil, it means the field is not part + // of the schema and we can safely ignore it. + if crdt == nil { + return nil + } - err = crdt.Clock().ProcessBlock(ctx, block, blockLink) - if err != nil { - return err + err = crdt.Clock().ProcessBlock(ctx, block, blockLink) + if err != nil { + return err + } } for _, link := range dagBlock.Links { diff --git a/tests/integration/encryption/peer_sec_index_test.go b/tests/integration/encryption/peer_sec_index_test.go index 3b2ea3dc34..eea77714af 100644 --- a/tests/integration/encryption/peer_sec_index_test.go +++ b/tests/integration/encryption/peer_sec_index_test.go @@ -79,6 +79,24 @@ func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecrypt }`, Asserter: testUtils.NewExplainAsserter().WithIndexFetches(2), }, + testUtils.Request{ + Request: ` + query { + User(filter: {age: {_eq: 21}}) { + name + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "name": "Andy", + }, + { + "name": "John", + }, + }, + }, + }, }, } diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 2e7a808a7e..0bf42b9312 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -322,8 +322,8 @@ func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecry Doc: `{"age": 2}`, }, testUtils.WaitForSync{ - //jEvent: immutable.Some(encryption.KeysRetrievedEventName), - //NodeIDs: []int{1}, + Event: immutable.Some(encryption.KeysRetrievedEventName), + NodeIDs: []int{1}, }, testUtils.Request{ NodeID: immutable.Some(1), diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 959263cd83..067d19787c 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -20,6 +20,7 @@ import ( func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -109,6 +110,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { func TestDocEncryptionPeer_IfPeerDidNotReceiveKey_ShouldNotFetch(t *testing.T) { test := testUtils.TestCase{ + TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), From 3deb0c36ad244b2fb1d65bcf310b32590b95d7c5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 15:28:48 +0200 Subject: [PATCH 65/88] Change KMS test setup, Rename p2p KMS to pubsub --- internal/kms/{p2p.go => pubsub.go} | 46 ++++++++++++------- internal/kms/service.go | 2 +- node/node.go | 4 +- tests/integration/acp.go | 8 +++- tests/integration/db.go | 4 +- .../encryption/peer_sec_index_test.go | 4 +- .../integration/encryption/peer_share_test.go | 14 +++--- tests/integration/encryption/peer_test.go | 4 +- tests/integration/test_case.go | 7 ++- tests/integration/utils.go | 14 ++++-- 10 files changed, 66 insertions(+), 41 deletions(-) rename internal/kms/{p2p.go => pubsub.go} (87%) diff --git a/internal/kms/p2p.go b/internal/kms/pubsub.go similarity index 87% rename from internal/kms/p2p.go rename to internal/kms/pubsub.go index 27d705681f..03b8eaca5f 100644 --- a/internal/kms/p2p.go +++ b/internal/kms/pubsub.go @@ -15,6 +15,7 @@ import ( "context" "crypto/ecdh" "encoding/base64" + "fmt" libpeer "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/client" @@ -32,14 +33,14 @@ import ( "google.golang.org/protobuf/proto" ) -const requestType = "encryption" +const pubsubTopic = "encryption" type PubSubServer interface { AddPubSubTopic(string, rpc.MessageHandler) error SendPubSubMessage(context.Context, string, []byte) (<-chan rpc.Response, error) } -type p2pService struct { +type pubSubService struct { ctx context.Context peerID libpeer.ID pubsub PubSubServer @@ -48,9 +49,9 @@ type p2pService struct { eventBus *event.Bus } -var _ Service = (*p2pService)(nil) +var _ Service = (*pubSubService)(nil) -func (s *p2pService) GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) { +func (s *pubSubService) GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) { res, ch := encryption.NewResults() err := s.requestEncryptionKey(ctx, keys, ch) @@ -61,21 +62,21 @@ func (s *p2pService) GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) ( return res, nil } -func NewP2PService( +func NewPubSubService( ctx context.Context, peerID libpeer.ID, pubsub PubSubServer, eventBus *event.Bus, encstore datastore.DSReaderWriter, -) (*p2pService, error) { - s := &p2pService{ +) (*pubSubService, error) { + s := &pubSubService{ ctx: ctx, peerID: peerID, pubsub: pubsub, encstore: encstore, eventBus: eventBus, } - err := pubsub.AddPubSubTopic(requestType, s.handleRequestFromPeer) + err := pubsub.AddPubSubTopic(pubsubTopic, s.handleRequestFromPeer) if err != nil { return nil, err } @@ -87,7 +88,7 @@ func NewP2PService( return s, nil } -func (s *p2pService) handleKeyRequestedEvent() { +func (s *pubSubService) handleKeyRequestedEvent() { for { msg, isOpen := <-s.keyRequestedSub.Message() if !isOpen { @@ -106,11 +107,22 @@ func (s *p2pService) handleKeyRequestedEvent() { _, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) for _, encItem := range encResult.Items { - err := encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) + encKey, err := encryptor.GetKey(encItem.StoreKey) + if err != nil { + fmt.Printf(">>>>>>>>>> Failed to get stored key %s\n", encItem.StoreKey.ToString()) + continue + } + if len(encKey) == 0 { + fmt.Printf(">>>>>>>>>> No key stored %s\n", encItem.StoreKey.ToString()) + } else { + fmt.Printf(">>>>>>>>>> Has already key stored %s\n", encItem.StoreKey.ToString()) + } + err = encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) if err != nil { log.ErrorContextE(s.ctx, "Failed to save encryption key", err) return } + fmt.Printf(">>>>>>>>>> Saved key %s\n", encItem.StoreKey.ToString()) } m := make(map[core.EncStoreDocKey][]byte) @@ -128,7 +140,7 @@ func (s *p2pService) handleKeyRequestedEvent() { } // handleEncryptionMessage handles incoming FetchEncryptionKeyRequest messages from the pubsub network. -func (s *p2pService) handleRequestFromPeer(peerID libpeer.ID, topic string, msg []byte) ([]byte, error) { +func (s *pubSubService) handleRequestFromPeer(peerID libpeer.ID, topic string, msg []byte) ([]byte, error) { // TODO: check how it makes sense and how much effort to separate net package so that it has // client-related and server-related code independently. Conceptually, they should no depend on each other. // Any common functionality (like hosting peer) can be shared. @@ -149,7 +161,7 @@ func (s *p2pService) handleRequestFromPeer(peerID libpeer.ID, topic string, msg return res.MarshalVT() } -func (s *p2pService) prepareFetchEncryptionKeyRequest( +func (s *pubSubService) prepareFetchEncryptionKeyRequest( encStoreKeys []core.EncStoreDocKey, ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { @@ -172,7 +184,7 @@ func (s *p2pService) prepareFetchEncryptionKeyRequest( } // requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *p2pService) requestEncryptionKey( +func (s *pubSubService) requestEncryptionKey( ctx context.Context, encStoreKeys []core.EncStoreDocKey, result chan<- encryption.Result, @@ -193,7 +205,7 @@ func (s *p2pService) requestEncryptionKey( return errors.Wrap("failed to marshal pubsub message", err) } - respChan, err := s.pubsub.SendPubSubMessage(ctx, requestType, data) + respChan, err := s.pubsub.SendPubSubMessage(ctx, pubsubTopic, data) //respChan, err := s.topic.Publish(ctx, data) if err != nil { return errors.Wrap("failed publishing to encryption thread", err) @@ -209,7 +221,7 @@ func (s *p2pService) requestEncryptionKey( } // handleFetchEncryptionKeyResponse handles incoming FetchEncryptionKeyResponse messages -func (s *p2pService) handleFetchEncryptionKeyResponse( +func (s *pubSubService) handleFetchEncryptionKeyResponse( resp rpc.Response, req *pb.FetchEncryptionKeyRequest, privateKey *ecdh.PrivateKey, @@ -271,7 +283,7 @@ func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) [] }, []byte{})) } -func (s *p2pService) TryGenEncryptionKey( +func (s *pubSubService) TryGenEncryptionKey( ctx context.Context, req *pb.FetchEncryptionKeyRequest, ) (*pb.FetchEncryptionKeyReply, error) { @@ -301,7 +313,7 @@ func (s *p2pService) TryGenEncryptionKey( // getEncryptionKeys retrieves the encryption keys for the given targets. // It returns the encryption keys and the targets for which the keys were found. -func (s *p2pService) getEncryptionKeys( +func (s *pubSubService) getEncryptionKeys( ctx context.Context, req *pb.FetchEncryptionKeyRequest, ) ([]byte, []*pb.EncryptionKeyTarget, error) { diff --git a/internal/kms/service.go b/internal/kms/service.go index 9ad3a365c8..55af0c97d3 100644 --- a/internal/kms/service.go +++ b/internal/kms/service.go @@ -25,7 +25,7 @@ var ( type ServiceType string const ( - P2PServiceType = "p2p" + PubSubServiceType ServiceType = "pubsub" ) type Service interface { diff --git a/node/node.go b/node/node.go index a5b9e16d8d..4cb1b9d88e 100644 --- a/node/node.go +++ b/node/node.go @@ -150,8 +150,8 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { if options.kmsType.HasValue() { switch options.kmsType.Value() { - case kms.P2PServiceType: - kmsService, err = kms.NewP2PService( + case kms.PubSubServiceType: + kmsService, err = kms.NewPubSubService( ctx, peer.PeerID(), peer.Server(), diff --git a/tests/integration/acp.go b/tests/integration/acp.go index ebba25527f..4e2615eb6d 100644 --- a/tests/integration/acp.go +++ b/tests/integration/acp.go @@ -60,10 +60,14 @@ var ( type KMSType string const ( - NoneKMSType = "none" - P2PKMSType = "p2p" + NoneKMSType KMSType = "none" + PubSubKMSType KMSType = "pubsub" ) +func getKMSTypes() []KMSType { + return []KMSType{PubSubKMSType} +} + func init() { acpType = ACPType(os.Getenv(acpTypeEnvName)) if acpType == "" { diff --git a/tests/integration/db.go b/tests/integration/db.go index dbc0b4efa3..0824a1fc75 100644 --- a/tests/integration/db.go +++ b/tests/integration/db.go @@ -182,8 +182,8 @@ func setupNode(s *state, opts ...node.Option) (*node.Node, string, error) { return nil, "", fmt.Errorf("invalid database type: %v", s.dbt) } - if s.kms == P2PKMSType { - opts = append(opts, node.WithKMS(kms.P2PServiceType)) + if s.kms == PubSubKMSType { + opts = append(opts, node.WithKMS(kms.PubSubServiceType)) } node, err := node.NewNode(s.ctx, opts...) diff --git a/tests/integration/encryption/peer_sec_index_test.go b/tests/integration/encryption/peer_sec_index_test.go index eea77714af..271a0eb56b 100644 --- a/tests/integration/encryption/peer_sec_index_test.go +++ b/tests/integration/encryption/peer_sec_index_test.go @@ -21,7 +21,7 @@ import ( func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecryption(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -105,7 +105,7 @@ func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecrypt func TestDocEncryptionPeer_IfDocDocHasEncryptedIndexedField_ShouldIndexAfterDecryption(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 0bf42b9312..cd9f8ef568 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -21,7 +21,7 @@ import ( func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -70,7 +70,7 @@ func TestDocEncryptionPeer_IfDocIsPublic_ShouldFetchKeyAndDecrypt(t *testing.T) func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -124,7 +124,7 @@ func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -179,7 +179,7 @@ func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeys func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -234,7 +234,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypt func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_ShouldFetchKeysAndDecrypt(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -288,7 +288,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_Should func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecryptAndCorrectlyMerge(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -350,7 +350,7 @@ func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecry func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_ShouldDecryptAndCorrectlyMerge(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 067d19787c..05c78540ac 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -20,7 +20,7 @@ import ( func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), @@ -110,7 +110,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { func TestDocEncryptionPeer_IfPeerDidNotReceiveKey_ShouldNotFetch(t *testing.T) { test := testUtils.TestCase{ - TargetKMSTypes: []testUtils.KMSType{testUtils.P2PKMSType}, + KMS: testUtils.KMS{Activated: true}, Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 48ef36a155..f40cdca4e5 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -53,7 +53,12 @@ type TestCase struct { // differences between acp types, or we need to temporarily document a bug. SupportedACPTypes immutable.Option[[]ACPType] - TargetKMSTypes []KMSType + KMS KMS +} + +type KMS struct { + Activated bool + ExcludedTypes []KMSType } // SetupComplete is a flag to explicitly notify the change detector at which point diff --git a/tests/integration/utils.go b/tests/integration/utils.go index fbe273f532..5653a60fa2 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -17,12 +17,12 @@ import ( "log/slog" "os" "reflect" + "slices" "strconv" "strings" "testing" "time" - "github.com/bxcodec/faker/support/slice" "github.com/fxamacker/cbor/v2" "github.com/sourcenetwork/corelog" "github.com/sourcenetwork/immutable" @@ -170,9 +170,13 @@ func ExecuteTestCase( } var kmsList []KMSType - if len(testCase.TargetKMSTypes) > 0 { - kmsList = testCase.TargetKMSTypes - } else { + if testCase.KMS.Activated { + kmsList = getKMSTypes() + for _, excluded := range testCase.KMS.ExcludedTypes { + kmsList = slices.DeleteFunc(kmsList, func(t KMSType) bool { return t == excluded }) + } + } + if len(kmsList) == 0 { kmsList = []KMSType{NoneKMSType} } @@ -398,7 +402,7 @@ func generateDocs(s *state, action GenerateDocs) { collections := getNodeCollections(action.NodeID, s.collections) defs := make([]client.CollectionDefinition, 0, len(collections[0])) for _, col := range collections[0] { - if len(action.ForCollections) == 0 || slice.Contains(action.ForCollections, col.Name().Value()) { + if len(action.ForCollections) == 0 || slices.Contains(action.ForCollections, col.Name().Value()) { defs = append(defs, col.Definition()) } } From 9dc089b4bce45288028c0cdeaab6f5538bc05726 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 17:44:44 +0200 Subject: [PATCH 66/88] Strong error types for crypto package --- crypto/aes.go | 3 +-- crypto/ecies.go | 25 ++++++++++---------- crypto/errors.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 15 deletions(-) create mode 100644 crypto/errors.go diff --git a/crypto/aes.go b/crypto/aes.go index 6fc27c54c6..9fa2bd8deb 100644 --- a/crypto/aes.go +++ b/crypto/aes.go @@ -13,7 +13,6 @@ package crypto import ( "crypto/aes" "crypto/cipher" - "fmt" ) // EncryptAES encrypts data using AES-GCM with a provided key and additional data. @@ -70,7 +69,7 @@ func EncryptAES(plainText, key, additionalData []byte, prependNonce bool) ([]byt func DecryptAES(nonce, cipherText, key, additionalData []byte) ([]byte, error) { if len(nonce) == 0 { if len(cipherText) < AESNonceSize { - return nil, fmt.Errorf("cipherText too short") + return nil, ErrCipherTextTooShort } nonce = cipherText[:AESNonceSize] cipherText = cipherText[AESNonceSize:] diff --git a/crypto/ecies.go b/crypto/ecies.go index e263a7ca9b..32ad228b23 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -15,7 +15,6 @@ import ( "crypto/hmac" "crypto/rand" "crypto/sha256" - "fmt" "golang.org/x/crypto/hkdf" ) @@ -59,28 +58,28 @@ func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData []byte) ([]byte, error) { ephemeralPrivate, err := GenerateX25519() if err != nil { - return nil, fmt.Errorf("failed to generate ephemeral key: %w", err) + return nil, NewErrFailedToGenerateEphemeralKey(err) } ephemeralPublic := ephemeralPrivate.PublicKey() sharedSecret, err := ephemeralPrivate.ECDH(publicKey) if err != nil { - return nil, fmt.Errorf("failed ECDH operation: %w", err) + return nil, NewErrFailedECDHOperation(err) } kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) aesKey := make([]byte, AESKeySize) hmacKey := make([]byte, HMACSize) if _, err := kdf.Read(aesKey); err != nil { - return nil, fmt.Errorf("failed KDF operation for AES key: %w", err) + return nil, NewErrFailedKDFOperationForAESKey(err) } if _, err := kdf.Read(hmacKey); err != nil { - return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) + return nil, NewErrFailedKDFOperationForHMACKey(err) } cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ephemeralPublic.Bytes(), associatedData), true) if err != nil { - return nil, fmt.Errorf("failed to encrypt: %w", err) + return nil, NewErrFailedToEncrypt(err) } mac := hmac.New(sha256.New, hmacKey) @@ -114,28 +113,28 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] // - Error if any step of the decryption process fails, including authentication failure func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData []byte) ([]byte, error) { if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { - return nil, fmt.Errorf("ciphertext too short") + return nil, ErrCipherTextTooShort } ephemeralPublicBytes := cipherText[:X25519PublicKeySize] ephemeralPublic, err := ecdh.X25519().NewPublicKey(ephemeralPublicBytes) if err != nil { - return nil, fmt.Errorf("failed to parse ephemeral public key: %w", err) + return nil, NewErrFailedToParseEphemeralPublicKey(err) } sharedSecret, err := privateKey.ECDH(ephemeralPublic) if err != nil { - return nil, fmt.Errorf("failed ECDH operation: %w", err) + return nil, NewErrFailedECDHOperation(err) } kdf := hkdf.New(sha256.New, sharedSecret, nil, nil) aesKey := make([]byte, AESKeySize) hmacKey := make([]byte, HMACSize) if _, err := kdf.Read(aesKey); err != nil { - return nil, fmt.Errorf("failed KDF operation for AES key: %w", err) + return nil, NewErrFailedKDFOperationForAESKey(err) } if _, err := kdf.Read(hmacKey); err != nil { - return nil, fmt.Errorf("failed KDF operation for HMAC key: %w", err) + return nil, NewErrFailedKDFOperationForHMACKey(err) } macSum := cipherText[len(cipherText)-HMACSize:] @@ -145,12 +144,12 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData mac.Write(cipherTextWithNonce) expectedMAC := mac.Sum(nil) if !hmac.Equal(macSum, expectedMAC) { - return nil, fmt.Errorf("verification with HMAC failed") + return nil, ErrVerificationWithHMACFailed } plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(ephemeralPublicBytes, associatedData)) if err != nil { - return nil, fmt.Errorf("failed to decrypt: %w", err) + return nil, NewErrFailedToDecrypt(err) } return plainText, nil diff --git a/crypto/errors.go b/crypto/errors.go new file mode 100644 index 0000000000..362d0584e9 --- /dev/null +++ b/crypto/errors.go @@ -0,0 +1,60 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package crypto + +import ( + "github.com/sourcenetwork/defradb/errors" +) + +const ( + errFailedToGenerateEphemeralKey string = "failed to generate ephemeral key" + errFailedECDHOperation string = "failed ECDH operation" + errFailedKDFOperationForAESKey string = "failed KDF operation for AES key" + errFailedKDFOperationForHMACKey string = "failed KDF operation for HMAC key" + errFailedToEncrypt string = "failed to encrypt" + errCipherTextTooShort string = "ciphertext too short" + errFailedToParseEphemeralPublicKey string = "failed to parse ephemeral public key" + errVerificationWithHMACFailed string = "verification with HMAC failed" + errFailedToDecrypt string = "failed to decrypt" +) + +var ( + ErrCipherTextTooShort = errors.New(errCipherTextTooShort) + ErrVerificationWithHMACFailed = errors.New(errVerificationWithHMACFailed) +) + +func NewErrFailedToGenerateEphemeralKey(inner error) error { + return errors.Wrap(errFailedToGenerateEphemeralKey, inner) +} + +func NewErrFailedECDHOperation(inner error) error { + return errors.Wrap(errFailedECDHOperation, inner) +} + +func NewErrFailedKDFOperationForAESKey(inner error) error { + return errors.Wrap(errFailedKDFOperationForAESKey, inner) +} + +func NewErrFailedKDFOperationForHMACKey(inner error) error { + return errors.Wrap(errFailedKDFOperationForHMACKey, inner) +} + +func NewErrFailedToEncrypt(inner error) error { + return errors.Wrap(errFailedToEncrypt, inner) +} + +func NewErrFailedToParseEphemeralPublicKey(inner error) error { + return errors.Wrap(errFailedToParseEphemeralPublicKey, inner) +} + +func NewErrFailedToDecrypt(inner error) error { + return errors.Wrap(errFailedToDecrypt, inner) +} From 302191cc61016884f12401dd22e9a0a211657a4b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 22:37:50 +0200 Subject: [PATCH 67/88] Use response chan instead of another event --- internal/db/db.go | 3 +- internal/db/merge.go | 19 ++++++- internal/db/messages.go | 12 ---- internal/encryption/event.go | 41 +------------- internal/kms/pubsub.go | 1 - .../encryption/peer_sec_index_test.go | 13 +---- .../integration/encryption/peer_share_test.go | 31 ++-------- tests/integration/events.go | 56 +------------------ tests/integration/p2p.go | 15 +---- tests/integration/state.go | 17 ++---- tests/integration/utils.go | 4 +- 11 files changed, 37 insertions(+), 175 deletions(-) diff --git a/internal/db/db.go b/internal/db/db.go index 5177d99edb..855450bf0c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -31,7 +31,6 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/core" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/request/graphql" ) @@ -136,7 +135,7 @@ func newDB( return nil, err } - sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName, encryption.KeysRetrievedEventName) + sub, err := db.events.Subscribe(event.MergeName, event.PeerInfoName) if err != nil { return nil, err } diff --git a/internal/db/merge.go b/internal/db/merge.go index 38ff74d14f..69170c71a4 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -77,7 +77,20 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { } if len(mp.pendingEncryptionKeyRequests) != 0 { - mp.sendPendingEncryptionRequest(dagMerge) + entResults := mp.sendPendingEncryptionRequest() + go func() { + res := <-entResults.Get() + if res.Error != nil { + fmt.Printf("Error fetching keys: %s\n", res.Error) + return + } + err = db.executeMerge(context.Background(), dagMerge) + if err != nil { + fmt.Printf("Error executing merge: %s\n", err) + } + }() + // if there are pending encryption keys, we discard the transaction + // and wait for the keys to be fetched return nil } @@ -292,12 +305,12 @@ func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName im mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, keyID)] = struct{}{} } -func (mp *mergeProcessor) sendPendingEncryptionRequest(mergeEvent event.Merge) *encryption.Results { +func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) for k := range mp.pendingEncryptionKeyRequests { storeKeys = append(storeKeys, k) } - msg, results := encryption.NewRequestKeysMessage(mp.col.SchemaRoot(), storeKeys, mergeEvent) + msg, results := encryption.NewRequestKeysMessage(storeKeys) mp.col.db.events.Publish(msg) return results } diff --git a/internal/db/messages.go b/internal/db/messages.go index 5e0ff36748..ae8274ca99 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -79,19 +79,7 @@ func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { log.ErrorContextE(ctx, "Failed to load replicators", err) } }) - - case encryption.KeyRetrievedEvent: - go func() { - if err := db.handleEncryptionKeysRetrievedEvent(ctx, evt); err != nil { - log.ErrorContextE(ctx, errFailedToHandleEncKeysReceivedEvent, err, corelog.Any("Event", evt)) - } - }() } } } } - -// handleEncryptionKeysRetrievedEvent handles the event when requested encryption keys are retrieved from other peers. -func (db *db) handleEncryptionKeysRetrievedEvent(ctx context.Context, evt encryption.KeyRetrievedEvent) error { - return db.executeMerge(ctx, evt.MergeEvent) -} diff --git a/internal/encryption/event.go b/internal/encryption/event.go index c617bc72bf..14e05cd727 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -16,22 +16,16 @@ import ( ) const RequestKeysEventName = event.Name("enc-keys-request") -const KeysRetrievedEventName = event.Name("enc-keys-retrieved") // RequestKeysEvent represents a request of a node to fetch an encryption key for a specific // docID/field // // It must only contain public elements not protected by ACP. type RequestKeysEvent struct { - // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. - SchemaRoot string - // Keys is a list of the keys that are being requested. Keys []core.EncStoreDocKey Resp chan<- Result - - MergeEvent event.Merge } // RequestedKeyEventData represents the data that was retrieved for a specific key. @@ -41,16 +35,6 @@ type RequestedKeyEventData struct { } // KeyRetrievedEvent represents a key that was retrieved. -type KeyRetrievedEvent struct { - // SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated. - SchemaRoot string - - // Keys is a map of the requested keys to the corresponding raw encryption keys. - Keys map[core.EncStoreDocKey][]byte - - MergeEvent event.Merge -} - type Item struct { StoreKey core.EncStoreDocKey EncryptionKey []byte @@ -78,29 +62,10 @@ func NewResults() (*Results, chan<- Result) { // NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field -func NewRequestKeysMessage( - schemaRoot string, - keys []core.EncStoreDocKey, - mergeEvent event.Merge, -) (event.Message, *Results) { +func NewRequestKeysMessage(keys []core.EncStoreDocKey) (event.Message, *Results) { res, ch := NewResults() return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ - SchemaRoot: schemaRoot, - Keys: keys, - Resp: ch, - MergeEvent: mergeEvent, + Keys: keys, + Resp: ch, }), res } - -// NewKeysRetrievedMessage creates a new event message for a key that was retrieved -func NewKeysRetrievedMessage( - schemaRoot string, - keys map[core.EncStoreDocKey][]byte, - mergeEvent event.Merge, -) event.Message { - return event.NewMessage(KeysRetrievedEventName, KeyRetrievedEvent{ - SchemaRoot: schemaRoot, - Keys: keys, - MergeEvent: mergeEvent, - }) -} diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index 03b8eaca5f..eb751761f0 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -129,7 +129,6 @@ func (s *pubSubService) handleKeyRequestedEvent() { for _, item := range encResult.Items { m[item.StoreKey] = item.EncryptionKey } - s.eventBus.Publish(encryption.NewKeysRetrievedMessage(keyReqEvent.SchemaRoot, m, keyReqEvent.MergeEvent)) keyReqEvent.Resp <- encResult close(keyReqEvent.Resp) }() diff --git a/tests/integration/encryption/peer_sec_index_test.go b/tests/integration/encryption/peer_sec_index_test.go index 271a0eb56b..e6fd3548cf 100644 --- a/tests/integration/encryption/peer_sec_index_test.go +++ b/tests/integration/encryption/peer_sec_index_test.go @@ -15,7 +15,6 @@ import ( "github.com/sourcenetwork/immutable" - "github.com/sourcenetwork/defradb/internal/encryption" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -65,11 +64,7 @@ func TestDocEncryptionPeer_IfEncryptedDocHasIndexedField_ShouldIndexAfterDecrypt Doc: john21Doc, IsDocEncrypted: true, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - Count: 2, - }, + testUtils.WaitForSync{}, testUtils.Request{ Request: ` query @explain(type: execute) { @@ -149,11 +144,7 @@ func TestDocEncryptionPeer_IfDocDocHasEncryptedIndexedField_ShouldIndexAfterDecr Doc: john21Doc, EncryptedFields: []string{"age"}, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - Count: 2, - }, + testUtils.WaitForSync{}, testUtils.Request{ Request: ` query @explain(type: execute) { diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index cd9f8ef568..54ad066bc8 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -15,7 +15,6 @@ import ( "github.com/sourcenetwork/immutable" - "github.com/sourcenetwork/defradb/internal/encryption" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -95,10 +94,7 @@ func TestDocEncryptionPeer_IfPublicDocHasEncryptedField_ShouldFetchKeyAndDecrypt Doc: john21Doc, EncryptedFields: []string{"age"}, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { @@ -150,10 +146,7 @@ func TestDocEncryptionPeer_IfEncryptedPublicDocHasEncryptedField_ShouldFetchKeys IsDocEncrypted: true, EncryptedFields: []string{"age"}, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { @@ -205,10 +198,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfEncryptedPublicDocAreIndividuallyEncrypt IsDocEncrypted: true, EncryptedFields: []string{"name", "age"}, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { @@ -259,10 +249,7 @@ func TestDocEncryptionPeer_IfAllFieldsOfPublicDocAreIndividuallyEncrypted_Should Doc: john21Doc, EncryptedFields: []string{"name", "age"}, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { @@ -321,10 +308,7 @@ func TestDocEncryptionPeer_WithUpdatesOnEncryptedDeltaBasedCRDTField_ShouldDecry NodeID: immutable.Some(0), Doc: `{"age": 2}`, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { @@ -383,10 +367,7 @@ func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_Should NodeID: immutable.Some(0), Doc: `{"age": 2}`, }, - testUtils.WaitForSync{ - Event: immutable.Some(encryption.KeysRetrievedEventName), - NodeIDs: []int{1}, - }, + testUtils.WaitForSync{}, testUtils.Request{ NodeID: immutable.Some(1), Request: `query { diff --git a/tests/integration/events.go b/tests/integration/events.go index 1a127ed497..bf004b99aa 100644 --- a/tests/integration/events.go +++ b/tests/integration/events.go @@ -12,8 +12,6 @@ package tests import ( "encoding/json" - "slices" - "sync" "time" "github.com/sourcenetwork/immutable" @@ -21,7 +19,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/encryption" ) // eventTimeout is the amount of time to wait @@ -352,57 +349,8 @@ func getEventsForCreateDoc(s *state, action CreateDoc) map[string]struct{} { return expect } -// waitForKeyRetrievedEvent waits for nodes to receive a key retrieved event. -// If targetNodeIDs is empty, all nodes will be waiting on the event sync. -// Otherwise, only the nodes with the specified IDs will be checked. -func waitForKeyRetrievedEvent(s *state, targetNodeIDs []int) { - for nodeID := 0; nodeID < len(s.nodes); nodeID++ { - // if we have target nodes and the current node is not in the list, skip it - if len(targetNodeIDs) > 0 && !slices.Contains(targetNodeIDs, nodeID) { - continue - } - - select { - case _, ok := <-s.nodeEvents[nodeID].encKeysRetrieved.Message(): - if !ok { - require.Fail(s.t, "subscription closed waiting for key retrieved event") - } - - case <-time.After(eventTimeout): - require.Fail(s.t, "timeout waiting for key retrieved event") - } - } -} - -func waitForSync(s *state, action WaitForSync) { - count := int(action.Count) - if count == 0 { - count = 1 - } - - syncFunc := func(s *state, action WaitForSync) { - if !action.Event.HasValue() || action.Event.Value() == event.MergeCompleteName { - waitForMergeEvents(s) - } else if action.Event.Value() == encryption.KeysRetrievedEventName { - waitForKeyRetrievedEvent(s, action.NodeIDs) - } else { - require.Fail(s.t, "unsupported event type: %s", action.Event.Value()) - } - } - - if count == 1 { - syncFunc(s, action) - } else { - var wg sync.WaitGroup - wg.Add(count) - for i := 0; i < count; i++ { - go func(s *state, action WaitForSync) { - defer wg.Done() - syncFunc(s, action) - }(s, action) - } - wg.Wait() - } +func waitForSync(s *state) { + waitForMergeEvents(s) } // getEventsForUpdateWithFilter returns a map of docIDs that should be diff --git a/tests/integration/p2p.go b/tests/integration/p2p.go index 6839242938..99c713bb79 100644 --- a/tests/integration/p2p.go +++ b/tests/integration/p2p.go @@ -13,10 +13,7 @@ package tests import ( "time" - "github.com/sourcenetwork/immutable" - "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/corelog" @@ -136,17 +133,7 @@ type GetAllP2PCollections struct { // // For example you will likely wish to `WaitForSync` after creating a document in node 0 before querying // node 1 to see if it has been replicated. -type WaitForSync struct { - // Event is the name of the event to wait for. - // If is not provided then the action will wait for MergeComplete event. - Event immutable.Option[event.Name] - // NodeIDs are the node IDs (indexes) of the nodes to wait for sync on. - // If not provided then the action will wait for sync on all nodes. - NodeIDs []int - // Count is the number of times the event should be received by each node. - // Default value (zero) is treated as 1, e.g. the event should be received once. - Count uint -} +type WaitForSync struct{} // connectPeers connects two existing, started, nodes as peers. It returns a channel // that will receive an empty struct upon sync completion of all expected peer-sync events. diff --git a/tests/integration/state.go b/tests/integration/state.go index 2263dd2a97..eb70b40448 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -21,7 +21,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/net" "github.com/sourcenetwork/defradb/node" "github.com/sourcenetwork/defradb/tests/clients" @@ -79,9 +78,6 @@ type eventState struct { // p2pTopic is the `event.P2PTopicCompletedName` subscription p2pTopic *event.Subscription - - // encKeysRetrieved is the `encryption.KeyRetrieved` subscription - encKeysRetrieved *event.Subscription } // newEventState returns an eventState with all required subscriptions. @@ -102,16 +98,11 @@ func newEventState(bus *event.Bus) (*eventState, error) { if err != nil { return nil, err } - encKeysRetrieved, err := bus.Subscribe(encryption.KeysRetrievedEventName) - if err != nil { - return nil, err - } return &eventState{ - merge: merge, - update: update, - replicator: replicator, - p2pTopic: p2pTopic, - encKeysRetrieved: encKeysRetrieved, + merge: merge, + update: update, + replicator: replicator, + p2pTopic: p2pTopic, }, nil } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 5653a60fa2..daf49496ef 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -221,7 +221,7 @@ func executeTestCase( } if kms != NoneKMSType { - logAttrs = append(logAttrs, corelog.Any("kms", kms)) + logAttrs = append(logAttrs, corelog.Any("KMS", kms)) } log.InfoContext(ctx, testCase.Description, logAttrs...) @@ -365,7 +365,7 @@ func performAction( assertClientIntrospectionResults(s, action) case WaitForSync: - waitForSync(s, action) + waitForSync(s) case Benchmark: benchmarkAction(s, actionIndex, action) From 4e35ca0e031a62774b96672247310c348c4b8466 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 22:42:40 +0200 Subject: [PATCH 68/88] Polish --- internal/core/key.go | 1 - internal/db/context.go | 4 ---- internal/db/merge.go | 14 +++++++------- internal/db/messages.go | 1 - 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/internal/core/key.go b/internal/core/key.go index 29f515babf..900a04ee11 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -808,7 +808,6 @@ var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. // Unset fieldName indicates that the key is for the whole document. -// blockHeight is the height of the block that the key is for. func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], keyID string) EncStoreDocKey { return EncStoreDocKey{DocID: docID, FieldName: fieldName, KeyID: keyID} } diff --git a/internal/db/context.go b/internal/db/context.go index 47f5759af8..464bf812f5 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -58,10 +58,6 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if ok { return SetContextTxn(ctx, &explicitTxn{txn}), &explicitTxn{txn}, nil } - return EnforceNewContextTxn(ctx, db, readOnly) -} - -func EnforceNewContextTxn(ctx context.Context, db transactionDB, readOnly bool) (context.Context, datastore.Txn, error) { txn, err := db.NewTxn(ctx, readOnly) if err != nil { return nil, txn, err diff --git a/internal/db/merge.go b/internal/db/merge.go index 69170c71a4..807f87c873 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -66,12 +66,12 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.loadBlocks(ctx, dagMerge.Cid, mt, false) + err = mp.loadComposites(ctx, dagMerge.Cid, mt, false) if err != nil { return err } - err = mp.mergeBlocks(ctx) + err = mp.mergeComposites(ctx) if err != nil { return err } @@ -190,9 +190,9 @@ func newMergeTarget() mergeTarget { } } -// loadBlocks retrieves and stores into the merge processor the blocks for the given +// loadComposites retrieves and stores into the merge processor the blocks for the given // CID until it reaches a block that has already been merged or until we reach the genesis block. -func (mp *mergeProcessor) loadBlocks( +func (mp *mergeProcessor) loadComposites( ctx context.Context, blockCid cid.Cid, mt mergeTarget, @@ -223,7 +223,7 @@ func (mp *mergeProcessor) loadBlocks( if block.Delta.GetPriority() >= mt.headHeight { mp.blocks.PushFront(block) for _, prevCid := range block.GetPrevBlockCids() { - err := mp.loadBlocks(ctx, prevCid, mt, willDecrypt) + err := mp.loadComposites(ctx, prevCid, mt, willDecrypt) if err != nil { return err } @@ -248,12 +248,12 @@ func (mp *mergeProcessor) loadBlocks( } } } - return mp.loadBlocks(ctx, blockCid, newMT, willDecrypt) + return mp.loadComposites(ctx, blockCid, newMT, willDecrypt) } return nil } -func (mp *mergeProcessor) mergeBlocks(ctx context.Context) error { +func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { for e := mp.blocks.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() diff --git a/internal/db/messages.go b/internal/db/messages.go index ae8274ca99..04b81cd210 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -19,7 +19,6 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/encryption" ) func (db *db) handleMessages(ctx context.Context, sub *event.Subscription) { From 88519eb4ca2eb99205cd1c4bb3ce6d2380c9ba2d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 10 Sep 2024 22:44:20 +0200 Subject: [PATCH 69/88] Remove unused method --- internal/db/merge.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 807f87c873..c58363c9bd 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -11,7 +11,6 @@ package db import ( - "bytes" "container/list" "context" "fmt" @@ -519,32 +518,6 @@ func loadBlockFromBlockStore(ctx context.Context, txn datastore.Txn, cid cid.Cid return block, nil } -// loadBlocksWithKeyIDFromBlockstore loads the blocks from the blockstore that have given encryption -// keyID until it reaches a block with a different keyID or without any. -// The returned blocks are ordered from the newest to the oldest. -func loadBlocksWithKeyIDFromBlockstore( - ctx context.Context, - txn datastore.Txn, - cids []cid.Cid, - keyID string, -) ([]*coreblock.Block, error) { - var blocks []*coreblock.Block - for len(cids) > 0 { - cid := cids[0] - block, err := loadBlockFromBlockStore(ctx, txn, cid) - if err != nil { - return nil, err - } - - if block.Encryption != nil && bytes.Equal(block.Encryption.KeyID, []byte(keyID)) { - blocks = append(blocks, block) - cids = append(cids, block.GetPrevBlockCids()...) - } - cids = cids[1:] - } - return blocks, nil -} - func syncIndexedDoc( ctx context.Context, docID client.DocID, From c4f8270b12950fb62ff3a2f9f4ac56eec3631e82 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 12 Sep 2024 07:50:29 +0200 Subject: [PATCH 70/88] Polish --- crypto/aes_test.go | 2 +- internal/db/merge.go | 38 +++++++++++++------------------------- internal/kms/pubsub.go | 15 --------------- 3 files changed, 14 insertions(+), 41 deletions(-) diff --git a/crypto/aes_test.go b/crypto/aes_test.go index 25009304ff..7218ca24b2 100644 --- a/crypto/aes_test.go +++ b/crypto/aes_test.go @@ -137,7 +137,7 @@ func TestDecryptAES(t *testing.T) { key: validKey, additionalData: validAAD, expectError: true, - errorContains: "cipherText too short", + errorContains: errCipherTextTooShort, }, { name: "Invalid additional data", diff --git a/internal/db/merge.go b/internal/db/merge.go index c58363c9bd..0e9a6ce895 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -13,7 +13,6 @@ package db import ( "container/list" "context" - "fmt" "sync" "github.com/ipfs/go-cid" @@ -65,7 +64,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - err = mp.loadComposites(ctx, dagMerge.Cid, mt, false) + err = mp.loadComposites(ctx, dagMerge.Cid, mt) if err != nil { return err } @@ -80,12 +79,12 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { go func() { res := <-entResults.Get() if res.Error != nil { - fmt.Printf("Error fetching keys: %s\n", res.Error) + log.ErrorContextE(ctx, "Error fetching keys", res.Error) return } err = db.executeMerge(context.Background(), dagMerge) if err != nil { - fmt.Printf("Error executing merge: %s\n", err) + log.ErrorContextE(ctx, "Error executing merge", err) } }() // if there are pending encryption keys, we discard the transaction @@ -154,8 +153,8 @@ type mergeProcessor struct { mCRDTs map[string]merklecrdt.MerkleCRDT col *collection dsKey core.DataStoreKey - // blocks is a list of blocks that need to be merged. - blocks *list.List + // composites is a list of composites that need to be merged. + composites *list.List // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge // and doesn't have locally, so they need to be requested from the network. pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} @@ -173,7 +172,7 @@ func (db *db) newMergeProcessor( mCRDTs: make(map[string]merklecrdt.MerkleCRDT), col: col, dsKey: dsKey, - blocks: list.New(), + composites: list.New(), pendingEncryptionKeyRequests: make(map[core.EncStoreDocKey]struct{}), }, nil } @@ -195,15 +194,10 @@ func (mp *mergeProcessor) loadComposites( ctx context.Context, blockCid cid.Cid, mt mergeTarget, - willDecrypt bool, ) error { - if b, ok := mt.heads[blockCid]; ok { - // the head is already known, but the block might be encrypted - // if this time we try to decrypt it, we load the block - if !b.IsEncrypted() || !willDecrypt { - // We've already processed this block. - return nil - } + if _, ok := mt.heads[blockCid]; ok { + // We've already processed this block. + return nil } nd, err := mp.lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, coreblock.SchemaPrototype) @@ -220,9 +214,9 @@ func (mp *mergeProcessor) loadComposites( // of the composite DAG. However, the new block and its children might have branched off from an older block. // In this case, we also need to walk back the merge target's DAG until we reach a common block. if block.Delta.GetPriority() >= mt.headHeight { - mp.blocks.PushFront(block) + mp.composites.PushFront(block) for _, prevCid := range block.GetPrevBlockCids() { - err := mp.loadComposites(ctx, prevCid, mt, willDecrypt) + err := mp.loadComposites(ctx, prevCid, mt) if err != nil { return err } @@ -247,13 +241,13 @@ func (mp *mergeProcessor) loadComposites( } } } - return mp.loadComposites(ctx, blockCid, newMT, willDecrypt) + return mp.loadComposites(ctx, blockCid, newMT) } return nil } func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { - for e := mp.blocks.Front(); e != nil; e = e.Next() { + for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) link, err := block.GenerateLink() if err != nil { @@ -315,7 +309,6 @@ func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { } // processBlock merges the block and its children to the datastore and sets the head accordingly. -// withDecryption is a flag that indicates this is the decryption pass instead of a normal merge. func (mp *mergeProcessor) processBlock( ctx context.Context, dagBlock *coreblock.Block, @@ -538,11 +531,6 @@ func syncIndexedDoc( return err } - // this can happen if we received an encrypted document that we haven't decrypted yet - if isNewDoc && isDeletedDoc { - return nil - } - if isNewDoc { return col.indexNewDoc(ctx, doc) } else if isDeletedDoc { diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index eb751761f0..b13aef9a6d 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -15,7 +15,6 @@ import ( "context" "crypto/ecdh" "encoding/base64" - "fmt" libpeer "github.com/libp2p/go-libp2p/core/peer" "github.com/sourcenetwork/defradb/client" @@ -107,22 +106,11 @@ func (s *pubSubService) handleKeyRequestedEvent() { _, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) for _, encItem := range encResult.Items { - encKey, err := encryptor.GetKey(encItem.StoreKey) - if err != nil { - fmt.Printf(">>>>>>>>>> Failed to get stored key %s\n", encItem.StoreKey.ToString()) - continue - } - if len(encKey) == 0 { - fmt.Printf(">>>>>>>>>> No key stored %s\n", encItem.StoreKey.ToString()) - } else { - fmt.Printf(">>>>>>>>>> Has already key stored %s\n", encItem.StoreKey.ToString()) - } err = encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) if err != nil { log.ErrorContextE(s.ctx, "Failed to save encryption key", err) return } - fmt.Printf(">>>>>>>>>> Saved key %s\n", encItem.StoreKey.ToString()) } m := make(map[core.EncStoreDocKey][]byte) @@ -205,13 +193,10 @@ func (s *pubSubService) requestEncryptionKey( } respChan, err := s.pubsub.SendPubSubMessage(ctx, pubsubTopic, data) - //respChan, err := s.topic.Publish(ctx, data) if err != nil { return errors.Wrap("failed publishing to encryption thread", err) } - //s.server.RequestEncryptionKey(ctx, req, ephPrivKey) - go func() { s.handleFetchEncryptionKeyResponse(<-respChan, req, ephPrivKey, result) }() From 4d97c067e4be99eaaf9d7c72a93fbeb2d1cb0ae5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 12:29:56 +0200 Subject: [PATCH 71/88] Make encryption key be store in dedicated IPLD block --- cli/collection_create.go | 3 - client/db.go | 2 +- client/mocks/db.go | 12 +- crypto/ecies.go | 32 +- crypto/ecies_test.go | 6 +- datastore/mocks/txn.go | 12 +- datastore/multi.go | 6 +- datastore/store.go | 2 +- http/client.go | 2 +- http/client_tx.go | 2 +- internal/core/block/block.go | 150 ++++--- internal/core/block/block_test.go | 143 ++----- internal/db/context.go | 2 - internal/db/db.go | 2 +- internal/db/merge.go | 120 +++--- internal/encryption/context.go | 17 +- internal/encryption/encryptor.go | 129 ++---- internal/encryption/encryptor_test.go | 220 ++++++---- internal/encryption/event.go | 10 +- internal/kms/enc_store.go | 65 +++ internal/kms/pubsub.go | 156 +++---- internal/kms/service.go | 4 +- internal/merkle/clock/clock.go | 105 +++-- internal/merkle/clock/clock_test.go | 3 +- internal/merkle/crdt/composite.go | 3 +- internal/merkle/crdt/counter.go | 2 +- internal/merkle/crdt/lwwreg.go | 2 +- internal/merkle/crdt/merklecrdt.go | 1 + internal/merkle/crdt/merklecrdt_test.go | 2 +- net/pb/net.pb.go | 332 +++++---------- net/pb/net.proto | 36 +- net/pb/net_grpc.pb.go | 50 +-- net/pb/net_vtproto.pb.go | 429 +++----------------- net/peer.go | 4 +- tests/bench/query/planner/utils.go | 2 +- tests/clients/cli/wrapper.go | 2 +- tests/clients/cli/wrapper_tx.go | 2 +- tests/clients/http/wrapper.go | 2 +- tests/clients/http/wrapper_tx.go | 2 +- tests/integration/encryption/commit_test.go | 10 +- tests/integration/encryption/peer_test.go | 10 +- 41 files changed, 793 insertions(+), 1303 deletions(-) create mode 100644 internal/kms/enc_store.go diff --git a/cli/collection_create.go b/cli/collection_create.go index 57c65d6226..3cb652fee8 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -121,9 +121,6 @@ func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptF return } ctx := cmd.Context() - if txn != nil { - ctx, _ = encryption.ContextWithStore(ctx, txn.Encstore()) - } ctx = encryption.SetContextConfigFromParams(ctx, shouldEncryptDoc, encryptFields) cmd.SetContext(ctx) } diff --git a/client/db.go b/client/db.go index d3ea9e63f9..d245804321 100644 --- a/client/db.go +++ b/client/db.go @@ -53,7 +53,7 @@ type DB interface { // Encstore returns the store, that contains all known encryption keys for documents and their fields. // // It sits within the rootstore returned by [Root]. - Encstore() datastore.DSReaderWriter + Encstore() datastore.Blockstore // Peerstore returns the peerstore where known host information is stored. // diff --git a/client/mocks/db.go b/client/mocks/db.go index 5f4c7ee78e..da5aed24f6 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -480,19 +480,19 @@ func (_c *DB_DeleteReplicator_Call) RunAndReturn(run func(context.Context, clien } // Encstore provides a mock function with given fields: -func (_m *DB) Encstore() datastore.DSReaderWriter { +func (_m *DB) Encstore() datastore.Blockstore { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Encstore") } - var r0 datastore.DSReaderWriter - if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + var r0 datastore.Blockstore + if rf, ok := ret.Get(0).(func() datastore.Blockstore); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(datastore.DSReaderWriter) + r0 = ret.Get(0).(datastore.Blockstore) } } @@ -516,12 +516,12 @@ func (_c *DB_Encstore_Call) Run(run func()) *DB_Encstore_Call { return _c } -func (_c *DB_Encstore_Call) Return(_a0 datastore.DSReaderWriter) *DB_Encstore_Call { +func (_c *DB_Encstore_Call) Return(_a0 datastore.Blockstore) *DB_Encstore_Call { _c.Call.Return(_a0) return _c } -func (_c *DB_Encstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *DB_Encstore_Call { +func (_c *DB_Encstore_Call) RunAndReturn(run func() datastore.Blockstore) *DB_Encstore_Call { _c.Call.Return(run) return _c } diff --git a/crypto/ecies.go b/crypto/ecies.go index 32ad228b23..a2d9fd787f 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -55,14 +55,18 @@ func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { // Returns: // - Byte slice containing the encrypted message and necessary metadata for decryption // - Error if any step of the encryption process fails -func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData []byte) ([]byte, error) { - ephemeralPrivate, err := GenerateX25519() - if err != nil { - return nil, NewErrFailedToGenerateEphemeralKey(err) +func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData []byte, ourPrivateKey *ecdh.PrivateKey) ([]byte, error) { + // TODO: apply option patter + if ourPrivateKey == nil { + var err error + ourPrivateKey, err = GenerateX25519() + if err != nil { + return nil, NewErrFailedToGenerateEphemeralKey(err) + } } - ephemeralPublic := ephemeralPrivate.PublicKey() + ourPublicKey := ourPrivateKey.PublicKey() - sharedSecret, err := ephemeralPrivate.ECDH(publicKey) + sharedSecret, err := ourPrivateKey.ECDH(publicKey) if err != nil { return nil, NewErrFailedECDHOperation(err) } @@ -77,7 +81,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] return nil, NewErrFailedKDFOperationForHMACKey(err) } - cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ephemeralPublic.Bytes(), associatedData), true) + cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ourPublicKey.Bytes(), associatedData), true) if err != nil { return nil, NewErrFailedToEncrypt(err) } @@ -86,7 +90,7 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] mac.Write(cipherText) macSum := mac.Sum(nil) - result := append(ephemeralPublic.Bytes(), cipherText...) + result := append(ourPublicKey.Bytes(), cipherText...) result = append(result, macSum...) return result, nil @@ -111,18 +115,20 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] // Returns: // - Byte slice containing the decrypted plaintext // - Error if any step of the decryption process fails, including authentication failure -func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData []byte) ([]byte, error) { +func DecryptECIES(cipherText []byte, ourPrivateKey *ecdh.PrivateKey, associatedData, publicKeyBytes []byte) ([]byte, error) { if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { return nil, ErrCipherTextTooShort } - ephemeralPublicBytes := cipherText[:X25519PublicKeySize] - ephemeralPublic, err := ecdh.X25519().NewPublicKey(ephemeralPublicBytes) + if publicKeyBytes == nil { + publicKeyBytes = cipherText[:X25519PublicKeySize] + } + publicKey, err := ecdh.X25519().NewPublicKey(publicKeyBytes) if err != nil { return nil, NewErrFailedToParseEphemeralPublicKey(err) } - sharedSecret, err := privateKey.ECDH(ephemeralPublic) + sharedSecret, err := ourPrivateKey.ECDH(publicKey) if err != nil { return nil, NewErrFailedECDHOperation(err) } @@ -147,7 +153,7 @@ func DecryptECIES(cipherText []byte, privateKey *ecdh.PrivateKey, associatedData return nil, ErrVerificationWithHMACFailed } - plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(ephemeralPublicBytes, associatedData)) + plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(publicKeyBytes, associatedData)) if err != nil { return nil, NewErrFailedToDecrypt(err) } diff --git a/crypto/ecies_test.go b/crypto/ecies_test.go index d973535f6a..8dfa639d66 100644 --- a/crypto/ecies_test.go +++ b/crypto/ecies_test.go @@ -37,7 +37,7 @@ func TestEncryptECIES_Errors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := EncryptECIES(tt.plainText, tt.publicKey, tt.associatedData) + _, err := EncryptECIES(tt.plainText, tt.publicKey, tt.associatedData, nil) if err == nil { t.Errorf("Expected an error, but got nil") } else if !strings.Contains(err.Error(), tt.expectError) { @@ -50,7 +50,7 @@ func TestEncryptECIES_Errors(t *testing.T) { func TestDecryptECIES_Errors(t *testing.T) { // Setup validPrivateKey, _ := GenerateX25519() - validCipherText, _ := EncryptECIES([]byte("test data"), validPrivateKey.PublicKey(), []byte("associated data")) + validCipherText, _ := EncryptECIES([]byte("test data"), validPrivateKey.PublicKey(), []byte("associated data"), nil) tests := []struct { name string @@ -91,7 +91,7 @@ func TestDecryptECIES_Errors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := DecryptECIES(tt.cipherText, tt.privateKey, tt.associatedData) + _, err := DecryptECIES(tt.cipherText, tt.privateKey, tt.associatedData, nil) if err == nil || !strings.Contains(err.Error(), tt.expectError) { t.Errorf("Expected error containing '%s', got %v", tt.expectError, err) } diff --git a/datastore/mocks/txn.go b/datastore/mocks/txn.go index 41606260ea..ea923d5de4 100644 --- a/datastore/mocks/txn.go +++ b/datastore/mocks/txn.go @@ -196,19 +196,19 @@ func (_c *Txn_Discard_Call) RunAndReturn(run func(context.Context)) *Txn_Discard } // Encstore provides a mock function with given fields: -func (_m *Txn) Encstore() datastore.DSReaderWriter { +func (_m *Txn) Encstore() datastore.Blockstore { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for Encstore") } - var r0 datastore.DSReaderWriter - if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + var r0 datastore.Blockstore + if rf, ok := ret.Get(0).(func() datastore.Blockstore); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(datastore.DSReaderWriter) + r0 = ret.Get(0).(datastore.Blockstore) } } @@ -232,12 +232,12 @@ func (_c *Txn_Encstore_Call) Run(run func()) *Txn_Encstore_Call { return _c } -func (_c *Txn_Encstore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Encstore_Call { +func (_c *Txn_Encstore_Call) Return(_a0 datastore.Blockstore) *Txn_Encstore_Call { _c.Call.Return(_a0) return _c } -func (_c *Txn_Encstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Encstore_Call { +func (_c *Txn_Encstore_Call) RunAndReturn(run func() datastore.Blockstore) *Txn_Encstore_Call { _c.Call.Return(run) return _c } diff --git a/datastore/multi.go b/datastore/multi.go index bcf0e7191f..cbbf80e23f 100644 --- a/datastore/multi.go +++ b/datastore/multi.go @@ -29,7 +29,7 @@ var ( type multistore struct { root DSReaderWriter data DSReaderWriter - enc DSReaderWriter + enc Blockstore head DSReaderWriter peer DSBatching system DSReaderWriter @@ -44,7 +44,7 @@ func MultiStoreFrom(rootstore ds.Datastore) MultiStore { ms := &multistore{ root: rootRW, data: prefix(rootRW, dataStoreKey), - enc: prefix(rootRW, encStoreKey), + enc: newBlockstore(prefix(rootRW, encStoreKey)), head: prefix(rootRW, headStoreKey), peer: namespace.Wrap(rootstore, peerStoreKey), system: prefix(rootRW, systemStoreKey), @@ -60,7 +60,7 @@ func (ms multistore) Datastore() DSReaderWriter { } // Encstore implements MultiStore. -func (ms multistore) Encstore() DSReaderWriter { +func (ms multistore) Encstore() Blockstore { return ms.enc } diff --git a/datastore/store.go b/datastore/store.go index 516bfe0b65..641cd10b1a 100644 --- a/datastore/store.go +++ b/datastore/store.go @@ -40,7 +40,7 @@ type MultiStore interface { // Encstore is a wrapped root DSReaderWriter under the /enc namespace // This store is used for storing symmetric encryption keys for doc encryption. // The store keys are comprised of docID + field name. - Encstore() DSReaderWriter + Encstore() Blockstore // Headstore is a wrapped root DSReaderWriter under the /head namespace Headstore() DSReaderWriter diff --git a/http/client.go b/http/client.go index 83bb57a99a..e06034f92f 100644 --- a/http/client.go +++ b/http/client.go @@ -447,7 +447,7 @@ func (c *Client) Blockstore() datastore.Blockstore { panic("client side database") } -func (c *Client) Encstore() datastore.DSReaderWriter { +func (c *Client) Encstore() datastore.Blockstore { panic("client side database") } diff --git a/http/client_tx.go b/http/client_tx.go index 5b99f5aaad..daacb4128e 100644 --- a/http/client_tx.go +++ b/http/client_tx.go @@ -91,7 +91,7 @@ func (c *Transaction) Datastore() datastore.DSReaderWriter { panic("client side transaction") } -func (c *Transaction) Encstore() datastore.DSReaderWriter { +func (c *Transaction) Encstore() datastore.Blockstore { panic("client side transaction") } diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 425bf61b59..baa5d2ff32 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -29,12 +29,15 @@ import ( // Schema is the IPLD schema type that represents a `Block`. var ( - Schema schema.Type - SchemaPrototype ipld.NodePrototype + Schema schema.Type + SchemaPrototype ipld.NodePrototype + EncryptionSchema schema.Type + EncryptionSchemaPrototype ipld.NodePrototype ) func init() { Schema, SchemaPrototype = mustSetSchema( + "Block", &Block{}, &DAGLink{}, &crdt.CRDT{}, @@ -42,6 +45,11 @@ func init() { &crdt.CompositeDAGDelta{}, &crdt.CounterDelta{}, ) + + EncryptionSchema, EncryptionSchemaPrototype = mustSetSchema( + "Encryption", + &Encryption{}, + ) } type schemaDefinition interface { @@ -49,7 +57,7 @@ type schemaDefinition interface { IPLDSchemaBytes() []byte } -func mustSetSchema(schemas ...schemaDefinition) (schema.Type, ipld.NodePrototype) { +func mustSetSchema(schemaName string, schemas ...schemaDefinition) (schema.Type, ipld.NodePrototype) { schemaBytes := make([][]byte, 0, len(schemas)) for _, s := range schemas { schemaBytes = append(schemaBytes, s.IPLDSchemaBytes()) @@ -59,12 +67,12 @@ func mustSetSchema(schemas ...schemaDefinition) (schema.Type, ipld.NodePrototype if err != nil { panic(err) } - blockSchemaType := ts.TypeByName("Block") + blockSchemaType := ts.TypeByName(schemaName) // Calling bindnode.Prototype here ensure that [Block] and all the types it contains // are compatible with the IPLD schema defined by blockSchemaType. // If [Block] and `blockSchematype` do not match, this will panic. - proto := bindnode.Prototype(&Block{}, blockSchemaType) + proto := bindnode.Prototype(schemas[0], blockSchemaType) return blockSchemaType, proto.Representation() } @@ -97,21 +105,16 @@ func NewDAGLink(name string, link cidlink.Link) DAGLink { } } -// EncryptionType represents the type (or level) of encryption applied to the block. -type EncryptionType int - -const ( - NotEncrypted EncryptionType = iota - DocumentEncrypted - FieldEncrypted -) - // Encryption contains the encryption information for the block's delta. type Encryption struct { - // Type indicates on what level encryption is applied. - Type EncryptionType - // CID of the key used for encryption. - KeyID []byte + // DocID is the ID of the document that is encrypted with the associated encryption key. + DocID []byte + // FieldName is the name of the field that is encrypted with the associated encryption key. + // It is set if encryption is applied to a field instead of the whole doc. + // It needs to be a pointer so that it can be translated from and to `optional` in the IPLD schema. + FieldName *string + // Encryption key. + Key []byte } // Block is a block that contains a CRDT delta and links to other blocks. @@ -122,18 +125,27 @@ type Block struct { Links []DAGLink // Encryption contains the encryption information for the block's delta. // It needs to be a pointer so that it can be translated from and to `optional` in the IPLD schema. - Encryption *Encryption + Encryption *cidlink.Link } // IsEncrypted returns true if the block is encrypted. -func (b *Block) IsEncrypted() bool { - return b.Encryption != nil && b.Encryption.Type != NotEncrypted +func (block *Block) IsEncrypted() bool { + return block.Encryption != nil +} + +// Clone returns a shallow copy of the block with cloned delta. +func (block *Block) Clone() *Block { + return &Block{ + Delta: block.Delta.Clone(), + Links: block.Links, + Encryption: block.Encryption, + } } // GetPrevBlockCids returns the CIDs of the previous blocks. It can be more than 1 with multiple heads. -func (b *Block) GetPrevBlockCids() []cid.Cid { +func (block *Block) GetPrevBlockCids() []cid.Cid { var heads []cid.Cid - for _, link := range b.Links { + for _, link := range block.Links { if link.Name == core.HEAD { heads = append(heads, link.Cid) } @@ -144,24 +156,26 @@ func (b *Block) GetPrevBlockCids() []cid.Cid { // IPLDSchemaBytes returns the IPLD schema representation for the block. // // This needs to match the [Block] struct or [mustSetSchema] will panic on init. -func (b Block) IPLDSchemaBytes() []byte { +func (block *Block) IPLDSchemaBytes() []byte { return []byte(` type Block struct { delta CRDT links [DAGLink] - encryption optional Encryption + encryption optional Link } - + `) +} + +// IPLDSchemaBytes returns the IPLD schema representation for the encryption block. +// +// This needs to match the [Encryption] struct or [mustSetSchema] will panic on init. +func (enc *Encryption) IPLDSchemaBytes() []byte { + return []byte(` type Encryption struct { - type EncryptionType - keyID Bytes + docID Bytes + fieldName optional String + key Bytes } - - type EncryptionType enum { - | NotEncrypted ("0") - | DocumentEncrypted ("1") - | FieldEncrypted ("2") - } representation int `) } @@ -198,6 +212,16 @@ func New(delta core.Delta, links []DAGLink, heads ...cid.Cid) *Block { } } +// GetFromBytes returns a block from encoded bytes. +func GetEncryptionBlockFromBytes(b []byte) (*Encryption, error) { + enc := &Encryption{} + err := enc.Unmarshal(b) + if err != nil { + return nil, err + } + return enc, nil +} + // GetFromBytes returns a block from encoded bytes. func GetFromBytes(b []byte) (*Block, error) { block := &Block{} @@ -217,8 +241,17 @@ func GetFromNode(node ipld.Node) (*Block, error) { return block, nil } +// GetFromNode returns a block from a node. +func GetEncryptionBlockFromNode(node ipld.Node) (*Encryption, error) { + encBlock, ok := bindnode.Unwrap(node).(*Encryption) + if !ok { + return nil, NewErrNodeToBlock(node) + } + return encBlock, nil +} + // Marshal encodes the delta using CBOR encoding. -func (block *Block) Marshal() (data []byte, err error) { +func (block *Block) Marshal() ([]byte, error) { b, err := ipld.Marshal(dagcbor.Encode, block, Schema) if err != nil { return nil, NewErrEncodingBlock(err) @@ -228,26 +261,41 @@ func (block *Block) Marshal() (data []byte, err error) { // Unmarshal decodes the delta from CBOR encoding. func (block *Block) Unmarshal(b []byte) error { - _, err := ipld.Unmarshal( - b, - dagcbor.Decode, - block, - Schema, - ) + _, err := ipld.Unmarshal(b, dagcbor.Decode, block, Schema) if err != nil { return NewErrUnmarshallingBlock(err) } - if err := block.Validate(); err != nil { - return err + return nil +} + +// Marshal encodes the delta using CBOR encoding. +func (enc *Encryption) Marshal() ([]byte, error) { + b, err := ipld.Marshal(dagcbor.Encode, enc, EncryptionSchema) + if err != nil { + return nil, NewErrEncodingBlock(err) + } + return b, nil +} + +// Unmarshal decodes the delta from CBOR encoding. +func (enc *Encryption) Unmarshal(b []byte) error { + _, err := ipld.Unmarshal(b, dagcbor.Decode, enc, EncryptionSchema) + if err != nil { + return NewErrUnmarshallingBlock(err) } return nil } // GenerateNode generates an IPLD node from the block in its representation form. -func (block *Block) GenerateNode() (node ipld.Node) { +func (block *Block) GenerateNode() ipld.Node { return bindnode.Wrap(block, Schema).Representation() } +// GenerateNode generates an IPLD node from the encryption block in its representation form. +func (enc *Encryption) GenerateNode() ipld.Node { + return bindnode.Wrap(enc, EncryptionSchema).Representation() +} + // GetLinkByName returns the link by name. It will return false if the link does not exist. func (block *Block) GetLinkByName(name string) (cidlink.Link, bool) { for _, link := range block.Links { @@ -286,17 +334,3 @@ func GetLinkPrototype() cidlink.LinkPrototype { MhLength: 32, }} } - -// Validate checks if the block is valid. -func (b *Block) Validate() error { - if b.Encryption != nil { - eType := b.Encryption.Type - if eType != NotEncrypted && eType != DocumentEncrypted && eType != FieldEncrypted { - return ErrInvalidBlockEncryptionType - } - if len(b.Encryption.KeyID) == 0 { - return ErrInvalidBlockEncryptionKeyID - } - } - return nil -} diff --git a/internal/core/block/block_test.go b/internal/core/block/block_test.go index f30c06451c..d7fe2d1bf0 100644 --- a/internal/core/block/block_test.go +++ b/internal/core/block/block_test.go @@ -180,13 +180,23 @@ func TestBlockDeltaPriority(t *testing.T) { require.Equal(t, uint64(2), block.Delta.GetPriority()) } -func TestBlockMarshal_IsEncryptedNotSet_ShouldNotContainIsEcryptedField(t *testing.T) { +func TestBlockMarshal_IfEncryptedNotSet_ShouldNotContainIsEncryptedField(t *testing.T) { lsys := cidlink.DefaultLinkSystem() store := memstore.Store{} lsys.SetReadStorage(&store) lsys.SetWriteStorage(&store) - fieldBlock := Block{ + encBlock := Encryption{ + DocID: []byte("docID"), + Key: []byte("keyID"), + } + + encBlockLink, err := lsys.Store(ipld.LinkContext{}, GetLinkPrototype(), encBlock.GenerateNode()) + require.NoError(t, err) + + link := encBlockLink.(cidlink.Link) + + block := Block{ Delta: crdt.CRDT{ LWWRegDelta: &crdt.LWWRegDelta{ DocID: []byte("docID"), @@ -196,11 +206,27 @@ func TestBlockMarshal_IsEncryptedNotSet_ShouldNotContainIsEcryptedField(t *testi Data: []byte("John"), }, }, + Encryption: &link, } - b, err := fieldBlock.Marshal() + blockLink, err := lsys.Store(ipld.LinkContext{}, GetLinkPrototype(), block.GenerateNode()) + require.NoError(t, err) + + nd, err := lsys.Load(ipld.LinkContext{}, blockLink, SchemaPrototype) + require.NoError(t, err) + + loadedBlock, err := GetFromNode(nd) + require.NoError(t, err) + + require.NotNil(t, loadedBlock.Encryption) + + nd, err = lsys.Load(ipld.LinkContext{}, loadedBlock.Encryption, EncryptionSchemaPrototype) + require.NoError(t, err) + + loadedEncBlock, err := GetEncryptionBlockFromNode(nd) require.NoError(t, err) - require.NotContains(t, string(b), "isEncrypted") + + require.Equal(t, encBlock, *loadedEncBlock) } func TestBlockMarshal_IsEncryptedNotSetWithLinkSystem_ShouldLoadWithNoError(t *testing.T) { @@ -259,106 +285,27 @@ func TestBlockUnmarshal_InvalidCBOR_Error(t *testing.T) { require.Error(t, err) } -func TestBlockUnmarshal_WithValidEncryption_Succeed(t *testing.T) { - encryptedBlock := Block{ - Delta: crdt.CRDT{ - LWWRegDelta: &crdt.LWWRegDelta{ - DocID: []byte("docID"), - FieldName: "name", - Priority: 1, - SchemaVersionID: "schemaVersionID", - Data: []byte("John"), - }, - }, - Encryption: &Encryption{ - Type: FieldEncrypted, - KeyID: []byte("keyID"), - }, - } - - marshaledData, err := encryptedBlock.Marshal() - require.NoError(t, err) - - var unmarshaledBlock Block - err = unmarshaledBlock.Unmarshal(marshaledData) - require.NoError(t, err) - - require.Equal(t, encryptedBlock, unmarshaledBlock) - require.NotNil(t, unmarshaledBlock.Encryption) - require.Equal(t, FieldEncrypted, unmarshaledBlock.Encryption.Type) - require.Equal(t, []byte("keyID"), unmarshaledBlock.Encryption.KeyID) +func TestEncryptionBlockUnmarshal_InvalidCBOR_Error(t *testing.T) { + invalidData := []byte("invalid CBOR data") + var encBlock Encryption + err := encBlock.Unmarshal(invalidData) + require.Error(t, err) } -func TestBlockUnmarshal_WithInvalidEncryption_Error(t *testing.T) { - encryptedBlock := Block{ - Delta: crdt.CRDT{ - LWWRegDelta: &crdt.LWWRegDelta{ - DocID: []byte("docID"), - FieldName: "name", - Priority: 1, - SchemaVersionID: "schemaVersionID", - Data: []byte("John"), - }, - }, - Encryption: &Encryption{ - Type: FieldEncrypted, - KeyID: []byte{}, - }, +func TestEncryptionBlockUnmarshal_ValidInput_Succeed(t *testing.T) { + fieldName := "fieldName" + encBlock := Encryption{ + DocID: []byte("docID"), + Key: []byte("keyID"), + FieldName: &fieldName, } - marshaledData, err := encryptedBlock.Marshal() + marshaledData, err := encBlock.Marshal() require.NoError(t, err) - var unmarshaledBlock Block + var unmarshaledBlock Encryption err = unmarshaledBlock.Unmarshal(marshaledData) - require.Error(t, err) -} - -func TestBlock_Validate(t *testing.T) { - tests := []struct { - name string - encryption *Encryption - expectedError error - }{ - { - name: "NotEncrypted type is valid", - encryption: &Encryption{Type: NotEncrypted, KeyID: []byte{1}}, - expectedError: nil, - }, - { - name: "DocumentEncrypted type is valid", - encryption: &Encryption{Type: DocumentEncrypted, KeyID: []byte{1}}, - expectedError: nil, - }, - { - name: "FieldEncrypted type is valid", - encryption: &Encryption{Type: FieldEncrypted, KeyID: []byte{1}}, - expectedError: nil, - }, - { - name: "Nil Encryption is valid", - encryption: nil, - expectedError: nil, - }, - { - name: "Invalid encryption type", - encryption: &Encryption{Type: EncryptionType(99), KeyID: []byte{1}}, - expectedError: ErrInvalidBlockEncryptionType, - }, - { - name: "Invalid encryption key id parameter", - encryption: &Encryption{Type: DocumentEncrypted, KeyID: []byte{}}, - expectedError: ErrInvalidBlockEncryptionKeyID, - }, - } + require.NoError(t, err) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &Block{ - Encryption: tt.encryption, - } - err := b.Validate() - require.Equal(t, tt.expectedError, err) - }) - } + require.Equal(t, encBlock, unmarshaledBlock) } diff --git a/internal/db/context.go b/internal/db/context.go index 464bf812f5..a2fa50507f 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -17,7 +17,6 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/internal/encryption" ) // txnContextKey is the key type for transaction context values. @@ -62,7 +61,6 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if err != nil { return nil, txn, err } - ctx, _ = encryption.ContextWithStore(ctx, txn.Encstore()) return SetContextTxn(ctx, txn), txn, nil } diff --git a/internal/db/db.go b/internal/db/db.go index 855450bf0c..d88c5920bc 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -167,7 +167,7 @@ func (db *db) Blockstore() datastore.Blockstore { } // Encstore returns the internal enc store which contains encryption key for documents and their fields. -func (db *db) Encstore() datastore.DSReaderWriter { +func (db *db) Encstore() datastore.Blockstore { return db.multistore.Encstore() } diff --git a/internal/db/merge.go b/internal/db/merge.go index 0e9a6ce895..02523279c4 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -74,21 +74,7 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - if len(mp.pendingEncryptionKeyRequests) != 0 { - entResults := mp.sendPendingEncryptionRequest() - go func() { - res := <-entResults.Get() - if res.Error != nil { - log.ErrorContextE(ctx, "Error fetching keys", res.Error) - return - } - err = db.executeMerge(context.Background(), dagMerge) - if err != nil { - log.ErrorContextE(ctx, "Error executing merge", err) - } - }() - // if there are pending encryption keys, we discard the transaction - // and wait for the keys to be fetched + if len(mp.failedEncryptionBlocks) > 0 { return nil } @@ -157,7 +143,10 @@ type mergeProcessor struct { composites *list.List // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge // and doesn't have locally, so they need to be requested from the network. - pendingEncryptionKeyRequests map[core.EncStoreDocKey]struct{} + pendingEncryptionKeyRequests map[cidlink.Link]struct{} + + failedEncryptionBlocks map[cidlink.Link]struct{} + fetchedEncryptionBlocks map[cidlink.Link]*coreblock.Encryption } func (db *db) newMergeProcessor( @@ -173,7 +162,9 @@ func (db *db) newMergeProcessor( col: col, dsKey: dsKey, composites: list.New(), - pendingEncryptionKeyRequests: make(map[core.EncStoreDocKey]struct{}), + pendingEncryptionKeyRequests: make(map[cidlink.Link]struct{}), + failedEncryptionBlocks: make(map[cidlink.Link]struct{}), + fetchedEncryptionBlocks: make(map[cidlink.Link]*coreblock.Encryption), }, nil } @@ -261,6 +252,38 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { return nil } +// TODO: check if we should send 1-by-1 +func (mp *mergeProcessor) getEncryptionBlock( + encLink cidlink.Link, +) (*coreblock.Encryption, error) { + if _, ok := mp.failedEncryptionBlocks[encLink]; ok { + return nil, nil + } + if encBlock, ok := mp.fetchedEncryptionBlocks[encLink]; ok { + return encBlock, nil + } + msg, results := encryption.NewRequestKeysMessage([]cidlink.Link{encLink}) + mp.col.db.events.Publish(msg) + + res := <-results.Get() + if res.Error != nil { + mp.failedEncryptionBlocks[encLink] = struct{}{} + return nil, nil + } + + delete(mp.failedEncryptionBlocks, encLink) + + var encBlock coreblock.Encryption + err := encBlock.Unmarshal(res.Items[0].Block) + if err != nil { + return nil, err + } + + mp.fetchedEncryptionBlocks[encLink] = &encBlock + + return &encBlock, nil +} + // processEncryptedBlock decrypts the block if it is encrypted and returns the decrypted block. // If the block is encrypted and we were not able to decrypt it, it returns true as the second return value // which indicates that the we should skip merging of the block. @@ -270,43 +293,39 @@ func (mp *mergeProcessor) processEncryptedBlock( dagBlock *coreblock.Block, ) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { - plainTextBlock, err := decryptBlock(ctx, dagBlock) + encBlock, err := mp.getEncryptionBlock(*dagBlock.Encryption) + if err != nil { + return nil, false, err + } + + if encBlock == nil { + return dagBlock, false, nil + } + + plainTextBlock, err := decryptBlock(ctx, dagBlock, encBlock) if err != nil { return nil, false, err } if plainTextBlock != nil { return plainTextBlock, true, nil - } else { - blockEnc := dagBlock.Encryption - // we weren't able to decrypt the block, so we request the encryption key unless it's a decryption pass - if (dagBlock.Delta.IsComposite() && blockEnc.Type == coreblock.DocumentEncrypted) || - blockEnc.Type == coreblock.FieldEncrypted { - docID := string(dagBlock.Delta.GetDocID()) - fieldName := immutable.None[string]() - if blockEnc.Type == coreblock.FieldEncrypted { - fieldName = immutable.Some(dagBlock.Delta.GetFieldName()) - } - mp.addPendingEncryptionRequest(docID, fieldName, string(blockEnc.KeyID)) - } - return dagBlock, false, nil } } return dagBlock, true, nil } -func (mp *mergeProcessor) addPendingEncryptionRequest(docID string, fieldName immutable.Option[string], keyID string) { - mp.pendingEncryptionKeyRequests[core.NewEncStoreDocKey(docID, fieldName, keyID)] = struct{}{} +/*func (mp *mergeProcessor) addPendingEncryptionRequest(link cidlink.Link) { + mp.pendingEncryptionKeyRequests[link] = struct{}{} } func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { - storeKeys := make([]core.EncStoreDocKey, 0, len(mp.pendingEncryptionKeyRequests)) + storeKeys := make([]cidlink.Link, 0, len(mp.pendingEncryptionKeyRequests)) for k := range mp.pendingEncryptionKeyRequests { storeKeys = append(storeKeys, k) } msg, results := encryption.NewRequestKeysMessage(storeKeys) mp.col.db.events.Publish(msg) return results -} +}*/ // processBlock merges the block and its children to the datastore and sets the head accordingly. func (mp *mergeProcessor) processBlock( @@ -360,43 +379,24 @@ func (mp *mergeProcessor) processBlock( return nil } -func decryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { - optFieldName := immutable.None[string]() - blockEnc := block.Encryption - if blockEnc.Type == coreblock.FieldEncrypted { - optFieldName = immutable.Some(block.Delta.GetFieldName()) - } - - encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), optFieldName, string(blockEnc.KeyID)) - - encryptor := encryption.GetEncryptorFromContext(ctx) - if encryptor == nil { - return nil, encryption.ErrContextHasNoEncryptor - } +func decryptBlock(ctx context.Context, block *coreblock.Block, encBlock *coreblock.Encryption) (*coreblock.Block, error) { + _, encryptor := encryption.EnsureContextWithEncryptor(ctx) if block.Delta.IsComposite() { // for composite blocks there is nothing to decrypt - // so we just check if we have the encryption key for child blocks - bytes, err := encryptor.GetKey(encStoreKey) - if err != nil { - return nil, err - } - if len(bytes) == 0 { - return nil, nil - } return block, nil } - clonedCRDT := block.Delta.Clone() - bytes, err := encryptor.Decrypt(encStoreKey, clonedCRDT.GetData()) + bytes, err := encryptor.Decrypt(block.Delta.GetData(), encBlock.Key) if err != nil { return nil, err } if len(bytes) == 0 { return nil, nil } - clonedCRDT.SetData(bytes) - return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil + newBlock := block.Clone() + newBlock.Delta.SetData(bytes) + return newBlock, nil } func (mp *mergeProcessor) initCRDTForType(field string) (merklecrdt.MerkleCRDT, error) { diff --git a/internal/encryption/context.go b/internal/encryption/context.go index cc3f9153bb..422bd97697 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -14,8 +14,6 @@ import ( "context" "github.com/sourcenetwork/immutable" - - "github.com/sourcenetwork/defradb/datastore" ) // docEncContextKey is the key type for document encryption context values. @@ -39,7 +37,11 @@ func setConfig(ctx context.Context, enc *DocEncryptor) { enc.ctx = ctx } -func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { +// EnsureContextWithEncryptor returns a context with a document encryptor and the +// document encryptor itself. If the context already has an encryptor, it +// returns the context and encryptor as is. Otherwise, it creates a new +// document encryptor and stores it in the context. +func EnsureContextWithEncryptor(ctx context.Context) (context.Context, *DocEncryptor) { enc := GetEncryptorFromContext(ctx) if enc == nil { enc = newDocEncryptor(ctx) @@ -48,14 +50,6 @@ func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncrypto return ctx, enc } -// ContextWithStore sets the store on the doc encryptor in the context and returns the updated -// context and doc encryptor. If the doc encryptor is not present, it will be created. -func ContextWithStore(ctx context.Context, encstore datastore.DSReaderWriter) (context.Context, *DocEncryptor) { - ctx, encryptor := ensureContextWithDocEnc(ctx) - encryptor.SetStore(encstore) - return ctx, encryptor -} - // GetContextConfig returns the doc encryption config from the given context. func GetContextConfig(ctx context.Context) immutable.Option[DocEncConfig] { encConfig, ok := ctx.Value(configContextKey{}).(DocEncConfig) @@ -67,6 +61,7 @@ func GetContextConfig(ctx context.Context) immutable.Option[DocEncConfig] { // SetContextConfig returns a new context with the doc encryption config set. func SetContextConfig(ctx context.Context, encConfig DocEncConfig) context.Context { + ctx, _ = EnsureContextWithEncryptor(ctx) return context.WithValue(ctx, configContextKey{}, encConfig) } diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 801eb7dbff..fdd2efec3d 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -13,18 +13,13 @@ package encryption import ( "context" "crypto/rand" - "errors" "io" "os" "strings" - ds "github.com/ipfs/go-datastore" - "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/crypto" - "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/internal/core" ) var generateEncryptionKeyFunc = generateEncryptionKey @@ -57,12 +52,16 @@ func generateTestEncryptionKey(docID string, fieldName immutable.Option[string]) type DocEncryptor struct { conf immutable.Option[DocEncConfig] ctx context.Context - store datastore.DSReaderWriter - generatedKeys []core.EncStoreDocKey + generatedKeys map[genK][]byte +} + +type genK struct { + docID string + fieldName immutable.Option[string] } func newDocEncryptor(ctx context.Context) *DocEncryptor { - return &DocEncryptor{ctx: ctx} + return &DocEncryptor{ctx: ctx, generatedKeys: make(map[genK][]byte)} } // SetConfig sets the configuration for the document encryptor. @@ -70,11 +69,6 @@ func (d *DocEncryptor) SetConfig(conf immutable.Option[DocEncConfig]) { d.conf = conf } -// SetStore sets the store for the document encryptor. -func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { - d.store = store -} - func shouldEncryptIndividualField(conf immutable.Option[DocEncConfig], fieldName immutable.Option[string]) bool { if !conf.HasValue() || !fieldName.HasValue() { return false @@ -108,19 +102,10 @@ func shouldEncryptDocField(conf immutable.Option[DocEncConfig], fieldName immuta // Encrypt encrypts the given plainText with the encryption key that is associated with the given docID, // fieldName and key id. func (d *DocEncryptor) Encrypt( - encStoreKey core.EncStoreDocKey, - plainText []byte, + plainText, encryptionKey []byte, ) ([]byte, error) { - if d.store == nil { - return nil, ErrNoStorageProvided - } - - encryptionKey, err := d.fetchByEncStoreKey(encStoreKey) - if err != nil { - return nil, err - } - var cipherText []byte + var err error if len(plainText) > 0 { cipherText, _, err = crypto.EncryptAES(plainText, encryptionKey, nil, true) } @@ -131,66 +116,20 @@ func (d *DocEncryptor) Encrypt( // Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. // If the corresponding encryption key is not found, it returns nil. func (d *DocEncryptor) Decrypt( - encStoreKey core.EncStoreDocKey, - cipherText []byte, + cipherText, encKey []byte, ) ([]byte, error) { - if d.store == nil { - return nil, ErrNoStorageProvided - } - encKey, err := d.fetchByEncStoreKey(encStoreKey) - if err != nil { - return nil, err - } if len(encKey) == 0 { return nil, nil } return crypto.DecryptAES(nil, cipherText, encKey, nil) } -func (d *DocEncryptor) fetchByEncStoreKey(storeKey core.EncStoreDocKey) ([]byte, error) { - encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) - isNotFound := errors.Is(err, ds.ErrNotFound) - if err != nil { - if isNotFound { - return nil, nil - } - return nil, err - } - - return encryptionKey, nil -} - -func (d *DocEncryptor) storeByEncStoreKey(storeKey core.EncStoreDocKey, encryptionKey []byte) error { - return d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) -} - -// GetKey returns the encryption key for the given docID, (optional) fieldName and block height. -func (d *DocEncryptor) GetKey(encStoreKey core.EncStoreDocKey) ([]byte, error) { - if d.store == nil { - return nil, ErrNoStorageProvided - } - encryptionKey, err := d.fetchByEncStoreKey(encStoreKey) - if err != nil { - return nil, err - } - return encryptionKey, nil -} - // getGeneratedKeyFor returns the generated key for the given docID and fieldName. func (d *DocEncryptor) getGeneratedKeyFor( docID string, fieldName immutable.Option[string], -) (immutable.Option[core.EncStoreDocKey], []byte) { - for _, key := range d.generatedKeys { - if key.DocID == docID && key.FieldName == fieldName { - fetchByEncStoreKey, err := d.fetchByEncStoreKey(key) - if err != nil { - return immutable.None[core.EncStoreDocKey](), nil - } - return immutable.Some(key), fetchByEncStoreKey - } - } - return immutable.None[core.EncStoreDocKey](), nil +) []byte { + return d.generatedKeys[genK{docID, fieldName}] } // GetOrGenerateEncryptionKey returns the generated encryption key for the given docID, (optional) fieldName. @@ -198,10 +137,10 @@ func (d *DocEncryptor) getGeneratedKeyFor( func (d *DocEncryptor) GetOrGenerateEncryptionKey( docID string, fieldName immutable.Option[string], -) (immutable.Option[core.EncStoreDocKey], []byte, error) { - encStoreKey, encryptionKey := d.getGeneratedKeyFor(docID, fieldName) - if encStoreKey.HasValue() { - return encStoreKey, encryptionKey, nil +) ([]byte, error) { + encryptionKey := d.getGeneratedKeyFor(docID, fieldName) + if len(encryptionKey) > 0 { + return encryptionKey, nil } return d.generateEncryptionKey(docID, fieldName) @@ -211,43 +150,23 @@ func (d *DocEncryptor) GetOrGenerateEncryptionKey( func (d *DocEncryptor) generateEncryptionKey( docID string, fieldName immutable.Option[string], -) (immutable.Option[core.EncStoreDocKey], []byte, error) { - encStoreKey := core.NewEncStoreDocKey(docID, fieldName, "") +) ([]byte, error) { if !shouldEncryptIndividualField(d.conf, fieldName) { - encStoreKey.FieldName = immutable.None[string]() + fieldName = immutable.None[string]() } - if !shouldEncryptDocField(d.conf, encStoreKey.FieldName) { - return immutable.None[core.EncStoreDocKey](), nil, nil - } - - encryptionKey, err := generateEncryptionKeyFunc(encStoreKey.DocID, encStoreKey.FieldName) - if err != nil { - return immutable.None[core.EncStoreDocKey](), nil, err - } - - keyID, err := crypto.GenerateCid(encryptionKey) - if err != nil { - return immutable.None[core.EncStoreDocKey](), nil, err + if !shouldEncryptDocField(d.conf, fieldName) { + return nil, nil } - encStoreKey.KeyID = keyID.String() - err = d.storeByEncStoreKey(encStoreKey, encryptionKey) + encryptionKey, err := generateEncryptionKeyFunc(docID, fieldName) if err != nil { - return immutable.None[core.EncStoreDocKey](), nil, err + return nil, err } - d.generatedKeys = append(d.generatedKeys, encStoreKey) - - return immutable.Some(encStoreKey), encryptionKey, nil -} + d.generatedKeys[genK{docID, fieldName}] = encryptionKey -// SaveKey saves the given encryption key for the given docID, (optional) fieldName and block height. -func (d *DocEncryptor) SaveKey(encStoreKey core.EncStoreDocKey, encryptionKey []byte) error { - if d.store == nil { - return ErrNoStorageProvided - } - return d.storeByEncStoreKey(encStoreKey, encryptionKey) + return encryptionKey, nil } // ShouldEncryptDocField returns true if the given field should be encrypted based on the context config. diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 8611c42046..3c34ed819d 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -12,150 +12,192 @@ package encryption import ( "context" - "errors" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/sourcenetwork/immutable" - - "github.com/sourcenetwork/defradb/crypto" - "github.com/sourcenetwork/defradb/datastore/mocks" - "github.com/sourcenetwork/defradb/internal/core" ) -var testErr = errors.New("test error") - -const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" +func TestContext_NoEncryptor_ReturnsNil(t *testing.T) { + ctx := context.Background() + enc := GetEncryptorFromContext(ctx) + assert.Nil(t, enc) +} -var fieldName = immutable.Some("name") -var noFieldName = immutable.None[string]() +func TestContext_WithEncryptor_ReturnsEncryptor(t *testing.T) { + ctx := context.Background() + enc := newDocEncryptor(ctx) + ctx = context.WithValue(ctx, docEncContextKey{}, enc) -func getPlainText() []byte { - return []byte("test") + retrievedEnc := GetEncryptorFromContext(ctx) + assert.NotNil(t, retrievedEnc) + assert.Equal(t, enc, retrievedEnc) } -func getEncKey(fieldName immutable.Option[string]) []byte { - key, _ := generateTestEncryptionKey(docID, fieldName) - return key -} +func TestContext_EnsureEncryptor_CreatesNew(t *testing.T) { + ctx := context.Background() + newCtx, enc := EnsureContextWithEncryptor(ctx) -func getKeyID(fieldName immutable.Option[string]) string { - key := getEncKey(fieldName) - cid, _ := crypto.GenerateCid(key) - return cid.String() -} + assert.NotNil(t, enc) + assert.NotEqual(t, ctx, newCtx) -func makeStoreKey(docID string, fieldName immutable.Option[string]) core.EncStoreDocKey { - return core.NewEncStoreDocKey(docID, fieldName, getKeyID(fieldName)) + retrievedEnc := GetEncryptorFromContext(newCtx) + assert.Equal(t, enc, retrievedEnc) } -func getCipherText(t *testing.T, fieldName immutable.Option[string]) []byte { - cipherText, _, err := crypto.EncryptAES(getPlainText(), getEncKey(fieldName), nil, true) - assert.NoError(t, err) - return cipherText +func TestContext_EnsureEncryptor_ReturnsExisting(t *testing.T) { + ctx := context.Background() + enc := newDocEncryptor(ctx) + ctx = context.WithValue(ctx, docEncContextKey{}, enc) + + newCtx, retrievedEnc := EnsureContextWithEncryptor(ctx) + assert.Equal(t, ctx, newCtx) + assert.Equal(t, enc, retrievedEnc) } -func newDefaultEncryptor(t *testing.T) (*DocEncryptor, *mocks.DSReaderWriter) { - return newEncryptorWithConfig(t, DocEncConfig{IsDocEncrypted: true}) +func TestConfig_GetFromContext_NoConfig(t *testing.T) { + ctx := context.Background() + config := GetContextConfig(ctx) + assert.False(t, config.HasValue()) } -func newEncryptorWithConfig(t *testing.T, conf DocEncConfig) (*DocEncryptor, *mocks.DSReaderWriter) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.SetConfig(immutable.Some(conf)) - enc.SetStore(st) - return enc, st +func TestConfig_GetFromContext_ReturnCurrentConfig(t *testing.T) { + ctx := context.Background() + expectedConfig := DocEncConfig{IsDocEncrypted: true, EncryptedFields: []string{"field1", "field2"}} + ctx = context.WithValue(ctx, configContextKey{}, expectedConfig) + + config := GetContextConfig(ctx) + assert.True(t, config.HasValue()) + assert.Equal(t, expectedConfig, config.Value()) } -func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { - enc, st := newDefaultEncryptor(t) +func TestConfig_SetContextConfig_StoreConfig(t *testing.T) { + ctx := context.Background() + config := DocEncConfig{IsDocEncrypted: true, EncryptedFields: []string{"field1", "field2"}} + + newCtx := SetContextConfig(ctx, config) + retrievedConfig := GetContextConfig(newCtx) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) + assert.True(t, retrievedConfig.HasValue()) + assert.Equal(t, config, retrievedConfig.Value()) +} - _, err := enc.Encrypt(makeStoreKey(docID, fieldName), []byte("test")) +func TestConfig_SetFromParamsWithDocEncryption_StoreConfig(t *testing.T) { + ctx := context.Background() + newCtx := SetContextConfigFromParams(ctx, true, []string{"field1", "field2"}) - assert.ErrorIs(t, err, testErr) + config := GetContextConfig(newCtx) + assert.True(t, config.HasValue()) + assert.True(t, config.Value().IsDocEncrypted) + assert.Equal(t, []string{"field1", "field2"}, config.Value().EncryptedFields) } -func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { - enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName.Value()}}) +func TestConfig_SetFromParamsWithFields_StoreConfig(t *testing.T) { + ctx := context.Background() + newCtx := SetContextConfigFromParams(ctx, false, []string{"field1", "field2"}) - storeKey := makeStoreKey(docID, fieldName) - st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) + config := GetContextConfig(newCtx) + assert.True(t, config.HasValue()) + assert.False(t, config.Value().IsDocEncrypted) + assert.Equal(t, []string{"field1", "field2"}, config.Value().EncryptedFields) +} - cipherText, err := enc.Encrypt(makeStoreKey(docID, fieldName), getPlainText()) +func TestConfig_SetFromParamsWithNoEncryptionSetting_NoConfig(t *testing.T) { + ctx := context.Background() + newCtx := SetContextConfigFromParams(ctx, false, nil) - assert.NoError(t, err) - assert.Equal(t, getCipherText(t, fieldName), cipherText) + config := GetContextConfig(newCtx) + assert.False(t, config.HasValue()) } -func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { - enc, st := newDefaultEncryptor(t) +func TestEncryptor_EncryptDecrypt_SuccessfulRoundTrip(t *testing.T) { + ctx := context.Background() + enc := newDocEncryptor(ctx) + enc.SetConfig(immutable.Some(DocEncConfig{EncryptedFields: []string{"field1"}})) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) + plainText := []byte("Hello, World!") + docID := "doc1" + fieldName := immutable.Some("field1") - cipherText, err := enc.Encrypt(makeStoreKey(docID, noFieldName), getPlainText()) + key, err := enc.GetOrGenerateEncryptionKey(docID, fieldName) + assert.NoError(t, err) + assert.NotNil(t, key) + cipherText, err := enc.Encrypt(plainText, key) assert.NoError(t, err) - assert.Equal(t, getCipherText(t, noFieldName), cipherText) -} + assert.NotEqual(t, plainText, cipherText) -func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *testing.T) { - enc, st := newDefaultEncryptor(t) + decryptedText, err := enc.Decrypt(cipherText, key) + assert.NoError(t, err) + assert.Equal(t, plainText, decryptedText) +} - st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) +func TestEncryptor_GetOrGenerateKey_ReturnsExistingKey(t *testing.T) { + ctx := context.Background() + enc := newDocEncryptor(ctx) + enc.SetConfig(immutable.Some(DocEncConfig{EncryptedFields: []string{"field1"}})) - _, _, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) - assert.ErrorIs(t, err, testErr) -} + docID := "doc1" + fieldName := immutable.Some("field1") -func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *testing.T) { - enc, _ := newDefaultEncryptor(t) - enc.SetConfig(immutable.None[DocEncConfig]()) + key1, err := enc.GetOrGenerateEncryptionKey(docID, fieldName) + assert.NoError(t, err) + assert.NotNil(t, key1) - encStoreKey, encryptionKey, err := enc.GetOrGenerateEncryptionKey(docID, noFieldName) + key2, err := enc.GetOrGenerateEncryptionKey(docID, fieldName) assert.NoError(t, err) - assert.Len(t, encryptionKey, 0) - assert.Equal(t, encStoreKey, immutable.None[core.EncStoreDocKey]()) + assert.Equal(t, key1, key2) } -func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { - enc, _ := newDefaultEncryptor(t) - enc.SetStore(nil) - - _, err := enc.Encrypt(makeStoreKey(docID, fieldName), getPlainText()) +func TestEncryptor_GenerateKey_DifferentKeysForDifferentFields(t *testing.T) { + ctx := context.Background() + enc := newDocEncryptor(ctx) + enc.SetConfig(immutable.Some(DocEncConfig{EncryptedFields: []string{"field1", "field2"}})) - assert.ErrorIs(t, err, ErrNoStorageProvided) -} + docID := "doc1" + fieldName1 := immutable.Some("field1") + fieldName2 := immutable.Some("field2") -func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { - enc, _ := newDefaultEncryptor(t) - enc.SetStore(nil) + key1, err := enc.GetOrGenerateEncryptionKey(docID, fieldName1) + assert.NoError(t, err) + assert.NotNil(t, key1) - _, err := enc.Decrypt(makeStoreKey(docID, fieldName), getPlainText()) + key2, err := enc.GetOrGenerateEncryptionKey(docID, fieldName2) + assert.NoError(t, err) + assert.NotNil(t, key2) - assert.ErrorIs(t, err, ErrNoStorageProvided) + assert.NotEqual(t, key1, key2) } -func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { - enc, st := newDefaultEncryptor(t) +func TestShouldEncryptField_WithDocEncryption_True(t *testing.T) { + config := DocEncConfig{IsDocEncrypted: true} + ctx := SetContextConfig(context.Background(), config) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) + assert.True(t, ShouldEncryptDocField(ctx, immutable.Some("field1"))) + assert.True(t, ShouldEncryptDocField(ctx, immutable.Some("field2"))) +} - _, err := enc.Decrypt(makeStoreKey(docID, fieldName), []byte("test")) +func TestShouldEncryptField_WithFieldEncryption_TrueForMatchingField(t *testing.T) { + config := DocEncConfig{EncryptedFields: []string{"field1"}} + ctx := SetContextConfig(context.Background(), config) - assert.ErrorIs(t, err, testErr) + assert.True(t, ShouldEncryptDocField(ctx, immutable.Some("field1"))) + assert.False(t, ShouldEncryptDocField(ctx, immutable.Some("field2"))) } -func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *testing.T) { - enc, st := newDefaultEncryptor(t) +func TestShouldEncryptIndividualField_WithDocEncryption_False(t *testing.T) { + config := DocEncConfig{IsDocEncrypted: true} + ctx := SetContextConfig(context.Background(), config) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(noFieldName), nil) + assert.False(t, ShouldEncryptIndividualField(ctx, immutable.Some("field1"))) + assert.False(t, ShouldEncryptIndividualField(ctx, immutable.Some("field2"))) +} - plainText, err := enc.Decrypt(makeStoreKey(docID, fieldName), getCipherText(t, noFieldName)) +func TestShouldEncryptIndividualField_WithFieldEncryption_TrueForMatchingField(t *testing.T) { + config := DocEncConfig{EncryptedFields: []string{"field1"}} + ctx := SetContextConfig(context.Background(), config) - assert.NoError(t, err) - assert.Equal(t, getPlainText(), plainText) + assert.True(t, ShouldEncryptIndividualField(ctx, immutable.Some("field1"))) + assert.False(t, ShouldEncryptIndividualField(ctx, immutable.Some("field2"))) } diff --git a/internal/encryption/event.go b/internal/encryption/event.go index 14e05cd727..324ebe99e1 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -11,8 +11,8 @@ package encryption import ( + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/core" ) const RequestKeysEventName = event.Name("enc-keys-request") @@ -23,7 +23,7 @@ const RequestKeysEventName = event.Name("enc-keys-request") // It must only contain public elements not protected by ACP. type RequestKeysEvent struct { // Keys is a list of the keys that are being requested. - Keys []core.EncStoreDocKey + Keys []cidlink.Link Resp chan<- Result } @@ -36,8 +36,8 @@ type RequestedKeyEventData struct { // KeyRetrievedEvent represents a key that was retrieved. type Item struct { - StoreKey core.EncStoreDocKey - EncryptionKey []byte + Link []byte + Block []byte } type Result struct { @@ -62,7 +62,7 @@ func NewResults() (*Results, chan<- Result) { // NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field -func NewRequestKeysMessage(keys []core.EncStoreDocKey) (event.Message, *Results) { +func NewRequestKeysMessage(keys []cidlink.Link) (event.Message, *Results) { res, ch := NewResults() return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ Keys: keys, diff --git a/internal/kms/enc_store.go b/internal/kms/enc_store.go new file mode 100644 index 0000000000..a9a76b4b90 --- /dev/null +++ b/internal/kms/enc_store.go @@ -0,0 +1,65 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package kms + +import ( + "context" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/sourcenetwork/defradb/datastore" + coreblock "github.com/sourcenetwork/defradb/internal/core/block" +) + +type ipldEncStorage struct { + encstore datastore.Blockstore +} + +func newIPLDEncryptionStorage(encstore datastore.Blockstore) *ipldEncStorage { + return &ipldEncStorage{encstore: encstore} +} + +func (s *ipldEncStorage) get(ctx context.Context, cidBytes []byte) (*coreblock.Encryption, error) { + lsys := cidlink.DefaultLinkSystem() + lsys.SetReadStorage(s.encstore.AsIPLDStorage()) + + _, blockCid, err := cid.CidFromBytes(cidBytes) + if err != nil { + return nil, err + } + + nd, err := lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, + coreblock.EncryptionSchemaPrototype) + if err != nil { + return nil, err + } + + return coreblock.GetEncryptionBlockFromNode(nd) +} + +func (s *ipldEncStorage) put(ctx context.Context, blockBytes []byte) ([]byte, error) { + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(s.encstore.AsIPLDStorage()) + + var encBlock coreblock.Encryption + err := encBlock.Unmarshal(blockBytes) + if err != nil { + return nil, err + } + + link, err := lsys.Store(linking.LinkContext{Ctx: ctx}, coreblock.GetLinkPrototype(), encBlock.GenerateNode()) + if err != nil { + return nil, err + } + + return []byte(link.String()), nil +} diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index b13aef9a6d..18a10b4b58 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -16,18 +16,16 @@ import ( "crypto/ecdh" "encoding/base64" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" libpeer "github.com/libp2p/go-libp2p/core/peer" - "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/net" pb "github.com/sourcenetwork/defradb/net/pb" rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" - "github.com/sourcenetwork/immutable" grpcpeer "google.golang.org/grpc/peer" "google.golang.org/protobuf/proto" ) @@ -44,16 +42,16 @@ type pubSubService struct { peerID libpeer.ID pubsub PubSubServer keyRequestedSub *event.Subscription - encstore datastore.DSReaderWriter eventBus *event.Bus + encStore *ipldEncStorage } var _ Service = (*pubSubService)(nil) -func (s *pubSubService) GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) { +func (s *pubSubService) GetKeys(ctx context.Context, cids ...cidlink.Link) (*encryption.Results, error) { res, ch := encryption.NewResults() - err := s.requestEncryptionKey(ctx, keys, ch) + err := s.requestEncryptionKeyFromPeers(ctx, cids, ch) if err != nil { return nil, err } @@ -66,14 +64,14 @@ func NewPubSubService( peerID libpeer.ID, pubsub PubSubServer, eventBus *event.Bus, - encstore datastore.DSReaderWriter, + encstore datastore.Blockstore, ) (*pubSubService, error) { s := &pubSubService{ ctx: ctx, peerID: peerID, pubsub: pubsub, - encstore: encstore, eventBus: eventBus, + encStore: newIPLDEncryptionStorage(encstore), } err := pubsub.AddPubSubTopic(pubsubTopic, s.handleRequestFromPeer) if err != nil { @@ -103,20 +101,14 @@ func (s *pubSubService) handleKeyRequestedEvent() { encResult := <-results.Get() - _, encryptor := encryption.ContextWithStore(s.ctx, s.encstore) - for _, encItem := range encResult.Items { - err = encryptor.SaveKey(encItem.StoreKey, encItem.EncryptionKey) + _, err = s.encStore.put(s.ctx, encItem.Block) if err != nil { log.ErrorContextE(s.ctx, "Failed to save encryption key", err) return } } - m := make(map[core.EncStoreDocKey][]byte) - for _, item := range encResult.Items { - m[item.StoreKey] = item.EncryptionKey - } keyReqEvent.Resp <- encResult close(keyReqEvent.Resp) }() @@ -140,7 +132,7 @@ func (s *pubSubService) handleRequestFromPeer(peerID libpeer.ID, topic string, m // TODO: check if this NewGRPCPeer can be abstracted away or copied in this package. ctx := grpcpeer.NewContext(s.ctx, net.NewGRPCPeer(peerID)) - res, err := s.TryGenEncryptionKey(ctx, req) + res, err := s.tryGenEncryptionKeyLocally(ctx, req) if err != nil { log.ErrorContextE(s.ctx, "failed attempt to get encryption key", err) return nil, errors.Wrap("failed attempt to get encryption key", err) @@ -149,31 +141,25 @@ func (s *pubSubService) handleRequestFromPeer(peerID libpeer.ID, topic string, m } func (s *pubSubService) prepareFetchEncryptionKeyRequest( - encStoreKeys []core.EncStoreDocKey, + cids []cidlink.Link, ephemeralPublicKey []byte, ) (*pb.FetchEncryptionKeyRequest, error) { req := &pb.FetchEncryptionKeyRequest{ EphemeralPublicKey: ephemeralPublicKey, } - for _, encStoreKey := range encStoreKeys { - encKey := &pb.EncryptionKeyTarget{ - DocID: []byte(encStoreKey.DocID), - KeyID: []byte(encStoreKey.KeyID), - } - if encStoreKey.FieldName.HasValue() { - encKey.FieldName = encStoreKey.FieldName.Value() - } - req.Targets = append(req.Targets, encKey) + req.Links = make([][]byte, len(cids)) + for i, cid := range cids { + req.Links[i] = cid.Bytes() } return req, nil } -// requestEncryptionKey publishes the given FetchEncryptionKeyRequest object on the PubSub network -func (s *pubSubService) requestEncryptionKey( +// requestEncryptionKeyFromPeers publishes the given FetchEncryptionKeyRequest object on the PubSub network +func (s *pubSubService) requestEncryptionKeyFromPeers( ctx context.Context, - encStoreKeys []core.EncStoreDocKey, + cids []cidlink.Link, result chan<- encryption.Result, ) error { ephPrivKey, err := crypto.GenerateX25519() @@ -182,7 +168,7 @@ func (s *pubSubService) requestEncryptionKey( } ephPubKeyBytes := ephPrivKey.PublicKey().Bytes() - req, err := s.prepareFetchEncryptionKeyRequest(encStoreKeys, ephPubKeyBytes) + req, err := s.prepareFetchEncryptionKeyRequest(cids, ephPubKeyBytes) if err != nil { return err } @@ -220,37 +206,25 @@ func (s *pubSubService) handleFetchEncryptionKeyResponse( return } - decryptedData, err := crypto.DecryptECIES( - keyResp.EncryptedKeys, - privateKey, - makeAssociatedData(req, resp.From), - ) - - if err != nil { - log.ErrorContextE(s.ctx, "Failed to decrypt encryption key", err) - result <- encryption.Result{Error: err} - return - } - - if len(decryptedData) != crypto.AESKeySize*len(keyResp.Targets) { - log.ErrorContext(s.ctx, "Invalid decrypted data length") - result <- encryption.Result{Error: errors.New("invalid decrypted data length")} - return - } + resultEncItems := make([]encryption.Item, 0, len(keyResp.Blocks)) + for i, block := range keyResp.Blocks { + decryptedData, err := crypto.DecryptECIES( + block, + privateKey, + makeAssociatedData(req, resp.From), + keyResp.EphemeralPublicKey, + ) - resultEncItems := make([]encryption.Item, 0, len(keyResp.Targets)) - for _, target := range keyResp.Targets { - optFieldName := immutable.None[string]() - if target.FieldName != "" { - optFieldName = immutable.Some(target.FieldName) + if err != nil { + log.ErrorContextE(s.ctx, "Failed to decrypt encryption key", err) + result <- encryption.Result{Error: err} + return } - encKey := decryptedData[:crypto.AESKeySize] - decryptedData = decryptedData[crypto.AESKeySize:] - resultEncItems = append(resultEncItems, encryption.Item{ - StoreKey: core.NewEncStoreDocKey(string(target.DocID), optFieldName, string(target.KeyID)), - EncryptionKey: encKey, + Link: keyResp.Links[i], + //Block: decryptedData[:crypto.AESKeySize], + Block: decryptedData, }) } @@ -267,12 +241,12 @@ func makeAssociatedData(req *pb.FetchEncryptionKeyRequest, peerID libpeer.ID) [] }, []byte{})) } -func (s *pubSubService) TryGenEncryptionKey( +func (s *pubSubService) tryGenEncryptionKeyLocally( ctx context.Context, req *pb.FetchEncryptionKeyRequest, ) (*pb.FetchEncryptionKeyReply, error) { - encryptionKeys, targets, err := s.getEncryptionKeys(ctx, req) - if err != nil || len(encryptionKeys) == 0 { + blocks, err := s.getEncryptionKeysLocally(ctx, req) + if err != nil || len(blocks) == 0 { return nil, err } @@ -281,15 +255,26 @@ func (s *pubSubService) TryGenEncryptionKey( return nil, errors.Wrap("failed to unmarshal ephemeral public key", err) } - encryptedKey, err := crypto.EncryptECIES(encryptionKeys, reqEphPubKey, makeAssociatedData(req, s.peerID)) + privKey, err := crypto.GenerateX25519() if err != nil { - return nil, errors.Wrap("failed to encrypt key for requester", err) + return nil, err } res := &pb.FetchEncryptionKeyReply{ - ReqEphemeralPublicKey: req.EphemeralPublicKey, - Targets: targets, - EncryptedKeys: encryptedKey, + Links: req.Links, + EphemeralPublicKey: privKey.PublicKey().Bytes(), + } + + res.Blocks = make([][]byte, 0, len(blocks)) + + for _, block := range blocks { + encryptedBlock, err := crypto.EncryptECIES(block, reqEphPubKey, makeAssociatedData(req, s.peerID), privKey) + if err != nil { + return nil, errors.Wrap("failed to encrypt key for requester", err) + } + + res.Blocks = append(res.Blocks, encryptedBlock) + //res.Blocks = append(res.Blocks, encryptedBlock[crypto.X25519PublicKeySize:]) } return res, nil @@ -297,41 +282,30 @@ func (s *pubSubService) TryGenEncryptionKey( // getEncryptionKeys retrieves the encryption keys for the given targets. // It returns the encryption keys and the targets for which the keys were found. -func (s *pubSubService) getEncryptionKeys( +func (s *pubSubService) getEncryptionKeysLocally( ctx context.Context, req *pb.FetchEncryptionKeyRequest, -) ([]byte, []*pb.EncryptionKeyTarget, error) { - encryptionKeys := make([]byte, 0, len(req.Targets)) - targets := make([]*pb.EncryptionKeyTarget, 0, len(req.Targets)) - for _, target := range req.Targets { - docID, err := client.NewDocIDFromString(string(target.DocID)) - if err != nil { - return nil, nil, err - } - - optFieldName := immutable.None[string]() - if target.FieldName != "" { - optFieldName = immutable.Some(target.FieldName) - } - _, encryptor := encryption.ContextWithStore(ctx, s.encstore) - if encryptor == nil { - return nil, nil, encryption.ErrContextHasNoEncryptor - } - encKey, err := encryptor.GetKey( - core.NewEncStoreDocKey(docID.String(), optFieldName, string(target.KeyID)), - ) +) ([][]byte, error) { + blocks := make([][]byte, 0, len(req.Links)) + for _, link := range req.Links { + encBlock, err := s.encStore.get(ctx, link) if err != nil { - return nil, nil, err + return nil, err } // TODO: we should test it somehow. For this this one peer should have some keys and // another one should have the others. https://github.com/sourcenetwork/defradb/issues/2895 - if len(encKey) == 0 { + if encBlock == nil { continue } - targets = append(targets, target) - encryptionKeys = append(encryptionKeys, encKey...) + + encBlockBytes, err := encBlock.Marshal() + if err != nil { + return nil, err + } + + blocks = append(blocks, encBlockBytes) } - return encryptionKeys, targets, nil + return blocks, nil } func encodeToBase64(data []byte) []byte { diff --git a/internal/kms/service.go b/internal/kms/service.go index 55af0c97d3..0047c2f1dd 100644 --- a/internal/kms/service.go +++ b/internal/kms/service.go @@ -13,8 +13,8 @@ package kms import ( "context" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/sourcenetwork/corelog" - "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/encryption" ) @@ -29,5 +29,5 @@ const ( ) type Service interface { - GetKeys(ctx context.Context, keys ...core.EncStoreDocKey) (*encryption.Results, error) + GetKeys(ctx context.Context, cids ...cidlink.Link) (*encryption.Results, error) } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index eda03c0bbf..2c44a28254 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -37,6 +37,7 @@ var ( type MerkleClock struct { headstore datastore.DSReaderWriter blockstore datastore.Blockstore + encstore datastore.Blockstore headset *heads crdt core.ReplicatedData } @@ -45,12 +46,14 @@ type MerkleClock struct { func NewMerkleClock( headstore datastore.DSReaderWriter, blockstore datastore.Blockstore, + encstore datastore.Blockstore, namespace core.HeadStoreKey, crdt core.ReplicatedData, ) *MerkleClock { return &MerkleClock{ headstore: headstore, blockstore: blockstore, + encstore: encstore, headset: NewHeadSet(headstore, namespace), crdt: crdt, } @@ -60,10 +63,23 @@ func (mc *MerkleClock) putBlock( ctx context.Context, block *coreblock.Block, ) (cidlink.Link, error) { - nd := block.GenerateNode() lsys := cidlink.DefaultLinkSystem() lsys.SetWriteStorage(mc.blockstore.AsIPLDStorage()) - link, err := lsys.Store(linking.LinkContext{Ctx: ctx}, coreblock.GetLinkPrototype(), nd) + link, err := lsys.Store(linking.LinkContext{Ctx: ctx}, coreblock.GetLinkPrototype(), block.GenerateNode()) + if err != nil { + return cidlink.Link{}, NewErrWritingBlock(err) + } + + return link.(cidlink.Link), nil +} + +func (mc *MerkleClock) putEncBlock( + ctx context.Context, + encBlock *coreblock.Encryption, +) (cidlink.Link, error) { + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(mc.encstore.AsIPLDStorage()) + link, err := lsys.Store(linking.LinkContext{Ctx: ctx}, coreblock.GetLinkPrototype(), encBlock.GenerateNode()) if err != nil { return cidlink.Link{}, NewErrWritingBlock(err) } @@ -91,17 +107,18 @@ func (mc *MerkleClock) AddDelta( if block.Delta.GetFieldName() != "" { fieldName = immutable.Some(block.Delta.GetFieldName()) } - blockEnc, err := mc.determineBlockEncryptionData(ctx, string(block.Delta.GetDocID()), fieldName, heads) + encBlock, encLink, err := mc.determineBlockEncryptionData(ctx, string(block.Delta.GetDocID()), fieldName, heads) if err != nil { return cidlink.Link{}, nil, err } dagBlock := block - if blockEnc != nil && blockEnc.Type != coreblock.NotEncrypted { - dagBlock, err = encryptBlock(ctx, block, blockEnc) + if encBlock != nil { + dagBlock, err = encryptBlock(ctx, block, encBlock) if err != nil { return cidlink.Link{}, nil, err } + dagBlock.Encryption = &encLink } link, err := mc.putBlock(ctx, dagBlock) @@ -123,83 +140,85 @@ func (mc *MerkleClock) AddDelta( return link, b, err } +// TODO: rename this to reflect storage func (mc *MerkleClock) determineBlockEncryptionData( ctx context.Context, docID string, fieldName immutable.Option[string], heads []cid.Cid, -) (*coreblock.Encryption, error) { +) (*coreblock.Encryption, cidlink.Link, error) { // if new encryption was requested by the user if encryption.ShouldEncryptDocField(ctx, fieldName) { - blockEnc := &coreblock.Encryption{} + encBlock := &coreblock.Encryption{DocID: []byte(docID)} if encryption.ShouldEncryptIndividualField(ctx, fieldName) { - blockEnc.Type = coreblock.FieldEncrypted - } else { - blockEnc.Type = coreblock.DocumentEncrypted + f := fieldName.Value() + encBlock.FieldName = &f } encryptor := encryption.GetEncryptorFromContext(ctx) - if encryptor == nil { - return nil, encryption.ErrContextHasNoEncryptor - } - encStoreKey, _, err := encryptor.GetOrGenerateEncryptionKey(docID, fieldName) - if err != nil { - return nil, err - } - if encStoreKey.HasValue() { - blockEnc.KeyID = []byte(encStoreKey.Value().KeyID) + if encryptor != nil { + encKey, err := encryptor.GetOrGenerateEncryptionKey(docID, fieldName) + if err != nil { + return nil, cidlink.Link{}, err + } + if len(encKey) > 0 { + encBlock.Key = encKey + } + + link, err := mc.putEncBlock(ctx, encBlock) + if err != nil { + return nil, cidlink.Link{}, err + } + return encBlock, link, nil } - return blockEnc, nil } // otherwise we use the same encryption as the previous block for _, headCid := range heads { - bytes, err := mc.blockstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) + prevBlockBytes, err := mc.blockstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) if err != nil { - return nil, NewErrCouldNotFindBlock(headCid, err) + return nil, cidlink.Link{}, NewErrCouldNotFindBlock(headCid, err) } - prevBlock, err := coreblock.GetFromBytes(bytes) + prevBlock, err := coreblock.GetFromBytes(prevBlockBytes) if err != nil { - return nil, err + return nil, cidlink.Link{}, err } if prevBlock.Encryption != nil { + prevBlockEncBytes, err := mc.encstore.AsIPLDStorage().Get(ctx, prevBlock.Encryption.Cid.KeyString()) + if err != nil { + return nil, cidlink.Link{}, NewErrCouldNotFindBlock(headCid, err) + } + prevEncBlock, err := coreblock.GetEncryptionBlockFromBytes(prevBlockEncBytes) + if err != nil { + return nil, cidlink.Link{}, err + } return &coreblock.Encryption{ - Type: prevBlock.Encryption.Type, - KeyID: prevBlock.Encryption.KeyID, - }, nil + DocID: prevEncBlock.DocID, + FieldName: prevEncBlock.FieldName, + Key: prevEncBlock.Key, + }, *prevBlock.Encryption, nil } } - return nil, nil + return nil, cidlink.Link{}, nil } func encryptBlock( ctx context.Context, block *coreblock.Block, - blockEnc *coreblock.Encryption, + encBlock *coreblock.Encryption, ) (*coreblock.Block, error) { - fieldName := immutable.None[string]() - if blockEnc.Type == coreblock.FieldEncrypted { - fieldName = immutable.Some(block.Delta.GetFieldName()) - } - - encStoreKey := core.NewEncStoreDocKey(string(block.Delta.GetDocID()), fieldName, string(blockEnc.KeyID)) - blockEnc.KeyID = []byte(encStoreKey.KeyID) if block.Delta.IsComposite() { - block.Encryption = blockEnc return block, nil } clonedCRDT := block.Delta.Clone() - encryptor := encryption.GetEncryptorFromContext(ctx) - if encryptor == nil { - return nil, encryption.ErrContextHasNoEncryptor - } - bytes, err := encryptor.Encrypt(encStoreKey, clonedCRDT.GetData()) + _, encryptor := encryption.EnsureContextWithEncryptor(ctx) + bytes, err := encryptor.Encrypt(clonedCRDT.GetData(), encBlock.Key) if err != nil { return nil, err } clonedCRDT.SetData(bytes) - return &coreblock.Block{Delta: clonedCRDT, Links: block.Links, Encryption: blockEnc}, nil + return &coreblock.Block{Delta: clonedCRDT, Links: block.Links}, nil } // ProcessBlock merges the delta CRDT and updates the state accordingly. diff --git a/internal/merkle/clock/clock_test.go b/internal/merkle/clock/clock_test.go index dbff19bc09..fe008971e4 100644 --- a/internal/merkle/clock/clock_test.go +++ b/internal/merkle/clock/clock_test.go @@ -37,6 +37,7 @@ func newTestMerkleClock() *MerkleClock { return NewMerkleClock( multistore.Headstore(), multistore.Blockstore(), + multistore.Encstore(), core.HeadStoreKey{DocID: request.DocIDArgName, FieldID: "1"}, reg, ) @@ -46,7 +47,7 @@ func TestNewMerkleClock(t *testing.T) { s := newDS() multistore := datastore.MultiStoreFrom(s) reg := crdt.NewLWWRegister(multistore.Rootstore(), core.CollectionSchemaVersionKey{}, core.DataStoreKey{}, "") - clk := NewMerkleClock(multistore.Headstore(), multistore.Blockstore(), core.HeadStoreKey{}, reg) + clk := NewMerkleClock(multistore.Headstore(), multistore.Blockstore(), multistore.Encstore(), core.HeadStoreKey{}, reg) if clk.headstore != multistore.Headstore() { t.Error("MerkleClock store not correctly set") diff --git a/internal/merkle/crdt/composite.go b/internal/merkle/crdt/composite.go index 26ab4134e5..f8211b9f0a 100644 --- a/internal/merkle/crdt/composite.go +++ b/internal/merkle/crdt/composite.go @@ -44,7 +44,8 @@ func NewMerkleCompositeDAG( fieldName, ) - clock := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), key.ToHeadStoreKey(), compositeDag) + clock := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), store.Encstore(), + key.ToHeadStoreKey(), compositeDag) base := &baseMerkleCRDT{clock: clock, crdt: compositeDag} return &MerkleCompositeDAG{ diff --git a/internal/merkle/crdt/counter.go b/internal/merkle/crdt/counter.go index 1ff6874b08..21b26785b6 100644 --- a/internal/merkle/crdt/counter.go +++ b/internal/merkle/crdt/counter.go @@ -39,7 +39,7 @@ func NewMerkleCounter( kind client.ScalarKind, ) *MerkleCounter { register := crdt.NewCounter(store.Datastore(), schemaVersionKey, key, fieldName, allowDecrement, kind) - clk := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), key.ToHeadStoreKey(), register) + clk := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), store.Encstore(), key.ToHeadStoreKey(), register) base := &baseMerkleCRDT{clock: clk, crdt: register} return &MerkleCounter{ baseMerkleCRDT: base, diff --git a/internal/merkle/crdt/lwwreg.go b/internal/merkle/crdt/lwwreg.go index 11e73089bf..00c70dc4a9 100644 --- a/internal/merkle/crdt/lwwreg.go +++ b/internal/merkle/crdt/lwwreg.go @@ -37,7 +37,7 @@ func NewMerkleLWWRegister( fieldName string, ) *MerkleLWWRegister { register := corecrdt.NewLWWRegister(store.Datastore(), schemaVersionKey, key, fieldName) - clk := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), key.ToHeadStoreKey(), register) + clk := clock.NewMerkleClock(store.Headstore(), store.Blockstore(), store.Encstore(), key.ToHeadStoreKey(), register) base := &baseMerkleCRDT{clock: clk, crdt: register} return &MerkleLWWRegister{ baseMerkleCRDT: base, diff --git a/internal/merkle/crdt/merklecrdt.go b/internal/merkle/crdt/merklecrdt.go index 27f92f21ca..457ba0f200 100644 --- a/internal/merkle/crdt/merklecrdt.go +++ b/internal/merkle/crdt/merklecrdt.go @@ -27,6 +27,7 @@ import ( type Stores interface { Datastore() datastore.DSReaderWriter Blockstore() datastore.Blockstore + Encstore() datastore.Blockstore Headstore() datastore.DSReaderWriter } diff --git a/internal/merkle/crdt/merklecrdt_test.go b/internal/merkle/crdt/merklecrdt_test.go index 29482b28bf..74f4814ca3 100644 --- a/internal/merkle/crdt/merklecrdt_test.go +++ b/internal/merkle/crdt/merklecrdt_test.go @@ -32,7 +32,7 @@ func newTestBaseMerkleCRDT() (*baseMerkleCRDT, datastore.DSReaderWriter) { multistore := datastore.MultiStoreFrom(s) reg := crdt.NewLWWRegister(multistore.Datastore(), core.CollectionSchemaVersionKey{}, core.DataStoreKey{}, "") - clk := clock.NewMerkleClock(multistore.Headstore(), multistore.Blockstore(), core.HeadStoreKey{}, reg) + clk := clock.NewMerkleClock(multistore.Headstore(), multistore.Blockstore(), multistore.Encstore(), core.HeadStoreKey{}, reg) return &baseMerkleCRDT{clock: clk, crdt: reg}, multistore.Rootstore() } diff --git a/net/pb/net.pb.go b/net/pb/net.pb.go index ed2ce4375b..dbac6829d0 100644 --- a/net/pb/net.pb.go +++ b/net/pb/net.pb.go @@ -344,74 +344,6 @@ func (x *PushLogRequest) GetBody() *PushLogRequest_Body { return nil } -// EncryptionKeyTarget is a struct containing information about required doc encryption key. -type EncryptionKeyTarget struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // docID is the ID of the document that the key is being requested for. - DocID []byte `protobuf:"bytes,1,opt,name=docID,proto3" json:"docID,omitempty"` - // fieldName if the name of the document field that the key is being requested for. - // If the fieldName is empty, the key is being requested for the whole document. - FieldName string `protobuf:"bytes,2,opt,name=fieldName,proto3" json:"fieldName,omitempty"` - // keyID is the hash (Cid) of the key that is being requested. - KeyID []byte `protobuf:"bytes,3,opt,name=keyID,proto3" json:"keyID,omitempty"` -} - -func (x *EncryptionKeyTarget) Reset() { - *x = EncryptionKeyTarget{} - if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EncryptionKeyTarget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EncryptionKeyTarget) ProtoMessage() {} - -func (x *EncryptionKeyTarget) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EncryptionKeyTarget.ProtoReflect.Descriptor instead. -func (*EncryptionKeyTarget) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{8} -} - -func (x *EncryptionKeyTarget) GetDocID() []byte { - if x != nil { - return x.DocID - } - return nil -} - -func (x *EncryptionKeyTarget) GetFieldName() string { - if x != nil { - return x.FieldName - } - return "" -} - -func (x *EncryptionKeyTarget) GetKeyID() []byte { - if x != nil { - return x.KeyID - } - return nil -} - // FetchEncryptionKeyRequest is a request to receive a doc encryption key // from a peer that holds it. type FetchEncryptionKeyRequest struct { @@ -419,18 +351,16 @@ type FetchEncryptionKeyRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // targets is the list of docs/fields for which encryption keys are being requested. - Targets []*EncryptionKeyTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` + // links is the list of cid links of the blocks containing encryption keys. + Links [][]byte `protobuf:"bytes,1,rep,name=links,proto3" json:"links,omitempty"` // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret EphemeralPublicKey []byte `protobuf:"bytes,2,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` - // signature is the requesting peer's signature of the request - Signature []byte `protobuf:"bytes,3,opt,name=signature,proto3" json:"signature,omitempty"` } func (x *FetchEncryptionKeyRequest) Reset() { *x = FetchEncryptionKeyRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -443,7 +373,7 @@ func (x *FetchEncryptionKeyRequest) String() string { func (*FetchEncryptionKeyRequest) ProtoMessage() {} func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[9] + mi := &file_net_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -456,12 +386,12 @@ func (x *FetchEncryptionKeyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyRequest.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{9} + return file_net_proto_rawDescGZIP(), []int{8} } -func (x *FetchEncryptionKeyRequest) GetTargets() []*EncryptionKeyTarget { +func (x *FetchEncryptionKeyRequest) GetLinks() [][]byte { if x != nil { - return x.Targets + return x.Links } return nil } @@ -473,13 +403,6 @@ func (x *FetchEncryptionKeyRequest) GetEphemeralPublicKey() []byte { return nil } -func (x *FetchEncryptionKeyRequest) GetSignature() []byte { - if x != nil { - return x.Signature - } - return nil -} - // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request // by a peer that holds the requested doc encryption key. type FetchEncryptionKeyReply struct { @@ -487,22 +410,19 @@ type FetchEncryptionKeyReply struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // targets is the list of docs/fields for which encryption keys are being requested. - Targets []*EncryptionKeyTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` - // encryptedKeys is an encrypted list of doc encryption keys. - // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM - // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. - EncryptedKeys []byte `protobuf:"bytes,2,opt,name=encryptedKeys,proto3" json:"encryptedKeys,omitempty"` - // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id - ReqEphemeralPublicKey []byte `protobuf:"bytes,3,opt,name=reqEphemeralPublicKey,proto3" json:"reqEphemeralPublicKey,omitempty"` - // signature is the responding peer's signature of the reply - Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3" json:"signature,omitempty"` + // links is the list of cid links of the blocks containing encryption keys. + Links [][]byte `protobuf:"bytes,1,rep,name=links,proto3" json:"links,omitempty"` + // blocks is the list of blocks containing encryption keys. The order of blocks should match the order of links. + // Every block is encrypted and contains a nonce. + Blocks [][]byte `protobuf:"bytes,2,rep,name=blocks,proto3" json:"blocks,omitempty"` + // ephemeralPublicKey is an ephemeral public of the responding peer for deriving shared secret + EphemeralPublicKey []byte `protobuf:"bytes,3,opt,name=ephemeralPublicKey,proto3" json:"ephemeralPublicKey,omitempty"` } func (x *FetchEncryptionKeyReply) Reset() { *x = FetchEncryptionKeyReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -515,7 +435,7 @@ func (x *FetchEncryptionKeyReply) String() string { func (*FetchEncryptionKeyReply) ProtoMessage() {} func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[10] + mi := &file_net_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -528,33 +448,26 @@ func (x *FetchEncryptionKeyReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchEncryptionKeyReply.ProtoReflect.Descriptor instead. func (*FetchEncryptionKeyReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{10} -} - -func (x *FetchEncryptionKeyReply) GetTargets() []*EncryptionKeyTarget { - if x != nil { - return x.Targets - } - return nil + return file_net_proto_rawDescGZIP(), []int{9} } -func (x *FetchEncryptionKeyReply) GetEncryptedKeys() []byte { +func (x *FetchEncryptionKeyReply) GetLinks() [][]byte { if x != nil { - return x.EncryptedKeys + return x.Links } return nil } -func (x *FetchEncryptionKeyReply) GetReqEphemeralPublicKey() []byte { +func (x *FetchEncryptionKeyReply) GetBlocks() [][]byte { if x != nil { - return x.ReqEphemeralPublicKey + return x.Blocks } return nil } -func (x *FetchEncryptionKeyReply) GetSignature() []byte { +func (x *FetchEncryptionKeyReply) GetEphemeralPublicKey() []byte { if x != nil { - return x.Signature + return x.EphemeralPublicKey } return nil } @@ -568,7 +481,7 @@ type GetHeadLogRequest struct { func (x *GetHeadLogRequest) Reset() { *x = GetHeadLogRequest{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -581,7 +494,7 @@ func (x *GetHeadLogRequest) String() string { func (*GetHeadLogRequest) ProtoMessage() {} func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[11] + mi := &file_net_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -594,7 +507,7 @@ func (x *GetHeadLogRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogRequest.ProtoReflect.Descriptor instead. func (*GetHeadLogRequest) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{11} + return file_net_proto_rawDescGZIP(), []int{10} } type PushLogReply struct { @@ -606,7 +519,7 @@ type PushLogReply struct { func (x *PushLogReply) Reset() { *x = PushLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -619,7 +532,7 @@ func (x *PushLogReply) String() string { func (*PushLogReply) ProtoMessage() {} func (x *PushLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[12] + mi := &file_net_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -632,7 +545,7 @@ func (x *PushLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogReply.ProtoReflect.Descriptor instead. func (*PushLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{12} + return file_net_proto_rawDescGZIP(), []int{11} } type GetHeadLogReply struct { @@ -644,7 +557,7 @@ type GetHeadLogReply struct { func (x *GetHeadLogReply) Reset() { *x = GetHeadLogReply{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -657,7 +570,7 @@ func (x *GetHeadLogReply) String() string { func (*GetHeadLogReply) ProtoMessage() {} func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[13] + mi := &file_net_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -670,7 +583,7 @@ func (x *GetHeadLogReply) ProtoReflect() protoreflect.Message { // Deprecated: Use GetHeadLogReply.ProtoReflect.Descriptor instead. func (*GetHeadLogReply) Descriptor() ([]byte, []int) { - return file_net_proto_rawDescGZIP(), []int{13} + return file_net_proto_rawDescGZIP(), []int{12} } type PushLogRequest_Body struct { @@ -693,7 +606,7 @@ type PushLogRequest_Body struct { func (x *PushLogRequest_Body) Reset() { *x = PushLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_net_proto_msgTypes[14] + mi := &file_net_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -706,7 +619,7 @@ func (x *PushLogRequest_Body) String() string { func (*PushLogRequest_Body) ProtoMessage() {} func (x *PushLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_net_proto_msgTypes[14] + mi := &file_net_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -783,68 +696,46 @@ var file_net_proto_rawDesc = []byte{ 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x52, - 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x5f, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, - 0x6f, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x64, 0x6f, 0x63, 0x49, - 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x6b, 0x65, 0x79, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x6b, 0x65, 0x79, 0x49, 0x44, 0x22, 0xa0, 0x01, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, 0x6e, - 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, + 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x61, 0x0a, 0x19, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x22, 0x77, 0x0a, 0x17, 0x46, 0x65, 0x74, 0x63, 0x68, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, + 0x12, 0x2e, 0x0a, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, - 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0xca, 0x01, 0x0a, 0x17, 0x46, 0x65, 0x74, - 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x12, 0x35, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x45, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x65, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4b, 0x65, 0x79, - 0x73, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, - 0x6c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x15, 0x72, 0x65, 0x71, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x75, - 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, - 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xae, 0x03, - 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, - 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x48, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x12, 0x1b, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, - 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, - 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x12, 0x39, 0x0a, 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, - 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, - 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5b, 0x0a, - 0x13, 0x54, 0x72, 0x79, 0x47, 0x65, 0x6e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x65, - 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, - 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x4b, 0x65, 0x79, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, - 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, - 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, - 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, - 0x5a, 0x08, 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x32, 0xd1, 0x02, 0x0a, 0x07, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x12, 0x1a, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x63, 0x47, + 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0c, 0x50, + 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x1b, 0x2e, 0x6e, 0x65, + 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x44, 0x6f, 0x63, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x12, + 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, + 0x07, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, + 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x14, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x48, + 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x2e, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, + 0x61, 0x64, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, + 0x2f, 0x3b, 0x6e, 0x65, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -859,7 +750,7 @@ func file_net_proto_rawDescGZIP() []byte { return file_net_proto_rawDescData } -var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_net_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_net_proto_goTypes = []any{ (*Log)(nil), // 0: net.pb.Log (*GetDocGraphRequest)(nil), // 1: net.pb.GetDocGraphRequest @@ -869,36 +760,31 @@ var file_net_proto_goTypes = []any{ (*GetLogRequest)(nil), // 5: net.pb.GetLogRequest (*GetLogReply)(nil), // 6: net.pb.GetLogReply (*PushLogRequest)(nil), // 7: net.pb.PushLogRequest - (*EncryptionKeyTarget)(nil), // 8: net.pb.EncryptionKeyTarget - (*FetchEncryptionKeyRequest)(nil), // 9: net.pb.FetchEncryptionKeyRequest - (*FetchEncryptionKeyReply)(nil), // 10: net.pb.FetchEncryptionKeyReply - (*GetHeadLogRequest)(nil), // 11: net.pb.GetHeadLogRequest - (*PushLogReply)(nil), // 12: net.pb.PushLogReply - (*GetHeadLogReply)(nil), // 13: net.pb.GetHeadLogReply - (*PushLogRequest_Body)(nil), // 14: net.pb.PushLogRequest.Body + (*FetchEncryptionKeyRequest)(nil), // 8: net.pb.FetchEncryptionKeyRequest + (*FetchEncryptionKeyReply)(nil), // 9: net.pb.FetchEncryptionKeyReply + (*GetHeadLogRequest)(nil), // 10: net.pb.GetHeadLogRequest + (*PushLogReply)(nil), // 11: net.pb.PushLogReply + (*GetHeadLogReply)(nil), // 12: net.pb.GetHeadLogReply + (*PushLogRequest_Body)(nil), // 13: net.pb.PushLogRequest.Body } var file_net_proto_depIdxs = []int32{ - 14, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body - 8, // 1: net.pb.FetchEncryptionKeyRequest.targets:type_name -> net.pb.EncryptionKeyTarget - 8, // 2: net.pb.FetchEncryptionKeyReply.targets:type_name -> net.pb.EncryptionKeyTarget - 0, // 3: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log - 1, // 4: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest - 3, // 5: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest - 5, // 6: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest - 7, // 7: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest - 9, // 8: net.pb.Service.TryGenEncryptionKey:input_type -> net.pb.FetchEncryptionKeyRequest - 11, // 9: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest - 2, // 10: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply - 4, // 11: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply - 6, // 12: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply - 12, // 13: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply - 10, // 14: net.pb.Service.TryGenEncryptionKey:output_type -> net.pb.FetchEncryptionKeyReply - 13, // 15: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply - 10, // [10:16] is the sub-list for method output_type - 4, // [4:10] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 13, // 0: net.pb.PushLogRequest.body:type_name -> net.pb.PushLogRequest.Body + 0, // 1: net.pb.PushLogRequest.Body.log:type_name -> net.pb.Log + 1, // 2: net.pb.Service.GetDocGraph:input_type -> net.pb.GetDocGraphRequest + 3, // 3: net.pb.Service.PushDocGraph:input_type -> net.pb.PushDocGraphRequest + 5, // 4: net.pb.Service.GetLog:input_type -> net.pb.GetLogRequest + 7, // 5: net.pb.Service.PushLog:input_type -> net.pb.PushLogRequest + 10, // 6: net.pb.Service.GetHeadLog:input_type -> net.pb.GetHeadLogRequest + 2, // 7: net.pb.Service.GetDocGraph:output_type -> net.pb.GetDocGraphReply + 4, // 8: net.pb.Service.PushDocGraph:output_type -> net.pb.PushDocGraphReply + 6, // 9: net.pb.Service.GetLog:output_type -> net.pb.GetLogReply + 11, // 10: net.pb.Service.PushLog:output_type -> net.pb.PushLogReply + 12, // 11: net.pb.Service.GetHeadLog:output_type -> net.pb.GetHeadLogReply + 7, // [7:12] is the sub-list for method output_type + 2, // [2:7] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_net_proto_init() } @@ -1004,18 +890,6 @@ func file_net_proto_init() { } } file_net_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*EncryptionKeyTarget); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*FetchEncryptionKeyRequest); i { case 0: return &v.state @@ -1027,7 +901,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*FetchEncryptionKeyReply); i { case 0: return &v.state @@ -1039,7 +913,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogRequest); i { case 0: return &v.state @@ -1051,7 +925,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*PushLogReply); i { case 0: return &v.state @@ -1063,7 +937,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*GetHeadLogReply); i { case 0: return &v.state @@ -1075,7 +949,7 @@ func file_net_proto_init() { return nil } } - file_net_proto_msgTypes[14].Exporter = func(v any, i int) any { + file_net_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*PushLogRequest_Body); i { case 0: return &v.state @@ -1094,7 +968,7 @@ func file_net_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_net_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/net/pb/net.proto b/net/pb/net.proto index 82a0799f03..8dc8fe8a46 100644 --- a/net/pb/net.proto +++ b/net/pb/net.proto @@ -38,41 +38,25 @@ message PushLogRequest { } } -// EncryptionKeyTarget is a struct containing information about required doc encryption key. -message EncryptionKeyTarget { - // docID is the ID of the document that the key is being requested for. - bytes docID = 1; - // fieldName if the name of the document field that the key is being requested for. - // If the fieldName is empty, the key is being requested for the whole document. - string fieldName = 2; - // keyID is the hash (Cid) of the key that is being requested. - bytes keyID = 3; -} - // FetchEncryptionKeyRequest is a request to receive a doc encryption key // from a peer that holds it. message FetchEncryptionKeyRequest { - // targets is the list of docs/fields for which encryption keys are being requested. - repeated EncryptionKeyTarget targets = 1; + // links is the list of cid links of the blocks containing encryption keys. + repeated bytes links = 1; // ephemeralPublicKey is an ephemeral public of the requesting peer for deriving shared secret bytes ephemeralPublicKey = 2; - // signature is the requesting peer's signature of the request - bytes signature = 3; } // FetchEncryptionKeyReply is a response to FetchEncryptionKeyRequest request // by a peer that holds the requested doc encryption key. message FetchEncryptionKeyReply { - // targets is the list of docs/fields for which encryption keys are being requested. - repeated EncryptionKeyTarget targets = 1; - // encryptedKeys is an encrypted list of doc encryption keys. - // It is prepended with the responder's ephemeral public key and a nonce for AES-GCM - // Each encryption key in the decrypted list corresponds to the key requested in the same index in the targets list. - bytes encryptedKeys = 2; - // reqEphemeralPublicKey is an ephemeral public of the requesting peer to be used as session id - bytes reqEphemeralPublicKey = 3; - // signature is the responding peer's signature of the reply - bytes signature = 4; + // links is the list of cid links of the blocks containing encryption keys. + repeated bytes links = 1; + // blocks is the list of blocks containing encryption keys. The order of blocks should match the order of links. + // Every block is encrypted and contains a nonce. + repeated bytes blocks = 2; + // ephemeralPublicKey is an ephemeral public of the responding peer for deriving shared secret + bytes ephemeralPublicKey = 3; } message GetHeadLogRequest {} @@ -91,8 +75,6 @@ service Service { rpc GetLog(GetLogRequest) returns (GetLogReply) {} // PushLog to this peer. rpc PushLog(PushLogRequest) returns (PushLogReply) {} - // TryGenEncryptionKey from this peer. - rpc TryGenEncryptionKey(FetchEncryptionKeyRequest) returns (FetchEncryptionKeyReply) {} // GetHeadLog from this peer rpc GetHeadLog(GetHeadLogRequest) returns (GetHeadLogReply) {} } diff --git a/net/pb/net_grpc.pb.go b/net/pb/net_grpc.pb.go index 8c0e2eab05..84564d6bec 100644 --- a/net/pb/net_grpc.pb.go +++ b/net/pb/net_grpc.pb.go @@ -19,12 +19,11 @@ import ( const _ = grpc.SupportPackageIsVersion8 const ( - Service_GetDocGraph_FullMethodName = "/net.pb.Service/GetDocGraph" - Service_PushDocGraph_FullMethodName = "/net.pb.Service/PushDocGraph" - Service_GetLog_FullMethodName = "/net.pb.Service/GetLog" - Service_PushLog_FullMethodName = "/net.pb.Service/PushLog" - Service_TryGenEncryptionKey_FullMethodName = "/net.pb.Service/TryGenEncryptionKey" - Service_GetHeadLog_FullMethodName = "/net.pb.Service/GetHeadLog" + Service_GetDocGraph_FullMethodName = "/net.pb.Service/GetDocGraph" + Service_PushDocGraph_FullMethodName = "/net.pb.Service/PushDocGraph" + Service_GetLog_FullMethodName = "/net.pb.Service/GetLog" + Service_PushLog_FullMethodName = "/net.pb.Service/PushLog" + Service_GetHeadLog_FullMethodName = "/net.pb.Service/GetHeadLog" ) // ServiceClient is the client API for Service service. @@ -41,8 +40,6 @@ type ServiceClient interface { GetLog(ctx context.Context, in *GetLogRequest, opts ...grpc.CallOption) (*GetLogReply, error) // PushLog to this peer. PushLog(ctx context.Context, in *PushLogRequest, opts ...grpc.CallOption) (*PushLogReply, error) - // TryGenEncryptionKey from this peer. - TryGenEncryptionKey(ctx context.Context, in *FetchEncryptionKeyRequest, opts ...grpc.CallOption) (*FetchEncryptionKeyReply, error) // GetHeadLog from this peer GetHeadLog(ctx context.Context, in *GetHeadLogRequest, opts ...grpc.CallOption) (*GetHeadLogReply, error) } @@ -95,16 +92,6 @@ func (c *serviceClient) PushLog(ctx context.Context, in *PushLogRequest, opts .. return out, nil } -func (c *serviceClient) TryGenEncryptionKey(ctx context.Context, in *FetchEncryptionKeyRequest, opts ...grpc.CallOption) (*FetchEncryptionKeyReply, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(FetchEncryptionKeyReply) - err := c.cc.Invoke(ctx, Service_TryGenEncryptionKey_FullMethodName, in, out, cOpts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *serviceClient) GetHeadLog(ctx context.Context, in *GetHeadLogRequest, opts ...grpc.CallOption) (*GetHeadLogReply, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetHeadLogReply) @@ -129,8 +116,6 @@ type ServiceServer interface { GetLog(context.Context, *GetLogRequest) (*GetLogReply, error) // PushLog to this peer. PushLog(context.Context, *PushLogRequest) (*PushLogReply, error) - // TryGenEncryptionKey from this peer. - TryGenEncryptionKey(context.Context, *FetchEncryptionKeyRequest) (*FetchEncryptionKeyReply, error) // GetHeadLog from this peer GetHeadLog(context.Context, *GetHeadLogRequest) (*GetHeadLogReply, error) mustEmbedUnimplementedServiceServer() @@ -152,9 +137,6 @@ func (UnimplementedServiceServer) GetLog(context.Context, *GetLogRequest) (*GetL func (UnimplementedServiceServer) PushLog(context.Context, *PushLogRequest) (*PushLogReply, error) { return nil, status.Errorf(codes.Unimplemented, "method PushLog not implemented") } -func (UnimplementedServiceServer) TryGenEncryptionKey(context.Context, *FetchEncryptionKeyRequest) (*FetchEncryptionKeyReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method TryGenEncryptionKey not implemented") -} func (UnimplementedServiceServer) GetHeadLog(context.Context, *GetHeadLogRequest) (*GetHeadLogReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetHeadLog not implemented") } @@ -243,24 +225,6 @@ func _Service_PushLog_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } -func _Service_TryGenEncryptionKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(FetchEncryptionKeyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ServiceServer).TryGenEncryptionKey(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Service_TryGenEncryptionKey_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServiceServer).TryGenEncryptionKey(ctx, req.(*FetchEncryptionKeyRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Service_GetHeadLog_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetHeadLogRequest) if err := dec(in); err != nil { @@ -302,10 +266,6 @@ var Service_ServiceDesc = grpc.ServiceDesc{ MethodName: "PushLog", Handler: _Service_PushLog_Handler, }, - { - MethodName: "TryGenEncryptionKey", - Handler: _Service_TryGenEncryptionKey_Handler, - }, { MethodName: "GetHeadLog", Handler: _Service_GetHeadLog_Handler, diff --git a/net/pb/net_vtproto.pb.go b/net/pb/net_vtproto.pb.go index 00b77f4f19..bf1c93e8e8 100644 --- a/net/pb/net_vtproto.pb.go +++ b/net/pb/net_vtproto.pb.go @@ -370,60 +370,6 @@ func (m *PushLogRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *EncryptionKeyTarget) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil - } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *EncryptionKeyTarget) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *EncryptionKeyTarget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) - } - if len(m.KeyID) > 0 { - i -= len(m.KeyID) - copy(dAtA[i:], m.KeyID) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.KeyID))) - i-- - dAtA[i] = 0x1a - } - if len(m.FieldName) > 0 { - i -= len(m.FieldName) - copy(dAtA[i:], m.FieldName) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FieldName))) - i-- - dAtA[i] = 0x12 - } - if len(m.DocID) > 0 { - i -= len(m.DocID) - copy(dAtA[i:], m.DocID) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DocID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *FetchEncryptionKeyRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -454,13 +400,6 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if len(m.Signature) > 0 { - i -= len(m.Signature) - copy(dAtA[i:], m.Signature) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) - i-- - dAtA[i] = 0x1a - } if len(m.EphemeralPublicKey) > 0 { i -= len(m.EphemeralPublicKey) copy(dAtA[i:], m.EphemeralPublicKey) @@ -468,14 +407,11 @@ func (m *FetchEncryptionKeyRequest) MarshalToSizedBufferVT(dAtA []byte) (int, er i-- dAtA[i] = 0x12 } - if len(m.Targets) > 0 { - for iNdEx := len(m.Targets) - 1; iNdEx >= 0; iNdEx-- { - size, err := m.Targets[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + if len(m.Links) > 0 { + for iNdEx := len(m.Links) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Links[iNdEx]) + copy(dAtA[i:], m.Links[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Links[iNdEx]))) i-- dAtA[i] = 0xa } @@ -513,35 +449,27 @@ func (m *FetchEncryptionKeyReply) MarshalToSizedBufferVT(dAtA []byte) (int, erro i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if len(m.Signature) > 0 { - i -= len(m.Signature) - copy(dAtA[i:], m.Signature) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) - i-- - dAtA[i] = 0x22 - } - if len(m.ReqEphemeralPublicKey) > 0 { - i -= len(m.ReqEphemeralPublicKey) - copy(dAtA[i:], m.ReqEphemeralPublicKey) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReqEphemeralPublicKey))) + if len(m.EphemeralPublicKey) > 0 { + i -= len(m.EphemeralPublicKey) + copy(dAtA[i:], m.EphemeralPublicKey) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EphemeralPublicKey))) i-- dAtA[i] = 0x1a } - if len(m.EncryptedKeys) > 0 { - i -= len(m.EncryptedKeys) - copy(dAtA[i:], m.EncryptedKeys) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.EncryptedKeys))) - i-- - dAtA[i] = 0x12 + if len(m.Blocks) > 0 { + for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Blocks[iNdEx]) + copy(dAtA[i:], m.Blocks[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Blocks[iNdEx]))) + i-- + dAtA[i] = 0x12 + } } - if len(m.Targets) > 0 { - for iNdEx := len(m.Targets) - 1; iNdEx >= 0; iNdEx-- { - size, err := m.Targets[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + if len(m.Links) > 0 { + for iNdEx := len(m.Links) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Links[iNdEx]) + copy(dAtA[i:], m.Links[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Links[iNdEx]))) i-- dAtA[i] = 0xa } @@ -766,37 +694,15 @@ func (m *PushLogRequest) SizeVT() (n int) { return n } -func (m *EncryptionKeyTarget) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.DocID) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - l = len(m.FieldName) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - l = len(m.KeyID) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - n += len(m.unknownFields) - return n -} - func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - if len(m.Targets) > 0 { - for _, e := range m.Targets { - l = e.SizeVT() + if len(m.Links) > 0 { + for _, b := range m.Links { + l = len(b) n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } @@ -804,10 +710,6 @@ func (m *FetchEncryptionKeyRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - l = len(m.Signature) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } n += len(m.unknownFields) return n } @@ -818,21 +720,19 @@ func (m *FetchEncryptionKeyReply) SizeVT() (n int) { } var l int _ = l - if len(m.Targets) > 0 { - for _, e := range m.Targets { - l = e.SizeVT() + if len(m.Links) > 0 { + for _, b := range m.Links { + l = len(b) n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } - l = len(m.EncryptedKeys) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - l = len(m.ReqEphemeralPublicKey) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + if len(m.Blocks) > 0 { + for _, b := range m.Blocks { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } } - l = len(m.Signature) + l = len(m.EphemeralPublicKey) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } @@ -1569,7 +1469,7 @@ func (m *PushLogRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { +func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1592,81 +1492,15 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: EncryptionKeyTarget: wiretype end group for non-group") + return fmt.Errorf("proto: FetchEncryptionKeyRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: EncryptionKeyTarget: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FetchEncryptionKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DocID", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.DocID = append(m.DocID[:0], dAtA[iNdEx:postIndex]...) - if m.DocID == nil { - m.DocID = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FieldName", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.FieldName = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field KeyID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Links", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1693,95 +1527,8 @@ func (m *EncryptionKeyTarget) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.KeyID = append(m.KeyID[:0], dAtA[iNdEx:postIndex]...) - if m.KeyID == nil { - m.KeyID = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: FetchEncryptionKeyRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: FetchEncryptionKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Targets", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Targets = append(m.Targets, &EncryptionKeyTarget{}) - if err := m.Targets[len(m.Targets)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Links = append(m.Links, make([]byte, postIndex-iNdEx)) + copy(m.Links[len(m.Links)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -1817,40 +1564,6 @@ func (m *FetchEncryptionKeyRequest) UnmarshalVT(dAtA []byte) error { m.EphemeralPublicKey = []byte{} } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) - if m.Signature == nil { - m.Signature = []byte{} - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -1904,41 +1617,7 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Targets", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Targets = append(m.Targets, &EncryptionKeyTarget{}) - if err := m.Targets[len(m.Targets)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EncryptedKeys", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Links", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1965,14 +1644,12 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.EncryptedKeys = append(m.EncryptedKeys[:0], dAtA[iNdEx:postIndex]...) - if m.EncryptedKeys == nil { - m.EncryptedKeys = []byte{} - } + m.Links = append(m.Links, make([]byte, postIndex-iNdEx)) + copy(m.Links[len(m.Links)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ReqEphemeralPublicKey", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1999,14 +1676,12 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ReqEphemeralPublicKey = append(m.ReqEphemeralPublicKey[:0], dAtA[iNdEx:postIndex]...) - if m.ReqEphemeralPublicKey == nil { - m.ReqEphemeralPublicKey = []byte{} - } + m.Blocks = append(m.Blocks, make([]byte, postIndex-iNdEx)) + copy(m.Blocks[len(m.Blocks)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field EphemeralPublicKey", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -2033,9 +1708,9 @@ func (m *FetchEncryptionKeyReply) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) - if m.Signature == nil { - m.Signature = []byte{} + m.EphemeralPublicKey = append(m.EphemeralPublicKey[:0], dAtA[iNdEx:postIndex]...) + if m.EphemeralPublicKey == nil { + m.EphemeralPublicKey = []byte{} } iNdEx = postIndex default: diff --git a/net/peer.go b/net/peer.go index 0d1559a80b..9a9e915414 100644 --- a/net/peer.go +++ b/net/peer.go @@ -47,7 +47,7 @@ import ( // to the underlying DefraDB instance. type Peer struct { blockstore datastore.Blockstore - encstore datastore.DSReaderWriter + encstore datastore.Blockstore bus *event.Bus updateSub *event.Subscription @@ -72,7 +72,7 @@ type Peer struct { func NewPeer( ctx context.Context, blockstore datastore.Blockstore, - encstore datastore.DSReaderWriter, + encstore datastore.Blockstore, bus *event.Bus, opts ...NodeOpt, ) (p *Peer, err error) { diff --git a/tests/bench/query/planner/utils.go b/tests/bench/query/planner/utils.go index b2a6e3c0d6..cbfb29a62f 100644 --- a/tests/bench/query/planner/utils.go +++ b/tests/bench/query/planner/utils.go @@ -134,7 +134,7 @@ type dummyTxn struct{} func (*dummyTxn) Rootstore() datastore.DSReaderWriter { return nil } func (*dummyTxn) Datastore() datastore.DSReaderWriter { return nil } -func (*dummyTxn) Encstore() datastore.DSReaderWriter { return nil } +func (*dummyTxn) Encstore() datastore.Blockstore { return nil } func (*dummyTxn) Headstore() datastore.DSReaderWriter { return nil } func (*dummyTxn) Peerstore() datastore.DSBatching { return nil } func (*dummyTxn) Blockstore() datastore.Blockstore { return nil } diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index eac9f12e1b..f306e557ca 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -510,7 +510,7 @@ func (w *Wrapper) Rootstore() datastore.Rootstore { return w.node.DB.Rootstore() } -func (w *Wrapper) Encstore() datastore.DSReaderWriter { +func (w *Wrapper) Encstore() datastore.Blockstore { return w.node.DB.Encstore() } diff --git a/tests/clients/cli/wrapper_tx.go b/tests/clients/cli/wrapper_tx.go index 46aefd000d..e3bf41d818 100644 --- a/tests/clients/cli/wrapper_tx.go +++ b/tests/clients/cli/wrapper_tx.go @@ -75,7 +75,7 @@ func (w *Transaction) Datastore() datastore.DSReaderWriter { return w.tx.Datastore() } -func (w *Transaction) Encstore() datastore.DSReaderWriter { +func (w *Transaction) Encstore() datastore.Blockstore { return w.tx.Encstore() } diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 3f2c7be0ec..e46f7edf5f 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -203,7 +203,7 @@ func (w *Wrapper) Rootstore() datastore.Rootstore { return w.node.DB.Rootstore() } -func (w *Wrapper) Encstore() datastore.DSReaderWriter { +func (w *Wrapper) Encstore() datastore.Blockstore { return w.node.DB.Encstore() } diff --git a/tests/clients/http/wrapper_tx.go b/tests/clients/http/wrapper_tx.go index e4b838a2e9..baf841871a 100644 --- a/tests/clients/http/wrapper_tx.go +++ b/tests/clients/http/wrapper_tx.go @@ -69,7 +69,7 @@ func (w *TxWrapper) Datastore() datastore.DSReaderWriter { return w.server.Datastore() } -func (w *TxWrapper) Encstore() datastore.DSReaderWriter { +func (w *TxWrapper) Encstore() datastore.Blockstore { return w.server.Encstore() } diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 4d1efbc2fa..da493e097f 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -48,7 +48,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", + "cid": "bafyreidkuvcdxxkyoeapnmttu6l2vk43qnm3zuzpxegbifpj6w24jrvrxq", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -58,7 +58,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", + "cid": "bafyreihdlv4fvvptetghxzyerxt4jc4zgprecybhoijrfjuyxqe55qw3x4", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -68,7 +68,7 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "links": []map[string]any{}, }, { - "cid": "bafyreiey25ljav736mpbs6ghkmvwstxil4lt4jrte33p6jnixg4nvzz264", + "cid": "bafyreie5jegw4c2hg56bbiv6cgxmfz336jruukjakbjuyapockfnn6b5le", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -77,11 +77,11 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", + "cid": "bafyreidkuvcdxxkyoeapnmttu6l2vk43qnm3zuzpxegbifpj6w24jrvrxq", "name": "age", }, { - "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", + "cid": "bafyreihdlv4fvvptetghxzyerxt4jc4zgprecybhoijrfjuyxqe55qw3x4", "name": "name", }, }, diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 05c78540ac..9f5b875586 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -61,7 +61,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { Results: map[string]any{ "commits": []map[string]any{ { - "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", + "cid": "bafyreidkuvcdxxkyoeapnmttu6l2vk43qnm3zuzpxegbifpj6w24jrvrxq", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": john21DocID, @@ -71,7 +71,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", + "cid": "bafyreihdlv4fvvptetghxzyerxt4jc4zgprecybhoijrfjuyxqe55qw3x4", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": john21DocID, @@ -81,7 +81,7 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreiey25ljav736mpbs6ghkmvwstxil4lt4jrte33p6jnixg4nvzz264", + "cid": "bafyreie5jegw4c2hg56bbiv6cgxmfz336jruukjakbjuyapockfnn6b5le", "collectionID": int64(1), "delta": nil, "docID": john21DocID, @@ -90,11 +90,11 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreibsfegxzo5isgcmwfhw4jpj4eo3atmykyrnnh3a52afxookdrrylu", + "cid": "bafyreidkuvcdxxkyoeapnmttu6l2vk43qnm3zuzpxegbifpj6w24jrvrxq", "name": "age", }, { - "cid": "bafyreifly6elh3267k6kbnbvsalu7suou7dbgkqm3dbpt5w7hnwqkdhnli", + "cid": "bafyreihdlv4fvvptetghxzyerxt4jc4zgprecybhoijrfjuyxqe55qw3x4", "name": "name", }, }, From f84a4d4708584705797acfb5363f2d08a259a162 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 13:30:46 +0200 Subject: [PATCH 72/88] Add mocks for encstore --- datastore/mocks/blockstore.go | 493 +++++++++++++++++++++++++++++++ datastore/mocks/utils.go | 11 + internal/db/indexed_docs_test.go | 2 +- tools/configs/mockery.yaml | 1 + 4 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 datastore/mocks/blockstore.go diff --git a/datastore/mocks/blockstore.go b/datastore/mocks/blockstore.go new file mode 100644 index 0000000000..6dab79de7c --- /dev/null +++ b/datastore/mocks/blockstore.go @@ -0,0 +1,493 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + + context "context" + + datastore "github.com/sourcenetwork/defradb/datastore" + + mock "github.com/stretchr/testify/mock" +) + +// Blockstore is an autogenerated mock type for the Blockstore type +type Blockstore struct { + mock.Mock +} + +type Blockstore_Expecter struct { + mock *mock.Mock +} + +func (_m *Blockstore) EXPECT() *Blockstore_Expecter { + return &Blockstore_Expecter{mock: &_m.Mock} +} + +// AllKeysChan provides a mock function with given fields: ctx +func (_m *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for AllKeysChan") + } + + var r0 <-chan cid.Cid + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (<-chan cid.Cid, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) <-chan cid.Cid); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan cid.Cid) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Blockstore_AllKeysChan_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllKeysChan' +type Blockstore_AllKeysChan_Call struct { + *mock.Call +} + +// AllKeysChan is a helper method to define mock.On call +// - ctx context.Context +func (_e *Blockstore_Expecter) AllKeysChan(ctx interface{}) *Blockstore_AllKeysChan_Call { + return &Blockstore_AllKeysChan_Call{Call: _e.mock.On("AllKeysChan", ctx)} +} + +func (_c *Blockstore_AllKeysChan_Call) Run(run func(ctx context.Context)) *Blockstore_AllKeysChan_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Blockstore_AllKeysChan_Call) Return(_a0 <-chan cid.Cid, _a1 error) *Blockstore_AllKeysChan_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Blockstore_AllKeysChan_Call) RunAndReturn(run func(context.Context) (<-chan cid.Cid, error)) *Blockstore_AllKeysChan_Call { + _c.Call.Return(run) + return _c +} + +// AsIPLDStorage provides a mock function with given fields: +func (_m *Blockstore) AsIPLDStorage() datastore.IPLDStorage { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for AsIPLDStorage") + } + + var r0 datastore.IPLDStorage + if rf, ok := ret.Get(0).(func() datastore.IPLDStorage); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.IPLDStorage) + } + } + + return r0 +} + +// Blockstore_AsIPLDStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AsIPLDStorage' +type Blockstore_AsIPLDStorage_Call struct { + *mock.Call +} + +// AsIPLDStorage is a helper method to define mock.On call +func (_e *Blockstore_Expecter) AsIPLDStorage() *Blockstore_AsIPLDStorage_Call { + return &Blockstore_AsIPLDStorage_Call{Call: _e.mock.On("AsIPLDStorage")} +} + +func (_c *Blockstore_AsIPLDStorage_Call) Run(run func()) *Blockstore_AsIPLDStorage_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Blockstore_AsIPLDStorage_Call) Return(_a0 datastore.IPLDStorage) *Blockstore_AsIPLDStorage_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Blockstore_AsIPLDStorage_Call) RunAndReturn(run func() datastore.IPLDStorage) *Blockstore_AsIPLDStorage_Call { + _c.Call.Return(run) + return _c +} + +// DeleteBlock provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) DeleteBlock(_a0 context.Context, _a1 cid.Cid) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for DeleteBlock") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Blockstore_DeleteBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteBlock' +type Blockstore_DeleteBlock_Call struct { + *mock.Call +} + +// DeleteBlock is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *Blockstore_Expecter) DeleteBlock(_a0 interface{}, _a1 interface{}) *Blockstore_DeleteBlock_Call { + return &Blockstore_DeleteBlock_Call{Call: _e.mock.On("DeleteBlock", _a0, _a1)} +} + +func (_c *Blockstore_DeleteBlock_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *Blockstore_DeleteBlock_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *Blockstore_DeleteBlock_Call) Return(_a0 error) *Blockstore_DeleteBlock_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Blockstore_DeleteBlock_Call) RunAndReturn(run func(context.Context, cid.Cid) error) *Blockstore_DeleteBlock_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) Get(_a0 context.Context, _a1 cid.Cid) (blocks.Block, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 blocks.Block + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (blocks.Block, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) blocks.Block); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(blocks.Block) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Blockstore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type Blockstore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *Blockstore_Expecter) Get(_a0 interface{}, _a1 interface{}) *Blockstore_Get_Call { + return &Blockstore_Get_Call{Call: _e.mock.On("Get", _a0, _a1)} +} + +func (_c *Blockstore_Get_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *Blockstore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *Blockstore_Get_Call) Return(_a0 blocks.Block, _a1 error) *Blockstore_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Blockstore_Get_Call) RunAndReturn(run func(context.Context, cid.Cid) (blocks.Block, error)) *Blockstore_Get_Call { + _c.Call.Return(run) + return _c +} + +// GetSize provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) GetSize(_a0 context.Context, _a1 cid.Cid) (int, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for GetSize") + } + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (int, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) int); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Blockstore_GetSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSize' +type Blockstore_GetSize_Call struct { + *mock.Call +} + +// GetSize is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *Blockstore_Expecter) GetSize(_a0 interface{}, _a1 interface{}) *Blockstore_GetSize_Call { + return &Blockstore_GetSize_Call{Call: _e.mock.On("GetSize", _a0, _a1)} +} + +func (_c *Blockstore_GetSize_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *Blockstore_GetSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *Blockstore_GetSize_Call) Return(_a0 int, _a1 error) *Blockstore_GetSize_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Blockstore_GetSize_Call) RunAndReturn(run func(context.Context, cid.Cid) (int, error)) *Blockstore_GetSize_Call { + _c.Call.Return(run) + return _c +} + +// Has provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) Has(_a0 context.Context, _a1 cid.Cid) (bool, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Has") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) (bool, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, cid.Cid) bool); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, cid.Cid) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Blockstore_Has_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Has' +type Blockstore_Has_Call struct { + *mock.Call +} + +// Has is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 cid.Cid +func (_e *Blockstore_Expecter) Has(_a0 interface{}, _a1 interface{}) *Blockstore_Has_Call { + return &Blockstore_Has_Call{Call: _e.mock.On("Has", _a0, _a1)} +} + +func (_c *Blockstore_Has_Call) Run(run func(_a0 context.Context, _a1 cid.Cid)) *Blockstore_Has_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(cid.Cid)) + }) + return _c +} + +func (_c *Blockstore_Has_Call) Return(_a0 bool, _a1 error) *Blockstore_Has_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Blockstore_Has_Call) RunAndReturn(run func(context.Context, cid.Cid) (bool, error)) *Blockstore_Has_Call { + _c.Call.Return(run) + return _c +} + +// HashOnRead provides a mock function with given fields: enabled +func (_m *Blockstore) HashOnRead(enabled bool) { + _m.Called(enabled) +} + +// Blockstore_HashOnRead_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HashOnRead' +type Blockstore_HashOnRead_Call struct { + *mock.Call +} + +// HashOnRead is a helper method to define mock.On call +// - enabled bool +func (_e *Blockstore_Expecter) HashOnRead(enabled interface{}) *Blockstore_HashOnRead_Call { + return &Blockstore_HashOnRead_Call{Call: _e.mock.On("HashOnRead", enabled)} +} + +func (_c *Blockstore_HashOnRead_Call) Run(run func(enabled bool)) *Blockstore_HashOnRead_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(bool)) + }) + return _c +} + +func (_c *Blockstore_HashOnRead_Call) Return() *Blockstore_HashOnRead_Call { + _c.Call.Return() + return _c +} + +func (_c *Blockstore_HashOnRead_Call) RunAndReturn(run func(bool)) *Blockstore_HashOnRead_Call { + _c.Call.Return(run) + return _c +} + +// Put provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) Put(_a0 context.Context, _a1 blocks.Block) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Put") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, blocks.Block) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Blockstore_Put_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Put' +type Blockstore_Put_Call struct { + *mock.Call +} + +// Put is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 blocks.Block +func (_e *Blockstore_Expecter) Put(_a0 interface{}, _a1 interface{}) *Blockstore_Put_Call { + return &Blockstore_Put_Call{Call: _e.mock.On("Put", _a0, _a1)} +} + +func (_c *Blockstore_Put_Call) Run(run func(_a0 context.Context, _a1 blocks.Block)) *Blockstore_Put_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(blocks.Block)) + }) + return _c +} + +func (_c *Blockstore_Put_Call) Return(_a0 error) *Blockstore_Put_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Blockstore_Put_Call) RunAndReturn(run func(context.Context, blocks.Block) error) *Blockstore_Put_Call { + _c.Call.Return(run) + return _c +} + +// PutMany provides a mock function with given fields: _a0, _a1 +func (_m *Blockstore) PutMany(_a0 context.Context, _a1 []blocks.Block) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for PutMany") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []blocks.Block) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Blockstore_PutMany_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutMany' +type Blockstore_PutMany_Call struct { + *mock.Call +} + +// PutMany is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 []blocks.Block +func (_e *Blockstore_Expecter) PutMany(_a0 interface{}, _a1 interface{}) *Blockstore_PutMany_Call { + return &Blockstore_PutMany_Call{Call: _e.mock.On("PutMany", _a0, _a1)} +} + +func (_c *Blockstore_PutMany_Call) Run(run func(_a0 context.Context, _a1 []blocks.Block)) *Blockstore_PutMany_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]blocks.Block)) + }) + return _c +} + +func (_c *Blockstore_PutMany_Call) Return(_a0 error) *Blockstore_PutMany_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Blockstore_PutMany_Call) RunAndReturn(run func(context.Context, []blocks.Block) error) *Blockstore_PutMany_Call { + _c.Call.Return(run) + return _c +} + +// NewBlockstore creates a new instance of Blockstore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewBlockstore(t interface { + mock.TestingT + Cleanup(func()) +}) *Blockstore { + mock := &Blockstore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/datastore/mocks/utils.go b/datastore/mocks/utils.go index af3c49fd0c..d6c69684be 100644 --- a/datastore/mocks/utils.go +++ b/datastore/mocks/utils.go @@ -24,6 +24,7 @@ type MultiStoreTxn struct { MockRootstore *DSReaderWriter MockDatastore *DSReaderWriter MockHeadstore *DSReaderWriter + MockEncstore *Blockstore MockDAGstore *DAGStore MockSystemstore *DSReaderWriter } @@ -36,6 +37,14 @@ func prepareDataStore(t *testing.T) *DSReaderWriter { return dataStore } +func prepareEncStore(t *testing.T) *Blockstore { + encStore := NewBlockstore(t) + encStore.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound).Maybe() + encStore.EXPECT().Put(mock.Anything, mock.Anything).Return(nil).Maybe() + encStore.EXPECT().Has(mock.Anything, mock.Anything).Return(true, nil).Maybe() + return encStore +} + func prepareRootstore(t *testing.T) *DSReaderWriter { return NewDSReaderWriter(t) } @@ -75,6 +84,7 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { t: t, MockRootstore: prepareRootstore(t), MockDatastore: prepareDataStore(t), + MockEncstore: prepareEncStore(t), MockHeadstore: prepareHeadStore(t), MockDAGstore: prepareDAGStore(t), MockSystemstore: prepareSystemStore(t), @@ -82,6 +92,7 @@ func NewTxnWithMultistore(t *testing.T) *MultiStoreTxn { txn.EXPECT().Rootstore().Return(result.MockRootstore).Maybe() txn.EXPECT().Datastore().Return(result.MockDatastore).Maybe() + txn.EXPECT().Encstore().Return(result.MockEncstore).Maybe() txn.EXPECT().Headstore().Return(result.MockHeadstore).Maybe() txn.EXPECT().Blockstore().Return(result.MockDAGstore).Maybe() txn.EXPECT().Systemstore().Return(result.MockSystemstore).Maybe() diff --git a/internal/db/indexed_docs_test.go b/internal/db/indexed_docs_test.go index 4c0a43e08e..9f4ea3fe72 100644 --- a/internal/db/indexed_docs_test.go +++ b/internal/db/indexed_docs_test.go @@ -309,7 +309,7 @@ func TestNonUnique_IfDocWithDescendingOrderIsAdded_ShouldBeIndexed(t *testing.T) assert.Len(t, data, 0) } -func TestNonUnique_IfFailsToStoredIndexedDoc_Error(t *testing.T) { +func TestNonUnique_IfFailsToStoreIndexedDoc_Error(t *testing.T) { f := newIndexTestFixture(t) defer f.db.Close() f.createUserCollectionIndexOnName() diff --git a/tools/configs/mockery.yaml b/tools/configs/mockery.yaml index 451ae55771..504dbd1be1 100644 --- a/tools/configs/mockery.yaml +++ b/tools/configs/mockery.yaml @@ -32,6 +32,7 @@ packages: # Packages and their interfaces to generate mocks for. DSReaderWriter: RootStore: Txn: + Blockstore: github.com/sourcenetwork/defradb/client: config: From a11d199b60d6bca74f617768d2318d12d1e9459e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 13:31:58 +0200 Subject: [PATCH 73/88] Remove unused files --- crypto/cid.go | 26 -------------------------- crypto/cid_test.go | 46 ---------------------------------------------- 2 files changed, 72 deletions(-) delete mode 100644 crypto/cid.go delete mode 100644 crypto/cid_test.go diff --git a/crypto/cid.go b/crypto/cid.go deleted file mode 100644 index a7cc4a952d..0000000000 --- a/crypto/cid.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2024 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package crypto - -import ( - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" -) - -// GenerateCid generates a CID from the given data. -func GenerateCid(data []byte) (cid.Cid, error) { - mh, err := multihash.Sum(data, multihash.SHA2_256, -1) - if err != nil { - return cid.Cid{}, err - } - - return cid.NewCidV1(cid.Raw, mh), nil -} diff --git a/crypto/cid_test.go b/crypto/cid_test.go deleted file mode 100644 index 11be7f04b8..0000000000 --- a/crypto/cid_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2024 Democratized Data Foundation -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -package crypto - -import ( - "testing" - - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" -) - -func TestGenerateCid_HappyPath(t *testing.T) { - testData := []byte("Hello, world!") - - generatedCid, err := GenerateCid(testData) - - require.NoError(t, err) - require.NotEmpty(t, generatedCid) -} - -func TestGenerateCid_ErrorPath(t *testing.T) { - // Define a custom GenerateCid function that uses a failing hash function - generateCidWithError := func(data []byte) (cid.Cid, error) { - _, err := multihash.Sum(data, 0xffff, -1) // Use an invalid hash function code - if err != nil { - return cid.Cid{}, err - } - return cid.Cid{}, nil // This line should never be reached - } - - testData := []byte("This data doesn't matter as we're forcing an error") - generatedCid, err := generateCidWithError(testData) - - require.Error(t, err) - require.Contains(t, err.Error(), "unknown multihash code") - require.Equal(t, cid.Cid{}, generatedCid) // Should be an empty CID -} From d2fec1fe3c9f7d91b97192021c3baea9825be8e7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 13:34:50 +0200 Subject: [PATCH 74/88] Fix lint --- crypto/ecies.go | 16 +++++++++++++--- internal/db/merge.go | 6 +++++- internal/encryption/event.go | 1 + internal/kms/enc_store.go | 1 + internal/kms/pubsub.go | 7 ++++--- internal/kms/service.go | 1 + net/errors.go | 2 +- 7 files changed, 26 insertions(+), 8 deletions(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index a2d9fd787f..059a31c818 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -55,8 +55,13 @@ func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { // Returns: // - Byte slice containing the encrypted message and necessary metadata for decryption // - Error if any step of the encryption process fails -func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData []byte, ourPrivateKey *ecdh.PrivateKey) ([]byte, error) { - // TODO: apply option patter +func EncryptECIES( + plainText []byte, + publicKey *ecdh.PublicKey, + associatedData []byte, + ourPrivateKey *ecdh.PrivateKey, +) ([]byte, error) { + // TODO: apply option patter if ourPrivateKey == nil { var err error ourPrivateKey, err = GenerateX25519() @@ -115,7 +120,12 @@ func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, associatedData [] // Returns: // - Byte slice containing the decrypted plaintext // - Error if any step of the decryption process fails, including authentication failure -func DecryptECIES(cipherText []byte, ourPrivateKey *ecdh.PrivateKey, associatedData, publicKeyBytes []byte) ([]byte, error) { +func DecryptECIES( + cipherText []byte, + ourPrivateKey *ecdh.PrivateKey, + associatedData, + publicKeyBytes []byte, +) ([]byte, error) { if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { return nil, ErrCipherTextTooShort } diff --git a/internal/db/merge.go b/internal/db/merge.go index 02523279c4..5ab54011a7 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -379,7 +379,11 @@ func (mp *mergeProcessor) processBlock( return nil } -func decryptBlock(ctx context.Context, block *coreblock.Block, encBlock *coreblock.Encryption) (*coreblock.Block, error) { +func decryptBlock( + ctx context.Context, + block *coreblock.Block, + encBlock *coreblock.Encryption, +) (*coreblock.Block, error) { _, encryptor := encryption.EnsureContextWithEncryptor(ctx) if block.Delta.IsComposite() { diff --git a/internal/encryption/event.go b/internal/encryption/event.go index 324ebe99e1..a7762b5716 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -12,6 +12,7 @@ package encryption import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/sourcenetwork/defradb/event" ) diff --git a/internal/kms/enc_store.go b/internal/kms/enc_store.go index a9a76b4b90..bd60592f26 100644 --- a/internal/kms/enc_store.go +++ b/internal/kms/enc_store.go @@ -16,6 +16,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/sourcenetwork/defradb/datastore" coreblock "github.com/sourcenetwork/defradb/internal/core/block" ) diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index 18a10b4b58..d0854db0aa 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -18,6 +18,10 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" libpeer "github.com/libp2p/go-libp2p/core/peer" + rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" + grpcpeer "google.golang.org/grpc/peer" + "google.golang.org/protobuf/proto" + "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/errors" @@ -25,9 +29,6 @@ import ( "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/net" pb "github.com/sourcenetwork/defradb/net/pb" - rpc "github.com/sourcenetwork/go-libp2p-pubsub-rpc" - grpcpeer "google.golang.org/grpc/peer" - "google.golang.org/protobuf/proto" ) const pubsubTopic = "encryption" diff --git a/internal/kms/service.go b/internal/kms/service.go index 0047c2f1dd..14589cd0ea 100644 --- a/internal/kms/service.go +++ b/internal/kms/service.go @@ -15,6 +15,7 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/sourcenetwork/corelog" + "github.com/sourcenetwork/defradb/internal/encryption" ) diff --git a/net/errors.go b/net/errors.go index 1a8c65a48f..e0e7c6ccaf 100644 --- a/net/errors.go +++ b/net/errors.go @@ -63,4 +63,4 @@ func NewErrTopicAlreadyExist(topic string) error { func NewErrTopicDoesNotExist(topic string) error { return errors.New(fmt.Sprintf(errTopicDoesNotExist, topic)) -} \ No newline at end of file +} From a9e286f90aa8ded17636b3f3e0eb423fee41385b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 17:27:46 +0200 Subject: [PATCH 75/88] Add options to ECIES --- crypto/ecies.go | 143 ++++++++++++++++++++++++++------ crypto/ecies_test.go | 179 ++++++++++++++++++++++++++++++++--------- crypto/errors.go | 4 +- internal/kms/pubsub.go | 13 ++- 4 files changed, 271 insertions(+), 68 deletions(-) diff --git a/crypto/ecies.go b/crypto/ecies.go index 059a31c818..f025e87823 100644 --- a/crypto/ecies.go +++ b/crypto/ecies.go @@ -35,34 +35,99 @@ func X25519PublicKeyFromBytes(publicKeyBytes []byte) (*ecdh.PublicKey, error) { return ecdh.X25519().NewPublicKey(publicKeyBytes) } +type ECIESOption func(*eciesOptions) + +type eciesOptions struct { + associatedData []byte + privateKey *ecdh.PrivateKey + publicKeyBytes []byte + noPubKeyPrepended bool +} + +// WithAAD sets the associated data to use for authentication. +func WithAAD(aad []byte) ECIESOption { + return func(o *eciesOptions) { + o.associatedData = aad + } +} + +// WithPrivKey sets the private key to use for encryption. +// +// If not set, a new ephemeral key will be generated. +// This option has no effect on decryption. +func WithPrivKey(privKey *ecdh.PrivateKey) ECIESOption { + return func(o *eciesOptions) { + o.privateKey = privKey + } +} + +// WithPubKeyBytes sets the public key bytes to use for decryption. +// +// If not set, the cipherText is assumed to have the public key X25519 prepended. +// This option has no effect on encryption. +func WithPubKeyBytes(pubKeyBytes []byte) ECIESOption { + return func(o *eciesOptions) { + o.publicKeyBytes = pubKeyBytes + } +} + +// WithPubKeyPrepended sets whether the public key should is prepended to the cipherText. +// +// Upon encryption, if set to true (default value), the public key is prepended to the cipherText. +// Otherwise it's not and in this case a private key should be provided with the WithPrivKey option. +// +// Upon decryption, if set to true (default value), the public key is expected to be prepended to the cipherText. +// Otherwise it's not and in this case the public key bytes should be provided with the WithPubKeyBytes option. +func WithPubKeyPrepended(prepended bool) ECIESOption { + return func(o *eciesOptions) { + o.noPubKeyPrepended = !prepended + } +} + // EncryptECIES encrypts plaintext using a custom Elliptic Curve Integrated Encryption Scheme (ECIES) // with X25519 for key agreement, HKDF for key derivation, AES for encryption, and HMAC for authentication. // // The function: -// - Generates an ephemeral X25519 key pair +// - Uses or generates an ephemeral X25519 key pair // - Performs ECDH with the provided public key // - Derives encryption and HMAC keys using HKDF // - Encrypts the plaintext using a custom AES encryption function // - Computes an HMAC over the ciphertext // -// The output format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// The default output format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// This can be modified using options. // // Parameters: // - plainText: The message to encrypt // - publicKey: The recipient's X25519 public key -// - associatedData: Optional associated data for additional authentication +// - opts: Optional ECIESOption functions to customize the encryption process +// +// Available options: +// - WithAAD(aad []byte): Sets the associated data for additional authentication +// - WithPrivKey(privKey *ecdh.PrivateKey): Uses the provided private key instead of generating a new one +// - WithPubKeyPrepended(prepended bool): Controls whether the public key is prepended to the ciphertext // // Returns: // - Byte slice containing the encrypted message and necessary metadata for decryption // - Error if any step of the encryption process fails -func EncryptECIES( - plainText []byte, - publicKey *ecdh.PublicKey, - associatedData []byte, - ourPrivateKey *ecdh.PrivateKey, -) ([]byte, error) { - // TODO: apply option patter +// +// Example usage: +// +// cipherText, err := EncryptECIES(plainText, recipientPublicKey, +// WithAAD(additionalData), +// WithPrivKey(senderPrivateKey), +// WithPubKeyPrepended(false)) +func EncryptECIES(plainText []byte, publicKey *ecdh.PublicKey, opts ...ECIESOption) ([]byte, error) { + options := &eciesOptions{} + for _, opt := range opts { + opt(options) + } + + ourPrivateKey := options.privateKey if ourPrivateKey == nil { + if options.noPubKeyPrepended { + return nil, ErrNoPublicKeyForDecryption + } var err error ourPrivateKey, err = GenerateX25519() if err != nil { @@ -86,7 +151,7 @@ func EncryptECIES( return nil, NewErrFailedKDFOperationForHMACKey(err) } - cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ourPublicKey.Bytes(), associatedData), true) + cipherText, _, err := EncryptAES(plainText, aesKey, makeAAD(ourPublicKey.Bytes(), options.associatedData), true) if err != nil { return nil, NewErrFailedToEncrypt(err) } @@ -95,7 +160,12 @@ func EncryptECIES( mac.Write(cipherText) macSum := mac.Sum(nil) - result := append(ourPublicKey.Bytes(), cipherText...) + var result []byte + if options.noPubKeyPrepended { + result = cipherText + } else { + result = append(ourPublicKey.Bytes(), cipherText...) + } result = append(result, macSum...) return result, nil @@ -104,34 +174,57 @@ func EncryptECIES( // DecryptECIES decrypts ciphertext encrypted with EncryptECIES using the provided private key. // // The function: -// - Extracts the ephemeral public key from the ciphertext +// - Extracts or uses the provided ephemeral public key // - Performs ECDH with the provided private key // - Derives decryption and HMAC keys using HKDF // - Verifies the HMAC // - Decrypts the message using a custom AES decryption function // -// The expected input format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// The default expected input format is: [ephemeral public key | encrypted data (including nonce) | HMAC] +// This can be modified using options. // // Parameters: // - cipherText: The encrypted message, including all necessary metadata // - privateKey: The recipient's X25519 private key -// - associatedData: Optional associated data used during encryption for additional authentication +// - opts: Optional ECIESOption functions to customize the decryption process +// +// Available options: +// - WithAAD(aad []byte): Sets the associated data used during encryption for additional authentication +// - WithPubKeyBytes(pubKeyBytes []byte): Provides the public key bytes if not prepended to the ciphertext +// - WithPubKeyPrepended(prepended bool): Indicates whether the public key is prepended to the ciphertext // // Returns: // - Byte slice containing the decrypted plaintext // - Error if any step of the decryption process fails, including authentication failure -func DecryptECIES( - cipherText []byte, - ourPrivateKey *ecdh.PrivateKey, - associatedData, - publicKeyBytes []byte, -) ([]byte, error) { - if len(cipherText) < X25519PublicKeySize+AESNonceSize+HMACSize+minCipherTextSize { +// +// Example usage: +// +// plainText, err := DecryptECIES(cipherText, recipientPrivateKey, +// WithAAD(additionalData), +// WithPubKeyBytes(senderPublicKeyBytes), +// WithPubKeyPrepended(false)) +func DecryptECIES(cipherText []byte, ourPrivateKey *ecdh.PrivateKey, opts ...ECIESOption) ([]byte, error) { + options := &eciesOptions{} + for _, opt := range opts { + opt(options) + } + + minLength := X25519PublicKeySize + AESNonceSize + HMACSize + minCipherTextSize + if options.noPubKeyPrepended { + minLength -= X25519PublicKeySize + } + + if len(cipherText) < minLength { return nil, ErrCipherTextTooShort } - if publicKeyBytes == nil { + publicKeyBytes := options.publicKeyBytes + if options.publicKeyBytes == nil { + if options.noPubKeyPrepended { + return nil, ErrNoPublicKeyForDecryption + } publicKeyBytes = cipherText[:X25519PublicKeySize] + cipherText = cipherText[X25519PublicKeySize:] } publicKey, err := ecdh.X25519().NewPublicKey(publicKeyBytes) if err != nil { @@ -154,7 +247,7 @@ func DecryptECIES( } macSum := cipherText[len(cipherText)-HMACSize:] - cipherTextWithNonce := cipherText[X25519PublicKeySize : len(cipherText)-HMACSize] + cipherTextWithNonce := cipherText[:len(cipherText)-HMACSize] mac := hmac.New(sha256.New, hmacKey) mac.Write(cipherTextWithNonce) @@ -163,7 +256,7 @@ func DecryptECIES( return nil, ErrVerificationWithHMACFailed } - plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(publicKeyBytes, associatedData)) + plainText, err := DecryptAES(nil, cipherTextWithNonce, aesKey, makeAAD(publicKeyBytes, options.associatedData)) if err != nil { return nil, NewErrFailedToDecrypt(err) } diff --git a/crypto/ecies_test.go b/crypto/ecies_test.go index 8dfa639d66..f4ed463c26 100644 --- a/crypto/ecies_test.go +++ b/crypto/ecies_test.go @@ -14,30 +14,41 @@ import ( "crypto/ecdh" "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEncryptECIES_Errors(t *testing.T) { validAssociatedData := []byte("associated data") + validPrivateKey, _ := GenerateX25519() tests := []struct { - name string - plainText []byte - publicKey *ecdh.PublicKey - associatedData []byte - expectError string + name string + plainText []byte + publicKey *ecdh.PublicKey + opts []ECIESOption + expectError string }{ { - name: "Invalid public key", - plainText: []byte("test data"), - publicKey: &ecdh.PublicKey{}, - associatedData: validAssociatedData, - expectError: "failed ECDH operation", + name: "Invalid public key", + plainText: []byte("test data"), + publicKey: &ecdh.PublicKey{}, + opts: []ECIESOption{WithAAD(validAssociatedData)}, + expectError: errFailedECDHOperation, + }, + { + name: "No public key prepended and no private key provided", + plainText: []byte("test data"), + publicKey: validPrivateKey.PublicKey(), + opts: []ECIESOption{WithPubKeyPrepended(false)}, + expectError: errNoPublicKeyForDecryption, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := EncryptECIES(tt.plainText, tt.publicKey, tt.associatedData, nil) + _, err := EncryptECIES(tt.plainText, tt.publicKey, tt.opts...) if err == nil { t.Errorf("Expected an error, but got nil") } else if !strings.Contains(err.Error(), tt.expectError) { @@ -48,53 +59,143 @@ func TestEncryptECIES_Errors(t *testing.T) { } func TestDecryptECIES_Errors(t *testing.T) { - // Setup validPrivateKey, _ := GenerateX25519() - validCipherText, _ := EncryptECIES([]byte("test data"), validPrivateKey.PublicKey(), []byte("associated data"), nil) + aad := []byte("associated data") + validCipherText, _ := EncryptECIES([]byte("test data test data"), validPrivateKey.PublicKey(), WithAAD(aad)) tests := []struct { - name string - cipherText []byte - privateKey *ecdh.PrivateKey - associatedData []byte - expectError string + name string + cipherText []byte + privateKey *ecdh.PrivateKey + opts []ECIESOption + expectError string }{ { - name: "Ciphertext too short", - cipherText: []byte("short"), - privateKey: validPrivateKey, - associatedData: []byte("associated data"), - expectError: "ciphertext too short", + name: "Ciphertext too short", + cipherText: []byte("short"), + privateKey: validPrivateKey, + opts: []ECIESOption{WithAAD(aad)}, + expectError: errCipherTextTooShort, }, { - name: "Invalid private key", - cipherText: validCipherText, - privateKey: &ecdh.PrivateKey{}, - associatedData: []byte("associated data"), - expectError: "failed ECDH operation", + name: "Invalid private key", + cipherText: validCipherText, + privateKey: &ecdh.PrivateKey{}, + opts: []ECIESOption{WithAAD(aad)}, + expectError: errFailedECDHOperation, }, { - name: "Tampered ciphertext", - cipherText: append(validCipherText, byte(0)), - privateKey: validPrivateKey, - associatedData: []byte("associated data"), - expectError: "verification with HMAC failed", + name: "Tampered ciphertext", + cipherText: append(validCipherText, byte(0)), + privateKey: validPrivateKey, + opts: []ECIESOption{WithAAD(aad)}, + expectError: errVerificationWithHMACFailed, }, { - name: "Wrong associated data", - cipherText: validCipherText, - privateKey: validPrivateKey, - associatedData: []byte("wrong data"), - expectError: "failed to decrypt", + name: "Wrong associated data", + cipherText: validCipherText, + privateKey: validPrivateKey, + opts: []ECIESOption{WithAAD([]byte("wrong data"))}, + expectError: errFailedToDecrypt, + }, + { + name: "No public key prepended and no public key bytes provided", + cipherText: validCipherText[X25519PublicKeySize:], + privateKey: validPrivateKey, + opts: []ECIESOption{WithAAD(aad), WithPubKeyPrepended(false)}, + expectError: errNoPublicKeyForDecryption, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := DecryptECIES(tt.cipherText, tt.privateKey, tt.associatedData, nil) + _, err := DecryptECIES(tt.cipherText, tt.privateKey, tt.opts...) if err == nil || !strings.Contains(err.Error(), tt.expectError) { t.Errorf("Expected error containing '%s', got %v", tt.expectError, err) } }) } } + +func TestEncryptDecryptECIES_DefaultOptions_Succeeds(t *testing.T) { + plainText := []byte("Hello, World!") + recipientPrivateKey := mustGenerateX25519(t) + + cipherText, err := EncryptECIES(plainText, recipientPrivateKey.PublicKey()) + require.NoError(t, err) + + decryptedText, err := DecryptECIES(cipherText, recipientPrivateKey) + require.NoError(t, err) + + assert.Equal(t, plainText, decryptedText) +} + +func TestEncryptDecryptECIES_WithAAD_Succeeds(t *testing.T) { + plainText := []byte("Secret message") + aad := []byte("extra authentication data") + recipientPrivateKey := mustGenerateX25519(t) + + cipherText, err := EncryptECIES(plainText, recipientPrivateKey.PublicKey(), WithAAD(aad)) + require.NoError(t, err) + + decryptedText, err := DecryptECIES(cipherText, recipientPrivateKey, WithAAD(aad)) + require.NoError(t, err) + + assert.Equal(t, plainText, decryptedText) +} + +func TestEncryptDecryptECIES_WithCustomPrivateKey_Succeeds(t *testing.T) { + plainText := []byte("Custom key message") + recipientPrivateKey := mustGenerateX25519(t) + senderPrivateKey := mustGenerateX25519(t) + + cipherText, err := EncryptECIES(plainText, recipientPrivateKey.PublicKey(), WithPrivKey(senderPrivateKey)) + require.NoError(t, err) + + require.Equal(t, senderPrivateKey.PublicKey().Bytes(), cipherText[:X25519PublicKeySize]) + + decryptedText, err := DecryptECIES(cipherText, recipientPrivateKey) + require.NoError(t, err) + + assert.Equal(t, plainText, decryptedText) +} + +func TestEncryptDecryptECIES_WithoutPublicKeyPrepended_Succeeds(t *testing.T) { + plainText := []byte("No prepended key") + recipientPrivateKey := mustGenerateX25519(t) + senderPrivateKey := mustGenerateX25519(t) + + cipherText, err := EncryptECIES(plainText, recipientPrivateKey.PublicKey(), + WithPubKeyPrepended(false), + WithPrivKey(senderPrivateKey)) + require.NoError(t, err) + + // In a real scenario, the public key would be transmitted separately + senderPublicKeyBytes := senderPrivateKey.PublicKey().Bytes() + + decryptedText, err := DecryptECIES(cipherText, recipientPrivateKey, + WithPubKeyPrepended(false), + WithPubKeyBytes(senderPublicKeyBytes)) + require.NoError(t, err) + + assert.Equal(t, plainText, decryptedText) +} + +func TestEncryptDecryptECIES_DifferentAAD_FailsToDecrypt(t *testing.T) { + plainText := []byte("AAD test message") + encryptAAD := []byte("encryption AAD") + decryptAAD := []byte("decryption AAD") + recipientPrivateKey := mustGenerateX25519(t) + + cipherText, err := EncryptECIES(plainText, recipientPrivateKey.PublicKey(), WithAAD(encryptAAD)) + require.NoError(t, err) + + _, err = DecryptECIES(cipherText, recipientPrivateKey, WithAAD(decryptAAD)) + assert.Error(t, err, "Decryption should fail with different AAD") +} + +func mustGenerateX25519(t *testing.T) *ecdh.PrivateKey { + key, err := GenerateX25519() + require.NoError(t, err) + return key +} diff --git a/crypto/errors.go b/crypto/errors.go index 362d0584e9..a6128f9860 100644 --- a/crypto/errors.go +++ b/crypto/errors.go @@ -20,15 +20,17 @@ const ( errFailedKDFOperationForAESKey string = "failed KDF operation for AES key" errFailedKDFOperationForHMACKey string = "failed KDF operation for HMAC key" errFailedToEncrypt string = "failed to encrypt" - errCipherTextTooShort string = "ciphertext too short" + errCipherTextTooShort string = "cipherText too short" errFailedToParseEphemeralPublicKey string = "failed to parse ephemeral public key" errVerificationWithHMACFailed string = "verification with HMAC failed" errFailedToDecrypt string = "failed to decrypt" + errNoPublicKeyForDecryption string = "no public key provided for decryption" ) var ( ErrCipherTextTooShort = errors.New(errCipherTextTooShort) ErrVerificationWithHMACFailed = errors.New(errVerificationWithHMACFailed) + ErrNoPublicKeyForDecryption = errors.New(errNoPublicKeyForDecryption) ) func NewErrFailedToGenerateEphemeralKey(inner error) error { diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index d0854db0aa..9bcf871a07 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -212,8 +212,9 @@ func (s *pubSubService) handleFetchEncryptionKeyResponse( decryptedData, err := crypto.DecryptECIES( block, privateKey, - makeAssociatedData(req, resp.From), - keyResp.EphemeralPublicKey, + crypto.WithAAD(makeAssociatedData(req, resp.From)), + crypto.WithPubKeyBytes(keyResp.EphemeralPublicKey), + crypto.WithPubKeyPrepended(false), ) if err != nil { @@ -269,7 +270,13 @@ func (s *pubSubService) tryGenEncryptionKeyLocally( res.Blocks = make([][]byte, 0, len(blocks)) for _, block := range blocks { - encryptedBlock, err := crypto.EncryptECIES(block, reqEphPubKey, makeAssociatedData(req, s.peerID), privKey) + encryptedBlock, err := crypto.EncryptECIES( + block, + reqEphPubKey, + crypto.WithAAD(makeAssociatedData(req, s.peerID)), + crypto.WithPrivKey(privKey), + crypto.WithPubKeyPrepended(false), + ) if err != nil { return nil, errors.Wrap("failed to encrypt key for requester", err) } From ab0578173f9944807a8c8d36bc9415bd71bc99d6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 23:02:20 +0200 Subject: [PATCH 76/88] Remove EncStoreKey --- internal/core/key.go | 34 ---------------------------------- net/errors.go | 5 ----- 2 files changed, 39 deletions(-) diff --git a/internal/core/key.go b/internal/core/key.go index 900a04ee11..f54e1d2401 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -792,37 +792,3 @@ func bytesPrefixEnd(b []byte) []byte { // maximal byte string (i.e. already \xff...). return b } - -// EncStoreDocKey is a key for the encryption store. -type EncStoreDocKey struct { - // DocID is the ID of the document that the key is for. - DocID string - // FieldName is the name of the field that the key is for. - // If unset, it indicates that the key is for the whole document. - FieldName immutable.Option[string] - // KeyID is a hash (Cid) of the of the encryption key. - KeyID string -} - -var _ Key = (*EncStoreDocKey)(nil) - -// NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. -// Unset fieldName indicates that the key is for the whole document. -func NewEncStoreDocKey(docID string, fieldName immutable.Option[string], keyID string) EncStoreDocKey { - return EncStoreDocKey{DocID: docID, FieldName: fieldName, KeyID: keyID} -} - -func (k EncStoreDocKey) ToString() string { - if k.FieldName.HasValue() { - return fmt.Sprintf("%s/%s/%s", k.DocID, k.FieldName.Value(), k.KeyID) - } - return fmt.Sprintf("%s/%s", k.DocID, k.KeyID) -} - -func (k EncStoreDocKey) Bytes() []byte { - return []byte(k.ToString()) -} - -func (k EncStoreDocKey) ToDS() ds.Key { - return ds.NewKey(k.ToString()) -} diff --git a/net/errors.go b/net/errors.go index e0e7c6ccaf..3a21c8e5c1 100644 --- a/net/errors.go +++ b/net/errors.go @@ -14,7 +14,6 @@ import ( "fmt" "github.com/sourcenetwork/defradb/errors" - "github.com/sourcenetwork/defradb/internal/core" ) const ( @@ -53,10 +52,6 @@ func NewErrPublishingToSchemaTopic(inner error, cid, docID string, kv ...errors. return errors.Wrap(fmt.Sprintf(errPublishingToSchemaTopic, cid, docID), inner, kv...) } -func NewErrRequestingEncryptionKeys(inner error, keys []core.EncStoreDocKey) error { - return errors.Wrap(fmt.Sprintf(errRequestingEncryptionKeys, keys), inner) -} - func NewErrTopicAlreadyExist(topic string) error { return errors.New(fmt.Sprintf(errTopicAlreadyExist, topic)) } From bae47d7d6584946e5246c3427d68c8ceba13c4cd Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 14 Sep 2024 23:03:11 +0200 Subject: [PATCH 77/88] Polish --- internal/core/crdt/lwwreg_test.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/internal/core/crdt/lwwreg_test.go b/internal/core/crdt/lwwreg_test.go index 136d5cd09d..5b56df7636 100644 --- a/internal/core/crdt/lwwreg_test.go +++ b/internal/core/crdt/lwwreg_test.go @@ -16,6 +16,7 @@ import ( "testing" ds "github.com/ipfs/go-datastore" + "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" @@ -31,11 +32,12 @@ func setupLWWRegister() LWWRegister { return NewLWWRegister(store, core.CollectionSchemaVersionKey{}, key, "") } -func setupLoadedLWWRegister(ctx context.Context) LWWRegister { +func setupLoadedLWWRegister(t *testing.T, ctx context.Context) LWWRegister { lww := setupLWWRegister() addDelta := lww.Set([]byte("test")) addDelta.SetPriority(1) - lww.Merge(ctx, addDelta) + err := lww.Merge(ctx, addDelta) + require.NoError(t, err) return lww } @@ -71,12 +73,13 @@ func TestLWWRegisterInitialMerge(t *testing.T) { } } -func TestLWWReisterFollowupMerge(t *testing.T) { +func TestLWWRegisterFollowupMerge(t *testing.T) { ctx := context.Background() - lww := setupLoadedLWWRegister(ctx) + lww := setupLoadedLWWRegister(t, ctx) addDelta := lww.Set([]byte("test2")) addDelta.SetPriority(2) - lww.Merge(ctx, addDelta) + err := lww.Merge(ctx, addDelta) + require.NoError(t, err) val, err := lww.Value(ctx) if err != nil { @@ -90,10 +93,11 @@ func TestLWWReisterFollowupMerge(t *testing.T) { func TestLWWRegisterOldMerge(t *testing.T) { ctx := context.Background() - lww := setupLoadedLWWRegister(ctx) + lww := setupLoadedLWWRegister(t, ctx) addDelta := lww.Set([]byte("test-1")) addDelta.SetPriority(0) - lww.Merge(ctx, addDelta) + err := lww.Merge(ctx, addDelta) + require.NoError(t, err) val, err := lww.Value(ctx) if err != nil { @@ -106,9 +110,7 @@ func TestLWWRegisterOldMerge(t *testing.T) { } func TestLWWRegisterDeltaInit(t *testing.T) { - delta := &LWWRegDelta{ - Data: []byte("test"), - } + delta := &LWWRegDelta{} var _ core.Delta = delta // checks if LWWRegDelta implements core.Delta (also checked in the implementation code, but w.e) } From 73e7069336a4a0b08943f17c8744f2a62fb0d767 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 15 Sep 2024 13:51:51 +0200 Subject: [PATCH 78/88] Request encBlocks' cids in batches --- internal/core/crdt/composite.go | 2 +- internal/db/merge.go | 112 ++++++++++++++++++++------------ 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/internal/core/crdt/composite.go b/internal/core/crdt/composite.go index 71a95b9aac..c730badcb6 100644 --- a/internal/core/crdt/composite.go +++ b/internal/core/crdt/composite.go @@ -111,7 +111,7 @@ func (c CompositeDAG) Merge(ctx context.Context, delta core.Delta) error { // We cannot rely on the dagDelta.Status here as it may have been deleted locally, this is not // reflected in `dagDelta.Status` if sourced via P2P. Updates synced via P2P should not undelete - // the local reperesentation of the document. + // the local representation of the document. versionKey := c.key.WithValueFlag().WithFieldID(core.DATASTORE_DOC_VERSION_FIELD_ID) objectMarker, err := c.store.Get(ctx, c.key.ToPrimaryDataStoreKey().ToDS()) hasObjectMarker := !errors.Is(err, ds.ErrNotFound) diff --git a/internal/db/merge.go b/internal/db/merge.go index 5ab54011a7..8ac9907df5 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -16,6 +16,7 @@ import ( "sync" "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -74,10 +75,6 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - if len(mp.failedEncryptionBlocks) > 0 { - return nil - } - err = syncIndexedDoc(ctx, docID, col) if err != nil { return err @@ -144,9 +141,10 @@ type mergeProcessor struct { // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge // and doesn't have locally, so they need to be requested from the network. pendingEncryptionKeyRequests map[cidlink.Link]struct{} - - failedEncryptionBlocks map[cidlink.Link]struct{} - fetchedEncryptionBlocks map[cidlink.Link]*coreblock.Encryption + // missingEncryptionBlocks is a list of blocks that we failed to fetch + missingEncryptionBlocks map[cidlink.Link]struct{} + // availableEncryptionBlocks is a list of blocks that we have successfully fetched + availableEncryptionBlocks map[cidlink.Link]*coreblock.Encryption } func (db *db) newMergeProcessor( @@ -163,8 +161,8 @@ func (db *db) newMergeProcessor( dsKey: dsKey, composites: list.New(), pendingEncryptionKeyRequests: make(map[cidlink.Link]struct{}), - failedEncryptionBlocks: make(map[cidlink.Link]struct{}), - fetchedEncryptionBlocks: make(map[cidlink.Link]*coreblock.Encryption), + missingEncryptionBlocks: make(map[cidlink.Link]struct{}), + availableEncryptionBlocks: make(map[cidlink.Link]*coreblock.Encryption), }, nil } @@ -179,7 +177,7 @@ func newMergeTarget() mergeTarget { } } -// loadComposites retrieves and stores into the merge processor the blocks for the given +// loadComposites retrieves and stores into the merge processor the composite blocks for the given // CID until it reaches a block that has already been merged or until we reach the genesis block. func (mp *mergeProcessor) loadComposites( ctx context.Context, @@ -249,39 +247,85 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { return err } } + + return mp.tryFetchMissingBlocksAndMerge(ctx) +} + +func (mp *mergeProcessor) tryFetchMissingBlocksAndMerge(ctx context.Context) error { + for len(mp.missingEncryptionBlocks) > 0 { + links := make([]cidlink.Link, 0, len(mp.missingEncryptionBlocks)) + for link := range mp.missingEncryptionBlocks { + links = append(links, link) + } + msg, results := encryption.NewRequestKeysMessage(links) + mp.col.db.events.Publish(msg) + + res := <-results.Get() + if res.Error != nil { + return res.Error + } + + clear(mp.missingEncryptionBlocks) + + for i := range res.Items { + _, link, err := cid.CidFromBytes(res.Items[i].Link) + if err != nil { + return err + } + var encBlock coreblock.Encryption + err = encBlock.Unmarshal(res.Items[i].Block) + if err != nil { + return err + } + + mp.availableEncryptionBlocks[cidlink.Link{Cid: link}] = &encBlock + } + + err := mp.mergeComposites(ctx) + if err != nil { + return err + } + } return nil } -// TODO: check if we should send 1-by-1 func (mp *mergeProcessor) getEncryptionBlock( + ctx context.Context, encLink cidlink.Link, ) (*coreblock.Encryption, error) { - if _, ok := mp.failedEncryptionBlocks[encLink]; ok { - return nil, nil - } - if encBlock, ok := mp.fetchedEncryptionBlocks[encLink]; ok { + if encBlock, ok := mp.availableEncryptionBlocks[encLink]; ok { return encBlock, nil } - msg, results := encryption.NewRequestKeysMessage([]cidlink.Link{encLink}) - mp.col.db.events.Publish(msg) - - res := <-results.Get() - if res.Error != nil { - mp.failedEncryptionBlocks[encLink] = struct{}{} + if _, ok := mp.pendingEncryptionKeyRequests[encLink]; ok { return nil, nil } - delete(mp.failedEncryptionBlocks, encLink) + lsys := cidlink.DefaultLinkSystem() + lsys.SetReadStorage(mp.txn.Encstore().AsIPLDStorage()) - var encBlock coreblock.Encryption - err := encBlock.Unmarshal(res.Items[0].Block) + _, blockCid, err := cid.CidFromBytes(encLink.Cid.Bytes()) if err != nil { return nil, err } - mp.fetchedEncryptionBlocks[encLink] = &encBlock + nd, err := lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, + coreblock.EncryptionSchemaPrototype) + if err != nil { + if errors.Is(err, ipld.ErrNotFound{}) { + mp.missingEncryptionBlocks[encLink] = struct{}{} + return nil, nil + } + return nil, err + } + + encBlock, err := coreblock.GetEncryptionBlockFromNode(nd) + if err != nil { + return nil, err + } - return &encBlock, nil + mp.availableEncryptionBlocks[encLink] = encBlock + + return encBlock, nil } // processEncryptedBlock decrypts the block if it is encrypted and returns the decrypted block. @@ -293,7 +337,7 @@ func (mp *mergeProcessor) processEncryptedBlock( dagBlock *coreblock.Block, ) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { - encBlock, err := mp.getEncryptionBlock(*dagBlock.Encryption) + encBlock, err := mp.getEncryptionBlock(ctx, *dagBlock.Encryption) if err != nil { return nil, false, err } @@ -313,20 +357,6 @@ func (mp *mergeProcessor) processEncryptedBlock( return dagBlock, true, nil } -/*func (mp *mergeProcessor) addPendingEncryptionRequest(link cidlink.Link) { - mp.pendingEncryptionKeyRequests[link] = struct{}{} -} - -func (mp *mergeProcessor) sendPendingEncryptionRequest() *encryption.Results { - storeKeys := make([]cidlink.Link, 0, len(mp.pendingEncryptionKeyRequests)) - for k := range mp.pendingEncryptionKeyRequests { - storeKeys = append(storeKeys, k) - } - msg, results := encryption.NewRequestKeysMessage(storeKeys) - mp.col.db.events.Publish(msg) - return results -}*/ - // processBlock merges the block and its children to the datastore and sets the head accordingly. func (mp *mergeProcessor) processBlock( ctx context.Context, From c7121a4864471244c6cc0ff60ebb5438328af593 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 15 Sep 2024 22:42:18 +0200 Subject: [PATCH 79/88] Minor refactor --- internal/db/merge.go | 49 +++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 8ac9907df5..4478abf461 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -138,9 +138,6 @@ type mergeProcessor struct { dsKey core.DataStoreKey // composites is a list of composites that need to be merged. composites *list.List - // pendingEncryptionKeyRequests is a set of encryption keys that the node encountered during the merge - // and doesn't have locally, so they need to be requested from the network. - pendingEncryptionKeyRequests map[cidlink.Link]struct{} // missingEncryptionBlocks is a list of blocks that we failed to fetch missingEncryptionBlocks map[cidlink.Link]struct{} // availableEncryptionBlocks is a list of blocks that we have successfully fetched @@ -154,15 +151,14 @@ func (db *db) newMergeProcessor( dsKey core.DataStoreKey, ) (*mergeProcessor, error) { return &mergeProcessor{ - txn: txn, - lsys: lsys, - mCRDTs: make(map[string]merklecrdt.MerkleCRDT), - col: col, - dsKey: dsKey, - composites: list.New(), - pendingEncryptionKeyRequests: make(map[cidlink.Link]struct{}), - missingEncryptionBlocks: make(map[cidlink.Link]struct{}), - availableEncryptionBlocks: make(map[cidlink.Link]*coreblock.Encryption), + txn: txn, + lsys: lsys, + mCRDTs: make(map[string]merklecrdt.MerkleCRDT), + col: col, + dsKey: dsKey, + composites: list.New(), + missingEncryptionBlocks: make(map[cidlink.Link]struct{}), + availableEncryptionBlocks: make(map[cidlink.Link]*coreblock.Encryption), }, nil } @@ -281,7 +277,7 @@ func (mp *mergeProcessor) tryFetchMissingBlocksAndMerge(ctx context.Context) err mp.availableEncryptionBlocks[cidlink.Link{Cid: link}] = &encBlock } - err := mp.mergeComposites(ctx) + err := mp.mergeComposites(context.Background()) if err != nil { return err } @@ -289,17 +285,10 @@ func (mp *mergeProcessor) tryFetchMissingBlocksAndMerge(ctx context.Context) err return nil } -func (mp *mergeProcessor) getEncryptionBlock( +func (mp *mergeProcessor) loadEncryptionBlock( ctx context.Context, encLink cidlink.Link, ) (*coreblock.Encryption, error) { - if encBlock, ok := mp.availableEncryptionBlocks[encLink]; ok { - return encBlock, nil - } - if _, ok := mp.pendingEncryptionKeyRequests[encLink]; ok { - return nil, nil - } - lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(mp.txn.Encstore().AsIPLDStorage()) @@ -318,7 +307,21 @@ func (mp *mergeProcessor) getEncryptionBlock( return nil, err } - encBlock, err := coreblock.GetEncryptionBlockFromNode(nd) + return coreblock.GetEncryptionBlockFromNode(nd) +} + +func (mp *mergeProcessor) tryGetEncryptionBlock( + ctx context.Context, + encLink cidlink.Link, +) (*coreblock.Encryption, error) { + if encBlock, ok := mp.availableEncryptionBlocks[encLink]; ok { + return encBlock, nil + } + if _, ok := mp.missingEncryptionBlocks[encLink]; ok { + return nil, nil + } + + encBlock, err := mp.loadEncryptionBlock(ctx, encLink) if err != nil { return nil, err } @@ -337,7 +340,7 @@ func (mp *mergeProcessor) processEncryptedBlock( dagBlock *coreblock.Block, ) (*coreblock.Block, bool, error) { if dagBlock.IsEncrypted() { - encBlock, err := mp.getEncryptionBlock(ctx, *dagBlock.Encryption) + encBlock, err := mp.tryGetEncryptionBlock(ctx, *dagBlock.Encryption) if err != nil { return nil, false, err } From 0a218abcd4bac4167e7cafb92c902386e707e4c1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 15 Sep 2024 23:39:11 +0200 Subject: [PATCH 80/88] Make KMS also wait on ctx.Done() --- internal/encryption/event.go | 3 +++ internal/kms/pubsub.go | 33 +++++++++++++++------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/encryption/event.go b/internal/encryption/event.go index a7762b5716..16c1442b64 100644 --- a/internal/encryption/event.go +++ b/internal/encryption/event.go @@ -54,6 +54,8 @@ func (r *Results) Get() <-chan Result { return r.output } +// NewResults creates a new Results object and a channel that can be used to send results to it. +// The Results object can be used to wait on the results, and the channel can be used to send results. func NewResults() (*Results, chan<- Result) { ch := make(chan Result, 1) return &Results{ @@ -63,6 +65,7 @@ func NewResults() (*Results, chan<- Result) { // NewRequestKeysMessage creates a new event message for a request of a node to fetch an encryption key // for a specific docID/field +// It returns the message and the results that that can be waited on. func NewRequestKeysMessage(keys []cidlink.Link) (event.Message, *Results) { res, ch := NewResults() return event.NewMessage(RequestKeysEventName, RequestKeysEvent{ diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index 9bcf871a07..4b78ff6416 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -100,18 +100,22 @@ func (s *pubSubService) handleKeyRequestedEvent() { log.ErrorContextE(s.ctx, "Failed to get encryption keys", err) } - encResult := <-results.Get() - - for _, encItem := range encResult.Items { - _, err = s.encStore.put(s.ctx, encItem.Block) - if err != nil { - log.ErrorContextE(s.ctx, "Failed to save encryption key", err) - return + defer close(keyReqEvent.Resp) + + select { + case <-s.ctx.Done(): + return + case encResult := <-results.Get(): + for _, encItem := range encResult.Items { + _, err = s.encStore.put(s.ctx, encItem.Block) + if err != nil { + log.ErrorContextE(s.ctx, "Failed to save encryption key", err) + return + } } - } - keyReqEvent.Resp <- encResult - close(keyReqEvent.Resp) + keyReqEvent.Resp <- encResult + } }() } else { log.ErrorContext(s.ctx, "Failed to cast event data to RequestKeysEvent") @@ -121,17 +125,12 @@ func (s *pubSubService) handleKeyRequestedEvent() { // handleEncryptionMessage handles incoming FetchEncryptionKeyRequest messages from the pubsub network. func (s *pubSubService) handleRequestFromPeer(peerID libpeer.ID, topic string, msg []byte) ([]byte, error) { - // TODO: check how it makes sense and how much effort to separate net package so that it has - // client-related and server-related code independently. Conceptually, they should no depend on each other. - // Any common functionality (like hosting peer) can be shared. - // This way we could make kms package depend only on client. The server would depend on kms. req := new(pb.FetchEncryptionKeyRequest) if err := proto.Unmarshal(msg, req); err != nil { log.ErrorContextE(s.ctx, "Failed to unmarshal pubsub message %s", err) return nil, err } - // TODO: check if this NewGRPCPeer can be abstracted away or copied in this package. ctx := grpcpeer.NewContext(s.ctx, net.NewGRPCPeer(peerID)) res, err := s.tryGenEncryptionKeyLocally(ctx, req) if err != nil { @@ -224,8 +223,7 @@ func (s *pubSubService) handleFetchEncryptionKeyResponse( } resultEncItems = append(resultEncItems, encryption.Item{ - Link: keyResp.Links[i], - //Block: decryptedData[:crypto.AESKeySize], + Link: keyResp.Links[i], Block: decryptedData, }) } @@ -282,7 +280,6 @@ func (s *pubSubService) tryGenEncryptionKeyLocally( } res.Blocks = append(res.Blocks, encryptedBlock) - //res.Blocks = append(res.Blocks, encryptedBlock[crypto.X25519PublicKeySize:]) } return res, nil From 16d1f8e8f48a58b06dc8792847d4c6a75d696214 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 16 Sep 2024 08:45:51 +0200 Subject: [PATCH 81/88] Polish --- internal/db/merge.go | 2 +- internal/kms/pubsub.go | 5 +++++ internal/kms/service.go | 6 ++++++ internal/merkle/clock/clock.go | 2 -- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 4478abf461..bce7b7e1b7 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -277,7 +277,7 @@ func (mp *mergeProcessor) tryFetchMissingBlocksAndMerge(ctx context.Context) err mp.availableEncryptionBlocks[cidlink.Link{Cid: link}] = &encBlock } - err := mp.mergeComposites(context.Background()) + err := mp.mergeComposites(ctx) if err != nil { return err } diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index 4b78ff6416..b66f7641ba 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -60,6 +60,11 @@ func (s *pubSubService) GetKeys(ctx context.Context, cids ...cidlink.Link) (*enc return res, nil } +// NewPubSubService creates a new instance of the KMS service that is connected to the given PubSubServer, +// event bus and encryption storage. +// +// The service will subscribe to the "encryption" topic on the PubSubServer and to the +// "enc-keys-request" event on the event bus. func NewPubSubService( ctx context.Context, peerID libpeer.ID, diff --git a/internal/kms/service.go b/internal/kms/service.go index 14589cd0ea..97985c9a43 100644 --- a/internal/kms/service.go +++ b/internal/kms/service.go @@ -26,9 +26,15 @@ var ( type ServiceType string const ( + // PubSubServiceType is the type of KMS that uses PubSub mechanism to exchange keys + // between peers. PubSubServiceType ServiceType = "pubsub" ) +// Service is interface for key management service (KMS) type Service interface { + // GetKeys retrieves the encryption blocks containing encryption keys for the given links. + // Blocks are fetched asynchronously, so the method returns an [encryption.Results] object + // that can be used to wait for the results. GetKeys(ctx context.Context, cids ...cidlink.Link) (*encryption.Results, error) } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 2c44a28254..6d80ccb7f0 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -222,8 +222,6 @@ func encryptBlock( } // ProcessBlock merges the delta CRDT and updates the state accordingly. -// If skipMerge is true, it will skip merging and update only the heads. -// If skipHeads is true, it will skip updating the heads. func (mc *MerkleClock) ProcessBlock( ctx context.Context, block *coreblock.Block, From 32f2471080aabbd48aba493f6994aeec5b2f269c Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 16 Sep 2024 09:45:25 +0200 Subject: [PATCH 82/88] Update go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2a0fd067fa..c56b74ec3d 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 - github.com/planetscale/vtprotobuf v0.6.0 + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/sourcenetwork/acp_core v0.0.0-20240607160510-47a5306b2ad2 github.com/sourcenetwork/badger/v4 v4.2.1-0.20231113215945-a63444ca5276 github.com/sourcenetwork/corelog v0.0.8 diff --git a/go.sum b/go.sum index 10fa713e66..88633958ca 100644 --- a/go.sum +++ b/go.sum @@ -1261,8 +1261,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA= -github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From b2c1f9d6e0bde4c00d790732fd5ee0df68320fcc Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 16 Sep 2024 09:49:20 +0200 Subject: [PATCH 83/88] Lint polish --- internal/kms/pubsub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index b66f7641ba..4e908bb99e 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -64,7 +64,7 @@ func (s *pubSubService) GetKeys(ctx context.Context, cids ...cidlink.Link) (*enc // event bus and encryption storage. // // The service will subscribe to the "encryption" topic on the PubSubServer and to the -// "enc-keys-request" event on the event bus. +// "enc-keys-request" event on the event bus. func NewPubSubService( ctx context.Context, peerID libpeer.ID, From ab44314c36584380006c62097ed9889c7d80d726 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 17 Sep 2024 08:39:22 +0200 Subject: [PATCH 84/88] Add comments --- tests/integration/acp.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/acp.go b/tests/integration/acp.go index 4e2615eb6d..f3dfaf8d7c 100644 --- a/tests/integration/acp.go +++ b/tests/integration/acp.go @@ -57,10 +57,13 @@ var ( acpType ACPType ) +// KMSType is the type of KMS to use. type KMSType string const ( - NoneKMSType KMSType = "none" + // NoneKMSType is the none KMS type. It is used to indicate that no KMS should be used. + NoneKMSType KMSType = "none" + // PubSubKMSType is the PubSub KMS type. PubSubKMSType KMSType = "pubsub" ) From 3a72abb09887e3090666fbe4b4defa43cdcf15c1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 17 Sep 2024 08:45:04 +0200 Subject: [PATCH 85/88] Fix lint --- tests/integration/test_case.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 0d87377825..9b0bce913b 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -59,7 +59,7 @@ type TestCase struct { // This is to only be used in the very rare cases where we really do want behavioural // differences between view types, or we need to temporarily document a bug. SupportedViewTypes immutable.Option[[]ViewType] - + // Configuration for KMS to be used in the test KMS KMS } @@ -67,7 +67,7 @@ type TestCase struct { // KMS contains the configuration for KMS to be used in the test type KMS struct { // Activated indicates if the KMS should be used in the test - Activated bool + Activated bool // ExcludedTypes specifies the KMS types that should be excluded from the test. // If none are specified all types will be used. ExcludedTypes []KMSType From ef9022f10e064f78f768833546084a865e5217ed Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 19 Sep 2024 10:21:27 +0200 Subject: [PATCH 86/88] PR fix up --- cli/collection_create.go | 7 ++--- internal/core/block/block.go | 4 +-- internal/db/merge.go | 55 ++++++++++++++++------------------ internal/merkle/clock/clock.go | 5 ++-- node/node.go | 2 +- 5 files changed, 33 insertions(+), 40 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index 3cb652fee8..002847d6ec 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -17,8 +17,6 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/datastore" - "github.com/sourcenetwork/defradb/internal/db" "github.com/sourcenetwork/defradb/internal/encryption" ) @@ -89,8 +87,7 @@ Example: create from stdin: return cmd.Usage() } - txn, _ := db.TryGetContextTxn(cmd.Context()) - setContextDocEncryption(cmd, shouldEncryptDoc, encryptedFields, txn) + setContextDocEncryption(cmd, shouldEncryptDoc, encryptedFields) if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) @@ -116,7 +113,7 @@ Example: create from stdin: } // setContextDocEncryption sets doc encryption for the current command context. -func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptFields []string, txn datastore.Txn) { +func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptFields []string) { if !shouldEncryptDoc && len(encryptFields) == 0 { return } diff --git a/internal/core/block/block.go b/internal/core/block/block.go index baa5d2ff32..1ec62fe939 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -142,8 +142,8 @@ func (block *Block) Clone() *Block { } } -// GetPrevBlockCids returns the CIDs of the previous blocks. It can be more than 1 with multiple heads. -func (block *Block) GetPrevBlockCids() []cid.Cid { +// GetHeadLinks returns the CIDs of the previous blocks. There can be more than 1 with multiple heads. +func (block *Block) GetHeadLinks() []cid.Cid { var heads []cid.Cid for _, link := range block.Links { if link.Name == core.HEAD { diff --git a/internal/db/merge.go b/internal/db/merge.go index bce7b7e1b7..58c89cfc4e 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -46,16 +46,13 @@ func (db *db) executeMerge(ctx context.Context, dagMerge event.Merge) error { return err } - ls := cidlink.DefaultLinkSystem() - ls.SetReadStorage(txn.Blockstore().AsIPLDStorage()) - docID, err := client.NewDocIDFromString(dagMerge.DocID) if err != nil { return err } dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(col.Description(), docID.String()) - mp, err := db.newMergeProcessor(txn, ls, col, dsKey) + mp, err := db.newMergeProcessor(txn, col, dsKey) if err != nil { return err } @@ -131,11 +128,12 @@ func (m *mergeQueue) done(docID string) { } type mergeProcessor struct { - txn datastore.Txn - lsys linking.LinkSystem - mCRDTs map[string]merklecrdt.MerkleCRDT - col *collection - dsKey core.DataStoreKey + txn datastore.Txn + blockLS linking.LinkSystem + encBlockLS linking.LinkSystem + mCRDTs map[string]merklecrdt.MerkleCRDT + col *collection + dsKey core.DataStoreKey // composites is a list of composites that need to be merged. composites *list.List // missingEncryptionBlocks is a list of blocks that we failed to fetch @@ -146,13 +144,19 @@ type mergeProcessor struct { func (db *db) newMergeProcessor( txn datastore.Txn, - lsys linking.LinkSystem, col *collection, dsKey core.DataStoreKey, ) (*mergeProcessor, error) { + blockLS := cidlink.DefaultLinkSystem() + blockLS.SetReadStorage(txn.Blockstore().AsIPLDStorage()) + + encBlockLS := cidlink.DefaultLinkSystem() + encBlockLS.SetReadStorage(txn.Encstore().AsIPLDStorage()) + return &mergeProcessor{ txn: txn, - lsys: lsys, + blockLS: blockLS, + encBlockLS: encBlockLS, mCRDTs: make(map[string]merklecrdt.MerkleCRDT), col: col, dsKey: dsKey, @@ -185,7 +189,7 @@ func (mp *mergeProcessor) loadComposites( return nil } - nd, err := mp.lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, coreblock.SchemaPrototype) + nd, err := mp.blockLS.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, coreblock.SchemaPrototype) if err != nil { return err } @@ -200,7 +204,7 @@ func (mp *mergeProcessor) loadComposites( // In this case, we also need to walk back the merge target's DAG until we reach a common block. if block.Delta.GetPriority() >= mt.headHeight { mp.composites.PushFront(block) - for _, prevCid := range block.GetPrevBlockCids() { + for _, prevCid := range block.GetHeadLinks() { err := mp.loadComposites(ctx, prevCid, mt) if err != nil { return err @@ -211,7 +215,7 @@ func (mp *mergeProcessor) loadComposites( for _, b := range mt.heads { for _, link := range b.Links { if link.Name == core.HEAD { - nd, err := mp.lsys.Load(linking.LinkContext{Ctx: ctx}, link.Link, coreblock.SchemaPrototype) + nd, err := mp.blockLS.Load(linking.LinkContext{Ctx: ctx}, link.Link, coreblock.SchemaPrototype) if err != nil { return err } @@ -289,16 +293,7 @@ func (mp *mergeProcessor) loadEncryptionBlock( ctx context.Context, encLink cidlink.Link, ) (*coreblock.Encryption, error) { - lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(mp.txn.Encstore().AsIPLDStorage()) - - _, blockCid, err := cid.CidFromBytes(encLink.Cid.Bytes()) - if err != nil { - return nil, err - } - - nd, err := lsys.Load(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: blockCid}, - coreblock.EncryptionSchemaPrototype) + nd, err := mp.encBlockLS.Load(linking.LinkContext{Ctx: ctx}, encLink, coreblock.EncryptionSchemaPrototype) if err != nil { if errors.Is(err, ipld.ErrNotFound{}) { mp.missingEncryptionBlocks[encLink] = struct{}{} @@ -326,15 +321,17 @@ func (mp *mergeProcessor) tryGetEncryptionBlock( return nil, err } - mp.availableEncryptionBlocks[encLink] = encBlock + if encBlock != nil { + mp.availableEncryptionBlocks[encLink] = encBlock + } return encBlock, nil } // processEncryptedBlock decrypts the block if it is encrypted and returns the decrypted block. -// If the block is encrypted and we were not able to decrypt it, it returns true as the second return value -// which indicates that the we should skip merging of the block. -// If we were able to decrypt the block, we return the decrypted block and false as the second return value. +// If the block is encrypted and we were not able to decrypt it, it returns false as the second return value +// which indicates that the we can't read the block. +// If we were able to decrypt the block, we return the decrypted block and true as the second return value. func (mp *mergeProcessor) processEncryptedBlock( ctx context.Context, dagBlock *coreblock.Block, @@ -394,7 +391,7 @@ func (mp *mergeProcessor) processBlock( continue } - nd, err := mp.lsys.Load(linking.LinkContext{Ctx: ctx}, link.Link, coreblock.SchemaPrototype) + nd, err := mp.blockLS.Load(linking.LinkContext{Ctx: ctx}, link.Link, coreblock.SchemaPrototype) if err != nil { return err } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 6d80ccb7f0..b5b55e1374 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -107,7 +107,7 @@ func (mc *MerkleClock) AddDelta( if block.Delta.GetFieldName() != "" { fieldName = immutable.Some(block.Delta.GetFieldName()) } - encBlock, encLink, err := mc.determineBlockEncryptionData(ctx, string(block.Delta.GetDocID()), fieldName, heads) + encBlock, encLink, err := mc.determineBlockEncryption(ctx, string(block.Delta.GetDocID()), fieldName, heads) if err != nil { return cidlink.Link{}, nil, err } @@ -140,8 +140,7 @@ func (mc *MerkleClock) AddDelta( return link, b, err } -// TODO: rename this to reflect storage -func (mc *MerkleClock) determineBlockEncryptionData( +func (mc *MerkleClock) determineBlockEncryption( ctx context.Context, docID string, fieldName immutable.Option[string], diff --git a/node/node.go b/node/node.go index 011c23fb9e..f3af07c9c8 100644 --- a/node/node.go +++ b/node/node.go @@ -151,7 +151,7 @@ func (n *Node) Start(ctx context.Context) error { if !n.options.disableP2P { // setup net node - n.Peer, err = net.NewPeer(ctx, n.DB.Blockstore(), n.DB.Events(), n.netOpts...) + n.Peer, err = net.NewPeer(ctx, n.DB.Blockstore(), n.DB.Encstore(), n.DB.Events(), n.netOpts...) if err != nil { return err } From bc335aff03da5c99c66368064ca66eca4d2a2f62 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 19 Sep 2024 11:50:10 +0200 Subject: [PATCH 87/88] Add tests for checking encryption of empty and nil values --- .../integration/encryption/peer_share_test.go | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/integration/encryption/peer_share_test.go b/tests/integration/encryption/peer_share_test.go index 54ad066bc8..c04d204a84 100644 --- a/tests/integration/encryption/peer_share_test.go +++ b/tests/integration/encryption/peer_share_test.go @@ -390,3 +390,141 @@ func TestDocEncryptionPeer_WithUpdatesOnDeltaBasedCRDTFieldOfEncryptedDoc_Should testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryptionPeer_WithUpdatesThatSetsEmptyString_ShouldDecryptAndCorrectlyMerge(t *testing.T) { + test := testUtils.TestCase{ + KMS: testUtils.KMS{Activated: true}, + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"name": ""}`, + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + {"name": ""}, + }, + }, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"name": "John"}`, + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + {"name": "John"}, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionPeer_WithUpdatesThatSetsStringToNull_ShouldDecryptAndCorrectlyMerge(t *testing.T) { + test := testUtils.TestCase{ + KMS: testUtils.KMS{Activated: true}, + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"name": null}`, + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + {"name": nil}, + }, + }, + }, + testUtils.UpdateDoc{ + NodeID: immutable.Some(0), + Doc: `{"name": "John"}`, + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + User { + name + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + {"name": "John"}, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From a492a1ddf5c4a1fc77ff0873fa48e74aaf4aaf91 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 23 Sep 2024 10:10:19 +0200 Subject: [PATCH 88/88] PR fixup --- internal/kms/pubsub.go | 18 +++++++++++++-- net/peer.go | 10 +-------- net/server.go | 50 +++++++++++++----------------------------- 3 files changed, 32 insertions(+), 46 deletions(-) diff --git a/internal/kms/pubsub.go b/internal/kms/pubsub.go index 4e908bb99e..ca67603a7c 100644 --- a/internal/kms/pubsub.go +++ b/internal/kms/pubsub.go @@ -27,7 +27,6 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/encryption" - "github.com/sourcenetwork/defradb/net" pb "github.com/sourcenetwork/defradb/net/pb" ) @@ -136,7 +135,7 @@ func (s *pubSubService) handleRequestFromPeer(peerID libpeer.ID, topic string, m return nil, err } - ctx := grpcpeer.NewContext(s.ctx, net.NewGRPCPeer(peerID)) + ctx := grpcpeer.NewContext(s.ctx, newGRPCPeer(peerID)) res, err := s.tryGenEncryptionKeyLocally(ctx, req) if err != nil { log.ErrorContextE(s.ctx, "failed attempt to get encryption key", err) @@ -323,3 +322,18 @@ func encodeToBase64(data []byte) []byte { base64.StdEncoding.Encode(encoded, data) return encoded } + +func newGRPCPeer(peerID libpeer.ID) *grpcpeer.Peer { + return &grpcpeer.Peer{ + Addr: addr{peerID}, + } +} + +// addr implements net.Addr and holds a libp2p peer ID. +type addr struct{ id libpeer.ID } + +// Network returns the name of the network that this address belongs to (libp2p). +func (a addr) Network() string { return "libp2p" } + +// String returns the peer ID of this address in string form (B58-encoded). +func (a addr) String() string { return a.id.String() } diff --git a/net/peer.go b/net/peer.go index 9a9e915414..24976ed388 100644 --- a/net/peer.go +++ b/net/peer.go @@ -27,13 +27,11 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - libpeer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" "github.com/sourcenetwork/corelog" "google.golang.org/grpc" - grpcpeer "google.golang.org/grpc/peer" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" @@ -229,12 +227,6 @@ func (p *Peer) Close() { } } -func NewGRPCPeer(peerID libpeer.ID) *grpcpeer.Peer { - return &grpcpeer.Peer{ - Addr: addr{peerID}, - } -} - // handleMessage loop manages the transition of messages // from the internal broadcaster to the external pubsub network func (p *Peer) handleMessageLoop() { @@ -278,7 +270,7 @@ func (p *Peer) RegisterNewDocument( schemaRoot string, ) error { // register topic - err := p.server.addPubSubTopic(docID.String(), !p.server.hasPubSubTopic(schemaRoot)) + err := p.server.addPubSubTopic(docID.String(), !p.server.hasPubSubTopic(schemaRoot), nil) if err != nil { log.ErrorE( "Failed to create new pubsub topic", diff --git a/net/server.go b/net/server.go index e8f433ca79..42ff15f5fb 100644 --- a/net/server.go +++ b/net/server.go @@ -146,7 +146,7 @@ func (s *server) PushLog(ctx context.Context, req *pb.PushLogRequest) (*pb.PushL // Once processed, subscribe to the DocID topic on the pubsub network unless we already // subscribed to the collection. if !s.hasPubSubTopic(string(req.Body.SchemaRoot)) { - err = s.addPubSubTopic(docID.String(), true) + err = s.addPubSubTopic(docID.String(), true, nil) if err != nil { return nil, err } @@ -172,7 +172,9 @@ func (s *server) GetHeadLog( } // addPubSubTopic subscribes to a topic on the pubsub network -func (s *server) addPubSubTopic(topic string, subscribe bool) error { +// A custom message handler can be provided to handle incoming messages. If not provided, +// the default message handler will be used. +func (s *server) addPubSubTopic(topic string, subscribe bool, handler rpc.MessageHandler) error { if s.peer.ps == nil { return nil } @@ -200,8 +202,12 @@ func (s *server) addPubSubTopic(topic string, subscribe bool) error { return err } + if handler == nil { + handler = s.pubSubMessageHandler + } + t.SetEventHandler(s.pubSubEventHandler) - t.SetMessageHandler(s.pubSubMessageHandler) + t.SetMessageHandler(handler) s.topics[topic] = pubsubTopic{ Topic: t, subscribed: subscribe, @@ -209,6 +215,10 @@ func (s *server) addPubSubTopic(topic string, subscribe bool) error { return nil } +func (s *server) AddPubSubTopic(topicName string, handler rpc.MessageHandler) error { + return s.addPubSubTopic(topicName, true, handler) +} + // hasPubSubTopic checks if we are subscribed to a topic. func (s *server) hasPubSubTopic(topic string) bool { s.mu.Lock() @@ -269,7 +279,7 @@ func (s *server) publishLog(ctx context.Context, topic string, req *pb.PushLogRe t, ok := s.topics[topic] s.mu.Unlock() if !ok { - err := s.addPubSubTopic(topic, false) + err := s.addPubSubTopic(topic, false, nil) if err != nil { return errors.Wrap(fmt.Sprintf("failed to created single use topic %s", topic), err) } @@ -347,7 +357,7 @@ func peerIDFromContext(ctx context.Context) (libpeer.ID, error) { func (s *server) updatePubSubTopics(evt event.P2PTopic) { for _, topic := range evt.ToAdd { - err := s.addPubSubTopic(topic, true) + err := s.addPubSubTopic(topic, true, nil) if err != nil { log.ErrorE("Failed to add pubsub topic.", err) } @@ -410,36 +420,6 @@ func (s *server) updateReplicators(evt event.Replicator) { s.peer.bus.Publish(event.NewMessage(event.ReplicatorCompletedName, nil)) } -func (s *server) AddPubSubTopic(topicName string, handler rpc.MessageHandler) error { - if s.peer.ps == nil { - return nil - } - - s.mu.Lock() - _, ok := s.topics[topicName] - s.mu.Unlock() - if ok { - return NewErrTopicAlreadyExist(topicName) - } - - t, err := rpc.NewTopic(s.peer.ctx, s.peer.ps, s.peer.host.ID(), topicName, true) - if err != nil { - return err - } - - t.SetEventHandler(s.pubSubEventHandler) - t.SetMessageHandler(handler) - - s.mu.Lock() - defer s.mu.Unlock() - - s.topics[topicName] = pubsubTopic{ - Topic: t, - subscribed: true, - } - return nil -} - func (s *server) SendPubSubMessage( ctx context.Context, topic string,