diff --git a/clients/tests/retrieval_client_test.go b/clients/tests/retrieval_client_test.go index d275870b4e..7b60e563bb 100644 --- a/clients/tests/retrieval_client_test.go +++ b/clients/tests/retrieval_client_test.go @@ -13,8 +13,10 @@ import ( "github.com/Layr-Labs/eigenda/core/encoding" coreindexer "github.com/Layr-Labs/eigenda/core/indexer" coremock "github.com/Layr-Labs/eigenda/core/mock" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" indexermock "github.com/Layr-Labs/eigenda/indexer/mock" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -25,7 +27,7 @@ import ( const numOperators = 10 func makeTestEncoder() (core.Encoder, error) { - config := &kzgEncoder.KzgConfig{ + config := &kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point", G2Path: "../../inabox/resources/kzg/g2.point", CacheDir: "../../inabox/resources/kzg/SRSTables", @@ -34,13 +36,19 @@ func makeTestEncoder() (core.Encoder, error) { NumWorker: uint64(runtime.GOMAXPROCS(0)), } - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, true) + kzgEncoderGroup, err := prover.NewProver(config, true) + if err != nil { + return nil, err + } + + kzgVerifierGroup, err := verifier.NewVerifier(config, true) if err != nil { return nil, err } return &encoding.Encoder{ - EncoderGroup: kzgEncoderGroup, + EncoderGroup: kzgEncoderGroup, + VerifierGroup: kzgVerifierGroup, }, nil } diff --git a/core/encoding.go b/core/encoding.go index 79dc4d636f..661d778ed5 100644 --- a/core/encoding.go +++ b/core/encoding.go @@ -3,7 +3,7 @@ package core import ( "fmt" - "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + encoder "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) diff --git a/core/encoding/cli.go b/core/encoding/cli.go index 8bc8634a08..89e1f7b8e1 100644 --- a/core/encoding/cli.go +++ b/core/encoding/cli.go @@ -4,7 +4,7 @@ import ( "runtime" "github.com/Layr-Labs/eigenda/common" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/urfave/cli" ) @@ -88,7 +88,7 @@ func CLIFlags(envPrefix string) []cli.Flag { } func ReadCLIConfig(ctx *cli.Context) EncoderConfig { - cfg := kzgEncoder.KzgConfig{} + cfg := kzgrs.KzgConfig{} cfg.G1Path = ctx.GlobalString(G1PathFlagName) cfg.G2Path = ctx.GlobalString(G2PathFlagName) cfg.CacheDir = ctx.GlobalString(CachePathFlagName) diff --git a/core/encoding/encoder.go b/core/encoding/encoder.go index 8b2917e92b..9646ba28fb 100644 --- a/core/encoding/encoder.go +++ b/core/encoding/encoder.go @@ -4,8 +4,11 @@ import ( "crypto/sha256" "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + encoder "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" lru "github.com/hashicorp/golang-lru/v2" ) @@ -15,20 +18,26 @@ func toEncParams(params core.EncodingParams) encoder.EncodingParams { } type EncoderConfig struct { - KzgConfig kzgEncoder.KzgConfig + KzgConfig kzgrs.KzgConfig CacheEncodedBlobs bool } type Encoder struct { - Config EncoderConfig - EncoderGroup *kzgEncoder.KzgEncoderGroup - Cache *lru.Cache[string, encodedValue] + Config EncoderConfig + EncoderGroup *prover.Prover + VerifierGroup *verifier.Verifier + Cache *lru.Cache[string, encodedValue] } var _ core.Encoder = &Encoder{} func NewEncoder(config EncoderConfig, loadG2Points bool) (*Encoder, error) { - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig, loadG2Points) + kzgEncoderGroup, err := prover.NewProver(&config.KzgConfig, loadG2Points) + if err != nil { + return nil, err + } + + kzgVerifierGroup, err := verifier.NewVerifier(&config.KzgConfig, loadG2Points) if err != nil { return nil, err } @@ -39,9 +48,10 @@ func NewEncoder(config EncoderConfig, loadG2Points bool) (*Encoder, error) { } return &Encoder{ - EncoderGroup: kzgEncoderGroup, - Cache: cache, - Config: config, + EncoderGroup: kzgEncoderGroup, + VerifierGroup: kzgVerifierGroup, + Cache: cache, + Config: config, }, nil } @@ -100,7 +110,7 @@ func (e *Encoder) Encode(data []byte, params core.EncodingParams) (core.BlobComm } func (e *Encoder) VerifyBlobLength(commitments core.BlobCommitments) error { - return e.EncoderGroup.VerifyCommit((*bn254.G2Point)(commitments.LengthCommitment), (*bn254.G2Point)(commitments.LengthProof), uint64(commitments.Length)) + return e.VerifierGroup.VerifyCommit((*bn254.G2Point)(commitments.LengthCommitment), (*bn254.G2Point)(commitments.LengthProof), uint64(commitments.Length)) } @@ -108,7 +118,7 @@ func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber, encParams := toEncParams(params) - verifier, err := e.EncoderGroup.GetKzgVerifier(encParams) + verifier, err := e.VerifierGroup.GetKzgVerifier(encParams) if err != nil { return err } @@ -116,7 +126,7 @@ func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber, for ind := range chunks { err = verifier.VerifyFrame( (*bn254.G1Point)(commitments.Commitment), - &kzgEncoder.Frame{ + &encoding.Frame{ Proof: chunks[ind].Proof, Coeffs: chunks[ind].Coeffs, }, @@ -133,21 +143,21 @@ func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber, } func (e *Encoder) VerifyCommitEquivalenceBatch(commitments []core.BlobCommitments) error { - commitmentsPair := make([]kzgEncoder.CommitmentPair, len(commitments)) + commitmentsPair := make([]verifier.CommitmentPair, len(commitments)) for i, c := range commitments { - commitmentsPair[i] = kzgEncoder.CommitmentPair{ + commitmentsPair[i] = verifier.CommitmentPair{ Commitment: (bn254.G1Point)(*c.Commitment), LengthCommitment: (bn254.G2Point)(*c.LengthCommitment), } } - return e.EncoderGroup.BatchVerifyCommitEquivalence(commitmentsPair) + return e.VerifierGroup.BatchVerifyCommitEquivalence(commitmentsPair) } // convert struct understandable by the crypto library func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCore []core.Sample, numBlobs int) error { encParams := toEncParams(params) - samples := make([]kzgEncoder.Sample, len(samplesCore)) + samples := make([]verifier.Sample, len(samplesCore)) for i, sc := range samplesCore { x, err := encoder.GetLeadingCosetIndex( @@ -158,7 +168,7 @@ func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCor return err } - sample := kzgEncoder.Sample{ + sample := verifier.Sample{ Commitment: (bn254.G1Point)(*sc.Commitment), Proof: sc.Chunk.Proof, RowIndex: sc.BlobIndex, @@ -168,15 +178,15 @@ func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCor samples[i] = sample } - return e.EncoderGroup.UniversalVerify(encParams, samples, numBlobs) + return e.VerifierGroup.UniversalVerify(encParams, samples, numBlobs) } // Decode takes in the chunks, indices, and encoding parameters and returns the decoded blob // The result is trimmed to the given maxInputSize. func (e *Encoder) Decode(chunks []*core.Chunk, indices []core.ChunkNumber, params core.EncodingParams, maxInputSize uint64) ([]byte, error) { - frames := make([]kzgEncoder.Frame, len(chunks)) + frames := make([]encoding.Frame, len(chunks)) for i := range chunks { - frames[i] = kzgEncoder.Frame{ + frames[i] = encoding.Frame{ Proof: chunks[i].Proof, Coeffs: chunks[i].Coeffs, } diff --git a/core/encoding/encoder_test.go b/core/encoding/encoder_test.go index ee4d7cb4bb..0f63b8d53d 100644 --- a/core/encoding/encoder_test.go +++ b/core/encoding/encoder_test.go @@ -8,7 +8,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/core/encoding" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/stretchr/testify/assert" ) @@ -28,7 +28,7 @@ func init() { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { - config := kzgEncoder.KzgConfig{ + config := kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point.300000", G2Path: "../../inabox/resources/kzg/g2.point.300000", CacheDir: "../../inabox/resources/kzg/SRSTables", diff --git a/core/test/core_test.go b/core/test/core_test.go index 380c45633d..be255679c8 100644 --- a/core/test/core_test.go +++ b/core/test/core_test.go @@ -12,10 +12,9 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/core/encoding" "github.com/Layr-Labs/eigenda/core/mock" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/gammazero/workerpool" "github.com/stretchr/testify/assert" - - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" ) var ( @@ -40,7 +39,7 @@ func setup(m *testing.M) { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { - config := kzgEncoder.KzgConfig{ + config := kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point", G2Path: "../../inabox/resources/kzg/g2.point", CacheDir: "../../inabox/resources/kzg/SRSTables", diff --git a/disperser/batcher/batcher_test.go b/disperser/batcher/batcher_test.go index 0c666fb970..59e8a7567a 100644 --- a/disperser/batcher/batcher_test.go +++ b/disperser/batcher/batcher_test.go @@ -21,7 +21,7 @@ import ( batchermock "github.com/Layr-Labs/eigenda/disperser/batcher/mock" "github.com/Layr-Labs/eigenda/disperser/common/inmem" dmock "github.com/Layr-Labs/eigenda/disperser/mock" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/assert" @@ -43,7 +43,7 @@ type batcherComponents struct { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { - config := kzgEncoder.KzgConfig{ + config := kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point", G2Path: "../../inabox/resources/kzg/g2.point", CacheDir: "../../inabox/resources/kzg/SRSTables", diff --git a/disperser/encoder/server_test.go b/disperser/encoder/server_test.go index a74aa80701..b91d89ae23 100644 --- a/disperser/encoder/server_test.go +++ b/disperser/encoder/server_test.go @@ -19,7 +19,7 @@ import ( "github.com/Layr-Labs/eigenda/core/encoding" coremock "github.com/Layr-Labs/eigenda/core/mock" pb "github.com/Layr-Labs/eigenda/disperser/api/grpc/encoder" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) @@ -30,7 +30,7 @@ var ( var logger = &cmock.Logger{} func makeTestEncoder(numPoint uint64) (*encoding.Encoder, ServerConfig) { - kzgConfig := kzgEncoder.KzgConfig{ + kzgConfig := kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point", G2Path: "../../inabox/resources/kzg/g2.point", CacheDir: "../../inabox/resources/kzg/SRSTables", diff --git a/encoding/encoding.go b/encoding/encoding.go new file mode 100644 index 0000000000..9651cfce0b --- /dev/null +++ b/encoding/encoding.go @@ -0,0 +1,36 @@ +package encoding + +import ( + "bytes" + "encoding/gob" + + "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" +) + +// Proof is the multireveal proof +// Coeffs is identical to input data converted into Fr element +type Frame struct { + Proof bn254.G1Point + Coeffs []bn254.Fr +} + +func (f *Frame) Encode() ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(f) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func Decode(b []byte) (Frame, error) { + var f Frame + buf := bytes.NewBuffer(b) + dec := gob.NewDecoder(buf) + err := dec.Decode(&f) + if err != nil { + return Frame{}, err + } + return f, nil +} diff --git a/encoding/kzgrs/kzgrs.go b/encoding/kzgrs/kzgrs.go new file mode 100644 index 0000000000..8d8905887c --- /dev/null +++ b/encoding/kzgrs/kzgrs.go @@ -0,0 +1,14 @@ +package kzgrs + +type KzgConfig struct { + G1Path string + G2Path string + G1PowerOf2Path string + G2PowerOf2Path string + CacheDir string + NumWorker uint64 + SRSOrder uint64 // Order is the total size of SRS + SRSNumberToLoad uint64 // Number of points to be loaded from the begining + Verbose bool + PreloadEncoder bool +} diff --git a/pkg/encoding/utils/pointsIO.go b/encoding/kzgrs/pointsIO.go similarity index 75% rename from pkg/encoding/utils/pointsIO.go rename to encoding/kzgrs/pointsIO.go index 45c3f18527..a74e804972 100644 --- a/pkg/encoding/utils/pointsIO.go +++ b/encoding/kzgrs/pointsIO.go @@ -1,10 +1,11 @@ -package utils +package kzgrs import ( "bufio" "fmt" "io" "log" + "math" "os" "time" @@ -36,6 +37,65 @@ func ReadDesiredBytes(reader *bufio.Reader, numBytesToRead uint64) ([]byte, erro return buf, nil } +// Read the n-th G1 point from SRS. +func ReadG1Point(n uint64, g *KzgConfig) (bls.G1Point, error) { + if n > g.SRSOrder { + return bls.G1Point{}, fmt.Errorf("requested power %v is larger than SRSOrder %v", n, g.SRSOrder) + } + + g1point, err := ReadG1PointSection(g.G1Path, n, n+1, 1) + if err != nil { + return bls.G1Point{}, err + } + + return g1point[0], nil +} + +// Read the n-th G2 point from SRS. +func ReadG2Point(n uint64, g *KzgConfig) (bls.G2Point, error) { + if n > g.SRSOrder { + return bls.G2Point{}, fmt.Errorf("requested power %v is larger than SRSOrder %v", n, g.SRSOrder) + } + + g2point, err := ReadG2PointSection(g.G2Path, n, n+1, 1) + if err != nil { + return bls.G2Point{}, err + } + return g2point[0], nil +} + +// Read g2 points from power of 2 file +func ReadG2PointOnPowerOf2(exponent uint64, g *KzgConfig) (bls.G2Point, error) { + + // the powerOf2 file, only [tau^exp] are stored. + // exponent 0, 1, 2, , .. + // actual pow [tau],[tau^2],[tau^4],.. (stored in the file) + // In our convention SRSOrder contains the total number of series of g1, g2 starting with generator + // i.e. [1] [tau] [tau^2].. + // So the actual power of tau is SRSOrder - 1 + // The mainnet SRS, the max power is 2^28-1, so the last power in powerOf2 file is [tau^(2^27)] + // For test case of 3000 SRS, the max power is 2999, so last power in powerOf2 file is [tau^2048] + // if a actual SRS order is 15, the file will contain four symbols (1,2,4,8) with indices [0,1,2,3] + // if a actual SRS order is 16, the file will contain five symbols (1,2,4,8,16) with indices [0,1,2,3,4] + + actualPowerOfTau := g.SRSOrder - 1 + largestPowerofSRS := uint64(math.Log2(float64(actualPowerOfTau))) + if exponent > largestPowerofSRS { + return bls.G2Point{}, fmt.Errorf("requested power %v is larger than largest power of SRS %v", + uint64(math.Pow(2, float64(exponent))), largestPowerofSRS) + } + + if len(g.G2PowerOf2Path) == 0 { + return bls.G2Point{}, fmt.Errorf("G2PathPowerOf2 path is empty") + } + + g2point, err := ReadG2PointSection(g.G2PowerOf2Path, exponent, exponent+1, 1) + if err != nil { + return bls.G2Point{}, err + } + return g2point[0], nil +} + func ReadG1Points(filepath string, n uint64, numWorker uint64) ([]bls.G1Point, error) { g1f, err := os.Open(filepath) if err != nil { @@ -290,7 +350,7 @@ func ReadG2PointSection(filepath string, from, to uint64, numWorker uint64) ([]b }() n := to - from - + g2r := bufio.NewReaderSize(g2f, int(n*G2PointBytes)) _, err = g2f.Seek(int64(from*G2PointBytes), 0) diff --git a/encoding/kzgrs/prover/data/SRSTable/dimE16.coset8 b/encoding/kzgrs/prover/data/SRSTable/dimE16.coset8 new file mode 100644 index 0000000000..4fdd63a2f8 Binary files /dev/null and b/encoding/kzgrs/prover/data/SRSTable/dimE16.coset8 differ diff --git a/encoding/kzgrs/prover/decode.go b/encoding/kzgrs/prover/decode.go new file mode 100644 index 0000000000..6b01e7d599 --- /dev/null +++ b/encoding/kzgrs/prover/decode.go @@ -0,0 +1,15 @@ +package prover + +import ( + enc "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/rs" +) + +func (g *ParametrizedProver) Decode(frames []enc.Frame, indices []uint64, maxInputSize uint64) ([]byte, error) { + rsFrames := make([]rs.Frame, len(frames)) + for ind, frame := range frames { + rsFrames[ind] = rs.Frame{Coeffs: frame.Coeffs} + } + + return g.Encoder.Decode(rsFrames, indices, maxInputSize) +} diff --git a/encoding/kzgrs/prover/decode_test.go b/encoding/kzgrs/prover/decode_test.go new file mode 100644 index 0000000000..5b6a9d9a63 --- /dev/null +++ b/encoding/kzgrs/prover/decode_test.go @@ -0,0 +1,39 @@ +package prover_test + +import ( + "testing" + + enc "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEncodeDecodeFrame_AreInverses(t *testing.T) { + teardownSuite := setupSuite(t) + defer teardownSuite(t) + + group, _ := prover.NewProver(kzgConfig, true) + + params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) + + p, err := group.NewKzgEncoder(params) + + require.Nil(t, err) + require.NotNil(t, p) + + _, _, _, frames, _, err := p.EncodeBytes(GETTYSBURG_ADDRESS_BYTES) + require.Nil(t, err) + require.NotNil(t, frames, err) + + b, err := frames[0].Encode() + require.Nil(t, err) + require.NotNil(t, b) + + frame, err := enc.Decode(b) + require.Nil(t, err) + require.NotNil(t, frame) + + assert.Equal(t, frame, frames[0]) +} diff --git a/encoding/kzgrs/prover/parametrized_prover.go b/encoding/kzgrs/prover/parametrized_prover.go new file mode 100644 index 0000000000..493332c8c2 --- /dev/null +++ b/encoding/kzgrs/prover/parametrized_prover.go @@ -0,0 +1,257 @@ +package prover + +import ( + "fmt" + "log" + "math" + "sync" + "time" + + enc "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/toeplitz" + kzg "github.com/Layr-Labs/eigenda/pkg/kzg" + bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" +) + +type ParametrizedProver struct { + *rs.Encoder + + *kzgrs.KzgConfig + Srs *kzg.SRS + G2Trailing []bls.G2Point + + Fs *kzg.FFTSettings + Ks *kzg.KZGSettings + SFs *kzg.FFTSettings // fft used for submatrix product helper + FFTPoints [][]bls.G1Point + FFTPointsT [][]bls.G1Point // transpose of FFTPoints +} + +type WorkerResult struct { + points []bls.G1Point + err error +} + +// just a wrapper to take bytes not Fr Element +func (g *ParametrizedProver) EncodeBytes(inputBytes []byte) (*bls.G1Point, *bls.G2Point, *bls.G2Point, []enc.Frame, []uint32, error) { + inputFr := rs.ToFrArray(inputBytes) + return g.Encode(inputFr) +} + +func (g *ParametrizedProver) Encode(inputFr []bls.Fr) (*bls.G1Point, *bls.G2Point, *bls.G2Point, []enc.Frame, []uint32, error) { + + startTime := time.Now() + poly, frames, indices, err := g.Encoder.Encode(inputFr) + if err != nil { + return nil, nil, nil, nil, nil, err + } + + if len(poly.Coeffs) > int(g.KzgConfig.SRSNumberToLoad) { + return nil, nil, nil, nil, nil, fmt.Errorf("poly Coeff length %v is greater than Loaded SRS points %v", len(poly.Coeffs), int(g.KzgConfig.SRSNumberToLoad)) + } + + // compute commit for the full poly + commit := g.Commit(poly.Coeffs) + lowDegreeCommitment := bls.LinCombG2(g.Srs.G2[:len(poly.Coeffs)], poly.Coeffs) + + intermediate := time.Now() + + polyDegreePlus1 := uint64(len(inputFr)) + + if g.Verbose { + log.Printf(" Commiting takes %v\n", time.Since(intermediate)) + intermediate = time.Now() + + log.Printf("shift %v\n", g.SRSOrder-polyDegreePlus1) + log.Printf("order %v\n", len(g.Srs.G2)) + log.Println("low degree verification info") + } + + shiftedSecret := g.G2Trailing[g.KzgConfig.SRSNumberToLoad-polyDegreePlus1:] + + //The proof of low degree is commitment of the polynomial shifted to the largest srs degree + lowDegreeProof := bls.LinCombG2(shiftedSecret, poly.Coeffs[:polyDegreePlus1]) + + //fmt.Println("kzgFFT lowDegreeProof", lowDegreeProof, "poly len ", len(fullCoeffsPoly), "order", len(g.Ks.SecretG2) ) + //ok := VerifyLowDegreeProof(&commit, lowDegreeProof, polyDegreePlus1-1, g.SRSOrder, g.Srs.G2) + //if !ok { + // log.Printf("Kzg FFT Cannot Verify low degree proof %v", lowDegreeProof) + // return nil, nil, nil, nil, errors.New("cannot verify low degree proof") + // } else { + // log.Printf("Kzg FFT Verify low degree proof PPPASSS %v", lowDegreeProof) + // } + + if g.Verbose { + log.Printf(" Generating Low Degree Proof takes %v\n", time.Since(intermediate)) + intermediate = time.Now() + } + + // compute proofs + paddedCoeffs := make([]bls.Fr, g.NumEvaluations()) + copy(paddedCoeffs, poly.Coeffs) + + proofs, err := g.ProveAllCosetThreads(paddedCoeffs, g.NumChunks, g.ChunkLen, g.NumWorker) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("could not generate proofs: %v", err) + } + + if g.Verbose { + log.Printf(" Proving takes %v\n", time.Since(intermediate)) + } + + kzgFrames := make([]enc.Frame, len(frames)) + for i, index := range indices { + kzgFrames[i] = enc.Frame{ + Proof: proofs[index], + Coeffs: frames[i].Coeffs, + } + } + + if g.Verbose { + log.Printf("Total encoding took %v\n", time.Since(startTime)) + } + return &commit, lowDegreeCommitment, lowDegreeProof, kzgFrames, indices, nil +} + +func (g *ParametrizedProver) Commit(polyFr []bls.Fr) bls.G1Point { + commit := g.Ks.CommitToPoly(polyFr) + return *commit +} + +func (p *ParametrizedProver) ProveAllCosetThreads(polyFr []bls.Fr, numChunks, chunkLen, numWorker uint64) ([]bls.G1Point, error) { + begin := time.Now() + // Robert: Standardizing this to use the same math used in precomputeSRS + dimE := numChunks + l := chunkLen + + sumVec := make([]bls.G1Point, dimE*2) + + jobChan := make(chan uint64, numWorker) + results := make(chan WorkerResult, numWorker) + + // create storage for intermediate fft outputs + coeffStore := make([][]bls.Fr, dimE*2) + for i := range coeffStore { + coeffStore[i] = make([]bls.Fr, l) + } + + for w := uint64(0); w < numWorker; w++ { + go p.proofWorker(polyFr, jobChan, l, dimE, coeffStore, results) + } + + for j := uint64(0); j < l; j++ { + jobChan <- j + } + close(jobChan) + + // return only first error + var err error + for w := uint64(0); w < numWorker; w++ { + wr := <-results + if wr.err != nil { + err = wr.err + } + } + + if err != nil { + return nil, fmt.Errorf("proof worker error: %v", err) + } + + t0 := time.Now() + + // compute proof by multi scaler mulplication + var wg sync.WaitGroup + for i := uint64(0); i < dimE*2; i++ { + wg.Add(1) + go func(k uint64) { + defer wg.Done() + sumVec[k] = *bls.LinCombG1(p.FFTPointsT[k], coeffStore[k]) + }(i) + } + + wg.Wait() + + t1 := time.Now() + + // only 1 ifft is needed + sumVecInv, err := p.Fs.FFTG1(sumVec, true) + if err != nil { + return nil, fmt.Errorf("fft error: %v", err) + } + + t2 := time.Now() + + // outputs is out of order - buttefly + proofs, err := p.Fs.FFTG1(sumVecInv[:dimE], false) + if err != nil { + return nil, err + } + + t3 := time.Now() + + fmt.Printf("mult-th %v, msm %v,fft1 %v, fft2 %v,\n", t0.Sub(begin), t1.Sub(t0), t2.Sub(t1), t3.Sub(t2)) + + //rb.ReverseBitOrderG1Point(proofs) + return proofs, nil +} + +func (p *ParametrizedProver) proofWorker( + polyFr []bls.Fr, + jobChan <-chan uint64, + l uint64, + dimE uint64, + coeffStore [][]bls.Fr, + results chan<- WorkerResult, +) { + + for j := range jobChan { + coeffs, err := p.GetSlicesCoeff(polyFr, dimE, j, l) + if err != nil { + results <- WorkerResult{ + points: nil, + err: err, + } + } + + for i := 0; i < len(coeffs); i++ { + coeffStore[i][j] = coeffs[i] + } + } + + results <- WorkerResult{ + err: nil, + } +} + +// output is in the form see primeField toeplitz +// +// phi ^ (coset size ) = 1 +// +// implicitly pad slices to power of 2 +func (p *ParametrizedProver) GetSlicesCoeff(polyFr []bls.Fr, dimE, j, l uint64) ([]bls.Fr, error) { + // there is a constant term + m := uint64(len(polyFr)) - 1 + dim := (m - j) / l + + toeV := make([]bls.Fr, 2*dimE-1) + for i := uint64(0); i < dim; i++ { + bls.CopyFr(&toeV[i], &polyFr[m-(j+i*l)]) + } + + // use precompute table + tm, err := toeplitz.NewToeplitz(toeV, p.SFs) + if err != nil { + return nil, err + } + return tm.GetFFTCoeff() +} + +/* +returns the power of 2 which is immediately bigger than the input +*/ +func CeilIntPowerOf2Num(d uint64) uint64 { + nextPower := math.Ceil(math.Log2(float64(d))) + return uint64(math.Pow(2.0, nextPower)) +} diff --git a/pkg/encoding/kzgEncoder/multiprover_test.go b/encoding/kzgrs/prover/parametrized_prover_test.go similarity index 66% rename from pkg/encoding/kzgEncoder/multiprover_test.go rename to encoding/kzgrs/prover/parametrized_prover_test.go index 11b6eefd7f..64a4a6fc4c 100644 --- a/pkg/encoding/kzgEncoder/multiprover_test.go +++ b/encoding/kzgrs/prover/parametrized_prover_test.go @@ -1,11 +1,13 @@ -package kzgEncoder_test +package prover_test import ( "fmt" "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -14,7 +16,7 @@ func TestProveAllCosetThreads(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + group, _ := prover.NewProver(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) @@ -37,8 +39,8 @@ func TestProveAllCosetThreads(t *testing.T) { fmt.Printf("frame %v leading coset %v\n", i, j) lc := enc.Fs.ExpandedRootsOfUnity[uint64(j)] - g2Atn, err := kzgRs.ReadG2Point(uint64(len(f.Coeffs)), kzgConfig) + g2Atn, err := kzgrs.ReadG2Point(uint64(len(f.Coeffs)), kzgConfig) require.Nil(t, err) - assert.True(t, f.Verify(enc.Ks, commit, &lc, &g2Atn), "Proof %v failed\n", i) + assert.True(t, verifier.VerifyFrame(&f, enc.Ks, commit, &lc, &g2Atn), "Proof %v failed\n", i) } } diff --git a/pkg/encoding/kzgEncoder/precomputeSRS.go b/encoding/kzgrs/prover/precompute.go similarity index 97% rename from pkg/encoding/kzgEncoder/precomputeSRS.go rename to encoding/kzgrs/prover/precompute.go index 0d83054264..327ada79a8 100644 --- a/pkg/encoding/kzgEncoder/precomputeSRS.go +++ b/encoding/kzgrs/prover/precompute.go @@ -1,4 +1,4 @@ -package kzgEncoder +package prover import ( "bufio" @@ -13,7 +13,7 @@ import ( "sync" "time" - "github.com/Layr-Labs/eigenda/pkg/encoding/utils" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" kzg "github.com/Layr-Labs/eigenda/pkg/kzg" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) @@ -227,7 +227,7 @@ func (p *SRSTable) TableReaderThreads(filePath string, dimE, l uint64, numWorker }() // 2 due to circular FFT mul - subTableSize := dimE * 2 * utils.G1PointBytes + subTableSize := dimE * 2 * kzgrs.G1PointBytes totalSubTableSize := subTableSize * l if numWorker > l { @@ -281,7 +281,7 @@ func (p *SRSTable) readWorker( for b := range jobChan { slicePoints := make([]bls.G1Point, dimE*2) for i := uint64(0); i < dimE*2; i++ { - g1 := buf[b.start+i*utils.G1PointBytes : b.start+(i+1)*utils.G1PointBytes] + g1 := buf[b.start+i*kzgrs.G1PointBytes : b.start+(i+1)*kzgrs.G1PointBytes] err := slicePoints[i].UnmarshalText(g1[:]) if err != nil { log.Printf("Error. From %v to %v. %v", b.start, b.end, err) diff --git a/pkg/encoding/kzgEncoder/precomputeSRS_test.go b/encoding/kzgrs/prover/precompute_test.go similarity index 66% rename from pkg/encoding/kzgEncoder/precomputeSRS_test.go rename to encoding/kzgrs/prover/precompute_test.go index 5520fdef51..35729bf814 100644 --- a/pkg/encoding/kzgEncoder/precomputeSRS_test.go +++ b/encoding/kzgrs/prover/precompute_test.go @@ -1,4 +1,4 @@ -package kzgEncoder_test +package prover_test import ( "testing" @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" - "github.com/Layr-Labs/eigenda/pkg/encoding/utils" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/rs" ) func TestNewSRSTable_PreComputeWorks(t *testing.T) { @@ -19,14 +19,14 @@ func TestNewSRSTable_PreComputeWorks(t *testing.T) { params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) require.NotNil(t, params) - s1, err := utils.ReadG1Points(kzgConfig.G1Path, kzgConfig.SRSOrder, kzgConfig.NumWorker) + s1, err := kzgrs.ReadG1Points(kzgConfig.G1Path, kzgConfig.SRSOrder, kzgConfig.NumWorker) require.Nil(t, err) require.NotNil(t, s1) - _, err = utils.ReadG2Points(kzgConfig.G2Path, kzgConfig.SRSOrder, kzgConfig.NumWorker) + _, err = kzgrs.ReadG2Points(kzgConfig.G2Path, kzgConfig.SRSOrder, kzgConfig.NumWorker) require.Nil(t, err) - subTable1, err := kzgRs.NewSRSTable(kzgConfig.CacheDir, s1, kzgConfig.NumWorker) + subTable1, err := prover.NewSRSTable(kzgConfig.CacheDir, s1, kzgConfig.NumWorker) require.Nil(t, err) require.NotNil(t, subTable1) @@ -34,7 +34,7 @@ func TestNewSRSTable_PreComputeWorks(t *testing.T) { require.Nil(t, err) require.NotNil(t, fftPoints1) - subTable2, err := kzgRs.NewSRSTable(kzgConfig.CacheDir, s1, kzgConfig.NumWorker) + subTable2, err := prover.NewSRSTable(kzgConfig.CacheDir, s1, kzgConfig.NumWorker) require.Nil(t, err) require.NotNil(t, subTable2) diff --git a/encoding/kzgrs/prover/prover.go b/encoding/kzgrs/prover/prover.go new file mode 100644 index 0000000000..566b1d5311 --- /dev/null +++ b/encoding/kzgrs/prover/prover.go @@ -0,0 +1,280 @@ +package prover + +import ( + "errors" + "fmt" + "log" + "math" + "os" + "runtime" + "strconv" + "strings" + "sync" + + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/rs" + kzg "github.com/Layr-Labs/eigenda/pkg/kzg" + + bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" + _ "go.uber.org/automaxprocs" +) + +type Prover struct { + *kzgrs.KzgConfig + Srs *kzg.SRS + G2Trailing []bls.G2Point + mu sync.Mutex + LoadG2Points bool + + ParametrizedProvers map[rs.EncodingParams]*ParametrizedProver +} + +func NewProver(config *kzgrs.KzgConfig, loadG2Points bool) (*Prover, error) { + + if config.SRSNumberToLoad > config.SRSOrder { + return nil, errors.New("SRSOrder is less than srsNumberToLoad") + } + + // read the whole order, and treat it as entire SRS for low degree proof + s1, err := kzgrs.ReadG1Points(config.G1Path, config.SRSNumberToLoad, config.NumWorker) + if err != nil { + log.Println("failed to read G1 points", err) + return nil, err + } + + s2 := make([]bls.G2Point, 0) + g2Trailing := make([]bls.G2Point, 0) + + // PreloadEncoder is by default not used by operator node, PreloadEncoder + if loadG2Points { + if len(config.G2Path) == 0 { + return nil, fmt.Errorf("G2Path is empty. However, object needs to load G2Points") + } + + s2, err = kzgrs.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker) + if err != nil { + log.Println("failed to read G2 points", err) + return nil, err + } + + g2Trailing, err = kzgrs.ReadG2PointSection( + config.G2Path, + config.SRSOrder-config.SRSNumberToLoad, + config.SRSOrder, // last exclusive + config.NumWorker, + ) + if err != nil { + return nil, err + } + } else { + // todo, there are better ways to handle it + if len(config.G2PowerOf2Path) == 0 { + return nil, fmt.Errorf("G2PowerOf2Path is empty. However, object needs to load G2Points") + } + } + + srs, err := kzg.NewSrs(s1, s2) + if err != nil { + log.Println("Could not create srs", err) + return nil, err + } + + fmt.Println("numthread", runtime.GOMAXPROCS(0)) + + encoderGroup := &Prover{ + KzgConfig: config, + Srs: srs, + G2Trailing: g2Trailing, + ParametrizedProvers: make(map[rs.EncodingParams]*ParametrizedProver), + LoadG2Points: loadG2Points, + } + + if config.PreloadEncoder { + // create table dir if not exist + err := os.MkdirAll(config.CacheDir, os.ModePerm) + if err != nil { + log.Println("Cannot make CacheDir", err) + return nil, err + } + + err = encoderGroup.PreloadAllEncoders() + if err != nil { + return nil, err + } + } + + return encoderGroup, nil + +} + +func (g *Prover) PreloadAllEncoders() error { + paramsAll, err := GetAllPrecomputedSrsMap(g.CacheDir) + if err != nil { + return nil + } + fmt.Printf("detect %v srs maps\n", len(paramsAll)) + for i := 0; i < len(paramsAll); i++ { + fmt.Printf(" %v. NumChunks: %v ChunkLen: %v\n", i, paramsAll[i].NumChunks, paramsAll[i].ChunkLen) + } + + if len(paramsAll) == 0 { + return nil + } + + for _, params := range paramsAll { + // get those encoders and store them + enc, err := g.GetKzgEncoder(params) + if err != nil { + return err + } + g.ParametrizedProvers[params] = enc + } + + return nil +} + +func (g *Prover) GetKzgEncoder(params rs.EncodingParams) (*ParametrizedProver, error) { + g.mu.Lock() + defer g.mu.Unlock() + enc, ok := g.ParametrizedProvers[params] + if ok { + return enc, nil + } + + enc, err := g.newKzgEncoder(params) + if err == nil { + g.ParametrizedProvers[params] = enc + } + + return enc, err +} + +func (g *Prover) NewKzgEncoder(params rs.EncodingParams) (*ParametrizedProver, error) { + g.mu.Lock() + defer g.mu.Unlock() + + return g.newKzgEncoder(params) +} + +func (g *Prover) newKzgEncoder(params rs.EncodingParams) (*ParametrizedProver, error) { + + // Check that the parameters are valid with respect to the SRS. + if params.ChunkLen*params.NumChunks >= g.SRSOrder { + return nil, fmt.Errorf("the supplied encoding parameters are not valid with respect to the SRS. ChunkLength: %d, NumChunks: %d, SRSOrder: %d", params.ChunkLen, params.NumChunks, g.SRSOrder) + } + + encoder, err := rs.NewEncoder(params, g.Verbose) + if err != nil { + log.Println("Could not create encoder: ", err) + return nil, err + } + + subTable, err := NewSRSTable(g.CacheDir, g.Srs.G1, g.NumWorker) + if err != nil { + log.Println("Could not create srs table:", err) + return nil, err + } + + fftPoints, err := subTable.GetSubTables(encoder.NumChunks, encoder.ChunkLen) + if err != nil { + log.Println("could not get sub tables", err) + return nil, err + } + + fftPointsT := make([][]bls.G1Point, len(fftPoints[0])) + for i := range fftPointsT { + fftPointsT[i] = make([]bls.G1Point, len(fftPoints)) + for j := uint64(0); j < encoder.ChunkLen; j++ { + fftPointsT[i][j] = fftPoints[j][i] + } + } + n := uint8(math.Log2(float64(encoder.NumEvaluations()))) + if encoder.ChunkLen == 1 { + n = uint8(math.Log2(float64(2 * encoder.NumChunks))) + } + fs := kzg.NewFFTSettings(n) + + ks, err := kzg.NewKZGSettings(fs, g.Srs) + if err != nil { + return nil, err + } + + t := uint8(math.Log2(float64(2 * encoder.NumChunks))) + sfs := kzg.NewFFTSettings(t) + + return &ParametrizedProver{ + Encoder: encoder, + KzgConfig: g.KzgConfig, + Srs: g.Srs, + G2Trailing: g.G2Trailing, + Fs: fs, + Ks: ks, + SFs: sfs, + FFTPoints: fftPoints, + FFTPointsT: fftPointsT, + }, nil +} + +// get Fiat-Shamir challenge +// func createFiatShamirChallenge(byteArray [][32]byte) *bls.Fr { +// alphaBytesTmp := make([]byte, 0) +// for i := 0; i < len(byteArray); i++ { +// for j := 0; j < len(byteArray[i]); j++ { +// alphaBytesTmp = append(alphaBytesTmp, byteArray[i][j]) +// } +// } +// alphaBytes := crypto.Keccak256(alphaBytesTmp) +// alpha := new(bls.Fr) +// bls.FrSetBytes(alpha, alphaBytes) +// +// return alpha +// } + +// invert the divisor, then multiply +// func polyFactorDiv(dst *bls.Fr, a *bls.Fr, b *bls.Fr) { +// // TODO: use divmod instead. +// var tmp bls.Fr +// bls.InvModFr(&tmp, b) +// bls.MulModFr(dst, &tmp, a) +// } + +// Detect the precomputed table from the specified directory +// the file name follow the name convention of +// +// dimE*.coset& +// +// where the first * specifies the dimension of the matrix which +// equals to the number of chunks +// where the second & specifies the length of each chunk +func GetAllPrecomputedSrsMap(tableDir string) ([]rs.EncodingParams, error) { + files, err := os.ReadDir(tableDir) + if err != nil { + log.Println("Error to list SRS Table directory", err) + return nil, err + } + + tables := make([]rs.EncodingParams, 0) + for _, file := range files { + filename := file.Name() + + tokens := strings.Split(filename, ".") + + dimEValue, err := strconv.Atoi(tokens[0][4:]) + if err != nil { + log.Println("Error to parse Dimension part of the Table", err) + return nil, err + } + cosetSizeValue, err := strconv.Atoi(tokens[1][5:]) + if err != nil { + log.Println("Error to parse Coset size of the Table", err) + return nil, err + } + + params := rs.EncodingParams{ + NumChunks: uint64(cosetSizeValue), + ChunkLen: uint64(dimEValue), + } + tables = append(tables, params) + } + return tables, nil +} diff --git a/pkg/encoding/kzgEncoder/encoder_fuzz_test.go b/encoding/kzgrs/prover/prover_fuzz_test.go similarity index 82% rename from pkg/encoding/kzgEncoder/encoder_fuzz_test.go rename to encoding/kzgrs/prover/prover_fuzz_test.go index 9551b0c526..2848a5765c 100644 --- a/pkg/encoding/kzgEncoder/encoder_fuzz_test.go +++ b/encoding/kzgrs/prover/prover_fuzz_test.go @@ -1,10 +1,10 @@ -package kzgEncoder_test +package prover_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" ) @@ -13,7 +13,7 @@ func FuzzOnlySystematic(f *testing.F) { f.Add(GETTYSBURG_ADDRESS_BYTES) f.Fuzz(func(t *testing.T, input []byte) { - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + group, _ := prover.NewProver(kzgConfig, true) params := rs.GetEncodingParams(10, 3, uint64(len(input))) enc, err := group.NewKzgEncoder(params) diff --git a/pkg/encoding/kzgEncoder/encoder_suite_test.go b/encoding/kzgrs/prover/prover_test.go similarity index 89% rename from pkg/encoding/kzgEncoder/encoder_suite_test.go rename to encoding/kzgrs/prover/prover_test.go index 98666f9ac7..76935ef4dd 100644 --- a/pkg/encoding/kzgEncoder/encoder_suite_test.go +++ b/encoding/kzgrs/prover/prover_test.go @@ -1,4 +1,4 @@ -package kzgEncoder_test +package prover_test import ( "log" @@ -7,7 +7,8 @@ import ( "runtime" "testing" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + enc "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" ) const ( @@ -16,7 +17,7 @@ const ( var ( GETTYSBURG_ADDRESS_BYTES = []byte("Fourscore and seven years ago our fathers brought forth, on this continent, a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived, and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting-place for those who here gave their lives, that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we cannot dedicate, we cannot consecrate—we cannot hallow—this ground. The brave men, living and dead, who struggled here, have consecrated it far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they here gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom, and that government of the people, by the people, for the people, shall not perish from the earth.") - kzgConfig *kzgRs.KzgConfig + kzgConfig *kzgrs.KzgConfig numNode uint64 numSys uint64 numPar uint64 @@ -25,7 +26,7 @@ var ( func setupSuite(t *testing.T) func(t *testing.T) { log.Println("Setting up suite") - kzgConfig = &kzgRs.KzgConfig{ + kzgConfig = &kzgrs.KzgConfig{ G1Path: "../../../inabox/resources/kzg/g1.point", G2Path: "../../../inabox/resources/kzg/g2.point", G2PowerOf2Path: "../../../inabox/resources/kzg/g2.point.powerOf2", @@ -47,8 +48,8 @@ func setupSuite(t *testing.T) func(t *testing.T) { } } -func sampleFrames(frames []kzgRs.Frame, num uint64) ([]kzgRs.Frame, []uint64) { - samples := make([]kzgRs.Frame, num) +func sampleFrames(frames []enc.Frame, num uint64) ([]enc.Frame, []uint64) { + samples := make([]enc.Frame, num) indices := rand.Perm(len(frames)) indices = indices[:num] diff --git a/pkg/encoding/kzgEncoder/batchCommitEquivalence.go b/encoding/kzgrs/verifier/batch_commit_equivalence.go similarity index 94% rename from pkg/encoding/kzgEncoder/batchCommitEquivalence.go rename to encoding/kzgrs/verifier/batch_commit_equivalence.go index d1fa88316c..4101d428d3 100644 --- a/pkg/encoding/kzgEncoder/batchCommitEquivalence.go +++ b/encoding/kzgrs/verifier/batch_commit_equivalence.go @@ -1,4 +1,4 @@ -package kzgEncoder +package verifier import ( "bytes" @@ -66,7 +66,7 @@ func CreateRandomnessVector(g1commits []bn254.G1Point, g2commits []bn254.G2Point return randomsFr, nil } -func (group *KzgEncoderGroup) BatchVerifyCommitEquivalence(commitmentsPair []CommitmentPair) error { +func (group *Verifier) BatchVerifyCommitEquivalence(commitmentsPair []CommitmentPair) error { g1commits := make([]bn254.G1Point, len(commitmentsPair)) g2commits := make([]bn254.G2Point, len(commitmentsPair)) diff --git a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go b/encoding/kzgrs/verifier/batch_commit_equivalence_test.go similarity index 57% rename from pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go rename to encoding/kzgrs/verifier/batch_commit_equivalence_test.go index 3f8834f649..58e57d90b7 100644 --- a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go +++ b/encoding/kzgrs/verifier/batch_commit_equivalence_test.go @@ -1,10 +1,11 @@ -package kzgEncoder_test +package verifier_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,7 +15,8 @@ func TestBatchEquivalence(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + group, _ := prover.NewProver(kzgConfig, true) + v, _ := verifier.NewVerifier(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) @@ -24,29 +26,29 @@ func TestBatchEquivalence(t *testing.T) { require.Nil(t, err) numBlob := 5 - commitPairs := make([]kzgRs.CommitmentPair, numBlob) + commitPairs := make([]verifier.CommitmentPair, numBlob) for z := 0; z < numBlob; z++ { - commitPairs[z] = kzgRs.CommitmentPair{ + commitPairs[z] = verifier.CommitmentPair{ Commitment: *commit, LengthCommitment: *g2commit, } } - assert.NoError(t, group.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed\n") + assert.NoError(t, v.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed\n") var modifiedCommit bn254.G1Point bn254.AddG1(&modifiedCommit, commit, commit) for z := 0; z < numBlob; z++ { - commitPairs[z] = kzgRs.CommitmentPair{ + commitPairs[z] = verifier.CommitmentPair{ Commitment: modifiedCommit, LengthCommitment: *g2commit, } } - assert.Error(t, group.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed\n") + assert.Error(t, v.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed\n") for z := 0; z < numBlob; z++ { - commitPairs[z] = kzgRs.CommitmentPair{ + commitPairs[z] = verifier.CommitmentPair{ Commitment: *commit, LengthCommitment: *g2commit, } @@ -54,5 +56,5 @@ func TestBatchEquivalence(t *testing.T) { bn254.AddG1(&commitPairs[numBlob/2].Commitment, &commitPairs[numBlob/2].Commitment, &commitPairs[numBlob/2].Commitment) - assert.Error(t, group.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed in outer loop\n") + assert.Error(t, v.BatchVerifyCommitEquivalence(commitPairs), "batch equivalence negative test failed in outer loop\n") } diff --git a/pkg/encoding/kzgEncoder/degree_test.go b/encoding/kzgrs/verifier/degree_test.go similarity index 56% rename from pkg/encoding/kzgEncoder/degree_test.go rename to encoding/kzgrs/verifier/degree_test.go index 6454a821d1..052bb771ef 100644 --- a/pkg/encoding/kzgEncoder/degree_test.go +++ b/encoding/kzgrs/verifier/degree_test.go @@ -1,10 +1,11 @@ -package kzgEncoder_test +package verifier_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,7 +14,8 @@ func TestLengthProof(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + group, _ := prover.NewProver(kzgConfig, true) + v, _ := verifier.NewVerifier(kzgConfig, true) params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) @@ -28,9 +30,9 @@ func TestLengthProof(t *testing.T) { require.Nil(t, err) length := len(inputFr) - assert.NoError(t, group.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)), "low degree verification failed\n") + assert.NoError(t, v.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)), "low degree verification failed\n") length = len(inputFr) - 10 - assert.Error(t, group.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)), "low degree verification failed\n") + assert.Error(t, v.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)), "low degree verification failed\n") } } diff --git a/encoding/kzgrs/verifier/frame_test.go b/encoding/kzgrs/verifier/frame_test.go new file mode 100644 index 0000000000..c75ff38e14 --- /dev/null +++ b/encoding/kzgrs/verifier/frame_test.go @@ -0,0 +1,44 @@ +package verifier_test + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" + kzg "github.com/Layr-Labs/eigenda/pkg/kzg" +) + +func TestVerify(t *testing.T) { + teardownSuite := setupSuite(t) + defer teardownSuite(t) + + group, _ := prover.NewProver(kzgConfig, true) + + params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) + + enc, err := group.NewKzgEncoder(params) + require.Nil(t, err) + require.NotNil(t, enc) + + commit, _, _, frames, _, err := enc.EncodeBytes(GETTYSBURG_ADDRESS_BYTES) + require.Nil(t, err) + require.NotNil(t, commit) + require.NotNil(t, frames) + + n := uint8(math.Log2(float64(params.NumEvaluations()))) + fs := kzg.NewFFTSettings(n) + require.NotNil(t, fs) + + lc := enc.Fs.ExpandedRootsOfUnity[uint64(0)] + require.NotNil(t, lc) + + g2Atn, err := kzgrs.ReadG2Point(uint64(len(frames[0].Coeffs)), kzgConfig) + require.Nil(t, err) + assert.True(t, verifier.VerifyFrame(&frames[0], enc.Ks, commit, &lc, &g2Atn)) +} diff --git a/pkg/encoding/kzgEncoder/multiframe.go b/encoding/kzgrs/verifier/multiframe.go similarity index 95% rename from pkg/encoding/kzgEncoder/multiframe.go rename to encoding/kzgrs/verifier/multiframe.go index 7adbc88f4b..f14649198b 100644 --- a/pkg/encoding/kzgEncoder/multiframe.go +++ b/encoding/kzgrs/verifier/multiframe.go @@ -1,4 +1,4 @@ -package kzgEncoder +package verifier import ( "bytes" @@ -7,7 +7,8 @@ import ( "fmt" "math" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/rs" kzg "github.com/Layr-Labs/eigenda/pkg/kzg" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) @@ -163,7 +164,7 @@ func genRhsG1(samples []Sample, randomsFr []bls.Fr, m int, params rs.EncodingPar // // The order of samples do not matter. // Each sample need not have unique row, it is possible that multiple chunks of the same blob are validated altogether -func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples []Sample, m int) error { +func (group *Verifier) UniversalVerify(params rs.EncodingParams, samples []Sample, m int) error { // precheck for i, s := range samples { if s.RowIndex >= m { @@ -204,11 +205,11 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples // lhs g2 exponent := uint64(math.Log2(float64(D))) - G2atD, err := ReadG2PointOnPowerOf2(exponent, group.KzgConfig) + G2atD, err := kzgrs.ReadG2PointOnPowerOf2(exponent, group.KzgConfig) if err != nil { // then try to access if there is a full list of g2 srs - G2atD, err = ReadG2Point(D, group.KzgConfig) + G2atD, err = kzgrs.ReadG2Point(D, group.KzgConfig) if err != nil { return err } diff --git a/pkg/encoding/kzgEncoder/multiframe_test.go b/encoding/kzgrs/verifier/multiframe_test.go similarity index 68% rename from pkg/encoding/kzgEncoder/multiframe_test.go rename to encoding/kzgrs/verifier/multiframe_test.go index 6839cfbc7d..ca39c241bb 100644 --- a/pkg/encoding/kzgEncoder/multiframe_test.go +++ b/encoding/kzgrs/verifier/multiframe_test.go @@ -1,10 +1,11 @@ -package kzgEncoder_test +package verifier_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,13 +14,15 @@ func TestUniversalVerify(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + group, _ := prover.NewProver(kzgConfig, true) + v, _ := verifier.NewVerifier(kzgConfig, true) + params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) require.Nil(t, err) numBlob := 5 - samples := make([]kzgRs.Sample, 0) + samples := make([]verifier.Sample, 0) for z := 0; z < numBlob; z++ { inputFr := rs.ToFrArray(GETTYSBURG_ADDRESS_BYTES) @@ -36,7 +39,7 @@ func TestUniversalVerify(t *testing.T) { assert.Equal(t, j, q, "leading coset inconsistency") - sample := kzgRs.Sample{ + sample := verifier.Sample{ Commitment: *commit, Proof: f.Proof, RowIndex: z, @@ -47,21 +50,27 @@ func TestUniversalVerify(t *testing.T) { } } - assert.True(t, group.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n") + assert.True(t, v.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n") } func TestUniversalVerifyWithPowerOf2G2(t *testing.T) { teardownSuite := setupSuite(t) defer teardownSuite(t) - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + kzgConfigCopy := *kzgConfig + group, err := prover.NewProver(&kzgConfigCopy, true) + assert.NoError(t, err) group.KzgConfig.G2Path = "" + + v, err := verifier.NewVerifier(kzgConfig, true) + assert.NoError(t, err) + params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) enc, err := group.NewKzgEncoder(params) - require.Nil(t, err) + assert.NoError(t, err) numBlob := 5 - samples := make([]kzgRs.Sample, 0) + samples := make([]verifier.Sample, 0) for z := 0; z < numBlob; z++ { inputFr := rs.ToFrArray(GETTYSBURG_ADDRESS_BYTES) @@ -78,7 +87,7 @@ func TestUniversalVerifyWithPowerOf2G2(t *testing.T) { assert.Equal(t, j, q, "leading coset inconsistency") - sample := kzgRs.Sample{ + sample := verifier.Sample{ Commitment: *commit, Proof: f.Proof, RowIndex: z, @@ -89,5 +98,5 @@ func TestUniversalVerifyWithPowerOf2G2(t *testing.T) { } } - assert.True(t, group.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n") + assert.True(t, v.UniversalVerify(params, samples, numBlob) == nil, "universal batch verification failed\n") } diff --git a/encoding/kzgrs/verifier/verifier.go b/encoding/kzgrs/verifier/verifier.go new file mode 100644 index 0000000000..992efa2d13 --- /dev/null +++ b/encoding/kzgrs/verifier/verifier.go @@ -0,0 +1,237 @@ +package verifier + +import ( + "errors" + "fmt" + "log" + "math" + "runtime" + "sync" + + enc "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/rs" + kzg "github.com/Layr-Labs/eigenda/pkg/kzg" + bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" + wbls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" +) + +type Verifier struct { + *kzgrs.KzgConfig + Srs *kzg.SRS + G2Trailing []bls.G2Point + mu sync.Mutex + LoadG2Points bool + + ParametrizedVerifiers map[rs.EncodingParams]*ParametrizedVerifier +} + +func NewVerifier(config *kzgrs.KzgConfig, loadG2Points bool) (*Verifier, error) { + + if config.SRSNumberToLoad > config.SRSOrder { + return nil, errors.New("SRSOrder is less than srsNumberToLoad") + } + + // read the whole order, and treat it as entire SRS for low degree proof + s1, err := kzgrs.ReadG1Points(config.G1Path, config.SRSNumberToLoad, config.NumWorker) + if err != nil { + log.Println("failed to read G1 points", err) + return nil, err + } + + s2 := make([]bls.G2Point, 0) + g2Trailing := make([]bls.G2Point, 0) + + // PreloadEncoder is by default not used by operator node, PreloadEncoder + if loadG2Points { + if len(config.G2Path) == 0 { + return nil, fmt.Errorf("G2Path is empty. However, object needs to load G2Points") + } + + s2, err = kzgrs.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker) + if err != nil { + log.Println("failed to read G2 points", err) + return nil, err + } + + g2Trailing, err = kzgrs.ReadG2PointSection( + config.G2Path, + config.SRSOrder-config.SRSNumberToLoad, + config.SRSOrder, // last exclusive + config.NumWorker, + ) + if err != nil { + return nil, err + } + } else { + // todo, there are better ways to handle it + if len(config.G2PowerOf2Path) == 0 { + return nil, fmt.Errorf("G2PowerOf2Path is empty. However, object needs to load G2Points") + } + } + + srs, err := kzg.NewSrs(s1, s2) + if err != nil { + log.Println("Could not create srs", err) + return nil, err + } + + fmt.Println("numthread", runtime.GOMAXPROCS(0)) + + encoderGroup := &Verifier{ + KzgConfig: config, + Srs: srs, + G2Trailing: g2Trailing, + ParametrizedVerifiers: make(map[rs.EncodingParams]*ParametrizedVerifier), + LoadG2Points: loadG2Points, + } + + return encoderGroup, nil + +} + +type ParametrizedVerifier struct { + *kzgrs.KzgConfig + Srs *kzg.SRS + + rs.EncodingParams + + Fs *kzg.FFTSettings + Ks *kzg.KZGSettings +} + +func (g *Verifier) GetKzgVerifier(params rs.EncodingParams) (*ParametrizedVerifier, error) { + g.mu.Lock() + defer g.mu.Unlock() + + if err := params.Validate(); err != nil { + return nil, err + } + + ver, ok := g.ParametrizedVerifiers[params] + if ok { + return ver, nil + } + + ver, err := g.newKzgVerifier(params) + if err == nil { + g.ParametrizedVerifiers[params] = ver + } + + return ver, err +} + +func (g *Verifier) NewKzgVerifier(params rs.EncodingParams) (*ParametrizedVerifier, error) { + g.mu.Lock() + defer g.mu.Unlock() + return g.newKzgVerifier(params) +} + +func (g *Verifier) newKzgVerifier(params rs.EncodingParams) (*ParametrizedVerifier, error) { + + if err := params.Validate(); err != nil { + return nil, err + } + + n := uint8(math.Log2(float64(params.NumEvaluations()))) + fs := kzg.NewFFTSettings(n) + ks, err := kzg.NewKZGSettings(fs, g.Srs) + + if err != nil { + return nil, err + } + + return &ParametrizedVerifier{ + KzgConfig: g.KzgConfig, + Srs: g.Srs, + EncodingParams: params, + Fs: fs, + Ks: ks, + }, nil +} + +// VerifyCommit verifies the low degree proof; since it doesn't depend on the encoding parameters +// we leave it as a method of the KzgEncoderGroup +func (v *Verifier) VerifyCommit(lengthCommit *wbls.G2Point, lowDegreeProof *wbls.G2Point, length uint64) error { + + g1Challenge, err := kzgrs.ReadG1Point(v.SRSOrder-length, v.KzgConfig) + if err != nil { + return err + } + + if !VerifyLowDegreeProof(lengthCommit, lowDegreeProof, &g1Challenge) { + return errors.New("low degree proof fails") + } + return nil + +} + +// The function verify low degree proof against a poly commitment +// We wish to show x^shift poly = shiftedPoly, with +// With shift = SRSOrder-1 - claimedDegree and +// proof = commit(shiftedPoly) on G1 +// so we can verify by checking +// e( commit_1, [x^shift]_2) = e( proof_1, G_2 ) +func VerifyLowDegreeProof(lengthCommit *bls.G2Point, proof *bls.G2Point, g1Challenge *bls.G1Point) bool { + return bls.PairingsVerify(g1Challenge, lengthCommit, &bls.GenG1, proof) +} + +func (v *ParametrizedVerifier) VerifyFrame(commit *wbls.G1Point, f *enc.Frame, index uint64) error { + + j, err := rs.GetLeadingCosetIndex( + uint64(index), + v.NumChunks, + ) + if err != nil { + return err + } + + g2Atn, err := kzgrs.ReadG2Point(uint64(len(f.Coeffs)), v.KzgConfig) + if err != nil { + return err + } + + if !VerifyFrame(f, v.Ks, commit, &v.Ks.ExpandedRootsOfUnity[j], &g2Atn) { + return errors.New("multireveal proof fails") + } + + return nil + +} + +// Verify function assumes the Data stored is coefficients of coset's interpolating poly +func VerifyFrame(f *enc.Frame, ks *kzg.KZGSettings, commitment *bls.G1Point, x *bls.Fr, g2Atn *bls.G2Point) bool { + var xPow bls.Fr + bls.CopyFr(&xPow, &bls.ONE) + + var tmp bls.Fr + for i := 0; i < len(f.Coeffs); i++ { + bls.MulModFr(&tmp, &xPow, x) + bls.CopyFr(&xPow, &tmp) + } + + // [x^n]_2 + var xn2 bls.G2Point + bls.MulG2(&xn2, &bls.GenG2, &xPow) + + // [s^n - x^n]_2 + var xnMinusYn bls.G2Point + + bls.SubG2(&xnMinusYn, g2Atn, &xn2) + + // [interpolation_polynomial(s)]_1 + is1 := bls.LinCombG1(ks.Srs.G1[:len(f.Coeffs)], f.Coeffs) + + // [commitment - interpolation_polynomial(s)]_1 = [commit]_1 - [interpolation_polynomial(s)]_1 + var commitMinusInterpolation bls.G1Point + bls.SubG1(&commitMinusInterpolation, commitment, is1) + + // Verify the pairing equation + // + // e([commitment - interpolation_polynomial(s)], [1]) = e([proof], [s^n - x^n]) + // equivalent to + // e([commitment - interpolation_polynomial]^(-1), [1]) * e([proof], [s^n - x^n]) = 1_T + // + + return bls.PairingsVerify(&commitMinusInterpolation, &bls.GenG2, &f.Proof, &xnMinusYn) +} diff --git a/encoding/kzgrs/verifier/verifier_test.go b/encoding/kzgrs/verifier/verifier_test.go new file mode 100644 index 0000000000..f7b3387db4 --- /dev/null +++ b/encoding/kzgrs/verifier/verifier_test.go @@ -0,0 +1,47 @@ +package verifier_test + +import ( + "log" + "os" + "runtime" + "testing" + + "github.com/Layr-Labs/eigenda/encoding/kzgrs" +) + +const ( + BYTES_PER_COEFFICIENT = 31 +) + +var ( + GETTYSBURG_ADDRESS_BYTES = []byte("Fourscore and seven years ago our fathers brought forth, on this continent, a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived, and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting-place for those who here gave their lives, that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we cannot dedicate, we cannot consecrate—we cannot hallow—this ground. The brave men, living and dead, who struggled here, have consecrated it far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they here gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom, and that government of the people, by the people, for the people, shall not perish from the earth.") + kzgConfig *kzgrs.KzgConfig + numNode uint64 + numSys uint64 + numPar uint64 +) + +func setupSuite(t *testing.T) func(t *testing.T) { + log.Println("Setting up suite") + + kzgConfig = &kzgrs.KzgConfig{ + G1Path: "../../../inabox/resources/kzg/g1.point", + G2Path: "../../../inabox/resources/kzg/g2.point", + G2PowerOf2Path: "../../../inabox/resources/kzg/g2.point.powerOf2", + CacheDir: "../../../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, + SRSNumberToLoad: 2900, + NumWorker: uint64(runtime.GOMAXPROCS(0)), + } + + numNode = uint64(4) + numSys = uint64(3) + numPar = numNode - numSys + + return func(t *testing.T) { + log.Println("Tearing down suite") + + // Some test may want to create a new SRS table so this should clean it up. + os.RemoveAll("./data") + } +} diff --git a/pkg/encoding/encoder/decode.go b/encoding/rs/decode.go similarity index 99% rename from pkg/encoding/encoder/decode.go rename to encoding/rs/decode.go index 7eefa707cc..fef231aae6 100644 --- a/pkg/encoding/encoder/decode.go +++ b/encoding/rs/decode.go @@ -1,4 +1,4 @@ -package encoder +package rs import ( "errors" diff --git a/pkg/encoding/encoder/encode.go b/encoding/rs/encode.go similarity index 97% rename from pkg/encoding/encoder/encode.go rename to encoding/rs/encode.go index a93c72c8ec..3bc05145bd 100644 --- a/pkg/encoding/encoder/encode.go +++ b/encoding/rs/encode.go @@ -1,11 +1,11 @@ -package encoder +package rs import ( "fmt" "log" "time" - rb "github.com/Layr-Labs/eigenda/pkg/encoding/utils/reverseBits" + rb "github.com/Layr-Labs/eigenda/encoding/utils/reverseBits" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) diff --git a/pkg/encoding/encoder/encode_test.go b/encoding/rs/encode_test.go similarity index 96% rename from pkg/encoding/encoder/encode_test.go rename to encoding/rs/encode_test.go index c6e060f90a..78da01925c 100644 --- a/pkg/encoding/encoder/encode_test.go +++ b/encoding/rs/encode_test.go @@ -1,4 +1,4 @@ -package encoder_test +package rs_test import ( "fmt" @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/rs" ) func TestEncodeDecode_InvertsWhenSamplingAllFrames(t *testing.T) { diff --git a/pkg/encoding/encoder/encoder.go b/encoding/rs/encoder.go similarity index 98% rename from pkg/encoding/encoder/encoder.go rename to encoding/rs/encoder.go index ca29f4a567..2d5ec57ca3 100644 --- a/pkg/encoding/encoder/encoder.go +++ b/encoding/rs/encoder.go @@ -1,4 +1,4 @@ -package encoder +package rs import ( "math" diff --git a/pkg/encoding/encoder/encoder_fuzz_test.go b/encoding/rs/encoder_fuzz_test.go similarity index 91% rename from pkg/encoding/encoder/encoder_fuzz_test.go rename to encoding/rs/encoder_fuzz_test.go index 3ee377c432..fd5840893a 100644 --- a/pkg/encoding/encoder/encoder_fuzz_test.go +++ b/encoding/rs/encoder_fuzz_test.go @@ -1,9 +1,9 @@ -package encoder_test +package rs_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" ) diff --git a/pkg/encoding/encoder/encoder_suite_test.go b/encoding/rs/encoder_suite_test.go similarity index 96% rename from pkg/encoding/encoder/encoder_suite_test.go rename to encoding/rs/encoder_suite_test.go index c5e15e2d86..16fdac6fa2 100644 --- a/pkg/encoding/encoder/encoder_suite_test.go +++ b/encoding/rs/encoder_suite_test.go @@ -1,4 +1,4 @@ -package encoder_test +package rs_test import ( "log" @@ -6,7 +6,7 @@ import ( "os" "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/rs" ) const ( diff --git a/pkg/encoding/encoder/frame.go b/encoding/rs/frame.go similarity index 97% rename from pkg/encoding/encoder/frame.go rename to encoding/rs/frame.go index ea8a64d960..05b7aff1e5 100644 --- a/pkg/encoding/encoder/frame.go +++ b/encoding/rs/frame.go @@ -1,4 +1,4 @@ -package encoder +package rs import ( "bytes" diff --git a/pkg/encoding/encoder/frame_test.go b/encoding/rs/frame_test.go similarity index 89% rename from pkg/encoding/encoder/frame_test.go rename to encoding/rs/frame_test.go index c76576e60e..7e21232007 100644 --- a/pkg/encoding/encoder/frame_test.go +++ b/encoding/rs/frame_test.go @@ -1,9 +1,9 @@ -package encoder_test +package rs_test import ( "testing" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/pkg/encoding/encoder/interpolation.go b/encoding/rs/interpolation.go similarity index 99% rename from pkg/encoding/encoder/interpolation.go rename to encoding/rs/interpolation.go index 8ee54d4c56..c206e0efec 100644 --- a/pkg/encoding/encoder/interpolation.go +++ b/encoding/rs/interpolation.go @@ -1,4 +1,4 @@ -package encoder +package rs import ( bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" diff --git a/pkg/encoding/encoder/params.go b/encoding/rs/params.go similarity index 98% rename from pkg/encoding/encoder/params.go rename to encoding/rs/params.go index 908d2bad5d..d3917e19a8 100644 --- a/pkg/encoding/encoder/params.go +++ b/encoding/rs/params.go @@ -1,4 +1,4 @@ -package encoder +package rs import ( "errors" diff --git a/pkg/encoding/encoder/utils.go b/encoding/rs/utils.go similarity index 95% rename from pkg/encoding/encoder/utils.go rename to encoding/rs/utils.go index 55116a209a..5d0306961d 100644 --- a/pkg/encoding/encoder/utils.go +++ b/encoding/rs/utils.go @@ -1,10 +1,10 @@ -package encoder +package rs import ( "errors" "math" - rb "github.com/Layr-Labs/eigenda/pkg/encoding/utils/reverseBits" + rb "github.com/Layr-Labs/eigenda/encoding/utils/reverseBits" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) diff --git a/pkg/encoding/encoder/utils_test.go b/encoding/rs/utils_test.go similarity index 94% rename from pkg/encoding/encoder/utils_test.go rename to encoding/rs/utils_test.go index e5fc0d656f..849f7f0fab 100644 --- a/pkg/encoding/encoder/utils_test.go +++ b/encoding/rs/utils_test.go @@ -1,4 +1,4 @@ -package encoder_test +package rs_test import ( "testing" @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" + "github.com/Layr-Labs/eigenda/encoding/rs" ) func TestGetEncodingParams(t *testing.T) { diff --git a/pkg/encoding/README.md b/encoding/test/README.md similarity index 100% rename from pkg/encoding/README.md rename to encoding/test/README.md diff --git a/pkg/encoding/main.go b/encoding/test/main.go similarity index 87% rename from pkg/encoding/main.go rename to encoding/test/main.go index fe32838d39..b777623012 100644 --- a/pkg/encoding/main.go +++ b/encoding/test/main.go @@ -9,8 +9,11 @@ import ( "runtime" "time" - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" + "github.com/Layr-Labs/eigenda/encoding/rs" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" ) @@ -24,7 +27,7 @@ func main() { } func readpoints() { - kzgConfig := &kzgRs.KzgConfig{ + kzgConfig := &kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point", G2Path: "../../inabox/resources/kzg/g2.point", CacheDir: "SRSTables", @@ -34,7 +37,7 @@ func readpoints() { } // create encoding object - kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + kzgGroup, _ := prover.NewProver(kzgConfig, true) fmt.Println("there are ", len(kzgGroup.Srs.G1), "points") for i := 0; i < len(kzgGroup.Srs.G1); i++ { fmt.Printf("%v %v\n", i, string(kzgGroup.Srs.G1[i].MarshalText())) @@ -56,7 +59,7 @@ func TestKzgRs() { fmt.Printf(" Num Par: %v\n", numPar) //fmt.Printf(" Data size(byte): %v\n", len(inputBytes)) - kzgConfig := &kzgRs.KzgConfig{ + kzgConfig := &kzgrs.KzgConfig{ G1Path: "g1.point", G2Path: "g2.point", CacheDir: "SRSTables", @@ -66,12 +69,12 @@ func TestKzgRs() { } // create encoding object - kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) + kzgGroup, _ := prover.NewProver(kzgConfig, true) params := rs.EncodingParams{NumChunks: 200, ChunkLen: 180} enc, _ := kzgGroup.NewKzgEncoder(params) - //inputFr := kzgRs.ToFrArray(inputBytes) + //inputFr := kzgrs.ToFrArray(inputBytes) inputSize := uint64(numSymbols) inputFr := make([]bls.Fr, inputSize) for i := uint64(0); i < inputSize; i++ { @@ -108,11 +111,11 @@ func TestKzgRs() { fmt.Printf("frame %v leading coset %v\n", i, j) lc := enc.Fs.ExpandedRootsOfUnity[uint64(j)] - g2Atn, err := kzgRs.ReadG2Point(uint64(len(f.Coeffs)), kzgConfig) + g2Atn, err := kzgrs.ReadG2Point(uint64(len(f.Coeffs)), kzgConfig) if err != nil { log.Fatalf("Load g2 %v failed\n", err) } - ok := f.Verify(enc.Ks, commit, &lc, &g2Atn) + ok := verifier.VerifyFrame(&f, enc.Ks, commit, &lc, &g2Atn) if !ok { log.Fatalf("Proof %v failed\n", i) } @@ -131,14 +134,14 @@ func TestKzgRs() { } //printFr(dataFr) - //dataFr, err := kzgRs.DecodeSys(samples, indices, inputSize) + //dataFr, err := kzgrs.DecodeSys(samples, indices, inputSize) //if err != nil { //log.Fatalf("%v", err) //} fmt.Println(dataFr) // printFr(dataFr) - //deData := kzgRs.ToByteArray(dataFr, inputByteSize) + //deData := kzgrs.ToByteArray(dataFr, inputByteSize) //fmt.Println("dataFr") // printFr(dataFr) //fmt.Println(deData) @@ -201,7 +204,7 @@ func TestKzgRs() { // v[i] = uint64(i + 1) // } // dataFr := makeFr(v) -// order := kzgRs.CeilIntPowerOf2Num(size) +// order := kzgrs.CeilIntPowerOf2Num(size) // fs := kzg.NewFFTSettings(uint8(order)) // polyFr, err := fs.FFT(dataFr, true) // if err != nil { @@ -232,8 +235,8 @@ func printFr(d []bls.Fr) { // fmt.Printf("\n") // } -func SampleFrames(frames []kzgRs.Frame, num uint64) ([]kzgRs.Frame, []uint64) { - samples := make([]kzgRs.Frame, num) +func SampleFrames(frames []encoding.Frame, num uint64) ([]encoding.Frame, []uint64) { + samples := make([]encoding.Frame, num) indices := rand.Perm(len(frames)) indices = indices[:num] diff --git a/pkg/encoding/testCrypto.go b/encoding/test/testCrypto.go similarity index 100% rename from pkg/encoding/testCrypto.go rename to encoding/test/testCrypto.go diff --git a/pkg/encoding/testKzgBn254.go b/encoding/test/testKzgBn254.go similarity index 100% rename from pkg/encoding/testKzgBn254.go rename to encoding/test/testKzgBn254.go diff --git a/pkg/encoding/utils/reverseBits/reverseBits.go b/encoding/utils/reverseBits/reverseBits.go similarity index 100% rename from pkg/encoding/utils/reverseBits/reverseBits.go rename to encoding/utils/reverseBits/reverseBits.go diff --git a/pkg/encoding/utils/toeplitz/cir.go b/encoding/utils/toeplitz/cir.go similarity index 100% rename from pkg/encoding/utils/toeplitz/cir.go rename to encoding/utils/toeplitz/cir.go diff --git a/pkg/encoding/utils/toeplitz/cir_test.go b/encoding/utils/toeplitz/cir_test.go similarity index 98% rename from pkg/encoding/utils/toeplitz/cir_test.go rename to encoding/utils/toeplitz/cir_test.go index b7d7a9a8fc..ff63da4ff4 100644 --- a/pkg/encoding/utils/toeplitz/cir_test.go +++ b/encoding/utils/toeplitz/cir_test.go @@ -3,7 +3,7 @@ package toeplitz_test import ( "testing" - "github.com/Layr-Labs/eigenda/pkg/encoding/utils/toeplitz" + "github.com/Layr-Labs/eigenda/encoding/utils/toeplitz" kzg "github.com/Layr-Labs/eigenda/pkg/kzg" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" "github.com/stretchr/testify/assert" diff --git a/pkg/encoding/utils/toeplitz/toeplitz.go b/encoding/utils/toeplitz/toeplitz.go similarity index 100% rename from pkg/encoding/utils/toeplitz/toeplitz.go rename to encoding/utils/toeplitz/toeplitz.go diff --git a/pkg/encoding/utils/toeplitz/toeplitz_test.go b/encoding/utils/toeplitz/toeplitz_test.go similarity index 98% rename from pkg/encoding/utils/toeplitz/toeplitz_test.go rename to encoding/utils/toeplitz/toeplitz_test.go index d8351d486a..13b932d2af 100644 --- a/pkg/encoding/utils/toeplitz/toeplitz_test.go +++ b/encoding/utils/toeplitz/toeplitz_test.go @@ -3,7 +3,7 @@ package toeplitz_test import ( "testing" - "github.com/Layr-Labs/eigenda/pkg/encoding/utils/toeplitz" + "github.com/Layr-Labs/eigenda/encoding/utils/toeplitz" kzg "github.com/Layr-Labs/eigenda/pkg/kzg" bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" "github.com/stretchr/testify/assert" diff --git a/inabox/tests/integration_suite_test.go b/inabox/tests/integration_suite_test.go index 60352b98ff..52b9a27297 100644 --- a/inabox/tests/integration_suite_test.go +++ b/inabox/tests/integration_suite_test.go @@ -19,9 +19,9 @@ import ( "github.com/Layr-Labs/eigenda/core/encoding" "github.com/Layr-Labs/eigenda/core/eth" coreindexer "github.com/Layr-Labs/eigenda/core/indexer" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/Layr-Labs/eigenda/inabox/deploy" "github.com/Layr-Labs/eigenda/indexer" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" gcommon "github.com/ethereum/go-ethereum/common" ethrpc "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo/v2" @@ -157,7 +157,7 @@ func setupRetrievalClient(testConfig *deploy.Config) error { return err } encoder, err := encoding.NewEncoder(encoding.EncoderConfig{ - KzgConfig: kzgEncoder.KzgConfig{ + KzgConfig: kzgrs.KzgConfig{ G1Path: testConfig.Retriever.RETRIEVER_G1_PATH, G2Path: testConfig.Retriever.RETRIEVER_G2_PATH, G2PowerOf2Path: testConfig.Retriever.RETRIEVER_G2_POWER_OF_2_PATH, diff --git a/node/grpc/server_test.go b/node/grpc/server_test.go index 6af185ff75..d584f0b996 100644 --- a/node/grpc/server_test.go +++ b/node/grpc/server_test.go @@ -16,9 +16,9 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/core/encoding" core_mock "github.com/Layr-Labs/eigenda/core/mock" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/Layr-Labs/eigenda/node" "github.com/Layr-Labs/eigenda/node/grpc" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" "github.com/Layr-Labs/eigensdk-go/metrics" "github.com/consensys/gnark-crypto/ecc/bn254/fp" "github.com/prometheus/client_golang/prometheus" @@ -43,7 +43,7 @@ func TestMain(m *testing.M) { // makeTestEncoder makes an encoder currently using the only supported backend. func makeTestEncoder() (core.Encoder, error) { - config := kzgEncoder.KzgConfig{ + config := kzgrs.KzgConfig{ G1Path: "../../inabox/resources/kzg/g1.point.300000", G2Path: "../../inabox/resources/kzg/g2.point.300000", CacheDir: "../../inabox/resources/kzg/SRSTables", diff --git a/pkg/encoding/kzgEncoder/decode.go b/pkg/encoding/kzgEncoder/decode.go deleted file mode 100644 index 7b88aba90e..0000000000 --- a/pkg/encoding/kzgEncoder/decode.go +++ /dev/null @@ -1,14 +0,0 @@ -package kzgEncoder - -import ( - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" -) - -func (g *KzgEncoder) Decode(frames []Frame, indices []uint64, maxInputSize uint64) ([]byte, error) { - rsFrames := make([]rs.Frame, len(frames)) - for ind, frame := range frames { - rsFrames[ind] = rs.Frame{Coeffs: frame.Coeffs} - } - - return g.Encoder.Decode(rsFrames, indices, maxInputSize) -} diff --git a/pkg/encoding/kzgEncoder/encoder.go b/pkg/encoding/kzgEncoder/encoder.go deleted file mode 100644 index 08995c7539..0000000000 --- a/pkg/encoding/kzgEncoder/encoder.go +++ /dev/null @@ -1,465 +0,0 @@ -package kzgEncoder - -import ( - "errors" - "fmt" - "log" - "math" - "os" - "runtime" - "strconv" - "strings" - "sync" - "time" - - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - "github.com/Layr-Labs/eigenda/pkg/encoding/utils" - kzg "github.com/Layr-Labs/eigenda/pkg/kzg" - - bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" - _ "go.uber.org/automaxprocs" -) - -type KzgConfig struct { - G1Path string - G2Path string - G1PowerOf2Path string - G2PowerOf2Path string - CacheDir string - NumWorker uint64 - SRSOrder uint64 // Order is the total size of SRS - SRSNumberToLoad uint64 // Number of points to be loaded from the begining - Verbose bool - PreloadEncoder bool -} - -type KzgEncoderGroup struct { - *KzgConfig - Srs *kzg.SRS - G2Trailing []bls.G2Point - mu sync.Mutex - LoadG2Points bool - - Encoders map[rs.EncodingParams]*KzgEncoder - Verifiers map[rs.EncodingParams]*KzgVerifier -} - -type KzgEncoder struct { - *rs.Encoder - - *KzgConfig - Srs *kzg.SRS - G2Trailing []bls.G2Point - - Fs *kzg.FFTSettings - Ks *kzg.KZGSettings - SFs *kzg.FFTSettings // fft used for submatrix product helper - FFTPoints [][]bls.G1Point - FFTPointsT [][]bls.G1Point // transpose of FFTPoints -} - -func NewKzgEncoderGroup(config *KzgConfig, loadG2Points bool) (*KzgEncoderGroup, error) { - - if config.SRSNumberToLoad > config.SRSOrder { - return nil, errors.New("SRSOrder is less than srsNumberToLoad") - } - - // read the whole order, and treat it as entire SRS for low degree proof - s1, err := utils.ReadG1Points(config.G1Path, config.SRSNumberToLoad, config.NumWorker) - if err != nil { - log.Println("failed to read G1 points", err) - return nil, err - } - - s2 := make([]bls.G2Point, 0) - g2Trailing := make([]bls.G2Point, 0) - - // PreloadEncoder is by default not used by operator node, PreloadEncoder - if loadG2Points { - if len(config.G2Path) == 0 { - return nil, fmt.Errorf("G2Path is empty. However, object needs to load G2Points") - } - - s2, err = utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker) - if err != nil { - log.Println("failed to read G2 points", err) - return nil, err - } - - g2Trailing, err = utils.ReadG2PointSection( - config.G2Path, - config.SRSOrder-config.SRSNumberToLoad, - config.SRSOrder, // last exclusive - config.NumWorker, - ) - if err != nil { - return nil, err - } - } else { - // todo, there are better ways to handle it - if len(config.G2PowerOf2Path) == 0 { - return nil, fmt.Errorf("G2PowerOf2Path is empty. However, object needs to load G2Points") - } - } - - srs, err := kzg.NewSrs(s1, s2) - if err != nil { - log.Println("Could not create srs", err) - return nil, err - } - - fmt.Println("numthread", runtime.GOMAXPROCS(0)) - - encoderGroup := &KzgEncoderGroup{ - KzgConfig: config, - Srs: srs, - G2Trailing: g2Trailing, - Encoders: make(map[rs.EncodingParams]*KzgEncoder), - Verifiers: make(map[rs.EncodingParams]*KzgVerifier), - LoadG2Points: loadG2Points, - } - - if config.PreloadEncoder { - // create table dir if not exist - err := os.MkdirAll(config.CacheDir, os.ModePerm) - if err != nil { - log.Println("Cannot make CacheDir", err) - return nil, err - } - - err = encoderGroup.PreloadAllEncoders() - if err != nil { - return nil, err - } - } - - return encoderGroup, nil - -} - -// Read the n-th G1 point from SRS. -func ReadG1Point(n uint64, g *KzgConfig) (bls.G1Point, error) { - if n > g.SRSOrder { - return bls.G1Point{}, fmt.Errorf("requested power %v is larger than SRSOrder %v", n, g.SRSOrder) - } - - g1point, err := utils.ReadG1PointSection(g.G1Path, n, n+1, 1) - if err != nil { - return bls.G1Point{}, err - } - - return g1point[0], nil -} - -// Read the n-th G2 point from SRS. -func ReadG2Point(n uint64, g *KzgConfig) (bls.G2Point, error) { - if n > g.SRSOrder { - return bls.G2Point{}, fmt.Errorf("requested power %v is larger than SRSOrder %v", n, g.SRSOrder) - } - - g2point, err := utils.ReadG2PointSection(g.G2Path, n, n+1, 1) - if err != nil { - return bls.G2Point{}, err - } - return g2point[0], nil -} - -// Read g2 points from power of 2 file -func ReadG2PointOnPowerOf2(exponent uint64, g *KzgConfig) (bls.G2Point, error) { - - // the powerOf2 file, only [tau^exp] are stored. - // exponent 0, 1, 2, , .. - // actual pow [tau],[tau^2],[tau^4],.. (stored in the file) - // In our convention SRSOrder contains the total number of series of g1, g2 starting with generator - // i.e. [1] [tau] [tau^2].. - // So the actual power of tau is SRSOrder - 1 - // The mainnet SRS, the max power is 2^28-1, so the last power in powerOf2 file is [tau^(2^27)] - // For test case of 3000 SRS, the max power is 2999, so last power in powerOf2 file is [tau^2048] - // if a actual SRS order is 15, the file will contain four symbols (1,2,4,8) with indices [0,1,2,3] - // if a actual SRS order is 16, the file will contain five symbols (1,2,4,8,16) with indices [0,1,2,3,4] - - actualPowerOfTau := g.SRSOrder - 1 - largestPowerofSRS := uint64(math.Log2(float64(actualPowerOfTau))) - if exponent > largestPowerofSRS { - return bls.G2Point{}, fmt.Errorf("requested power %v is larger than largest power of SRS %v", - uint64(math.Pow(2, float64(exponent))), largestPowerofSRS) - } - - if len(g.G2PowerOf2Path) == 0 { - return bls.G2Point{}, fmt.Errorf("G2PathPowerOf2 path is empty") - } - - g2point, err := utils.ReadG2PointSection(g.G2PowerOf2Path, exponent, exponent+1, 1) - if err != nil { - return bls.G2Point{}, err - } - return g2point[0], nil -} - -func (g *KzgEncoderGroup) PreloadAllEncoders() error { - paramsAll, err := GetAllPrecomputedSrsMap(g.CacheDir) - if err != nil { - return nil - } - fmt.Printf("detect %v srs maps\n", len(paramsAll)) - for i := 0; i < len(paramsAll); i++ { - fmt.Printf(" %v. NumChunks: %v ChunkLen: %v\n", i, paramsAll[i].NumChunks, paramsAll[i].ChunkLen) - } - - if len(paramsAll) == 0 { - return nil - } - - for _, params := range paramsAll { - // get those encoders and store them - enc, err := g.GetKzgEncoder(params) - if err != nil { - return err - } - g.Encoders[params] = enc - } - - return nil -} - -func (g *KzgEncoderGroup) GetKzgEncoder(params rs.EncodingParams) (*KzgEncoder, error) { - g.mu.Lock() - defer g.mu.Unlock() - enc, ok := g.Encoders[params] - if ok { - return enc, nil - } - - enc, err := g.newKzgEncoder(params) - if err == nil { - g.Encoders[params] = enc - } - - return enc, err -} - -func (g *KzgEncoderGroup) NewKzgEncoder(params rs.EncodingParams) (*KzgEncoder, error) { - g.mu.Lock() - defer g.mu.Unlock() - - return g.newKzgEncoder(params) -} - -func (g *KzgEncoderGroup) newKzgEncoder(params rs.EncodingParams) (*KzgEncoder, error) { - - // Check that the parameters are valid with respect to the SRS. - if params.ChunkLen*params.NumChunks >= g.SRSOrder { - return nil, fmt.Errorf("the supplied encoding parameters are not valid with respect to the SRS. ChunkLength: %d, NumChunks: %d, SRSOrder: %d", params.ChunkLen, params.NumChunks, g.SRSOrder) - } - - encoder, err := rs.NewEncoder(params, g.Verbose) - if err != nil { - log.Println("Could not create encoder: ", err) - return nil, err - } - - subTable, err := NewSRSTable(g.CacheDir, g.Srs.G1, g.NumWorker) - if err != nil { - log.Println("Could not create srs table:", err) - return nil, err - } - - fftPoints, err := subTable.GetSubTables(encoder.NumChunks, encoder.ChunkLen) - if err != nil { - log.Println("could not get sub tables", err) - return nil, err - } - - fftPointsT := make([][]bls.G1Point, len(fftPoints[0])) - for i := range fftPointsT { - fftPointsT[i] = make([]bls.G1Point, len(fftPoints)) - for j := uint64(0); j < encoder.ChunkLen; j++ { - fftPointsT[i][j] = fftPoints[j][i] - } - } - n := uint8(math.Log2(float64(encoder.NumEvaluations()))) - if encoder.ChunkLen == 1 { - n = uint8(math.Log2(float64(2 * encoder.NumChunks))) - } - fs := kzg.NewFFTSettings(n) - - ks, err := kzg.NewKZGSettings(fs, g.Srs) - if err != nil { - return nil, err - } - - t := uint8(math.Log2(float64(2 * encoder.NumChunks))) - sfs := kzg.NewFFTSettings(t) - - return &KzgEncoder{ - Encoder: encoder, - KzgConfig: g.KzgConfig, - Srs: g.Srs, - G2Trailing: g.G2Trailing, - Fs: fs, - Ks: ks, - SFs: sfs, - FFTPoints: fftPoints, - FFTPointsT: fftPointsT, - }, nil -} - -// just a wrapper to take bytes not Fr Element -func (g *KzgEncoder) EncodeBytes(inputBytes []byte) (*bls.G1Point, *bls.G2Point, *bls.G2Point, []Frame, []uint32, error) { - inputFr := rs.ToFrArray(inputBytes) - return g.Encode(inputFr) -} - -func (g *KzgEncoder) Encode(inputFr []bls.Fr) (*bls.G1Point, *bls.G2Point, *bls.G2Point, []Frame, []uint32, error) { - - startTime := time.Now() - poly, frames, indices, err := g.Encoder.Encode(inputFr) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - if len(poly.Coeffs) > int(g.KzgConfig.SRSNumberToLoad) { - return nil, nil, nil, nil, nil, fmt.Errorf("poly Coeff length %v is greater than Loaded SRS points %v", len(poly.Coeffs), int(g.KzgConfig.SRSNumberToLoad)) - } - - // compute commit for the full poly - commit := g.Commit(poly.Coeffs) - lowDegreeCommitment := bls.LinCombG2(g.Srs.G2[:len(poly.Coeffs)], poly.Coeffs) - - intermediate := time.Now() - - polyDegreePlus1 := uint64(len(inputFr)) - - if g.Verbose { - log.Printf(" Commiting takes %v\n", time.Since(intermediate)) - intermediate = time.Now() - - log.Printf("shift %v\n", g.SRSOrder-polyDegreePlus1) - log.Printf("order %v\n", len(g.Srs.G2)) - log.Println("low degree verification info") - } - - shiftedSecret := g.G2Trailing[g.KzgConfig.SRSNumberToLoad-polyDegreePlus1:] - - //The proof of low degree is commitment of the polynomial shifted to the largest srs degree - lowDegreeProof := bls.LinCombG2(shiftedSecret, poly.Coeffs[:polyDegreePlus1]) - - //fmt.Println("kzgFFT lowDegreeProof", lowDegreeProof, "poly len ", len(fullCoeffsPoly), "order", len(g.Ks.SecretG2) ) - //ok := VerifyLowDegreeProof(&commit, lowDegreeProof, polyDegreePlus1-1, g.SRSOrder, g.Srs.G2) - //if !ok { - // log.Printf("Kzg FFT Cannot Verify low degree proof %v", lowDegreeProof) - // return nil, nil, nil, nil, errors.New("cannot verify low degree proof") - // } else { - // log.Printf("Kzg FFT Verify low degree proof PPPASSS %v", lowDegreeProof) - // } - - if g.Verbose { - log.Printf(" Generating Low Degree Proof takes %v\n", time.Since(intermediate)) - intermediate = time.Now() - } - - // compute proofs - paddedCoeffs := make([]bls.Fr, g.NumEvaluations()) - copy(paddedCoeffs, poly.Coeffs) - - proofs, err := g.ProveAllCosetThreads(paddedCoeffs, g.NumChunks, g.ChunkLen, g.NumWorker) - if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("could not generate proofs: %v", err) - } - - if g.Verbose { - log.Printf(" Proving takes %v\n", time.Since(intermediate)) - } - - kzgFrames := make([]Frame, len(frames)) - for i, index := range indices { - kzgFrames[i] = Frame{ - Proof: proofs[index], - Coeffs: frames[i].Coeffs, - } - } - - if g.Verbose { - log.Printf("Total encoding took %v\n", time.Since(startTime)) - } - return &commit, lowDegreeCommitment, lowDegreeProof, kzgFrames, indices, nil -} - -func (g *KzgEncoder) Commit(polyFr []bls.Fr) bls.G1Point { - commit := g.Ks.CommitToPoly(polyFr) - return *commit -} - -// The function verify low degree proof against a poly commitment -// We wish to show x^shift poly = shiftedPoly, with -// With shift = SRSOrder-1 - claimedDegree and -// proof = commit(shiftedPoly) on G1 -// so we can verify by checking -// e( commit_1, [x^shift]_2) = e( proof_1, G_2 ) -func VerifyLowDegreeProof(lengthCommit *bls.G2Point, proof *bls.G2Point, g1Challenge *bls.G1Point) bool { - return bls.PairingsVerify(g1Challenge, lengthCommit, &bls.GenG1, proof) -} - -// get Fiat-Shamir challenge -// func createFiatShamirChallenge(byteArray [][32]byte) *bls.Fr { -// alphaBytesTmp := make([]byte, 0) -// for i := 0; i < len(byteArray); i++ { -// for j := 0; j < len(byteArray[i]); j++ { -// alphaBytesTmp = append(alphaBytesTmp, byteArray[i][j]) -// } -// } -// alphaBytes := crypto.Keccak256(alphaBytesTmp) -// alpha := new(bls.Fr) -// bls.FrSetBytes(alpha, alphaBytes) -// -// return alpha -// } - -// invert the divisor, then multiply -// func polyFactorDiv(dst *bls.Fr, a *bls.Fr, b *bls.Fr) { -// // TODO: use divmod instead. -// var tmp bls.Fr -// bls.InvModFr(&tmp, b) -// bls.MulModFr(dst, &tmp, a) -// } - -// Detect the precomputed table from the specified directory -// the file name follow the name convention of -// -// dimE*.coset& -// -// where the first * specifies the dimension of the matrix which -// equals to the number of chunks -// where the second & specifies the length of each chunk -func GetAllPrecomputedSrsMap(tableDir string) ([]rs.EncodingParams, error) { - files, err := os.ReadDir(tableDir) - if err != nil { - log.Println("Error to list SRS Table directory", err) - return nil, err - } - - tables := make([]rs.EncodingParams, 0) - for _, file := range files { - filename := file.Name() - - tokens := strings.Split(filename, ".") - - dimEValue, err := strconv.Atoi(tokens[0][4:]) - if err != nil { - log.Println("Error to parse Dimension part of the Table", err) - return nil, err - } - cosetSizeValue, err := strconv.Atoi(tokens[1][5:]) - if err != nil { - log.Println("Error to parse Coset size of the Table", err) - return nil, err - } - - params := rs.EncodingParams{ - NumChunks: uint64(cosetSizeValue), - ChunkLen: uint64(dimEValue), - } - tables = append(tables, params) - } - return tables, nil -} diff --git a/pkg/encoding/kzgEncoder/frame.go b/pkg/encoding/kzgEncoder/frame.go deleted file mode 100644 index ef5fe5bc10..0000000000 --- a/pkg/encoding/kzgEncoder/frame.go +++ /dev/null @@ -1,74 +0,0 @@ -package kzgEncoder - -import ( - "bytes" - "encoding/gob" - - kzg "github.com/Layr-Labs/eigenda/pkg/kzg" - bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" -) - -// Proof is the multireveal proof -// Coeffs is identical to input data converted into Fr element -type Frame struct { - Proof bls.G1Point - Coeffs []bls.Fr -} - -func (f *Frame) Encode() ([]byte, error) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - err := enc.Encode(f) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func Decode(b []byte) (Frame, error) { - var f Frame - buf := bytes.NewBuffer(b) - dec := gob.NewDecoder(buf) - err := dec.Decode(&f) - if err != nil { - return Frame{}, err - } - return f, nil -} - -// Verify function assumes the Data stored is coefficients of coset's interpolating poly -func (f *Frame) Verify(ks *kzg.KZGSettings, commitment *bls.G1Point, x *bls.Fr, g2Atn *bls.G2Point) bool { - var xPow bls.Fr - bls.CopyFr(&xPow, &bls.ONE) - - var tmp bls.Fr - for i := 0; i < len(f.Coeffs); i++ { - bls.MulModFr(&tmp, &xPow, x) - bls.CopyFr(&xPow, &tmp) - } - - // [x^n]_2 - var xn2 bls.G2Point - bls.MulG2(&xn2, &bls.GenG2, &xPow) - - // [s^n - x^n]_2 - var xnMinusYn bls.G2Point - - bls.SubG2(&xnMinusYn, g2Atn, &xn2) - - // [interpolation_polynomial(s)]_1 - is1 := bls.LinCombG1(ks.Srs.G1[:len(f.Coeffs)], f.Coeffs) - - // [commitment - interpolation_polynomial(s)]_1 = [commit]_1 - [interpolation_polynomial(s)]_1 - var commitMinusInterpolation bls.G1Point - bls.SubG1(&commitMinusInterpolation, commitment, is1) - - // Verify the pairing equation - // - // e([commitment - interpolation_polynomial(s)], [1]) = e([proof], [s^n - x^n]) - // equivalent to - // e([commitment - interpolation_polynomial]^(-1), [1]) * e([proof], [s^n - x^n]) = 1_T - // - - return bls.PairingsVerify(&commitMinusInterpolation, &bls.GenG2, &f.Proof, &xnMinusYn) -} diff --git a/pkg/encoding/kzgEncoder/frame_test.go b/pkg/encoding/kzgEncoder/frame_test.go deleted file mode 100644 index 8934c680ea..0000000000 --- a/pkg/encoding/kzgEncoder/frame_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package kzgEncoder_test - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" - kzg "github.com/Layr-Labs/eigenda/pkg/kzg" -) - -func TestEncodeDecodeFrame_AreInverses(t *testing.T) { - teardownSuite := setupSuite(t) - defer teardownSuite(t) - - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) - - params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) - - enc, err := group.NewKzgEncoder(params) - - require.Nil(t, err) - require.NotNil(t, enc) - - _, _, _, frames, _, err := enc.EncodeBytes(GETTYSBURG_ADDRESS_BYTES) - require.Nil(t, err) - require.NotNil(t, frames, err) - - b, err := frames[0].Encode() - require.Nil(t, err) - require.NotNil(t, b) - - frame, err := kzgRs.Decode(b) - require.Nil(t, err) - require.NotNil(t, frame) - - assert.Equal(t, frame, frames[0]) -} - -func TestVerify(t *testing.T) { - teardownSuite := setupSuite(t) - defer teardownSuite(t) - - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) - - params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) - - enc, err := group.NewKzgEncoder(params) - require.Nil(t, err) - require.NotNil(t, enc) - - commit, _, _, frames, _, err := enc.EncodeBytes(GETTYSBURG_ADDRESS_BYTES) - require.Nil(t, err) - require.NotNil(t, commit) - require.NotNil(t, frames) - - n := uint8(math.Log2(float64(params.NumEvaluations()))) - fs := kzg.NewFFTSettings(n) - require.NotNil(t, fs) - - lc := enc.Fs.ExpandedRootsOfUnity[uint64(0)] - require.NotNil(t, lc) - - g2Atn, err := kzgRs.ReadG2Point(uint64(len(frames[0].Coeffs)), kzgConfig) - require.Nil(t, err) - assert.True(t, frames[0].Verify(enc.Ks, commit, &lc, &g2Atn)) -} diff --git a/pkg/encoding/kzgEncoder/multiprover.go b/pkg/encoding/kzgEncoder/multiprover.go deleted file mode 100644 index a66851c9d7..0000000000 --- a/pkg/encoding/kzgEncoder/multiprover.go +++ /dev/null @@ -1,152 +0,0 @@ -package kzgEncoder - -import ( - "fmt" - "math" - "sync" - "time" - - "github.com/Layr-Labs/eigenda/pkg/encoding/utils/toeplitz" - bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" -) - -type WorkerResult struct { - points []bls.G1Point - err error -} - -func (p *KzgEncoder) ProveAllCosetThreads(polyFr []bls.Fr, numChunks, chunkLen, numWorker uint64) ([]bls.G1Point, error) { - begin := time.Now() - // Robert: Standardizing this to use the same math used in precomputeSRS - dimE := numChunks - l := chunkLen - - sumVec := make([]bls.G1Point, dimE*2) - - jobChan := make(chan uint64, numWorker) - results := make(chan WorkerResult, numWorker) - - // create storage for intermediate fft outputs - coeffStore := make([][]bls.Fr, dimE*2) - for i := range coeffStore { - coeffStore[i] = make([]bls.Fr, l) - } - - for w := uint64(0); w < numWorker; w++ { - go p.proofWorker(polyFr, jobChan, l, dimE, coeffStore, results) - } - - for j := uint64(0); j < l; j++ { - jobChan <- j - } - close(jobChan) - - // return only first error - var err error - for w := uint64(0); w < numWorker; w++ { - wr := <-results - if wr.err != nil { - err = wr.err - } - } - - if err != nil { - return nil, fmt.Errorf("proof worker error: %v", err) - } - - t0 := time.Now() - - // compute proof by multi scaler mulplication - var wg sync.WaitGroup - for i := uint64(0); i < dimE*2; i++ { - wg.Add(1) - go func(k uint64) { - defer wg.Done() - sumVec[k] = *bls.LinCombG1(p.FFTPointsT[k], coeffStore[k]) - }(i) - } - - wg.Wait() - - t1 := time.Now() - - // only 1 ifft is needed - sumVecInv, err := p.Fs.FFTG1(sumVec, true) - if err != nil { - return nil, fmt.Errorf("fft error: %v", err) - } - - t2 := time.Now() - - // outputs is out of order - buttefly - proofs, err := p.Fs.FFTG1(sumVecInv[:dimE], false) - if err != nil { - return nil, err - } - - t3 := time.Now() - - fmt.Printf("mult-th %v, msm %v,fft1 %v, fft2 %v,\n", t0.Sub(begin), t1.Sub(t0), t2.Sub(t1), t3.Sub(t2)) - - //rb.ReverseBitOrderG1Point(proofs) - return proofs, nil -} - -func (p *KzgEncoder) proofWorker( - polyFr []bls.Fr, - jobChan <-chan uint64, - l uint64, - dimE uint64, - coeffStore [][]bls.Fr, - results chan<- WorkerResult, -) { - - for j := range jobChan { - coeffs, err := p.GetSlicesCoeff(polyFr, dimE, j, l) - if err != nil { - results <- WorkerResult{ - points: nil, - err: err, - } - } - - for i := 0; i < len(coeffs); i++ { - coeffStore[i][j] = coeffs[i] - } - } - - results <- WorkerResult{ - err: nil, - } -} - -// output is in the form see primeField toeplitz -// -// phi ^ (coset size ) = 1 -// -// implicitly pad slices to power of 2 -func (p *KzgEncoder) GetSlicesCoeff(polyFr []bls.Fr, dimE, j, l uint64) ([]bls.Fr, error) { - // there is a constant term - m := uint64(len(polyFr)) - 1 - dim := (m - j) / l - - toeV := make([]bls.Fr, 2*dimE-1) - for i := uint64(0); i < dim; i++ { - bls.CopyFr(&toeV[i], &polyFr[m-(j+i*l)]) - } - - // use precompute table - tm, err := toeplitz.NewToeplitz(toeV, p.SFs) - if err != nil { - return nil, err - } - return tm.GetFFTCoeff() -} - -/* -returns the power of 2 which is immediately bigger than the input -*/ -func CeilIntPowerOf2Num(d uint64) uint64 { - nextPower := math.Ceil(math.Log2(float64(d))) - return uint64(math.Pow(2.0, nextPower)) -} diff --git a/pkg/encoding/kzgEncoder/verifier.go b/pkg/encoding/kzgEncoder/verifier.go deleted file mode 100644 index 6897b13d65..0000000000 --- a/pkg/encoding/kzgEncoder/verifier.go +++ /dev/null @@ -1,108 +0,0 @@ -package kzgEncoder - -import ( - "errors" - "math" - - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzg "github.com/Layr-Labs/eigenda/pkg/kzg" - wbls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" -) - -type KzgVerifier struct { - *KzgConfig - Srs *kzg.SRS - - rs.EncodingParams - - Fs *kzg.FFTSettings - Ks *kzg.KZGSettings -} - -func (g *KzgEncoderGroup) GetKzgVerifier(params rs.EncodingParams) (*KzgVerifier, error) { - g.mu.Lock() - defer g.mu.Unlock() - - if err := params.Validate(); err != nil { - return nil, err - } - - ver, ok := g.Verifiers[params] - if ok { - return ver, nil - } - - ver, err := g.newKzgVerifier(params) - if err == nil { - g.Verifiers[params] = ver - } - - return ver, err -} - -func (g *KzgEncoderGroup) NewKzgVerifier(params rs.EncodingParams) (*KzgVerifier, error) { - g.mu.Lock() - defer g.mu.Unlock() - return g.newKzgVerifier(params) -} - -func (g *KzgEncoderGroup) newKzgVerifier(params rs.EncodingParams) (*KzgVerifier, error) { - - if err := params.Validate(); err != nil { - return nil, err - } - - n := uint8(math.Log2(float64(params.NumEvaluations()))) - fs := kzg.NewFFTSettings(n) - ks, err := kzg.NewKZGSettings(fs, g.Srs) - - if err != nil { - return nil, err - } - - return &KzgVerifier{ - KzgConfig: g.KzgConfig, - Srs: g.Srs, - EncodingParams: params, - Fs: fs, - Ks: ks, - }, nil -} - -// VerifyCommit verifies the low degree proof; since it doesn't depend on the encoding parameters -// we leave it as a method of the KzgEncoderGroup -func (v *KzgEncoderGroup) VerifyCommit(lengthCommit *wbls.G2Point, lowDegreeProof *wbls.G2Point, length uint64) error { - - g1Challenge, err := ReadG1Point(v.SRSOrder-length, v.KzgConfig) - if err != nil { - return err - } - - if !VerifyLowDegreeProof(lengthCommit, lowDegreeProof, &g1Challenge) { - return errors.New("low degree proof fails") - } - return nil - -} - -func (v *KzgVerifier) VerifyFrame(commit *wbls.G1Point, f *Frame, index uint64) error { - - j, err := rs.GetLeadingCosetIndex( - uint64(index), - v.NumChunks, - ) - if err != nil { - return err - } - - g2Atn, err := ReadG2Point(uint64(len(f.Coeffs)), v.KzgConfig) - if err != nil { - return err - } - if !f.Verify(v.Ks, commit, &v.Ks.ExpandedRootsOfUnity[j], &g2Atn) { - return errors.New("multireveal proof fails") - } - - return nil - -} diff --git a/pkg/encoding/kzgEncoder/zero_padding_test.go b/pkg/encoding/kzgEncoder/zero_padding_test.go deleted file mode 100644 index af450e2fe9..0000000000 --- a/pkg/encoding/kzgEncoder/zero_padding_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package kzgEncoder_test - -import ( - "testing" - - rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" - kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestProveZeroPadding(t *testing.T) { - teardownSuite := setupSuite(t) - defer teardownSuite(t) - - group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true) - - params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES))) - enc, err := group.NewKzgEncoder(params) - require.Nil(t, err) - - inputFr := rs.ToFrArray(GETTYSBURG_ADDRESS_BYTES) - - _, _, _, _, _, err = enc.Encode(inputFr) - require.Nil(t, err) - - assert.True(t, true, "Proof %v failed\n") -} diff --git a/retriever/server_test.go b/retriever/server_test.go index c144b5c65a..2217f4edda 100644 --- a/retriever/server_test.go +++ b/retriever/server_test.go @@ -13,7 +13,9 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/core/encoding" coremock "github.com/Layr-Labs/eigenda/core/mock" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/prover" + "github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier" "github.com/Layr-Labs/eigenda/retriever" "github.com/Layr-Labs/eigenda/retriever/mock" "github.com/stretchr/testify/assert" @@ -31,7 +33,7 @@ var ( ) func makeTestEncoder() (core.Encoder, error) { - config := &kzgEncoder.KzgConfig{ + config := &kzgrs.KzgConfig{ G1Path: "../inabox/resources/kzg/g1.point", G2Path: "../inabox/resources/kzg/g2.point", G2PowerOf2Path: "../inabox/resources/kzg/g2.point.powerOf2", @@ -41,13 +43,19 @@ func makeTestEncoder() (core.Encoder, error) { NumWorker: uint64(runtime.GOMAXPROCS(0)), } - kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, false) + kzgEncoderGroup, err := prover.NewProver(config, true) + if err != nil { + return nil, err + } + + kzgVerifierGroup, err := verifier.NewVerifier(config, true) if err != nil { return nil, err } return &encoding.Encoder{ - EncoderGroup: kzgEncoderGroup, + EncoderGroup: kzgEncoderGroup, + VerifierGroup: kzgVerifierGroup, }, nil } func newTestServer(t *testing.T) *retriever.Server { diff --git a/test/integration_test.go b/test/integration_test.go index 7fdd2cb4ae..7f6e25b4f9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/Layr-Labs/eigenda/common/pubip" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" "github.com/consensys/gnark-crypto/ecc/bn254/fp" clientsmock "github.com/Layr-Labs/eigenda/clients/mock" @@ -40,7 +41,6 @@ import ( "github.com/Layr-Labs/eigenda/disperser/common/inmem" "github.com/Layr-Labs/eigenda/node" nodegrpc "github.com/Layr-Labs/eigenda/node/grpc" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" "github.com/Layr-Labs/eigenda/pkg/kzg/bn254" nodepb "github.com/Layr-Labs/eigenda/api/grpc/node" @@ -77,7 +77,7 @@ func init() { // makeTestEncoder makes an encoder currently using the only supported backend. func mustMakeTestEncoder() core.Encoder { - config := kzgEncoder.KzgConfig{ + config := kzgrs.KzgConfig{ G1Path: "../inabox/resources/kzg/g1.point", G2Path: "../inabox/resources/kzg/g2.point", CacheDir: "../inabox/resources/kzg/SRSTables", diff --git a/test/synthetic-test/synthetic_client_test.go b/test/synthetic-test/synthetic_client_test.go index a6581aae3f..10111155e7 100644 --- a/test/synthetic-test/synthetic_client_test.go +++ b/test/synthetic-test/synthetic_client_test.go @@ -34,7 +34,7 @@ import ( coremock "github.com/Layr-Labs/eigenda/core/mock" "github.com/Layr-Labs/eigenda/core/thegraph" encoder_rpc "github.com/Layr-Labs/eigenda/disperser/api/grpc/encoder" - "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder" + "github.com/Layr-Labs/eigenda/encoding/kzgrs" gcommon "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -221,7 +221,7 @@ func setupRetrievalClient(ethClient common.EthClient, retrievalClientConfig *Ret return err } encoder, err := encoding.NewEncoder(encoding.EncoderConfig{ - KzgConfig: kzgEncoder.KzgConfig{ + KzgConfig: kzgrs.KzgConfig{ G1Path: retrievalClientConfig.RetrieverG1Path, G2Path: retrievalClientConfig.RetrieverG2Path, CacheDir: retrievalClientConfig.RetrieverCachePath, @@ -578,7 +578,7 @@ func TestEncodeBlob(t *testing.T) { } // Test Assumes below params set for Encoder - kzgConfig := kzgEncoder.KzgConfig{ + kzgConfig := kzgrs.KzgConfig{ G1Path: "/data/kzg/g1.point", G2Path: "/data/kzg/g2.point", CacheDir: "/data/kzg/SRSTables",