Refactor encoding (1 of N) (#266)

clients/tests/retrieval_client_test.go

@@ -13,8 +13,10 @@ import (
 	"github.com/Layr-Labs/eigenda/core/encoding"
 	coreindexer "github.com/Layr-Labs/eigenda/core/indexer"
 	coremock "github.com/Layr-Labs/eigenda/core/mock"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs/prover"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier"
 	indexermock "github.com/Layr-Labs/eigenda/indexer/mock"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
 	"github.com/consensys/gnark-crypto/ecc/bn254"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -25,7 +27,7 @@ import (
 const numOperators = 10
 
 func makeTestEncoder() (core.Encoder, error) {
-	config := &kzgEncoder.KzgConfig{
+	config := &kzgrs.KzgConfig{
 		G1Path:          "../../inabox/resources/kzg/g1.point",
 		G2Path:          "../../inabox/resources/kzg/g2.point",
 		CacheDir:        "../../inabox/resources/kzg/SRSTables",
@@ -34,13 +36,19 @@ func makeTestEncoder() (core.Encoder, error) {
 		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, true)
+	kzgEncoderGroup, err := prover.NewProver(config, true)
+	if err != nil {
+		return nil, err
+	}
+
+	kzgVerifierGroup, err := verifier.NewVerifier(config, true)
 	if err != nil {
 		return nil, err
 	}
 
 	return &encoding.Encoder{
-		EncoderGroup: kzgEncoderGroup,
+		EncoderGroup:  kzgEncoderGroup,
+		VerifierGroup: kzgVerifierGroup,
 	}, nil
 }
 

core/encoding.go

@@ -3,7 +3,7 @@ package core
 import (
 	"fmt"
 
-	"github.com/Layr-Labs/eigenda/pkg/encoding/encoder"
+	encoder "github.com/Layr-Labs/eigenda/encoding/rs"
 	"github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 )
 

core/encoding/cli.go

@@ -4,7 +4,7 @@ import (
 	"runtime"
 
 	"github.com/Layr-Labs/eigenda/common"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
 	"github.com/urfave/cli"
 )
 
@@ -88,7 +88,7 @@ func CLIFlags(envPrefix string) []cli.Flag {
 }
 
 func ReadCLIConfig(ctx *cli.Context) EncoderConfig {
-	cfg := kzgEncoder.KzgConfig{}
+	cfg := kzgrs.KzgConfig{}
 	cfg.G1Path = ctx.GlobalString(G1PathFlagName)
 	cfg.G2Path = ctx.GlobalString(G2PathFlagName)
 	cfg.CacheDir = ctx.GlobalString(CachePathFlagName)

core/encoding/encoder.go

@@ -4,8 +4,11 @@ import (
 	"crypto/sha256"
 
 	"github.com/Layr-Labs/eigenda/core"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/encoder"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/encoding"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs/prover"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs/verifier"
+	encoder "github.com/Layr-Labs/eigenda/encoding/rs"
 	"github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 	lru "github.com/hashicorp/golang-lru/v2"
 )
@@ -15,20 +18,26 @@ func toEncParams(params core.EncodingParams) encoder.EncodingParams {
 }
 
 type EncoderConfig struct {
-	KzgConfig         kzgEncoder.KzgConfig
+	KzgConfig         kzgrs.KzgConfig
 	CacheEncodedBlobs bool
 }
 
 type Encoder struct {
-	Config       EncoderConfig
-	EncoderGroup *kzgEncoder.KzgEncoderGroup
-	Cache        *lru.Cache[string, encodedValue]
+	Config        EncoderConfig
+	EncoderGroup  *prover.Prover
+	VerifierGroup *verifier.Verifier
+	Cache         *lru.Cache[string, encodedValue]
 }
 
 var _ core.Encoder = &Encoder{}
 
 func NewEncoder(config EncoderConfig, loadG2Points bool) (*Encoder, error) {
-	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig, loadG2Points)
+	kzgEncoderGroup, err := prover.NewProver(&config.KzgConfig, loadG2Points)
+	if err != nil {
+		return nil, err
+	}
+
+	kzgVerifierGroup, err := verifier.NewVerifier(&config.KzgConfig, loadG2Points)
 	if err != nil {
 		return nil, err
 	}
@@ -39,9 +48,10 @@ func NewEncoder(config EncoderConfig, loadG2Points bool) (*Encoder, error) {
 	}
 
 	return &Encoder{
-		EncoderGroup: kzgEncoderGroup,
-		Cache:        cache,
-		Config:       config,
+		EncoderGroup:  kzgEncoderGroup,
+		VerifierGroup: kzgVerifierGroup,
+		Cache:         cache,
+		Config:        config,
 	}, nil
 }
 
@@ -100,23 +110,23 @@ func (e *Encoder) Encode(data []byte, params core.EncodingParams) (core.BlobComm
 }
 
 func (e *Encoder) VerifyBlobLength(commitments core.BlobCommitments) error {
-	return e.EncoderGroup.VerifyCommit((*bn254.G2Point)(commitments.LengthCommitment), (*bn254.G2Point)(commitments.LengthProof), uint64(commitments.Length))
+	return e.VerifierGroup.VerifyCommit((*bn254.G2Point)(commitments.LengthCommitment), (*bn254.G2Point)(commitments.LengthProof), uint64(commitments.Length))
 
 }
 
 func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber, commitments core.BlobCommitments, params core.EncodingParams) error {
 
 	encParams := toEncParams(params)
 
-	verifier, err := e.EncoderGroup.GetKzgVerifier(encParams)
+	verifier, err := e.VerifierGroup.GetKzgVerifier(encParams)
 	if err != nil {
 		return err
 	}
 
 	for ind := range chunks {
 		err = verifier.VerifyFrame(
 			(*bn254.G1Point)(commitments.Commitment),
-			&kzgEncoder.Frame{
+			&encoding.Frame{
 				Proof:  chunks[ind].Proof,
 				Coeffs: chunks[ind].Coeffs,
 			},
@@ -133,21 +143,21 @@ func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber,
 }
 
 func (e *Encoder) VerifyCommitEquivalenceBatch(commitments []core.BlobCommitments) error {
-	commitmentsPair := make([]kzgEncoder.CommitmentPair, len(commitments))
+	commitmentsPair := make([]verifier.CommitmentPair, len(commitments))
 
 	for i, c := range commitments {
-		commitmentsPair[i] = kzgEncoder.CommitmentPair{
+		commitmentsPair[i] = verifier.CommitmentPair{
 			Commitment:       (bn254.G1Point)(*c.Commitment),
 			LengthCommitment: (bn254.G2Point)(*c.LengthCommitment),
 		}
 	}
-	return e.EncoderGroup.BatchVerifyCommitEquivalence(commitmentsPair)
+	return e.VerifierGroup.BatchVerifyCommitEquivalence(commitmentsPair)
 }
 
 // convert struct understandable by the crypto library
 func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCore []core.Sample, numBlobs int) error {
 	encParams := toEncParams(params)
-	samples := make([]kzgEncoder.Sample, len(samplesCore))
+	samples := make([]verifier.Sample, len(samplesCore))
 
 	for i, sc := range samplesCore {
 		x, err := encoder.GetLeadingCosetIndex(
@@ -158,7 +168,7 @@ func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCor
 			return err
 		}
 
-		sample := kzgEncoder.Sample{
+		sample := verifier.Sample{
 			Commitment: (bn254.G1Point)(*sc.Commitment),
 			Proof:      sc.Chunk.Proof,
 			RowIndex:   sc.BlobIndex,
@@ -168,15 +178,15 @@ func (e *Encoder) UniversalVerifySubBatch(params core.EncodingParams, samplesCor
 		samples[i] = sample
 	}
 
-	return e.EncoderGroup.UniversalVerify(encParams, samples, numBlobs)
+	return e.VerifierGroup.UniversalVerify(encParams, samples, numBlobs)
 }
 
 // Decode takes in the chunks, indices, and encoding parameters and returns the decoded blob
 // The result is trimmed to the given maxInputSize.
 func (e *Encoder) Decode(chunks []*core.Chunk, indices []core.ChunkNumber, params core.EncodingParams, maxInputSize uint64) ([]byte, error) {
-	frames := make([]kzgEncoder.Frame, len(chunks))
+	frames := make([]encoding.Frame, len(chunks))
 	for i := range chunks {
-		frames[i] = kzgEncoder.Frame{
+		frames[i] = encoding.Frame{
 			Proof:  chunks[i].Proof,
 			Coeffs: chunks[i].Coeffs,
 		}

core/encoding/encoder_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/Layr-Labs/eigenda/core"
 	"github.com/Layr-Labs/eigenda/core/encoding"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,7 +28,7 @@ func init() {
 
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
-	config := kzgEncoder.KzgConfig{
+	config := kzgrs.KzgConfig{
 		G1Path:          "../../inabox/resources/kzg/g1.point.300000",
 		G2Path:          "../../inabox/resources/kzg/g2.point.300000",
 		CacheDir:        "../../inabox/resources/kzg/SRSTables",

core/test/core_test.go

@@ -12,10 +12,9 @@ import (
 	"github.com/Layr-Labs/eigenda/core"
 	"github.com/Layr-Labs/eigenda/core/encoding"
 	"github.com/Layr-Labs/eigenda/core/mock"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
 	"github.com/gammazero/workerpool"
 	"github.com/stretchr/testify/assert"
-
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
 )
 
 var (
@@ -40,7 +39,7 @@ func setup(m *testing.M) {
 
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
-	config := kzgEncoder.KzgConfig{
+	config := kzgrs.KzgConfig{
 		G1Path:          "../../inabox/resources/kzg/g1.point",
 		G2Path:          "../../inabox/resources/kzg/g2.point",
 		CacheDir:        "../../inabox/resources/kzg/SRSTables",

disperser/batcher/batcher_test.go

@@ -21,7 +21,7 @@ import (
 	batchermock "github.com/Layr-Labs/eigenda/disperser/batcher/mock"
 	"github.com/Layr-Labs/eigenda/disperser/common/inmem"
 	dmock "github.com/Layr-Labs/eigenda/disperser/mock"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
 	gethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/stretchr/testify/assert"
@@ -43,7 +43,7 @@ type batcherComponents struct {
 
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
-	config := kzgEncoder.KzgConfig{
+	config := kzgrs.KzgConfig{
 		G1Path:          "../../inabox/resources/kzg/g1.point",
 		G2Path:          "../../inabox/resources/kzg/g2.point",
 		CacheDir:        "../../inabox/resources/kzg/SRSTables",

disperser/encoder/server_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/Layr-Labs/eigenda/core/encoding"
 	coremock "github.com/Layr-Labs/eigenda/core/mock"
 	pb "github.com/Layr-Labs/eigenda/disperser/api/grpc/encoder"
-	"github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/encoding/kzgrs"
 	"github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 )
 
@@ -30,7 +30,7 @@ var (
 var logger = &cmock.Logger{}
 
 func makeTestEncoder(numPoint uint64) (*encoding.Encoder, ServerConfig) {
-	kzgConfig := kzgEncoder.KzgConfig{
+	kzgConfig := kzgrs.KzgConfig{
 		G1Path:          "../../inabox/resources/kzg/g1.point",
 		G2Path:          "../../inabox/resources/kzg/g2.point",
 		CacheDir:        "../../inabox/resources/kzg/SRSTables",

encoding/encoding.go

@@ -0,0 +1,36 @@
+package encoding
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
+)
+
+// Proof is the multireveal proof
+// Coeffs is identical to input data converted into Fr element
+type Frame struct {
+	Proof  bn254.G1Point
+	Coeffs []bn254.Fr
+}
+
+func (f *Frame) Encode() ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(f)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func Decode(b []byte) (Frame, error) {
+	var f Frame
+	buf := bytes.NewBuffer(b)
+	dec := gob.NewDecoder(buf)
+	err := dec.Decode(&f)
+	if err != nil {
+		return Frame{}, err
+	}
+	return f, nil
+}

encoding/kzgrs/kzgrs.go

@@ -0,0 +1,14 @@
+package kzgrs
+
+type KzgConfig struct {
+	G1Path          string
+	G2Path          string
+	G1PowerOf2Path  string
+	G2PowerOf2Path  string
+	CacheDir        string
+	NumWorker       uint64
+	SRSOrder        uint64 // Order is the total size of SRS
+	SRSNumberToLoad uint64 // Number of points to be loaded from the begining
+	Verbose         bool
+	PreloadEncoder  bool
+}