Adding basic project and update docs (wip) (#1)

* commons (config + key utils)

* core pacakge (wip)

* cmd for keygen

* main (wip) + client (placeholder) cmds

* sanity test

* infra:

- docker
- gh workflows
- makefile
- protoc (wip)
- config examples

* updates and fixes

* update docs doc

.github/workflows/lint.yml

@@ -0,0 +1,25 @@
+name: Lint 
+
+on:
+  push:
+    branches: [ "main", "dev" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+
+  golint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: '1.20'
+
+    - name: Format
+      run: make fmt 
+
+    - name: Lint
+      run: make lint

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+
+on:
+  push:
+    branches: [ "main", "dev" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+
+  gotest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: '1.20'
+
+    - name: Test
+      run: make test

.gitignore

@@ -19,3 +19,8 @@
 
 # Go workspace file
 go.work
+
+bin
+
+cover.out
+cover.html

DONS.md

@@ -2,21 +2,40 @@
 
 `p2pmq` enables to compose a network of peers cross DONs (Decentralized Oracles Network), acting as a decentralized message bus for DON to DON communication.
 
-The following diagram illustrates the composition of a network of `p2pmq` peers across DONs:
+## Overview
+
+`p2pmq` agents can run as a sidecar to the DON's nodes, and enables to gossip messages over topics with optimal latency, while enabling a decoupled message validation to avoid introducing additional dependencies for the agent e.g. public keys, persistent storage of reports, etc.
+
+Gossiping OCR reports enables to achieve optimal latency and throughput, while maintaining an optimal network topology, w/o external components or ledgers.
+
+Scoring and msg validation are used to protect the network from bad actors and ensure integrity.
+
+The following diagram visualizes the composition of a network of `p2pmq` agents across DONs:
 
 ![p2pmq DON Composition](./resources/img/composer-p2pmq.png)
 
 <br />
 
 ## Messaging
 
-DONs communication is based on OCR reports, which are broadcasted over some topic rather than on-chain transmission.
+DONs communication is based on OCR reports, which are broadcasted over some pubsub topic rather than on-chain transmission.
 
-The reports MUST be signed by a quorum of the DON's nodes, otherwise they are considered invalid and any nodes that broadcast them are penalized.
+The reports MUST be signed by a quorum of the DON's nodes, otherwise they are considered invalid and nodes that broadcast them are penalized.
 
-**NOTE** `p2pmq` enables to aid in a custom validation before processing and propagating messages to the network.
+### Message Validation
 
-**TBD** signature validation cross DONs.
+`p2pmq` enables to aid in a custom, decoupled validation before processing 
+and propagating messages to the network. 
+The validation is done on top of open gRPC duplex stream to ensure 
+high throughput and low latency as possible.
 
-<br />
+The actual validation needs to verify that a given report was originated by some DON, where at least a quorum of nodes have confirmed it. 
 
+**TBD** public key sharing cross DONs.
+
+In addition, sequence number is used to ensure message order and penalize bad actors
+that sends unrealistic sequence numbers.
+
+The disincentivation of sending and propogating invalid messages across the network, helps to protect the network from bad actors. Enabling a trustless environment for DONs to communicate.
+
+<br />

Dockerfile

@@ -0,0 +1,33 @@
+# Build
+
+FROM golang:1.20 AS builder
+
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+  git make g++ gcc-aarch64-linux-gnu wget \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG APP_VERSION
+ARG APP_NAME
+ARG BUILD_TARGET
+
+WORKDIR /p2pmq
+
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+
+RUN GOOS=linux CGO_ENABLED=0 go build -tags netgo -a -v -o ./bin/${BUILD_TARGET} ./cmd/${BUILD_TARGET}
+
+# Runtime
+
+FROM alpine:latest as runner
+
+ARG BUILD_TARGET
+
+RUN apk --no-cache --upgrade add ca-certificates bash
+
+WORKDIR /p2pmq
+
+COPY --from=builder /p2pmq/.env* ./
+COPY --from=builder /p2pmq/bin/${BUILD_TARGET} ./app

Makefile

@@ -0,0 +1,39 @@
+APP_NAME?=p2pmq
+BUILD_TARGET?=${APP_NAME}
+BUILD_IMG?=${APP_NAME}
+APP_VERSION?=$(git describe --tags $(git rev-list --tags --max-count=1) 2> /dev/null || echo "nightly")
+CFG_PATH?=/route-p2p/router.json
+TEST_PKG?=./core/...
+TEST_TIMEOUT?=2m
+
+lint:
+	@docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v1.54 golangci-lint run -v --timeout=5m ./...
+
+fmt:
+	@go fmt ./...
+
+test:
+	@go test -v -race -timeout=${TEST_TIMEOUT} `go list ./... | grep -v -E "cmd|scripts|resources"`
+
+test-pkg:
+	@go test -v -race -timeout=${TEST_TIMEOUT} ${TEST_PKG}
+
+test-cov:
+	@go test -v -race -timeout=${TEST_TIMEOUT} -coverprofile cover.out `go list ./... | grep -v -E "cmd|scripts|resources"`
+
+test-open-cov:
+	@make test-cov
+	@go tool cover -html cover.out -o cover.html
+	open cover.html
+
+keygen:
+	@go run ./cmd/keygen/main.go
+
+build:
+	@go build -o "./bin/${BUILD_TARGET}" "./cmd/${BUILD_TARGET}"
+
+docker-build:
+	@docker build -t "${APP_NAME}" --build-arg APP_VERSION="${APP_VERSION}" --build-arg APP_NAME="${APP_NAME}" --build-arg BUILD_TARGET="${BUILD_TARGET}" .
+
+docker-run:
+	@docker run -d --restart unless-stopped --name "${APP_NAME}" -v "${PWD}/data/${APP_NAME}/:/p2pmq/.data" -p "${TCP_PORT}":"${TCP_PORT}" -p "${GRPC_PORT}":"${GRPC_PORT}" -e "GRPC_PORT=${GRPC_PORT}" -it "${BUILD_IMG}" /p2pmq/app -config=${CFG_PATH}

cmd/keygen/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/amirylm/p2pmq/commons"
+)
+
+func main() {
+	_, skB64, err := commons.GetOrGeneratePrivateKey("")
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%s\n", skB64)
+}

cmd/p2pmq/main.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/amirylm/p2pmq/commons"
+	"github.com/amirylm/p2pmq/core"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := &cli.App{
+		Name:  "route-p2p",
+		Usage: "p2p router",
+		Flags: []cli.Flag{
+			// cli.IntFlag{
+			// 	Name:   "grpc-port",
+			// 	EnvVar: "GRPC_PORT",
+			// 	Value:  12001,
+			// },
+			// cli.IntFlag{
+			// 	Name:   "monitor-port",
+			// 	EnvVar: "MONITOR_PORT",
+			// },
+			cli.StringFlag{
+				Name:   "config",
+				EnvVar: "P2PMQ_CONFIG",
+				Value:  "/p2pmq/p2pmq.json",
+			},
+			cli.StringFlag{
+				Name:   "loglevel",
+				EnvVar: "P2PMQ_LOGLEVEL",
+				Value:  "info",
+			},
+			cli.StringFlag{
+				Name:   "libp2p-loglevel",
+				EnvVar: "LIBP2P_LOGLEVEL",
+				Value:  "info",
+			},
+		},
+		Action: func(c *cli.Context) (err error) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			_ = logging.SetLogLevelRegex("p2p:.*", c.String("libp2p-loglevel"))
+			_ = logging.SetLogLevelRegex("p2pmq.*", c.String("loglevel"))
+
+			var d *core.Daemon
+			if cfgPath := c.String("config"); len(cfgPath) > 0 {
+				cfg, err := commons.ReadConfig(cfgPath)
+				if err != nil {
+					return err
+				}
+				d, err = core.NewDaemon(ctx, *cfg, nil, "node")
+				if err != nil {
+					return err
+				}
+				if err := d.Start(ctx); err != nil {
+					return err
+				}
+				defer func() {
+					_ = d.Close()
+				}()
+			}
+
+			if d == nil {
+				return fmt.Errorf("could not create daemon instance, please provide a config file")
+			}
+
+			<-ctx.Done()
+
+			logging.Logger("p2pmq/cli").Info("closing node")
+
+			return nil
+
+			// svc := service.NewGrpc(ctx, c.String("name"))
+			// defer svc.Close()
+
+			// if monitorPort := c.Int("monitor-port"); monitorPort > 0 {
+			// 	mux := http.NewServeMux()
+			// 	monitoring.WithMetrics(mux)
+			// 	monitoring.WithProfiling(mux)
+			// 	monitoring.WithHealthCheck(mux, func() []error {
+			// 		err := ctx.Err()
+			// 		if err != nil {
+			// 			return []error{err}
+			// 		}
+			// 		return nil
+			// 	})
+			// 	go func() {
+			// 		err := http.ListenAndServe(fmt.Sprintf(":%d", monitorPort), mux)
+			// 		if err != nil {
+			// 			panic(err)
+			// 		}
+			// 	}()
+			// }
+
+			// s := svc.GrpcServer()
+			// return service.ListenGrpc(s, c.Int("grpc-port"))
+		},
+		Commands: []cli.Command{},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		panic(err)
+	}
+}

cmd/pqclient/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+
+}