Merge pull request #8 from butaneprotocol/better-dkg

Better dkg

README.md

@@ -2,9 +2,13 @@
 
 ## Setup
 
+### Pick a key directory
+
+The oracle uses several sets of private/public keys. By default, these are stored in a `keys` directory relative to your PWD, but you can set a `KEYS_DIRECTORY` env var to change that.
+
 ### Generate a private key
 
-Generate an ED25519 private key, stored in PEM format. The docker-compose file expects it in `keys/private.pem`.
+Generate an ED25519 private key, stored in PEM format. The oracle will look for this in `${KEYS_DIRECTORY}/private.pem`.
 
 ```sh
 openssl genpkey -algorithm ed25519 -out ./keys/private.pem
@@ -24,15 +28,15 @@ If you want, you can also override any default config settings (which are define
 
 ### Generate FROST keys
 
-The oracle needs a FROST key pair in order to sign payloads. The docker-compose file expects the private key in `keys/frost/private`, and the shared public key in `keys/frost/public`. You have two ways to get these keys:
+The oracle needs a FROST key pair in order to sign payloads. These keys are stored in `KEYS_DIRECTORY`. You have two ways to generate these keys:
 
 #### With DKG
 
 This oracle supports a "keygen mode". If every node is running in "keygen mode", they will collectively generate a new set up keys and store them to disk.
 
 To initiate DKG, set `keygen.enabled` to `true` in your `config.yaml`. Make sure that every node agrees on the value of `keygen.min_signers`, or DKG will fail.
 
-Once every node is online, they will finish DKG and store the keys to disk. Once those files exist, it is safe to restart your node with keygen mode disabled.
+Once every node is online, they will finish DKG and store the keys to disk. Once those files exist, you should update your configuration with a new `frost_address` and disable keygen mode (by setting `keygen.enabled` to `false`).
 
 #### Without DKG 
 
@@ -42,6 +46,8 @@ You can generate a set of FROST keys with the keygen command:
 cargo run --bin keygen -- --min-signers 2 --max-signers 3
 ```
 
+These will be saved in subdirectories of `KEYS_DIRECTORY`.
+
 ### Set up Maestro
 
 Querying prices from Maestro requires an API key. To query Maestro, create a `.env` file with your API key like so:

config.example.yaml

@@ -1,6 +1,9 @@
 keygen:
   enabled: false # Set this to true to generate frost keys instead of serving traffic
   min_signers: 2 # How many signers do we need to publish a package? Every node must agree on this.
+frost_address: addr11111111111
+  # This is the address of your current FROST public key. The key generation process will print it.
+  # Every node must agree on this.
 peers:
   - address: 127.0.0.1:8001
     label: Descriptive label

docker-compose.yaml

@@ -4,11 +4,9 @@ services:
     command: ["./oracles", "-c", "/app/config.yaml"]
     volumes:
       - ./config.yaml:/app/config.yaml
-      - ./keys/frost:/app/frost
+      - ./keys:/app/keys
     environment:
-      PRIVATE_KEY_PATH: /run/secrets/private_key
-      FROST_KEY_PATH: /app/frost/frost_key
-      FROST_PUBLIC_KEY_PATH: /app/frost/frost_public_key
+      KEY_DIRECTORY: /app/keys
     env_file:
       - path: .env
         required: false
@@ -21,8 +19,6 @@ services:
       - 31415:31415
       - 18000:18000
     restart: unless-stopped
-    secrets:
-      - private_key
   cardano-node:
     command: [
       run,
@@ -58,6 +54,3 @@ services:
       - ./mainnet-config:/config
       - ./volumes/kupo-db:/db
       - ${IPC_DIR:-./volumes/node-ipc}:/ipc
-secrets:
-  private_key:
-    file: keys/private.pem

src/bin/keygen.rs

@@ -2,7 +2,8 @@ use std::{env::current_dir, fs};
 
 use anyhow::Result;
 use clap::Parser;
-use frost_ed25519::keys::{self, IdentifierList, KeyPackage};
+use frost_ed25519::keys::{self as frost_keys, IdentifierList, KeyPackage};
+use oracles::keys;
 use rand::thread_rng;
 
 #[derive(Parser, Debug)]
@@ -18,25 +19,27 @@ pub fn main() -> Result<()> {
     let args = Args::parse();
 
     let rng = thread_rng();
-    let (shares, pubkey_package) = keys::generate_with_dealer(
+    let (shares, pubkey_package) = frost_keys::generate_with_dealer(
         args.max_signers,
         args.min_signers,
         IdentifierList::Default,
         rng,
     )?;
 
     let keys_path = current_dir()?.join("keys");
-
-    fs::create_dir_all(&keys_path)?;
-
-    let pubkey_path = keys_path.join("frost_public");
-    fs::write(pubkey_path, pubkey_package.serialize()?)?;
-
+    let mut address = None;
     for (index, share) in shares.into_values().enumerate() {
-        let privkey_path = keys_path.join(format!("frost_private_{}", index));
+        let keys_dir = keys_path.join(format!("node{}", index));
+        fs::create_dir_all(&keys_dir)?;
         let privkey_package: KeyPackage = share.try_into()?;
-        fs::write(privkey_path, privkey_package.serialize()?)?;
+        address = Some(keys::write_frost_keys(
+            &keys_dir,
+            privkey_package,
+            pubkey_package.clone(),
+        )?);
     }
 
+    println!("The new frost address is: {}", address.unwrap());
+
     Ok(())
 }

src/config.rs

@@ -15,6 +15,7 @@ pub struct OracleConfig {
     pub heartbeat_ms: u64,
     pub timeout_ms: u64,
     pub logs: LogConfig,
+    pub frost_address: Option<String>,
     pub keygen: KeygenConfig,
     pub synthetics: Vec<SyntheticConfig>,
     pub collateral: Vec<CollateralConfig>,

src/keygen.rs → src/dkg.rs

@@ -1,7 +1,8 @@
-use std::{collections::BTreeMap, env, fs, sync::Arc, time::Duration};
+use std::{collections::BTreeMap, sync::Arc, time::Duration};
 
 use crate::{
     config::OracleConfig,
+    keys::{self, get_keys_directory},
     network::{Network, NodeId},
 };
 use anyhow::{anyhow, Context, Result};
@@ -35,10 +36,13 @@ pub async fn run(config: &OracleConfig) -> Result<()> {
     let mut network = Network::new(config)?;
     let id = network.id.clone();
 
-    // Fetch the paths to store the frost keys early, so we can fail fast on misconfiguration.
-    let key_path = env::var("FROST_KEY_PATH").context("FROST_KEY_PATH not set")?;
-    let public_key_path =
-        env::var("FROST_PUBLIC_KEY_PATH").context("FROST_PUBLIC_KEY_PATH not set")?;
+    let keys_dir = get_keys_directory()?;
+
+    // Log where the keys will be saved
+    info!(
+        "Running in DKG mode. Will generate frost keys into {}.",
+        keys_dir.display()
+    );
 
     let identifier_lookup: Arc<DashMap<Identifier, NodeId>> = Arc::new(DashMap::new());
 
@@ -141,10 +145,10 @@ pub async fn run(config: &OracleConfig) -> Result<()> {
                     let (key_package, public_key_package) =
                         part3(round2_secret_package, &round1_packages, &round2_packages)?;
                     info!("Key generation complete!");
-                    fs::write(&key_path, key_package.serialize()?)?;
-                    info!("Frost private key saved to {}", key_path);
-                    fs::write(&public_key_path, public_key_package.serialize()?)?;
-                    info!("Frost public key saved to {}", public_key_path);
+                    let address =
+                        keys::write_frost_keys(&keys_dir, key_package, public_key_package)
+                            .context("Could not save frost keys")?;
+                    info!("The new frost address is: {}", address);
                 }
             }
         }

src/keys.rs

@@ -0,0 +1,73 @@
+use anyhow::{Context, Result};
+use bech32::{Bech32, Hrp};
+use frost_ed25519::keys::{KeyPackage, PublicKeyPackage};
+use pallas_crypto::hash::Hasher;
+use std::{
+    env::{self, VarError},
+    fs,
+    path::{Path, PathBuf},
+};
+
+pub fn get_keys_directory() -> Result<PathBuf> {
+    let dir: PathBuf = match env::var("KEYS_DIRECTORY") {
+        Ok(path) => path.into(),
+        Err(VarError::NotUnicode(path)) => path.into(),
+        Err(VarError::NotPresent) => "keys".into(),
+    };
+    if dir.is_absolute() {
+        return Ok(dir);
+    }
+    let pwd = env::current_dir().context("could not find keys directory")?;
+    Ok(pwd.join(dir))
+}
+
+pub fn read_frost_keys(
+    keys_dir: &Path,
+    public_key_hash: &str,
+) -> Result<(KeyPackage, PublicKeyPackage)> {
+    let frost_key_path = keys_dir.join(public_key_hash);
+
+    let private_key_path = frost_key_path.join("frost.skey");
+    let private_key_bytes = fs::read(&private_key_path).context(format!(
+        "Could not find frost private key in {}",
+        private_key_path.display()
+    ))?;
+    let private_key =
+        KeyPackage::deserialize(&private_key_bytes).context("Could not decode private key")?;
+
+    let public_key_path = frost_key_path.join("frost.vkey");
+    let public_key_bytes = fs::read(&public_key_path).context(format!(
+        "Could not find frost public key in {}",
+        public_key_path.display()
+    ))?;
+    let public_key =
+        PublicKeyPackage::deserialize(&public_key_bytes).context("Could not decode public key")?;
+
+    Ok((private_key, public_key))
+}
+
+pub fn write_frost_keys(
+    keys_dir: &Path,
+    private_key: KeyPackage,
+    public_key: PublicKeyPackage,
+) -> Result<String> {
+    let verifying_key = public_key.verifying_key();
+    let address = compute_address(&verifying_key.serialize())?;
+    let frost_key_path = keys_dir.join(&address);
+    fs::create_dir_all(&frost_key_path)?;
+
+    let private_key_bytes = private_key.serialize()?;
+    let public_key_bytes = public_key.serialize()?;
+    fs::write(frost_key_path.join("frost.skey"), private_key_bytes)?;
+    fs::write(frost_key_path.join("frost.vkey"), public_key_bytes)?;
+    Ok(address)
+}
+
+const ADDR_HRP: Hrp = Hrp::parse_unchecked("addr");
+
+fn compute_address(verifying_key: &[u8]) -> Result<String> {
+    let blake2b = Hasher::<224>::hash(verifying_key);
+    let mut bytes = vec![0x61];
+    bytes.extend_from_slice(blake2b.as_ref());
+    Ok(bech32::encode::<Bech32>(ADDR_HRP, &bytes)?)
+}

src/lib.rs

@@ -0,0 +1,11 @@
+pub mod apis;
+pub mod config;
+pub mod dkg;
+pub mod health;
+pub mod keys;
+pub mod network;
+pub mod price_aggregator;
+pub mod price_feed;
+pub mod publisher;
+pub mod raft;
+pub mod signature_aggregator;

src/main.rs

@@ -2,13 +2,16 @@ use std::{str::FromStr, sync::Arc, time::Duration};
 
 use anyhow::Result;
 use clap::Parser;
-use config::{load_config, LogConfig, OracleConfig};
-use health::{HealthServer, HealthSink};
-use network::Network;
-use price_aggregator::PriceAggregator;
-use publisher::Publisher;
-use raft::{Raft, RaftLeader};
-use signature_aggregator::SignatureAggregator;
+use oracles::{
+    config::{load_config, LogConfig, OracleConfig},
+    dkg,
+    health::{HealthServer, HealthSink},
+    network::Network,
+    price_aggregator::PriceAggregator,
+    publisher::Publisher,
+    raft::{Raft, RaftLeader},
+    signature_aggregator::SignatureAggregator,
+};
 use tokio::{
     sync::{mpsc, watch},
     task::{JoinError, JoinSet},
@@ -17,17 +20,6 @@ use tokio::{
 use tracing::{info, info_span, Instrument, Level, Span};
 use tracing_subscriber::FmtSubscriber;
 
-pub mod apis;
-pub mod config;
-pub mod health;
-pub mod keygen;
-pub mod network;
-pub mod price_aggregator;
-pub mod price_feed;
-pub mod publisher;
-pub mod raft;
-pub mod signature_aggregator;
-
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
 struct Args {
@@ -69,7 +61,7 @@ impl Node {
         let (result_tx, result_rx) = mpsc::channel(10);
 
         let signature_aggregator = if config.consensus {
-            SignatureAggregator::consensus(&mut network, pa_rx, leader_rx, result_tx)?
+            SignatureAggregator::consensus(&config, &mut network, pa_rx, leader_rx, result_tx)?
         } else {
             SignatureAggregator::single(pa_rx, leader_rx, result_tx)?
         };
@@ -218,7 +210,7 @@ async fn main() -> Result<()> {
     span.in_scope(|| info!("Node starting..."));
 
     if config.keygen.enabled {
-        keygen::run(&config).instrument(span).await?;
+        dkg::run(&config).instrument(span).await?;
         return Ok(());
     }
 

src/network.rs

@@ -4,7 +4,7 @@ use tokio::{sync::mpsc, task::JoinSet};
 use tracing::{info, warn, Instrument};
 
 use crate::config::OracleConfig;
-use crate::keygen::KeygenMessage;
+use crate::dkg::KeygenMessage;
 use crate::raft::RaftMessage;
 use crate::signature_aggregator::signer::SignerMessage;
 pub use channel::{NetworkChannel, NetworkReceiver, NetworkSender};

src/network/core.rs

@@ -32,6 +32,7 @@ type Nonce = chacha20poly1305::aead::generic_array::GenericArray<u8, chacha20pol
 
 use crate::{
     config::{OracleConfig, PeerConfig},
+    keys::get_keys_directory,
     raft::RaftMessage,
 };
 
@@ -563,8 +564,11 @@ fn parse_peer(config: &PeerConfig) -> Result<Peer> {
 }
 
 fn read_private_key() -> Result<PrivateKey> {
-    let key_path = env::var("PRIVATE_KEY_PATH").context("PRIVATE_KEY_PATH not set")?;
-    let key_pem_file = fs::read_to_string(key_path).context("could not load private key")?;
+    let key_path = get_keys_directory()?.join("private.pem");
+    let key_pem_file = fs::read_to_string(&key_path).context(format!(
+        "Could not load private key from {}",
+        key_path.display()
+    ))?;
     let decoded = KeypairBytes::from_pkcs8_pem(&key_pem_file)?;
     let private_key = PrivateKey::from_bytes(&decoded.secret_key);
     Ok(private_key)

src/raft.rs

@@ -121,6 +121,10 @@ impl RaftState {
             RaftMessage::Disconnect => {
                 info!("Peer disconnected {}", from);
                 self.peers.remove(&from);
+                if self.peers.len() < self.quorum - 1 {
+                    info!("Too few peers connected, raft status unknown");
+                    self.clear_status();
+                }
                 vec![]
             }
             RaftMessage::Heartbeat { term } => {
@@ -306,6 +310,13 @@ impl RaftState {
         }
     }
 
+    fn clear_status(&mut self) {
+        self.set_status(RaftStatus::Follower {
+            leader: None,
+            voted_for: None,
+        });
+    }
+
     fn set_status(&mut self, status: RaftStatus) {
         let leader = match &status {
             RaftStatus::Leader => RaftLeader::Myself,