-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge key-manager to operator (#152)
* Merge key-manager to operator * Add PG Deps * Merge commands * Fix * Move tests * Move tests --------- Signed-off-by: antares-sw <[email protected]>
- Loading branch information
1 parent
5a3118a
commit 4d7f39f
Showing
19 changed files
with
1,202 additions
and
153 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,3 +12,4 @@ GIT_SHA | |
| build | ||
| database | ||
| *.db | ||
| .DS_Store | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| import json | ||
| from pathlib import Path | ||
|
|
||
| import click | ||
| import yaml | ||
| from eth_typing import HexStr | ||
|
|
||
| from src.common.validators import validate_db_uri, validate_eth_address | ||
| from src.key_manager.database import Database, check_db_connection | ||
|
|
||
| VALIDATOR_DEFINITIONS_FILENAME = 'validator_definitions.yml' | ||
| SIGNER_KEYS_FILENAME = 'signer_keys.yml' | ||
| PROPOSER_CONFIG_FILENAME = 'proposer_config.json' | ||
|
|
||
|
|
||
| @click.option( | ||
| '--validator-index', | ||
| help='The validator index to generate the configuration files.', | ||
| prompt='Enter the validator index to generate the configuration files', | ||
| type=int, | ||
| ) | ||
| @click.option( | ||
| '--total-validators', | ||
| help='The total number of validators connected to the web3signer.', | ||
| prompt='Enter the total number of validators connected to the web3signer', | ||
| type=click.IntRange(min=1), | ||
| ) | ||
| @click.option( | ||
| '--db-url', | ||
| help='The database connection address.', | ||
| prompt="Enter the database connection string, ex. 'postgresql://username:pass@hostname/dbname'", | ||
| callback=validate_db_uri, | ||
| ) | ||
| @click.option( | ||
| '--web3signer-endpoint', | ||
| help='The endpoint of the web3signer service.', | ||
| prompt='Enter the endpoint of the web3signer service', | ||
| type=str, | ||
| ) | ||
| @click.option( | ||
| '--fee-recipient', | ||
| help='The recipient address for MEV & priority fees.', | ||
| prompt='Enter the recipient address for MEV & priority fees', | ||
| type=str, | ||
| callback=validate_eth_address, | ||
| ) | ||
| @click.option( | ||
| '--disable-proposal-builder', | ||
| is_flag=True, | ||
| default=False, | ||
| help='Disable proposal builder for Teku and Prysm clients.', | ||
| ) | ||
| @click.option( | ||
| '--output-dir', | ||
| required=False, | ||
| help='The directory to save configuration files. Defaults to ./data/configs.', | ||
| default='./data/configs', | ||
| type=click.Path(exists=False, file_okay=False, dir_okay=True), | ||
| ) | ||
| @click.command( | ||
| help='Creates validator configuration files for Lighthouse, ' | ||
| 'Prysm, and Teku clients to sign data using keys from database.' | ||
| ) | ||
| # pylint: disable-next=too-many-arguments,too-many-locals | ||
| def sync_validator( | ||
| validator_index: int, | ||
| total_validators: int, | ||
| db_url: str, | ||
| web3signer_endpoint: str, | ||
| fee_recipient: str, | ||
| disable_proposal_builder: bool, | ||
| output_dir: str, | ||
| ) -> None: | ||
| check_db_connection(db_url) | ||
| check_validator_index(validator_index, total_validators) | ||
|
|
||
| database = Database(db_url=db_url) | ||
| public_keys_count = database.fetch_public_keys_count() | ||
|
|
||
| keys_per_validator = public_keys_count // total_validators | ||
|
|
||
| start_index = keys_per_validator * validator_index | ||
| if validator_index == total_validators - 1: | ||
| end_index = public_keys_count | ||
| else: | ||
| end_index = start_index + keys_per_validator | ||
|
|
||
| public_keys = database.fetch_public_keys_by_range(start_index=start_index, end_index=end_index) | ||
|
|
||
| if not public_keys: | ||
| raise click.ClickException('Database does not contain in range') | ||
|
|
||
| Path.mkdir(Path(output_dir), exist_ok=True, parents=True) | ||
|
|
||
| # lighthouse | ||
| validator_definitions_filepath = str(Path(output_dir, VALIDATOR_DEFINITIONS_FILENAME)) | ||
| _generate_lighthouse_config( | ||
| public_keys=public_keys, | ||
| web3signer_url=web3signer_endpoint, | ||
| fee_recipient=fee_recipient, | ||
| filepath=validator_definitions_filepath, | ||
| ) | ||
|
|
||
| # teku/prysm | ||
| signer_keys_filepath = str(Path(output_dir, SIGNER_KEYS_FILENAME)) | ||
| _generate_signer_keys_config(public_keys=public_keys, filepath=signer_keys_filepath) | ||
|
|
||
| proposer_config_filepath = str(Path(output_dir, PROPOSER_CONFIG_FILENAME)) | ||
| _generate_proposer_config( | ||
| fee_recipient=fee_recipient, | ||
| proposal_builder_enabled=not disable_proposal_builder, | ||
| filepath=proposer_config_filepath, | ||
| ) | ||
|
|
||
| click.clear() | ||
| click.secho( | ||
| f'Done. ' | ||
| f'Generated configs with {len(public_keys)} keys for validator #{validator_index}.\n' | ||
| f'Validator definitions for Lighthouse saved to {validator_definitions_filepath} file.\n' | ||
| f'Signer keys for Teku\\Prysm saved to {signer_keys_filepath} file.\n' | ||
| f'Proposer config for Teku\\Prysm saved to {proposer_config_filepath} file.\n', | ||
| bold=True, | ||
| fg='green', | ||
| ) | ||
|
|
||
|
|
||
| def _generate_lighthouse_config( | ||
| public_keys: list[HexStr], | ||
| web3signer_url: str, | ||
| fee_recipient: str, | ||
| filepath: str, | ||
| ) -> None: | ||
| """ | ||
| Generate config for Lighthouse clients | ||
| """ | ||
| items = [ | ||
| { | ||
| 'enabled': True, | ||
| 'voting_public_key': public_key, | ||
| 'type': 'web3signer', | ||
| 'url': web3signer_url, | ||
| 'suggested_fee_recipient': fee_recipient, | ||
| } | ||
| for public_key in public_keys | ||
| ] | ||
|
|
||
| with open(filepath, 'w', encoding='utf-8') as f: | ||
| yaml.dump(items, f, explicit_start=True) | ||
|
|
||
|
|
||
| def _generate_signer_keys_config(public_keys: list[HexStr], filepath: str) -> None: | ||
| """ | ||
| Generate config for Teku and Prysm clients | ||
| """ | ||
| keys = ','.join([f'"{public_key}"' for public_key in public_keys]) | ||
| config = f"""validators-external-signer-public-keys: [{keys}]""" | ||
| with open(filepath, 'w', encoding='utf-8') as f: | ||
| f.write(config) | ||
|
|
||
|
|
||
| def _generate_proposer_config( | ||
| fee_recipient: str, | ||
| proposal_builder_enabled: bool, | ||
| filepath: str, | ||
| ) -> None: | ||
| """ | ||
| Generate proposal config for Teku and Prysm clients | ||
| """ | ||
| config = { | ||
| 'default_config': { | ||
| 'fee_recipient': fee_recipient, | ||
| 'builder': { | ||
| 'enabled': proposal_builder_enabled, | ||
| }, | ||
| }, | ||
| } | ||
| with open(filepath, 'w', encoding='utf-8') as f: | ||
| json.dump(config, f, ensure_ascii=False, indent=4) | ||
|
|
||
|
|
||
| def check_validator_index(validator_index, total_validators): | ||
| if not total_validators or total_validators <= validator_index: | ||
| raise click.BadParameter('validator index must be less than total validators') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| import glob | ||
| import os | ||
| from os import mkdir | ||
| from os.path import exists | ||
| from typing import List | ||
|
|
||
| import click | ||
| import yaml | ||
| from eth_utils import add_0x_prefix | ||
| from web3 import Web3 | ||
| from web3.types import HexStr | ||
|
|
||
| from src.common.contrib import is_lists_equal | ||
| from src.common.validators import validate_db_uri, validate_env_name | ||
| from src.key_manager.database import Database, check_db_connection | ||
| from src.key_manager.encryptor import Encryptor | ||
|
|
||
| DECRYPTION_KEY_ENV = 'DECRYPTION_KEY' | ||
|
|
||
|
|
||
| @click.option( | ||
| '--db-url', | ||
| help='The database connection address.', | ||
| prompt="Enter the database connection string, ex. 'postgresql://username:pass@hostname/dbname'", | ||
| callback=validate_db_uri, | ||
| ) | ||
| @click.option( | ||
| '--output-dir', | ||
| help='The folder where web3signer keystores will be saved.', | ||
| prompt='Enter the folder where web3signer keystores will be saved', | ||
| type=click.Path(exists=False, file_okay=False, dir_okay=True), | ||
| ) | ||
| @click.option( | ||
| '--decryption-key-env', | ||
| help='The environment variable with the decryption key for private keys in the database.', | ||
| default=DECRYPTION_KEY_ENV, | ||
| callback=validate_env_name, | ||
| ) | ||
| @click.command(help='Synchronizes web3signer private keys from the database') | ||
| # pylint: disable-next=too-many-locals | ||
| def sync_web3signer(db_url: str, output_dir: str, decryption_key_env: str) -> None: | ||
| """ | ||
| The command is running by the init container in web3signer pods. | ||
| Fetch and decrypt keys for web3signer and store them as keypairs in the output_dir. | ||
| """ | ||
| check_db_connection(db_url) | ||
|
|
||
| database = Database(db_url=db_url) | ||
| keys_records = database.fetch_keys() | ||
|
|
||
| # decrypt private keys | ||
| decryption_key = os.environ[decryption_key_env] | ||
| decryptor = Encryptor(decryption_key) | ||
| private_keys: List[str] = [] | ||
| for key_record in keys_records: | ||
| key = decryptor.decrypt(data=key_record.private_key, nonce=key_record.nonce) | ||
| key_hex = Web3.to_hex(int(key)) | ||
| # pylint: disable-next=unsubscriptable-object | ||
| key_hex = HexStr(key_hex[2:].zfill(64)) # pad missing leading zeros | ||
| private_keys.append(add_0x_prefix(key_hex)) | ||
|
|
||
| if not exists(output_dir): | ||
| mkdir(output_dir) | ||
|
|
||
| # check current keys | ||
| current_keys = [] | ||
| for filename in glob.glob(os.path.join(output_dir, '*.yaml')): | ||
| with open(filename, 'r', encoding='utf-8') as f: | ||
| content = yaml.safe_load(f.read()) | ||
| current_keys.append(content.get('privateKey')) | ||
|
|
||
| if is_lists_equal(current_keys, private_keys): | ||
| click.secho( | ||
| 'Keys already synced to the last version.\n', | ||
| bold=True, | ||
| fg='green', | ||
| ) | ||
| return | ||
|
|
||
| # save key files | ||
| for index, private_key in enumerate(private_keys): | ||
| filename = f'key_{index}.yaml' | ||
| with open(os.path.join(output_dir, filename), 'w', encoding='utf-8') as f: | ||
| f.write(_generate_key_file(private_key)) | ||
|
|
||
| click.secho( | ||
| f'Web3Signer now uses {len(private_keys)} private keys.\n', | ||
| bold=True, | ||
| fg='green', | ||
| ) | ||
|
|
||
|
|
||
| def _generate_key_file(private_key: str) -> str: | ||
| item = { | ||
| 'type': 'file-raw', | ||
| 'keyType': 'BLS', | ||
| 'privateKey': private_key, | ||
| } | ||
| return yaml.dump(item) |
Oops, something went wrong.