Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(zkstack): Split chain and ecosystem layers #3199

Open
wants to merge 56 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
f56e9c2
Migrate server to chain
matias-gonz Oct 13, 2024
409e768
Migrate contract verifier to chain
matias-gonz Oct 13, 2024
2d4f8fc
Migrate consensus to chain
matias-gonz Oct 13, 2024
ba8b707
fmt
matias-gonz Oct 13, 2024
8d238d5
Fix consensus proto
matias-gonz Oct 13, 2024
5b314a6
Fix consensus command
matias-gonz Oct 15, 2024
8f7d9f7
Add ZkStackConfig enum
matias-gonz Oct 15, 2024
ca2f0e8
fmt
matias-gonz Oct 15, 2024
8f046fd
Refactor deploy paymaster
matias-gonz Oct 17, 2024
62e69be
Refactor chain commands
matias-gonz Oct 17, 2024
06ddc2c
rollback consensus
matias-gonz Oct 17, 2024
1b12e6e
Refactor chain consensus setup
matias-gonz Oct 18, 2024
c2f326c
Merge branch 'main' into matias/refactor-3-layered-chain
matias-gonz Oct 18, 2024
49e971c
Merge branch 'main' of github.com:matter-labs/zksync-era into matias/…
matias-gonz Oct 30, 2024
cb00b8d
Update tests
matias-gonz Oct 30, 2024
c26c886
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Oct 30, 2024
3252224
fmt
matias-gonz Oct 30, 2024
17302bb
Merge branch 'matias/refactor-chain-commands' of github.com:matter-la…
matias-gonz Oct 30, 2024
e5978a4
fmt
matias-gonz Oct 30, 2024
88c2e99
generate autocompletion
matias-gonz Oct 30, 2024
efed818
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Oct 31, 2024
88bdf4d
lint autocomplete
matias-gonz Oct 31, 2024
b87969a
generate autocomplete
matias-gonz Oct 31, 2024
142a327
lint
matias-gonz Oct 31, 2024
95d82b1
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Oct 31, 2024
281c158
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 5, 2024
6d94d1b
Fix ci
matias-gonz Nov 5, 2024
942e335
Make contract_verifier:init:run async
matias-gonz Nov 7, 2024
1a6f628
Remove use messages:self
matias-gonz Nov 7, 2024
f6fb8bf
Remove _zkstack.zsh
matias-gonz Nov 7, 2024
a01eea8
Load chain in chain:run and pass it to the next commands
matias-gonz Nov 7, 2024
6c2f2b8
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 7, 2024
18403f1
Add ChainConfig::from_internal
matias-gonz Nov 7, 2024
bf1f8da
Rename ZkStackConfig::load_current_chain to current_chain
matias-gonz Nov 11, 2024
b2d7388
Add ZkStackConfig::ecosystem
matias-gonz Nov 11, 2024
6406717
Make EcosystemConfig::from_file private
matias-gonz Nov 11, 2024
2c5cf58
Make ChainConfigInternal::from_file private
matias-gonz Nov 11, 2024
f69f271
Make ChainConfig::from_internal private
matias-gonz Nov 11, 2024
d4632ef
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 11, 2024
0700795
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 12, 2024
d394301
Merge branch 'main' of github.com:matter-labs/zksync-era into matias/…
matias-gonz Nov 13, 2024
df2cc9e
fmt
matias-gonz Nov 13, 2024
e792afe
Add autocompletion files
matias-gonz Nov 13, 2024
c55067c
Add zsh autocomplete file
matias-gonz Nov 13, 2024
5b53a64
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 13, 2024
00d7649
Fix ci
matias-gonz Nov 13, 2024
b3773bb
Fix contract-verifier ci
matias-gonz Nov 13, 2024
f73f399
Fix contract-verifier ci
matias-gonz Nov 13, 2024
60054e7
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 15, 2024
850efc7
Merge branch 'main' of github.com:matter-labs/zksync-era into matias/…
matias-gonz Nov 19, 2024
a43a770
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 25, 2024
90fb3cc
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 25, 2024
b15b532
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 26, 2024
19c46c6
feat(zk_toolbox): Make chain create independent from ecosystem (#3210)
matias-gonz Nov 26, 2024
d6c3446
feat(zk_toolbox): Make chain init independent from ecosystem (#3231)
matias-gonz Nov 26, 2024
9f9ab7d
Merge branch 'main' into matias/refactor-chain-commands
matias-gonz Nov 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions .github/workflows/ci-core-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ jobs:
run: ci_run yarn l1-contracts test

- name: Download compilers for contract verifier tests
run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
run: ci_run zkstack chain contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era

- name: Rust unit tests
run: |
Expand Down Expand Up @@ -156,7 +156,7 @@ jobs:
- name: Run server
run: |
ci_run zkstack dev config-writer --path ${{ matrix.vm_mode == 'NEW' && 'etc/env/file_based/overrides/tests/loadtest-new.yaml' || 'etc/env/file_based/overrides/tests/loadtest-old.yaml' }} --chain legacy
ci_run zkstack server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
ci_run zkstack chain server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
ci_run sleep 60

- name: Perform loadtest
Expand Down Expand Up @@ -363,38 +363,38 @@ jobs:

- name: Build tested binaries
run: |
ci_run zkstack server build
ci_run zkstack chain server build
ci_run zkstack external-node build
ci_run zkstack contract-verifier build
ci_run zkstack chain contract-verifier build

- name: Initialize Contract verifier
run: |
ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
ci_run zkstack contract-verifier wait --chain era --verbose
ci_run zkstack chain contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
ci_run zkstack chain contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
ci_run zkstack chain contract-verifier wait --chain era --verbose

- name: Run servers
run: |
# Override config for part of chains to test the default config as well
ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain era
ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain validium

ci_run zkstack server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
ci_run zkstack server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log &
ci_run zkstack server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log &
ci_run zkstack server --ignore-prerequisites --chain consensus \
ci_run zkstack chain server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
ci_run zkstack chain server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log &
ci_run zkstack chain server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log &
ci_run zkstack chain server --ignore-prerequisites --chain consensus \
--components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
&> ${{ env.SERVER_LOGS_DIR }}/consensus.log &

ci_run zkstack server wait --ignore-prerequisites --verbose --chain era
ci_run zkstack server wait --ignore-prerequisites --verbose --chain validium
ci_run zkstack server wait --ignore-prerequisites --verbose --chain custom_token
ci_run zkstack server wait --ignore-prerequisites --verbose --chain consensus
ci_run zkstack chain server wait --ignore-prerequisites --verbose --chain era
ci_run zkstack chain server wait --ignore-prerequisites --verbose --chain validium
ci_run zkstack chain server wait --ignore-prerequisites --verbose --chain custom_token
ci_run zkstack chain server wait --ignore-prerequisites --verbose --chain consensus

- name: Set up attester committee for the consensus chain
run: |
ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log
ci_run zkstack chain consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
ci_run zkstack chain consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log

- name: Run integration tests
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci-prover-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
ci_run zkstack prover setup-keys --mode=download --region=us --verbose
- name: Run server
run: |
ci_run zkstack server --uring --chain=proving_chain --components=api,tree,eth,state_keeper,commitment_generator,proof_data_handler,vm_runner_protective_reads,vm_runner_bwip &>prover_logs/server.log &
ci_run zkstack chain server --uring --chain=proving_chain --components=api,tree,eth,state_keeper,commitment_generator,proof_data_handler,vm_runner_protective_reads,vm_runner_bwip &>prover_logs/server.log &
- name: Run Gateway
run: |
ci_run zkstack prover run --component=gateway --docker=false &>prover_logs/gateway.log &
Expand Down
4 changes: 2 additions & 2 deletions core/lib/contract_verifier/src/tests/real.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Tests using real compiler toolchains. Should be prepared by calling `zkstack contract-verifier init`
//! Tests using real compiler toolchains. Should be prepared by calling `zkstack chain contract-verifier init`
//! with at least one `solc` and `zksolc` version. If there are no compilers, the tests will be ignored
//! unless the `RUN_CONTRACT_VERIFICATION_TEST` env var is set to `true`, in which case the tests will fail.

Expand Down Expand Up @@ -90,7 +90,7 @@ fn assert_no_compilers_expected() {
env::var("RUN_CONTRACT_VERIFICATION_TEST").ok().as_deref(),
Some("true"),
"Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but they are not installed. \
Use `zkstack contract-verifier init` to install compilers"
Use `zkstack chain contract-verifier init` to install compilers"
);
println!("No compilers found, skipping the test");
}
Expand Down
2 changes: 1 addition & 1 deletion core/tests/revert-test/tests/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ export function runServerInBackground({
}): ChildProcessWithoutNullStreams {
let command = '';
if (useZkStack) {
command = 'zkstack server';
command = 'zkstack chain server';
if (chain) {
command += ` --chain ${chain}`;
}
Expand Down
2 changes: 1 addition & 1 deletion core/tests/ts-integration/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export function runServerInBackground({
}): ChildProcessWithoutNullStreams {
let command = '';
if (useZkStack) {
command = 'zkstack server';
command = 'zkstack chain server';
if (chain) {
command += ` --chain ${chain}`;
}
Expand Down
2 changes: 1 addition & 1 deletion core/tests/upgrade-test/tests/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export function runServerInBackground({
let command = '';

if (useZkStack) {
command = 'zkstack server';
command = 'zkstack chain server';
command += chain ? ` --chain ${chain}` : '';
} else {
command = 'cd $ZKSYNC_HOME && cargo run --bin zksync_server --release --';
Expand Down
2 changes: 1 addition & 1 deletion docs/src/guides/advanced/01_initialization.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ If everything goes well, you should see that L1 blocks are being produced.
Now we can start the main server:

```bash
zkstack server
zkstack chain server
```

This will actually run a cargo binary (`zksync_server`).
Expand Down
6 changes: 3 additions & 3 deletions docs/src/guides/launch.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ documentation for safe ways to customize your setup.
Run server:

```bash
zkstack server
zkstack chain server
```

The server's configuration files can be found in `/chains/<chain_name>/configs` directory. These files are created when
Expand All @@ -123,7 +123,7 @@ To manually modify configuration files:
5. Restart the relevant services for changes to take effect:

```bash
zkstack server
zkstack chain server
```

> NOTE: Manual changes to configuration files may be overwritten if the ecosystem is reinitialized or the chain is
Expand Down Expand Up @@ -176,7 +176,7 @@ cargo run --release --bin zksync_commitment_generator
## Running the contract verifier

```bash
zkstack contract-verifier run
zkstack chain contract-verifier run
```

## Troubleshooting
Expand Down
2 changes: 1 addition & 1 deletion prover/docs/src/03_launch.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ We will be running a bunch of binaries, it's recommended to run each in a separa
### Server

```bash
zkstack server --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,da_dispatcher,proof_data_handler,vm_runner_protective_reads,vm_runner_bwip
zkstack chain server --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,da_dispatcher,proof_data_handler,vm_runner_protective_reads,vm_runner_bwip
```

### Prover gateway
Expand Down
6 changes: 3 additions & 3 deletions zkstack_cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ by a third party).
To run the chain:

```bash
zkstack server
zkstack chain server
```

You can specify the component you want to run using `--components` flag
Expand Down Expand Up @@ -214,13 +214,13 @@ For `witness-generator`, specify the round with `--round <round>`. Rounds:
Download required binaries (`solc`, `zksolc`, `vyper`, `zkvyper`):

```bash
zkstack contract-verifier init
zkstack chain contract-verifier init
```

Run the contract verifier:

```bash
zkstack contract-verifier run
zkstack chain contract-verifier run
```

### External Node
Expand Down
67 changes: 66 additions & 1 deletion zkstack_cli/crates/config/src/chain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::{
path::{Path, PathBuf},
};

use anyhow::{bail, Context};
use serde::{Deserialize, Serialize, Serializer};
use types::{BaseToken, L1BatchCommitmentMode, L1Network, ProverMode, WalletCreation};
use xshell::Shell;
Expand All @@ -18,7 +19,8 @@ use crate::{
FileConfigWithDefaultName, ReadConfig, ReadConfigWithBasePath, SaveConfig,
SaveConfigWithBasePath, ZkStackConfig,
},
ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig,
utils::{find_file, get_preexisting_ecosystem_contracts_path},
ContractsConfig, EcosystemConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig,
};

/// Chain configuration file. This file is created in the chain
Expand All @@ -33,6 +35,8 @@ pub struct ChainConfigInternal {
pub prover_version: ProverMode,
pub configs: PathBuf,
pub rocks_db_path: PathBuf,
pub l1_network: Option<L1Network>,
pub link_to_code: Option<PathBuf>,
pub external_node_config_path: Option<PathBuf>,
pub artifacts_path: Option<PathBuf>,
pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode,
Expand Down Expand Up @@ -100,6 +104,7 @@ impl ChainConfig {
}
anyhow::bail!("Wallets configs has not been found");
}

pub fn get_contracts_config(&self) -> anyhow::Result<ContractsConfig> {
ContractsConfig::read_with_base_path(self.get_shell(), &self.configs)
}
Expand Down Expand Up @@ -146,6 +151,10 @@ impl ChainConfig {
config.save_with_base_path(shell, path)
}

pub fn get_preexisting_ecosystem_contracts_path(&self) -> PathBuf {
get_preexisting_ecosystem_contracts_path(&self.link_to_code, self.l1_network)
}

fn get_internal(&self) -> ChainConfigInternal {
ChainConfigInternal {
id: self.id,
Expand All @@ -156,13 +165,69 @@ impl ChainConfig {
rocks_db_path: self.rocks_db_path.clone(),
external_node_config_path: self.external_node_config_path.clone(),
artifacts_path: Some(self.artifacts.clone()),
l1_network: Some(self.l1_network),
link_to_code: Some(self.link_to_code.clone()),
l1_batch_commit_data_generator_mode: self.l1_batch_commit_data_generator_mode,
base_token: self.base_token.clone(),
wallet_creation: self.wallet_creation,
legacy_bridge: self.legacy_bridge,
evm_emulator: self.evm_emulator,
}
}

pub(crate) fn from_internal(
chain_internal: ChainConfigInternal,
shell: Shell,
) -> anyhow::Result<Self> {
let l1_network = chain_internal.l1_network.context("L1 Network not found")?;
let link_to_code = chain_internal
.link_to_code
.context("Link to code not found")?;
let artifacts = chain_internal
.artifacts_path
.context("Artifacts path not found")?;

Ok(Self {
id: chain_internal.id,
name: chain_internal.name,
chain_id: chain_internal.chain_id,
prover_version: chain_internal.prover_version,
configs: chain_internal.configs,
rocks_db_path: chain_internal.rocks_db_path,
external_node_config_path: chain_internal.external_node_config_path,
l1_network,
l1_batch_commit_data_generator_mode: chain_internal.l1_batch_commit_data_generator_mode,
base_token: chain_internal.base_token,
wallet_creation: chain_internal.wallet_creation,
legacy_bridge: chain_internal.legacy_bridge,
link_to_code,
artifacts,
evm_emulator: chain_internal.evm_emulator,
shell: shell.into(),
})
}
}

impl ChainConfigInternal {
pub(crate) fn from_file(shell: &Shell) -> anyhow::Result<ChainConfigInternal> {
let Ok(path) = find_file(shell, shell.current_dir(), CONFIG_NAME) else {
bail!("Chain config not found")
};

shell.change_dir(&path);

match ChainConfigInternal::read(shell, CONFIG_NAME) {
Ok(config) => Ok(config),
Err(err) => {
if let Ok(ecosystem) = EcosystemConfig::read(shell, CONFIG_NAME) {
let chain = ecosystem.load_current_chain()?;
Ok(chain.get_internal())
} else {
Err(err)
}
}
}
}
}

impl FileConfigWithDefaultName for ChainConfigInternal {
Expand Down
4 changes: 2 additions & 2 deletions zkstack_cli/crates/config/src/consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ pub(crate) const LOCAL_APPS_PATH: &str = "apps/";
pub(crate) const LOCAL_CHAINS_PATH: &str = "chains/";
pub(crate) const LOCAL_CONFIGS_PATH: &str = "configs/";
pub(crate) const LOCAL_GENERATED_PATH: &str = ".generated/";
pub(crate) const LOCAL_DB_PATH: &str = "db/";
pub(crate) const LOCAL_ARTIFACTS_PATH: &str = "artifacts/";
pub const LOCAL_DB_PATH: &str = "db/";
pub const LOCAL_ARTIFACTS_PATH: &str = "artifacts/";

/// Name of apps config file
pub const APPS_CONFIG_FILE: &str = "apps.yaml";
Expand Down
Loading
Loading