Skip to content

Commit

Permalink
generic upstreams introduced (#317)
Browse files Browse the repository at this point in the history
- added base implementation of upstreams/multistreams
- connectors were generalised, chain specific logic was extracted to ChainSpecific class
- StarkNet network support based on generic upstreams added
  • Loading branch information
a10zn8 authored Oct 9, 2023
1 parent 222e046 commit f3bf347
Showing 56 changed files with 864 additions and 364 deletions.
13 changes: 8 additions & 5 deletions buildSrc/src/main/kotlin/chainsconfig.codegen.gradle.kts
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy
import io.emeraldpay.dshackle.config.ChainsConfig
import io.emeraldpay.dshackle.config.ChainsConfigReader
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import java.math.BigInteger

open class CodeGen(private val config: ChainsConfig) {
companion object {
@@ -17,25 +18,27 @@ open class CodeGen(private val config: ChainsConfig) {
builder.addEnumConstant(
"UNSPECIFIED",
TypeSpec.anonymousClassBuilder()
.addSuperclassConstructorParameter("%L, %S, %S, %S, %L, %L", 0, "UNSPECIFIED", "Unknown", "0x0", 0, "emptyList()")
.addSuperclassConstructorParameter("%L, %S, %S, %S, %L, %L", 0, "UNSPECIFIED", "Unknown", "0x0", "BigInteger.ZERO", "emptyList()")
.build(),
)
for (chain in config) {
builder.addEnumConstant(
chain.blockchain.uppercase().replace('-', '_') + "__" + chain.id.uppercase().replace('-', '_'),
chain.blockchain.uppercase().replace('-', '_') + "__" + chain.id.uppercase().replace('-', '_')
.replace(' ', '_'),
TypeSpec.anonymousClassBuilder()
.addSuperclassConstructorParameter(
"%L, %S, %S, %S, %L, %L",
chain.grpcId,
chain.code,
chain.blockchain.replaceFirstChar { it.uppercase() } + " " + chain.id.replaceFirstChar { it.uppercase() },
chain.chainId,
chain.netVersion,
"BigInteger(\"" + chain.netVersion + "\")",
"listOf(" + chain.shortNames.map { "\"${it}\"" }.joinToString() + ")",
)
.build(),
)
}

return builder
}

@@ -60,7 +63,7 @@ open class CodeGen(private val config: ChainsConfig) {
.addParameter("chainCode", String::class)
.addParameter("chainName", String::class)
.addParameter("chainId", String::class)
.addParameter("netVersion", Long::class)
.addParameter("netVersion", BigInteger::class)
.addParameter("shortNames", List::class.asClassName().parameterizedBy(String::class.asClassName()))
.build(),
)
@@ -75,7 +78,7 @@ open class CodeGen(private val config: ChainsConfig) {
.build(),
)
.addProperty(
PropertySpec.builder("netVersion", Long::class)
PropertySpec.builder("netVersion", BigInteger::class)
.initializer("netVersion")
.build(),
)
2 changes: 1 addition & 1 deletion emerald-grpc
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.emeraldpay.dshackle.config

import io.emeraldpay.dshackle.foundation.ChainOptions
import java.math.BigInteger
import java.time.Duration

data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<ChainsConfig.ChainConfig> {
@@ -22,7 +23,7 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
val laggingLagSize: Int,
val options: ChainOptions.PartialOptions,
val chainId: String,
val netVersion: Long,
val netVersion: BigInteger,
val grpcId: Int,
val code: String,
val shortNames: List<String>,
@@ -38,7 +39,7 @@ data class ChainsConfig(private val chains: List<ChainConfig>) : Iterable<Chains
1,
ChainOptions.PartialOptions(),
"0x0",
0,
BigInteger.ZERO,
0,
"UNKNOWN",
emptyList(),
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ import org.yaml.snakeyaml.nodes.MappingNode
import org.yaml.snakeyaml.nodes.NodeTuple
import org.yaml.snakeyaml.nodes.ScalarNode
import org.yaml.snakeyaml.nodes.Tag
import java.math.BigInteger

class ChainsConfigReader(
private val chainsOptionsReader: ChainOptionsReader,
@@ -74,7 +75,7 @@ class ChainsConfigReader(
?: throw IllegalArgumentException("undefined code for $blockchain")
val grpcId = getValueAsInt(node, "grpcId")
?: throw IllegalArgumentException("undefined code for $blockchain")
val netVersion = getValueAsLong(node, "net-version") ?: chainId.drop(2).toLong(radix = 16)
val netVersion = getValueAsLong(node, "net-version")?.toBigInteger() ?: BigInteger(chainId.drop(2), 16)
val shortNames = getListOfString(node, "short-names")
?: throw IllegalArgumentException("undefined shortnames for $blockchain")
return ChainsConfig.ChainConfig(
27 changes: 27 additions & 0 deletions foundation/src/main/resources/chains.yaml
Original file line number Diff line number Diff line change
@@ -551,3 +551,30 @@ chain-settings:
short-names: [kava-testnet]
chain-id: 0x8ad
grpcId: 10033
- id: starknet
label: StarkNet
settings:
currency: ETH
expected-block-time: 20s
lags:
syncing: 5
lagging: 1
chains:
- id: Mainnet
priority: 100
code: STARKNET_MAINNET
short-names: [starknet]
chain-id: 0x534e5f4d41494e
grpcId: 1014
- id: Testnet
priority: 2
code: STARKNET_TESTNET
short-names: [starknet-testnet]
chain-id: 0x534e5f474f45524c49
grpcId: 10019
- id: Testnet 2
priority: 1
code: STARKNET_TESTNET2
short-names: [starknet-testnet2]
chain-id: 0x534e5f474f45524c4932
grpcId: 10020
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import io.emeraldpay.dshackle.config.ChainsConfigReader
import io.emeraldpay.dshackle.foundation.ChainOptionsReader
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import java.math.BigInteger

internal class ChainsConfigReaderTest {

@@ -19,7 +20,7 @@ internal class ChainsConfigReaderTest {

val ethc = config.resolve("ethereum-classic")
assertEquals(ethc.chainId, "0x3d")
assertEquals(ethc.netVersion, 1)
assertEquals(ethc.netVersion, BigInteger.valueOf(1))
assertEquals(ethc.code, "ETC")
assertEquals(ethc.grpcId, 101)
assertEquals(ethc.expectedBlockTime.seconds, 12)
@@ -39,6 +40,6 @@ internal class ChainsConfigReaderTest {
assertEquals(config.laggingLagSize, 4)
assertEquals(config.code, "FTA")
assertEquals(config.grpcId, 102)
assertEquals(config.netVersion, 251)
assertEquals(config.netVersion, BigInteger.valueOf(251))
}
}
6 changes: 5 additions & 1 deletion src/main/kotlin/io/emeraldpay/dshackle/BlockchainType.kt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.emeraldpay.dshackle

enum class BlockchainType {
BITCOIN, EVM_POW, EVM_POS;
BITCOIN, EVM_POW, EVM_POS, STARKNET;

companion object {

@@ -10,12 +10,16 @@ enum class BlockchainType {
)
val bitcoin = setOf(Chain.BITCOIN__MAINNET, Chain.BITCOIN__TESTNET)

val starknet = setOf(Chain.STARKNET__MAINNET, Chain.STARKNET__TESTNET, Chain.STARKNET__TESTNET_2)

@JvmStatic
fun from(chain: Chain): BlockchainType {
return if (pow.contains(chain)) {
EVM_POW
} else if (bitcoin.contains(chain)) {
BITCOIN
} else if (starknet.contains(chain)) {
STARKNET
} else {
EVM_POS
}
34 changes: 15 additions & 19 deletions src/main/kotlin/io/emeraldpay/dshackle/config/UpstreamsConfig.kt
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
package io.emeraldpay.dshackle.config

import io.emeraldpay.dshackle.foundation.ChainOptions
import io.emeraldpay.dshackle.upstream.ethereum.connectors.EthereumConnectorFactory.ConnectorMode
import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory.ConnectorMode
import java.net.URI
import java.util.Arrays
import java.util.Locale
@@ -58,23 +58,11 @@ data class UpstreamsConfig(

open class UpstreamConnection

open class RpcConnection(
open var rpc: HttpEndpoint? = null,
) : UpstreamConnection()

class GrpcConnection : UpstreamConnection() {
var host: String? = null
var port: Int = 0
var auth: AuthConfig.ClientTlsAuth? = null
var tokenAuth: AuthConfig.ClientTokenAuth? = null
var upstreamRating: Int = 0
}

data class EthereumConnection(
override var rpc: HttpEndpoint? = null,
data class RpcConnection(
var rpc: HttpEndpoint? = null,
var ws: WsEndpoint? = null,
var connectorMode: String? = null,
) : RpcConnection(rpc) {
) : UpstreamConnection() {

fun resolveMode(): ConnectorMode {
return if (connectorMode == null) {
@@ -91,14 +79,22 @@ data class UpstreamsConfig(
}
}

class GrpcConnection : UpstreamConnection() {
var host: String? = null
var port: Int = 0
var auth: AuthConfig.ClientTlsAuth? = null
var tokenAuth: AuthConfig.ClientTokenAuth? = null
var upstreamRating: Int = 0
}

data class BitcoinConnection(
override var rpc: HttpEndpoint? = null,
var rpc: HttpEndpoint? = null,
var esplora: HttpEndpoint? = null,
var zeroMq: BitcoinZeroMq? = null,
) : RpcConnection()
) : UpstreamConnection()

data class EthereumPosConnection(
var execution: EthereumConnection? = null,
var execution: RpcConnection? = null,
var upstreamRating: Int = 0,
) : UpstreamConnection()

Original file line number Diff line number Diff line change
@@ -86,9 +86,13 @@ class UpstreamsConfigReader(

getList<MappingNode>(input, "upstreams")?.value?.forEach { upNode ->
val connNode = getMapping(upNode, "connection")
if (hasAny(connNode, "ethereum")) {
if (hasAny(connNode, "generic")) {
readUpstream(config, upNode) {
readEthereumConnection(getMapping(connNode, "ethereum")!!)
readRpcConnection(getMapping(connNode, "generic")!!)
}
} else if (hasAny(connNode, "ethereum")) {
readUpstream(config, upNode) {
readRpcConnection(getMapping(connNode, "ethereum")!!)
}
} else if (hasAny(connNode, "bitcoin")) {
readUpstream(config, upNode) {
@@ -175,16 +179,16 @@ class UpstreamsConfigReader(
private fun readEthereumPosConnection(connConfigNode: MappingNode): UpstreamsConfig.EthereumPosConnection {
val connection = UpstreamsConfig.EthereumPosConnection()
getMapping(connConfigNode, "execution")?.let {
connection.execution = readEthereumConnection(it)
connection.execution = readRpcConnection(it)
}
getValueAsInt(connConfigNode, "upstream-rating")?.let {
connection.upstreamRating = it
}
return connection
}

private fun readEthereumConnection(connConfigNode: MappingNode): UpstreamsConfig.EthereumConnection {
val connection = UpstreamsConfig.EthereumConnection()
private fun readRpcConnection(connConfigNode: MappingNode): UpstreamsConfig.RpcConnection {
val connection = UpstreamsConfig.RpcConnection()
.apply { rpc = readRpcConfig(connConfigNode) }

getValueAsString(connConfigNode, "connector-mode")?.let {
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
package io.emeraldpay.dshackle.config.context

import io.emeraldpay.dshackle.BlockchainType
import io.emeraldpay.dshackle.BlockchainType.BITCOIN
import io.emeraldpay.dshackle.BlockchainType.EVM_POS
import io.emeraldpay.dshackle.BlockchainType.EVM_POW
import io.emeraldpay.dshackle.BlockchainType.STARKNET
import io.emeraldpay.dshackle.Chain
import io.emeraldpay.dshackle.cache.CachesFactory
import io.emeraldpay.dshackle.upstream.CallTargetsHolder
import io.emeraldpay.dshackle.upstream.Multistream
import io.emeraldpay.dshackle.upstream.bitcoin.BitcoinMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumMultistream
import io.emeraldpay.dshackle.upstream.ethereum.EthereumPosMultiStream
import io.emeraldpay.dshackle.upstream.generic.GenericMultistream
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory
import org.springframework.cloud.sleuth.Tracer
@@ -30,13 +35,28 @@ open class MultistreamsConfig(val beanFactory: ConfigurableListableBeanFactory)
.filterNot { it == Chain.UNSPECIFIED }
.map { chain ->
when (BlockchainType.from(chain)) {
BlockchainType.EVM_POS -> ethereumPosMultistream(chain, cachesFactory, headScheduler, tracer)
BlockchainType.EVM_POW -> ethereumMultistream(chain, cachesFactory, headScheduler, tracer)
BlockchainType.BITCOIN -> bitcoinMultistream(chain, cachesFactory, headScheduler)
EVM_POS -> ethereumPosMultistream(chain, cachesFactory, headScheduler, tracer)
EVM_POW -> ethereumMultistream(chain, cachesFactory, headScheduler, tracer)
BITCOIN -> bitcoinMultistream(chain, cachesFactory, headScheduler)
STARKNET -> genericMultistream(chain, cachesFactory, headScheduler)
}
}
}

private fun genericMultistream(
chain: Chain,
cachesFactory: CachesFactory,
headScheduler: Scheduler,
): Multistream {
val name = "multi-$chain"
return GenericMultistream(
chain,
CopyOnWriteArrayList(),
cachesFactory.getCaches(chain),
headScheduler,
).also { register(it, name) }
}

private fun ethereumMultistream(
chain: Chain,
cachesFactory: CachesFactory,
7 changes: 6 additions & 1 deletion src/main/kotlin/io/emeraldpay/dshackle/data/BlockId.kt
Original file line number Diff line number Diff line change
@@ -47,7 +47,12 @@ class BlockId(
} else {
id
}
val bytes = Hex.decode(clean)
val even = if (clean.length % 2 != 0) {
"0$clean"
} else {
clean
}
val bytes = Hex.decode(even)
return BlockId(bytes)
}
}
Loading

0 comments on commit f3bf347

Please sign in to comment.