validate upstream version

gradle/libs.versions.toml

@@ -53,6 +53,7 @@ jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", ver
 jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", version.ref = "jackson" }
 jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" }
 jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" }
+jackson-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" }
 
 java-websocket = "org.java-websocket:Java-WebSocket:1.5.1"
 
@@ -124,7 +125,7 @@ mockito-inline = "org.mockito:mockito-inline:4.0.0"
 apache-commons = ["commons-io", "apache-commons-lang3", "apache-commons-collections4"]
 grpc = ["grpc-protobuf", "grpc-stub", "grpc-netty", "grpc-proto-util", "grpc-services"]
 httpcomponents = ["httpcomponents-httpmime", "httpcomponents-httpclient"]
-jackson = ["jackson-core", "jackson-databind", "jackson-datatype-jdk8", "jackson-datatype-jsr310", "jackson-module-kotlin"]
+jackson = ["jackson-core", "jackson-databind", "jackson-datatype-jdk8", "jackson-datatype-jsr310", "jackson-module-kotlin", "jackson-yaml"]
 kotlin = ["kotlin-stdlib-jdk8", "kotlin-reflect"]
 netty = ["netty-common", "netty-transport", "netty-handler-core", "netty-handler-proxy", "netty-resolver-core", "netty-resolver-dns", "netty-codec-core", "netty-codec-http", "netty-codec-http2", "netty-buffer", "netty-tcnative-core"]
 reactor = ["reactor-core", "reactor-netty", "reactor-extra", "reactor-kotlin"]

src/main/kotlin/io/emeraldpay/dshackle/Global.kt

@@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.Version
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.module.SimpleModule
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.kotlin.registerKotlinModule
@@ -64,6 +65,9 @@ class Global {
         @JvmStatic
         val objectMapper: ObjectMapper = createObjectMapper()
 
+        @JvmStatic
+        val yamlMapper: ObjectMapper = ObjectMapper(YAMLFactory())
+
         var version: String = "DEV"
 
         val control: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor()

src/main/kotlin/io/emeraldpay/dshackle/config/context/HotConfig.kt

@@ -0,0 +1,28 @@
+package io.emeraldpay.dshackle.config.context
+
+import io.emeraldpay.dshackle.config.hot.AutoReloadbleConfig
+import io.emeraldpay.dshackle.config.hot.CompatibleVersionsRules
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.core.io.ClassPathResource
+import java.util.function.Supplier
+
+@Configuration
+open class HotConfig {
+    @Bean
+    open fun hotVersionsConfig(
+        @Value("\${compatibility.url}")
+        url: String,
+        @Value("\${compatibility.enabled}")
+        enabled: Boolean,
+    ): Supplier<CompatibleVersionsRules?> {
+        return if (enabled) {
+            val initialContent =
+                ClassPathResource("public/compatible-clients.yaml").inputStream.readBytes().toString(Charsets.UTF_8)
+            AutoReloadbleConfig(initialContent, url, CompatibleVersionsRules::class.java).also { it.start() }
+        } else {
+            Supplier { null }
+        }
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/hot/AutoReloadbleConfig.kt

@@ -0,0 +1,50 @@
+package io.emeraldpay.dshackle.config.hot
+
+import io.emeraldpay.dshackle.Global
+import org.slf4j.LoggerFactory
+import org.springframework.web.client.RestTemplate
+import reactor.core.publisher.Flux
+import java.time.Duration
+import java.util.concurrent.atomic.AtomicReference
+import java.util.function.Supplier
+
+class AutoReloadbleConfig<T>(
+    val initialContent: String,
+    val configUrl: String,
+    private val type: Class<T>,
+) : Supplier<T?> {
+    companion object {
+        private val log = LoggerFactory.getLogger(AutoReloadbleConfig::class.java)
+    }
+
+    private val restTemplate = RestTemplate()
+    private val instance = AtomicReference<T>()
+
+    fun reload() {
+        try {
+            val response = restTemplate.getForObject(configUrl, String::class.java)
+            if (response != null) {
+                instance.set(parseConfig(response))
+            }
+        } catch (e: Exception) {
+            log.error("Failed to reload config from $configUrl for type $type", e)
+        }
+    }
+
+    fun start() {
+        instance.set(parseConfig(initialContent))
+        Flux.interval(
+            Duration.ofSeconds(600),
+        ).subscribe {
+            reload()
+        }
+    }
+
+    private fun parseConfig(config: String): T {
+        return Global.yamlMapper.readValue(config, type)
+    }
+
+    override fun get(): T {
+        return instance.get()
+    }
+}

src/main/kotlin/io/emeraldpay/dshackle/config/hot/CompatibleVersionsRules.kt

@@ -0,0 +1,20 @@
+package io.emeraldpay.dshackle.config.hot
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class CompatibleVersionsRules(
+    @JsonProperty("rules")
+    val rules: List<CompatibleVersionsRule>,
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class CompatibleVersionsRule(
+    @JsonProperty("client")
+    val client: String,
+    @JsonProperty("blacklist")
+    val blacklist: List<String>?,
+    @JsonProperty("whitelist")
+    val whitelist: List<String>?,
+)

src/main/kotlin/io/emeraldpay/dshackle/rpc/NativeCall.kt

@@ -520,7 +520,7 @@ open class NativeCall(
                     "reason",
                     reason,
                     "chain",
-                    ctx.upstream.chain.chainCode,
+                    ctx.upstream.getChain().chainCode,
                 ).increment()
             }
         }

src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeChainStatus.kt

@@ -29,7 +29,7 @@ class SubscribeChainStatus(
             // we need to track not only multistreams with upstreams but all of them
             // because upstreams can be added in runtime with hot config reload
             multistreamHolder.all()
-                .filter { Common.ChainRef.forNumber(it.chain.id) != null }
+                .filter { Common.ChainRef.forNumber(it.getChain().id) != null }
                 .map { ms ->
                     Flux.concat(
                         // the first event must be filled with all fields
@@ -52,7 +52,7 @@ class SubscribeChainStatus(
             .map { events ->
                 val response = BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
                 val chainDescription = BlockchainOuterClass.ChainDescription.newBuilder()
-                    .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                    .setChain(Common.ChainRef.forNumber(ms.getChain().id))
 
                 events.forEach {
                     chainDescription.addChainEvent(processMsEvent(it))
@@ -100,7 +100,7 @@ class SubscribeChainStatus(
                 BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
                     .setChainDescription(
                         BlockchainOuterClass.ChainDescription.newBuilder()
-                            .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                            .setChain(Common.ChainRef.forNumber(ms.getChain().id))
                             .addChainEvent(chainEventMapper.mapHead(it))
                             .build(),
                     )
@@ -112,7 +112,7 @@ class SubscribeChainStatus(
         return BlockchainOuterClass.SubscribeChainStatusResponse.newBuilder()
             .setChainDescription(
                 BlockchainOuterClass.ChainDescription.newBuilder()
-                    .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                    .setChain(Common.ChainRef.forNumber(ms.getChain().id))
                     .addChainEvent(chainEventMapper.chainStatus(ms.getStatus()))
                     .addChainEvent(chainEventMapper.mapHead(head))
                     .addChainEvent(chainEventMapper.supportedMethods(ms.getMethods().getSupportedMethods()))

src/main/kotlin/io/emeraldpay/dshackle/rpc/SubscribeNodeStatus.kt

@@ -120,7 +120,7 @@ class SubscribeNodeStatus(
                     .setDescription(
                         NodeDescription.newBuilder()
                             .setNodeId(upstream.nodeId().toInt())
-                            .setChain(Common.ChainRef.forNumber(ms.chain.id))
+                            .setChain(Common.ChainRef.forNumber(ms.getChain().id))
                             .build(),
                     )
                     .setNodeId(upstream.getId())
@@ -148,7 +148,7 @@ class SubscribeNodeStatus(
 
     private fun buildDescription(ms: Multistream, up: Upstream): NodeDescription.Builder {
         val builder = NodeDescription.newBuilder()
-            .setChain(Common.ChainRef.forNumber(ms.chain.id))
+            .setChain(Common.ChainRef.forNumber(ms.getChain().id))
             .setNodeId(up.nodeId().toInt())
             .addAllNodeLabels(
                 up.getLabels().map { nodeLabels ->

src/main/kotlin/io/emeraldpay/dshackle/startup/configure/EthereumUpstreamCreator.kt

@@ -4,17 +4,20 @@ import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.IndexConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
+import io.emeraldpay.dshackle.config.hot.CompatibleVersionsRules
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.upstream.CallTargetsHolder
 import org.springframework.stereotype.Component
+import java.util.function.Supplier
 
 @Component
 class EthereumUpstreamCreator(
     chainsConfig: ChainsConfig,
     indexConfig: IndexConfig,
     callTargets: CallTargetsHolder,
     connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver,
-) : GenericUpstreamCreator(chainsConfig, indexConfig, callTargets, connectorFactoryCreatorResolver) {
+    versionRules: Supplier<CompatibleVersionsRules?>,
+) : GenericUpstreamCreator(chainsConfig, indexConfig, callTargets, connectorFactoryCreatorResolver, versionRules) {
 
     override fun createUpstream(
         upstreamsConfig: UpstreamsConfig.Upstream<*>,

src/main/kotlin/io/emeraldpay/dshackle/startup/configure/GenericUpstreamCreator.kt

@@ -4,6 +4,7 @@ import io.emeraldpay.dshackle.Chain
 import io.emeraldpay.dshackle.config.ChainsConfig
 import io.emeraldpay.dshackle.config.IndexConfig
 import io.emeraldpay.dshackle.config.UpstreamsConfig
+import io.emeraldpay.dshackle.config.hot.CompatibleVersionsRules
 import io.emeraldpay.dshackle.foundation.ChainOptions
 import io.emeraldpay.dshackle.startup.QuorumForLabels
 import io.emeraldpay.dshackle.upstream.BlockValidator
@@ -13,13 +14,15 @@ import io.emeraldpay.dshackle.upstream.generic.ChainSpecificRegistry
 import io.emeraldpay.dshackle.upstream.generic.GenericUpstream
 import io.emeraldpay.dshackle.upstream.generic.connectors.GenericConnectorFactory
 import org.springframework.stereotype.Component
+import java.util.function.Supplier
 
 @Component
 open class GenericUpstreamCreator(
     chainsConfig: ChainsConfig,
     indexConfig: IndexConfig,
     callTargets: CallTargetsHolder,
     private val connectorFactoryCreatorResolver: ConnectorFactoryCreatorResolver,
+    private val versionRules: Supplier<CompatibleVersionsRules?>,
 ) : UpstreamCreator(chainsConfig, indexConfig, callTargets) {
     private val hashes: MutableMap<Byte, Boolean> = HashMap()
 
@@ -85,6 +88,7 @@ open class GenericUpstreamCreator(
             buildMethodsFun,
             cs::lowerBoundService,
             cs::finalizationDetectorBuilder,
+            versionRules,
         )
 
         upstream.start()

src/main/kotlin/io/emeraldpay/dshackle/startup/configure/UpstreamFactory.kt

@@ -43,6 +43,6 @@ class UpstreamFactory(
         config: UpstreamsConfig.Upstream<UpstreamsConfig.GrpcConnection>,
         chainsConfig: ChainsConfig,
     ): GrpcUpstreams {
-        return grpcUpstreamCreator.creatGrpcUpstream(config, chainsConfig)
+        return grpcUpstreamCreator.createGrpcUpstream(config, chainsConfig)
     }
 }

src/main/kotlin/io/emeraldpay/dshackle/upstream/CurrentMultistreamHolder.kt

@@ -28,7 +28,7 @@ open class CurrentMultistreamHolder(
 
     private val log = LoggerFactory.getLogger(CurrentMultistreamHolder::class.java)
 
-    private val chainMapping = multistreams.associateBy { it.chain }
+    private val chainMapping = multistreams.associateBy { it.getChain() }
 
     override fun getUpstream(chain: Chain): Multistream {
         return chainMapping.getValue(chain)
@@ -37,7 +37,7 @@ open class CurrentMultistreamHolder(
     override fun getAvailable(): List<Chain> {
         return chainMapping.values.asSequence()
             .filter { it.haveUpstreams() }
-            .map { it.chain }
+            .map { it.getChain() }
             .toList()
     }
 

src/main/kotlin/io/emeraldpay/dshackle/upstream/DefaultUpstream.kt

@@ -175,4 +175,8 @@ abstract class DefaultUpstream(
     }
 
     data class Status(val lag: Long?, val avail: UpstreamAvailability, val status: UpstreamAvailability)
+
+    override fun getChain(): Chain {
+        return chain
+    }
 }