From 85451e09eddf28eb758178004b845074ad2528d4 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 21 Jun 2023 19:48:12 +0200 Subject: [PATCH] switch address to ergo-tree --- .../uexplorer/plugin/alert/Analyzer.scala | 14 ++--- .../uexplorer/plugin/alert/Detector.scala | 14 ++--- .../uexplorer/plugin/alert/Discord.scala | 2 +- .../plugin/alert/HighValueDetector.scala | 40 ++++++------- .../plugin/alert/SourceAnalyzer.scala | 2 +- .../entity/CassandraHeadersReader.scala | 2 +- .../uexplorer/indexer/Application.scala | 1 - .../uexplorer/indexer/Const.scala | 2 +- .../indexer/chain/BlockIndexer.scala | 2 +- .../indexer/config/ChainIndexerConf.scala | 1 - .../indexer/mempool/MempoolStateHolder.scala | 2 +- .../indexer/plugin/PluginManager.scala | 2 +- .../org/ergoplatform/uexplorer/Const.scala | 24 ++++---- .../uexplorer/ProtocolSettings.scala | 10 ++-- .../org/ergoplatform/uexplorer/Storage.scala | 8 +-- ...essingModel.scala => BlockEvolution.scala} | 18 ------ .../ergoplatform/uexplorer/db/BlockInfo.scala | 2 +- .../uexplorer/db/OutputBuilder.scala | 12 +++- .../uexplorer/db/RewardCalculator.scala | 2 + .../uexplorer/db/UtxoTracker.scala | 31 ++++++---- .../uexplorer/node/NodeModel.scala | 2 +- .../uexplorer/parser/ErgoTreeParser.scala | 30 ++++++---- .../uexplorer/plugin/Plugin.scala | 2 +- .../ergoplatform/uexplorer/uexplorer.scala | 25 ++++----- .../janusgraph/JanusGraphBackend.scala | 2 +- .../uexplorer/janusgraph/TxGraphWriter.scala | 32 +++++------ .../janusgraph/JanusGraphPlayground.scala | 2 +- .../uexplorer/mvstore/SuperNodeMvMap.scala | 2 +- .../uexplorer/storage/Implicits.scala | 10 ++-- .../uexplorer/storage/MvStorage.scala | 56 +++++++++---------- .../storage/kryo/BlockIdsCodec.scala | 2 +- .../storage/kryo/BlockInfoCodec.scala | 1 - .../uexplorer/storage/kryo/CounterCodec.scala | 1 - ...ressCodec.scala => ErgoTreeHexCodec.scala} | 10 ++-- .../storage/kryo/KryoSerialization.scala | 2 - .../storage/kryo/ValueByBoxCodec.scala | 2 +- 36 files changed, 186 insertions(+), 186 deletions(-) rename modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/{ProcessingModel.scala => BlockEvolution.scala} (76%) rename modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/{AddressCodec.scala => ErgoTreeHexCodec.scala} (62%) diff --git a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Analyzer.scala b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Analyzer.scala index d5b8d5ec..ea88188f 100644 --- a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Analyzer.scala +++ b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Analyzer.scala @@ -1,21 +1,21 @@ package org.ergoplatform.uexplorer.plugin.alert import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource -import org.ergoplatform.uexplorer.{SortedTopAddressMap, Storage} +import org.ergoplatform.uexplorer.Storage import org.ergoplatform.uexplorer.plugin.alert.HighValueDetector.{BlockMatch, TxMatch} trait Analyzer { def trackTx( - txMatch: TxMatch, - utxoState: Storage, - graphTraversalSource: Option[GraphTraversalSource] + txMatch: TxMatch, + utxoState: Storage, + graphTraversalSource: Option[GraphTraversalSource] ): Option[TxMatch] def trackBlock( - blockMatch: BlockMatch, - utxoState: Storage, - graphTraversalSource: Option[GraphTraversalSource] + blockMatch: BlockMatch, + utxoState: Storage, + graphTraversalSource: Option[GraphTraversalSource] ): Option[BlockMatch] } diff --git a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Detector.scala b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Detector.scala index e3f92f89..425e3278 100644 --- a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Detector.scala +++ b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Detector.scala @@ -1,24 +1,24 @@ package org.ergoplatform.uexplorer.plugin.alert import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource +import org.ergoplatform.uexplorer.Storage import org.ergoplatform.uexplorer.db.BestBlockInserted import org.ergoplatform.uexplorer.node.ApiTransaction import org.ergoplatform.uexplorer.plugin.alert.Detector.AlertMessage import org.ergoplatform.uexplorer.plugin.alert.HighValueDetector.{BlockMatch, TxMatch} -import org.ergoplatform.uexplorer.{Address, BoxId, SortedTopAddressMap, Storage, TopAddressMap} trait Detector { def inspectNewPoolTx( - tx: ApiTransaction, - utxoState: Storage, - graphTraversalSource: Option[GraphTraversalSource] + tx: ApiTransaction, + utxoState: Storage, + graphTraversalSource: Option[GraphTraversalSource] ): List[TxMatch] def inspectNewBlock( - newBlock: BestBlockInserted, - utxoState: Storage, - graphTraversalSource: Option[GraphTraversalSource] + newBlock: BestBlockInserted, + utxoState: Storage, + graphTraversalSource: Option[GraphTraversalSource] ): List[BlockMatch] } diff --git a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Discord.scala b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Discord.scala index cb51fce0..0d557359 100644 --- a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Discord.scala +++ b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/Discord.scala @@ -7,7 +7,7 @@ import discord4j.core.{DiscordClient, GatewayDiscordClient} import discord4j.core.`object`.entity.Message import discord4j.core.`object`.entity.channel.MessageChannel import org.ergoplatform.uexplorer.db.FullBlock -import org.ergoplatform.uexplorer.{Address, BoxId, TxId} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BoxId, TxId} import org.ergoplatform.uexplorer.node.ApiTransaction import org.ergoplatform.uexplorer.plugin.Plugin import org.slf4j.{Logger, LoggerFactory} diff --git a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/HighValueDetector.scala b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/HighValueDetector.scala index ee1938b5..05cdc64b 100644 --- a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/HighValueDetector.scala +++ b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/HighValueDetector.scala @@ -21,29 +21,29 @@ class HighValueDetector(txErgValueThreshold: Long, blockErgValueThreshold: Long) storage: Storage, graphTraversalSource: Option[GraphTraversalSource] ): List[TxMatch] = { - def sumAddressValues(addresses: Set[Address]): Map[Address, Address.State] = - addresses - .flatMap(a => storage.getUtxosByAddress(a).map(a -> _)) - .foldLeft(Map.empty[Address, Address.State]) { case (acc, (address, valueByBox)) => - acc.updated(address, Address.State(valueByBox.values.asScala.sum)) + def sumErgoTreeValues(ergoTrees: Set[ErgoTreeHex]): Map[ErgoTreeHex, ErgoTreeHex.State] = + ergoTrees + .flatMap(a => storage.getUtxosByErgoTreeHex(a).map(a -> _)) + .foldLeft(Map.empty[ErgoTreeHex, ErgoTreeHex.State]) { case (acc, (ergoTree, valueByBox)) => + acc.updated(ergoTree, ErgoTreeHex.State(valueByBox.values.asScala.sum)) } val outputsWithoutPaybacksAndFees = tx.outputs.filterNot(o => tx.inputs.exists(i => - storage.getAddressByUtxo(i.boxId).contains(o.address) - ) || o.address == Const.FeeContract.address + storage.getErgoTreeHexByUtxo(i.boxId).contains(o.ergoTree) + ) || o.ergoTree == Const.FeeContract.address ) Option(outputsWithoutPaybacksAndFees.iterator.map(_.value).sum) .filter(_ >= txErgValueThreshold * Const.NanoOrder) .map { value => - val inputAddresses = tx.inputs.iterator.map(_.boxId).flatMap(storage.getAddressByUtxo).toSet - val inputValueByAddress = sumAddressValues(inputAddresses) - val outputAddresses = outputsWithoutPaybacksAndFees.iterator.map(_.address).toSet - val outputValueByAddress = sumAddressValues(outputAddresses) - val txMatch = TxMatch(tx, value, inputValueByAddress, outputValueByAddress) + val inputErgoTrees = tx.inputs.iterator.map(_.boxId).flatMap(storage.getErgoTreeHexByUtxo).toSet + val inputValueByErgoTree = sumErgoTreeValues(inputErgoTrees) + val outputErgoTrees = outputsWithoutPaybacksAndFees.iterator.map(_.ergoTree).toSet + val outputValueByErgoTree = sumErgoTreeValues(outputErgoTrees) + val txMatch = TxMatch(tx, value, inputValueByErgoTree, outputValueByErgoTree) Either.cond( - inputValueByAddress.valuesIterator.map(_._1).sum >= value, + inputValueByErgoTree.valuesIterator.map(_._1).sum >= value, txMatch, txMatch ) @@ -72,15 +72,15 @@ object HighValueDetector { case class BlockMatch( block: FullBlock, blockValue: Value, - inputs: Map[Address, Address.State], - outputs: Map[Address, Address.State] + inputs: Map[ErgoTreeHex, ErgoTreeHex.State], + outputs: Map[ErgoTreeHex, ErgoTreeHex.State] ) case class TxMatch( tx: ApiTransaction, txValue: Value, - inputs: Map[Address, Address.State], - outputs: Map[Address, Address.State] + inputs: Map[ErgoTreeHex, ErgoTreeHex.State], + outputs: Map[ErgoTreeHex, ErgoTreeHex.State] ) { override def toString: AlertMessage = { @@ -88,10 +88,10 @@ object HighValueDetector { val fmtInputAddrSum = valueFormat.format(inputs.valuesIterator.map(_._1).sum / Const.NanoOrder) val fmtOutputAddrSum = valueFormat.format(outputs.valuesIterator.map(_._1).sum / Const.NanoOrder) - def stringify(tuple: (Address, Address.State)) = tuple match { - case (address, Address.State(value)) => + def stringify(tuple: (ErgoTreeHex, ErgoTreeHex.State)) = tuple match { + case (ergoTree, ErgoTreeHex.State(value)) => val fmtVal = valueFormat.format(value / Const.NanoOrder) - s"${address.asInstanceOf[String].take(5)} $fmtVal" + s"${ergoTree.asInstanceOf[String].take(5)} $fmtVal" } val addressDetails = diff --git a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/SourceAnalyzer.scala b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/SourceAnalyzer.scala index 49d3f8a8..5b1ca6ca 100644 --- a/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/SourceAnalyzer.scala +++ b/modules/alert-plugin/src/main/scala/org/ergoplatform/uexplorer/plugin/alert/SourceAnalyzer.scala @@ -2,7 +2,7 @@ package org.ergoplatform.uexplorer.plugin.alert import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter -import org.ergoplatform.uexplorer.{SortedTopAddressMap, Storage} +import org.ergoplatform.uexplorer.Storage import org.ergoplatform.uexplorer.plugin.alert.HighValueDetector.{BlockMatch, TxMatch} import java.io.FileOutputStream diff --git a/modules/cassandra/src/main/scala/org/ergoplatform/uexplorer/cassandra/entity/CassandraHeadersReader.scala b/modules/cassandra/src/main/scala/org/ergoplatform/uexplorer/cassandra/entity/CassandraHeadersReader.scala index 78ddb1e1..59fe182d 100644 --- a/modules/cassandra/src/main/scala/org/ergoplatform/uexplorer/cassandra/entity/CassandraHeadersReader.scala +++ b/modules/cassandra/src/main/scala/org/ergoplatform/uexplorer/cassandra/entity/CassandraHeadersReader.scala @@ -9,7 +9,7 @@ import com.typesafe.scalalogging.LazyLogging import eu.timepit.refined.auto.* import org.ergoplatform.uexplorer.{MapPimp, MutableMapPimp} import org.ergoplatform.uexplorer.cassandra.{CassandraBackend, CassandraPersistenceSupport} -import org.ergoplatform.uexplorer.{Address, BlockId, BoxId} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BlockId, BoxId} import org.ergoplatform.uexplorer.cassandra import scala.collection.compat.immutable.ArraySeq import scala.collection.immutable.{ArraySeq, TreeMap, TreeSet} diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Application.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Application.scala index 8fcc61c1..8c8bc0c2 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Application.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Application.scala @@ -14,7 +14,6 @@ import org.ergoplatform.uexplorer.indexer.mempool.MempoolStateHolder.* import org.ergoplatform.uexplorer.indexer.mempool.{MempoolStateHolder, MempoolSyncer} import org.ergoplatform.uexplorer.indexer.plugin.PluginManager import org.ergoplatform.uexplorer.plugin.Plugin -import org.ergoplatform.uexplorer.{Address, BoxId, Const} import org.slf4j.LoggerFactory import java.io.{PrintWriter, StringWriter} diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Const.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Const.scala index df6b870e..1e962d29 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Const.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/Const.scala @@ -1,7 +1,7 @@ package org.ergoplatform.uexplorer.indexer import org.ergoplatform.uexplorer.Const.MinerRewardDelta -import org.ergoplatform.uexplorer.{Address, HexString} +import org.ergoplatform.uexplorer.{ErgoTreeHex, HexString} import org.ergoplatform.ErgoScriptPredef import scorex.util.encode.{Base16, Base58} diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockIndexer.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockIndexer.scala index b1e6edbd..b797941a 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockIndexer.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockIndexer.scala @@ -87,7 +87,7 @@ class BlockIndexer( def addBestBlock(block: LinkedBlock): Try[BestBlockInserted] = for { - lb <- UtxoTracker(block, storage.getAddressByUtxo, storage.getUtxoValueByAddress) + lb <- UtxoTracker(block, storage.getErgoTreeHexByUtxo, storage.getUtxoValueByErgoTreeHex) _ <- storage.persistNewBlock(lb) _ <- if (lb.info.height % mvStoreConf.heightCompactRate == 0) compact(true) else Success(()) } yield BestBlockInserted(lb, None) // TODO we forgot about FullBlock ! diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/config/ChainIndexerConf.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/config/ChainIndexerConf.scala index 276d835e..8a48c556 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/config/ChainIndexerConf.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/config/ChainIndexerConf.scala @@ -9,7 +9,6 @@ import eu.timepit.refined.refineV import org.ergoplatform.ErgoAddressEncoder import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.settings.MonetarySettings -import org.ergoplatform.uexplorer.{Address, NetworkPrefix} import pureconfig.ConfigReader.Result import pureconfig.error.CannotConvert import pureconfig.{ConfigReader, ConfigSource} diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/mempool/MempoolStateHolder.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/mempool/MempoolStateHolder.scala index 5a755a72..e969e936 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/mempool/MempoolStateHolder.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/mempool/MempoolStateHolder.scala @@ -8,7 +8,7 @@ import com.typesafe.scalalogging.LazyLogging import org.ergoplatform.uexplorer.http.BlockHttpClient import org.ergoplatform.uexplorer.indexer.mempool.MempoolStateHolder.* import org.ergoplatform.uexplorer.node.ApiTransaction -import org.ergoplatform.uexplorer.{Address, BoxId, TxId} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BoxId, TxId} import scala.collection.immutable.{ArraySeq, ListMap} import scala.concurrent.Future diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/plugin/PluginManager.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/plugin/PluginManager.scala index b8806a8c..eaf92bb1 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/plugin/PluginManager.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/plugin/PluginManager.scala @@ -9,7 +9,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo import org.ergoplatform.uexplorer.db.{BestBlockInserted, FullBlock} import org.ergoplatform.uexplorer.indexer.mempool.MempoolStateHolder.MempoolStateChanges import org.ergoplatform.uexplorer.plugin.Plugin -import org.ergoplatform.uexplorer.{SortedTopAddressMap, Storage} +import org.ergoplatform.uexplorer.Storage import java.util.ServiceLoader import scala.concurrent.ExecutionContext.Implicits.global diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Const.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Const.scala index b067889b..e4523559 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Const.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Const.scala @@ -23,7 +23,9 @@ object Const { val address: Address = Address.fromStringUnsafe( "2Z4YBkDsDvQj8BX7xiySFewjitqp2ge9c99jfes2whbtKitZTxdBYqbrVZUvZvKv6aqn9by4kp3LE1c26LCyosFnVnm6b6U1JYvWpYmL2ZnixJbXLjWAWuBThV1D6dLpqZJYQHYDznJCk49g5TUiS4q8khpag2aNmHwREV7JSsypHdHLgJT7MGaw51aJfNubyzSKxZ4AJXFS27EfXwyCLzW1K6GVqwkJtCoPvrcLqmqwacAWJPkmh78nke9H4oT88XmSbRt2n9aWZjosiZCafZ4osUDxmZcc5QVEeTWn8drSraY3eFKe8Mu9MSCcVU" ) - def ergoTreeHex(implicit enc: ErgoAddressEncoder): Try[HexString] = ErgoTreeParser.base58Address2ErgoTreeHex(address) + val ergoTree: HexString = HexString.fromStringUnsafe( + "101004020e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a7017300730110010204020404040004c0fd4f05808c82f5f6030580b8c9e5ae040580f882ad16040204c0944004c0f407040004000580f882ad16d19683030191a38cc7a7019683020193c2b2a57300007473017302830108cdeeac93a38cc7b2a573030001978302019683040193b1a5730493c2a7c2b2a573050093958fa3730673079973089c73097e9a730a9d99a3730b730c0599c1a7c1b2a5730d00938cc7b2a5730e0001a390c1a7730f" + ) } object Foundation { @@ -34,15 +36,16 @@ object Const { val address: Address = Address.fromStringUnsafe( "4L1ktFSzm3SH1UioDuUf5hyaraHird4D2dEACwQ1qHGjSKtA6KaNvSzRCZXZGf9jkfNAEC1SrYaZmCuvb2BKiXk5zW9xuvrXFT7FdNe2KqbymiZvo5UQLAm5jQY8ZBRhTZ4AFtZa1UF5nd4aofwPiL7YkJuyiL5hDHMZL1ZnyL746tHmRYMjAhCgE7d698dRhkdSeVy" ) - - def ergoTreeHex(implicit enc: ErgoAddressEncoder): Try[HexString] = ErgoTreeParser.base58Address2ErgoTreeHex(address) + val ergoTree: HexString = HexString.fromStringUnsafe( + "100e040004c094400580809cde91e7b0010580acc7f03704be944004808948058080c7b7e4992c0580b4c4c32104fe884804c0fd4f0580bcc1960b04befd4f05000400ea03d192c1b2a5730000958fa373019a73029c73037e997304a305958fa373059a73069c73077e997308a305958fa373099c730a7e99730ba305730cd193c2a7c2b2a5730d00d5040800" + ) } object NoPremine { - val initialNanoErgs: Long = 1 * CoinsInOneErgo - val box: BoxId = BoxId("b8ce8cfe331e5eadfb0783bdc375c94413433f65e1e45857d71550d42e4d83bd") - val address: Address = Address.fromStringUnsafe("4MQyMKvMbnCJG3aJ") - def ergoTreeHex(implicit enc: ErgoAddressEncoder): Try[HexString] = ErgoTreeParser.base58Address2ErgoTreeHex(address) + val initialNanoErgs: Long = 1 * CoinsInOneErgo + val box: BoxId = BoxId("b8ce8cfe331e5eadfb0783bdc375c94413433f65e1e45857d71550d42e4d83bd") + val address: Address = Address.fromStringUnsafe("4MQyMKvMbnCJG3aJ") + val ergoTree: HexString = HexString.fromStringUnsafe("10010100d17300") } } @@ -55,10 +58,9 @@ object Const { "2iHkR7CWvD1R4j1yZg5bkeDRQavjAaVPeTDFGGLZduHyfWMuYpmhHocX8GJoaieTx78FntzJbCBVL6rf96ocJoZdmWBL2fci7NqWgAirppPQmZ7fN9V6z13Ay6brPriBKYqLp1bT2Fk4FkFLCfdPpe" ) - def ergoTree(implicit enc: ErgoAddressEncoder): Try[HexString] = ErgoTreeParser.base58Address2ErgoTreeHex(FeeContract.address) - - val ergoTreeTemplateHash = - "5b710d70f207f03745a8bb713006f235446f0104f89e977ae066ae184c0494fa" + val ergoTree: HexString = HexString.fromStringUnsafe( + "1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304" + ) } val EpochLength = 1024 diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/ProtocolSettings.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/ProtocolSettings.scala index 54095bde..0adde514 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/ProtocolSettings.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/ProtocolSettings.scala @@ -4,13 +4,13 @@ import org.ergoplatform.settings.MonetarySettings import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.ErgoAddressEncoder import pureconfig.ConfigReader -import pureconfig._ -import pureconfig.generic.derivation.default._ +import pureconfig.* +import pureconfig.generic.derivation.default.* final case class ProtocolSettings( - networkPrefix: NetworkPrefix, - genesisAddress: Address, - ) derives ConfigReader { + networkPrefix: NetworkPrefix, + genesisAddress: Address +) derives ConfigReader { val monetary = MonetarySettings() val emission = new EmissionRules(monetary) diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala index 296ca260..7917e2e2 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala @@ -1,7 +1,7 @@ package org.ergoplatform.uexplorer import org.ergoplatform.uexplorer.db.BlockInfo -import org.ergoplatform.uexplorer.{Address, BlockId, BoxId, Height, Value} +import org.ergoplatform.uexplorer.{BlockId, BoxId, ErgoTreeHex, Height, Value} import scala.collection.immutable.TreeSet import scala.collection.concurrent @@ -27,9 +27,9 @@ trait Storage { def getBlocksByHeight(atHeight: Height): Map[BlockId, BlockInfo] - def getAddressByUtxo(boxId: BoxId): Option[Address] + def getErgoTreeHexByUtxo(boxId: BoxId): Option[ErgoTreeHex] - def getUtxosByAddress(address: Address): Option[java.util.Map[BoxId, Value]] + def getUtxosByErgoTreeHex(ergoTreeHex: ErgoTreeHex): Option[java.util.Map[BoxId, Value]] - def getUtxoValueByAddress(address: Address, utxo: BoxId): Option[Value] + def getUtxoValueByErgoTreeHex(ergoTreeHex: ErgoTreeHex, utxo: BoxId): Option[Value] } diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/ProcessingModel.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockEvolution.scala similarity index 76% rename from modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/ProcessingModel.scala rename to modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockEvolution.scala index fa9af05e..329b403b 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/ProcessingModel.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockEvolution.scala @@ -6,24 +6,6 @@ import org.ergoplatform.uexplorer.node.* import scala.collection.immutable.ArraySeq import org.ergoplatform.uexplorer.* -case class OutputRecord( - txId: TxId, - boxId: BoxId, - ergoTree: HexString, - templateHashHex: TemplateHashHex, - address: Address, - value: Value, - additionalRegisters: Map[RegisterId, ExpandedRegister] -) - -case class InputRecord( - txId: TxId, - boxId: BoxId, - address: Address, - value: Value -) -case class MinerRewardInfo(reward: MinerReward, fee: MinerFee, address: Address) - case class BlockWithReward( b: ApiFullBlock, minerRewardInfo: MinerRewardInfo diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockInfo.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockInfo.scala index eb4cbf8c..99a3de0e 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockInfo.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/BlockInfo.scala @@ -2,7 +2,7 @@ package org.ergoplatform.uexplorer.db import org.ergoplatform.uexplorer.Const.Protocol import org.ergoplatform.uexplorer.Const.Protocol.Emission -import org.ergoplatform.uexplorer.{Address, BlockId, ProtocolSettings, Revision} +import org.ergoplatform.uexplorer.{Address, BlockId, ErgoTreeHex, ProtocolSettings, Revision} final case class BlockInfo( revision: Revision, diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala index 6c976478..804cf371 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala @@ -11,6 +11,15 @@ import sigmastate.serialization.{GroupElementSerializer, SigmaSerializer} import scala.collection.immutable.ArraySeq import scala.util.Try +case class OutputRecord( + txId: TxId, + boxId: BoxId, + ergoTree: ErgoTreeHex, + templateHashHex: Option[TemplateHashHex], + value: Value, + additionalRegisters: Map[RegisterId, ExpandedRegister] +) + object OutputBuilder { private def getOutputRecords(block: BlockWithReward)(implicit enc: ErgoAddressEncoder): Try[ArraySeq[OutputRecord]] = Try { @@ -18,10 +27,9 @@ object OutputBuilder { tx.outputs.map { o => ( for { - address <- ErgoTreeParser.ergoTreeHex2Base58Address(o.ergoTree) scriptTemplateHash <- ErgoTreeParser.ergoTreeHex2ErgoTreeTemplateSha256Hex(o.ergoTree) additionalRegisters = o.additionalRegisters.view.mapValues(hex => RegistersParser.parseAny(hex)).toMap - } yield OutputRecord(tx.id, o.boxId, o.ergoTree, scriptTemplateHash, address, o.value, additionalRegisters) + } yield OutputRecord(tx.id, o.boxId, o.ergoTree, scriptTemplateHash, o.value, additionalRegisters) ).get } } diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/RewardCalculator.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/RewardCalculator.scala index 8ff36295..29ef12f7 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/RewardCalculator.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/RewardCalculator.scala @@ -12,6 +12,8 @@ import eu.timepit.refined.auto.* import scala.collection.immutable.ArraySeq import scala.util.Try +case class MinerRewardInfo(reward: MinerReward, fee: MinerFee, address: Address) + object RewardCalculator { // CPU greedy (2% of all runtime) diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/UtxoTracker.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/UtxoTracker.scala index 2bb5f513..1142b235 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/UtxoTracker.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/UtxoTracker.scala @@ -7,32 +7,39 @@ import org.ergoplatform.uexplorer.* import scala.collection.compat.immutable.ArraySeq import scala.util.Try +case class InputRecord( + txId: TxId, + boxId: BoxId, + ergoTree: ErgoTreeHex, + value: Value +) + object UtxoTracker { def apply( b: LinkedBlock, - addressByUtxo: BoxId => Option[Address], - utxoValueByAddress: (Address, BoxId) => Option[Value] + ergoTreeHexByUtxo: BoxId => Option[ErgoTreeHex], + utxoValueByErgoTreeHex: (ErgoTreeHex, BoxId) => Option[Value] ): Try[BlockWithInputs] = Try { val outputLookup = b.outputRecords.iterator - .map(o => (o.boxId, (o.address, o.value))) + .map(o => (o.boxId, (o.ergoTree, o.value))) .toMap - def getInputAddressAndValue(inputBoxId: BoxId): (Address, Value) = + def getInputErgoTreeAndValue(inputBoxId: BoxId): (ErgoTreeHex, Value) = outputLookup.getOrElse( inputBoxId, { - val inputAddress = addressByUtxo(inputBoxId).getOrElse( + val inputErgoTree = ergoTreeHexByUtxo(inputBoxId).getOrElse( throw new IllegalStateException( s"Input boxId $inputBoxId of block ${b.b.header.id} at height ${b.info.height} not found in utxo state" ) ) - val value = utxoValueByAddress(inputAddress, inputBoxId).getOrElse( + val value = utxoValueByErgoTreeHex(inputErgoTree, inputBoxId).getOrElse( throw new IllegalStateException( - s"Address $inputAddress of block ${b.b.header.id} at height ${b.info.height} not found in utxo state" + s"Address $inputErgoTree of block ${b.b.header.id} at height ${b.info.height} not found in utxo state" ) ) - inputAddress -> value + inputErgoTree -> value } ) @@ -53,13 +60,13 @@ object UtxoTracker { .flatMap { tx => tx match { case tx if tx.id == Emission.tx => - Iterator(InputRecord(tx.id, Emission.inputBox, Emission.address, Emission.initialNanoErgs)) + Iterator(InputRecord(tx.id, Emission.inputBox, Emission.ergoTree, Emission.initialNanoErgs)) case tx if tx.id == Foundation.tx => - Iterator(InputRecord(tx.id, Foundation.box, Foundation.address, Foundation.initialNanoErgs)) + Iterator(InputRecord(tx.id, Foundation.box, Foundation.ergoTree, Foundation.initialNanoErgs)) case tx => tx.inputs.iterator.map { i => - val (inputAddress, inputValue) = getInputAddressAndValue(i.boxId) - InputRecord(tx.id, i.boxId, inputAddress, inputValue) + val (inputErgoTree, inputValue) = getInputErgoTreeAndValue(i.boxId) + InputRecord(tx.id, i.boxId, inputErgoTree, inputValue) } } } diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/node/NodeModel.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/node/NodeModel.scala index 9cf17e54..e912657a 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/node/NodeModel.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/node/NodeModel.scala @@ -63,7 +63,7 @@ final case class ApiOutput( boxId: BoxId, value: Long, creationHeight: Int, - ergoTree: HexString, + ergoTree: ErgoTreeHex, assets: List[ApiAsset], additionalRegisters: Map[RegisterId, HexString] ) diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/parser/ErgoTreeParser.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/parser/ErgoTreeParser.scala index 2afe3221..814543d8 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/parser/ErgoTreeParser.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/parser/ErgoTreeParser.scala @@ -2,11 +2,11 @@ package org.ergoplatform.uexplorer.parser import com.google.bitcoin.core.Base58 import com.typesafe.scalalogging.LazyLogging -import org.ergoplatform.uexplorer.{Address, HexString, TemplateHashHex} -import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, Pay2SAddress} +import org.ergoplatform.uexplorer.{Address, ErgoTreeHex, HexString, TemplateHashHex} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, P2PKAddress, Pay2SAddress, Pay2SHAddress} import scorex.crypto.hash.{Blake2b256, Digest32, Sha256} import scorex.util.encode.Base16 -import sigmastate.Values.{ErgoTree, FalseLeaf} +import sigmastate.Values.{ErgoTree, FalseLeaf, SigmaPropConstant, Value} import sigmastate.* import sigmastate.serialization.ErgoTreeSerializer @@ -14,6 +14,7 @@ import scala.util.{Failure, Success, Try} import eu.timepit.refined.auto.* import io.circe.{Decoder, DecodingFailure} import eu.timepit.refined.auto.* +import sigmastate.basics.DLogProtocol.ProveDlogProp object ErgoTreeParser extends LazyLogging { @@ -22,16 +23,23 @@ object ErgoTreeParser extends LazyLogging { @inline def ergoTreeHex2ErgoTree(ergoTree: HexString): Try[Values.ErgoTree] = Base16.decode(ergoTree).map(treeSerializer.deserializeErgoTree) - @inline def ergoTreeHex2ErgoTreeTemplateSha256Hex(ergoTree: HexString): Try[TemplateHashHex] = + @inline def ergoTreeHex2ErgoTreeTemplateSha256Hex( + ergoTree: HexString + )(implicit enc: ErgoAddressEncoder): Try[Option[TemplateHashHex]] = ergoTreeHex2ErgoTree(ergoTree) .map { tree => - TemplateHashHex.fromStringUnsafe(Base16.encode(Sha256.hash(tree.template))) - } - - @inline def ergoTreeHex2ErgoTreeTemplateBlake2b256Hex(ergoTree: HexString): Try[TemplateHashHex] = - ergoTreeHex2ErgoTree(ergoTree) - .map { tree => - TemplateHashHex.fromStringUnsafe(Base16.encode(Blake2b256.hash(tree.template))) + val templateOpt = + tree.root match { + case Right(SigmaPropConstant(ProveDlogProp(_))) => + None + case Right(enc.IsPay2SHAddress(_)) => + Option(tree.template) + case Right(b: Value[SSigmaProp.type] @unchecked) if b.tpe == SSigmaProp => + Option(tree.template) + case _ => + None + } + templateOpt.map(t => TemplateHashHex.fromStringUnsafe(Base16.encode(Sha256.hash(t)))) } // http://213.239.193.208:9053/blocks/at/545684 diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/plugin/Plugin.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/plugin/Plugin.scala index 66d482ac..cc951fe0 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/plugin/Plugin.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/plugin/Plugin.scala @@ -3,7 +3,7 @@ package org.ergoplatform.uexplorer.plugin import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource import org.ergoplatform.uexplorer.db.{BestBlockInserted, FullBlock} import org.ergoplatform.uexplorer.node.ApiTransaction -import org.ergoplatform.uexplorer.{Address, BoxId, SortedTopAddressMap, Storage, TopAddressMap, TxId, Value} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BoxId, Storage, TxId, Value} import scala.collection.immutable.ListMap import scala.concurrent.Future diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/uexplorer.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/uexplorer.scala index 202c6688..c13be1c1 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/uexplorer.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/uexplorer.scala @@ -23,28 +23,31 @@ package object uexplorer { type MinerReward = Long type MinerFee = Long - type BoxCount = Int - type TxCount = Int - type LastHeight = Int - type TopAddressMap = Map[Address, Address.Stats] - type SortedTopAddressMap = ListMap[Address, Address.Stats] + type BoxCount = Int + type TxCount = Int + type LastHeight = Int type Revision = Long type Base58Spec = MatchesRegex["[1-9A-HJ-NP-Za-km-z]+"] type Address = String Refined Base58Spec + type ErgoTreeHex = String Refined HexStringSpec type NetworkPrefix = String Refined ValidByte object Address { import eu.timepit.refined.auto.autoUnwrap - case class Stats(lastTxHeight: LastHeight, txCount: TxCount, boxCount: BoxCount) { - def this() = this(0, 0, 0) // kryo needs a no-arg constructor - } case class State(value: Value) extension (x: Address) def unwrapped: String = x def fromStringUnsafe(s: String): Address = unsafeWrap(refineV[Base58Spec].unsafeFrom(s)) } + object ErgoTreeHex { + import eu.timepit.refined.auto.autoUnwrap + case class State(value: Value) + extension (x: ErgoTreeHex) def unwrapped: String = x + def fromStringUnsafe(s: String): ErgoTreeHex = unsafeWrap(refineV[HexStringSpec].unsafeFrom(s)) + } + object NetworkPrefix { def fromStringUnsafe(s: String): NetworkPrefix = unsafeWrap(refineV[ValidByte].unsafeFrom(s)) } @@ -72,12 +75,6 @@ package object uexplorer { def fromStringUnsafe(s: String): TemplateHashHex = unsafeWrap(HexString.fromStringUnsafe(s)) } - type ErgoTree = String Refined HexStringSpec - - object ErgoTree { - def fromStringUnsafe(s: String): ErgoTree = unsafeWrap(refineV[HexStringSpec].unsafeFrom(s)) - } - opaque type TxId = String object TxId { diff --git a/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphBackend.scala b/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphBackend.scala index a9e9af79..5f00ced3 100644 --- a/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphBackend.scala +++ b/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphBackend.scala @@ -8,7 +8,7 @@ import com.typesafe.scalalogging.LazyLogging import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource import org.apache.tinkerpop.gremlin.structure.Transaction import org.ergoplatform.uexplorer.db.FullBlock -import org.ergoplatform.uexplorer.{Height, TopAddressMap} +import org.ergoplatform.uexplorer.Height import org.janusgraph.core.JanusGraphFactory import org.janusgraph.graphdb.database.StandardJanusGraph diff --git a/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/TxGraphWriter.scala b/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/TxGraphWriter.scala index b53f3e7d..dc850a5a 100644 --- a/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/TxGraphWriter.scala +++ b/modules/janusgraph/src/main/scala/org/ergoplatform/uexplorer/janusgraph/TxGraphWriter.scala @@ -18,7 +18,7 @@ object TxGraphWriter extends LazyLogging { private val blackListBoxes = Set(Protocol.Emission.inputBox, Protocol.NoPremine.box, Protocol.Foundation.box) private val blackListAddresses = - Set(FeeContract.address, Protocol.Emission.address, Protocol.NoPremine.address, Protocol.Foundation.address) + Set(FeeContract.ergoTree, Protocol.Emission.ergoTree, Protocol.NoPremine.ergoTree, Protocol.Foundation.ergoTree) def writeGraph( txId: TxId, @@ -31,37 +31,37 @@ object TxGraphWriter extends LazyLogging { newTxVertex.property("txId", txId) newTxVertex.property("height", height) newTxVertex.property("timestamp", timestamp) - val inputsByAddress = + val inputsByErgoTrees = inputs .filterNot(t => - blackListBoxes.contains(t.boxId) || blackListAddresses.contains(t.address) || t.value < CoinsInOneErgo + blackListBoxes.contains(t.boxId) || blackListAddresses.contains(t.ergoTree) || t.value < CoinsInOneErgo ) - .groupBy(_.address) + .groupBy(_.ergoTree) - inputsByAddress - .foreach { case (address, inputs) => - val inputAddressVertexIt = g.vertices(Utils.vertexHash(address, g)) + inputsByErgoTrees + .foreach { case (ergoTree, inputs) => + val inputAddressVertexIt = g.vertices(Utils.vertexHash(ergoTree, g)) if (!inputAddressVertexIt.hasNext) { - logger.error(s"inputAddress $address from height $height lacks corresponding vertex") + logger.error(s"inputAddress $ergoTree from height $height lacks corresponding vertex") } - newTxVertex.addEdge("from", inputAddressVertexIt.next(), "value", 0L) // TODO inputs.iterator.map(_.value).sum + newTxVertex.addEdge("from", inputAddressVertexIt.next(), "value", inputs.iterator.map(_.value).sum) } outputs .filterNot(t => - inputsByAddress.contains(t.address) || blackListBoxes.contains(t.boxId) || blackListAddresses.contains( - t.address + inputsByErgoTrees.contains(t.ergoTree) || blackListBoxes.contains(t.boxId) || blackListAddresses.contains( + t.ergoTree ) || t.value < CoinsInOneErgo ) - .groupBy(_.address) - .foreach { case (address, outputs) => - val outputAddressVertexIt = g.vertices(Utils.vertexHash(address, g)) + .groupBy(_.ergoTree) + .foreach { case (ergoTree, outputs) => + val outputAddressVertexIt = g.vertices(Utils.vertexHash(ergoTree, g)) val newOutputAddressVertex = if (outputAddressVertexIt.hasNext) { outputAddressVertexIt.next() } else { - val newOutputAddressVertex = g.addVertex(T.id, Utils.vertexHash(address, g), T.label, "address") - newOutputAddressVertex.property("address", address) + val newOutputAddressVertex = g.addVertex(T.id, Utils.vertexHash(ergoTree, g), T.label, "address") + newOutputAddressVertex.property("address", ergoTree) newOutputAddressVertex } newTxVertex.addEdge("to", newOutputAddressVertex, "value", outputs.iterator.map(_.value).sum) diff --git a/modules/janusgraph/src/test/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphPlayground.scala b/modules/janusgraph/src/test/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphPlayground.scala index 5b1ea216..32e8a5b0 100644 --- a/modules/janusgraph/src/test/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphPlayground.scala +++ b/modules/janusgraph/src/test/scala/org/ergoplatform/uexplorer/janusgraph/JanusGraphPlayground.scala @@ -7,7 +7,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.{AnonymousTraversalSource, import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__ as _g import org.apache.tinkerpop.gremlin.structure.util.GraphFactory import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph -import org.ergoplatform.uexplorer.Address +import org.ergoplatform.uexplorer.ErgoTreeHex import org.janusgraph.core.Multiplicity import org.janusgraph.graphdb.database.StandardJanusGraph import org.janusgraph.graphdb.transaction.StandardJanusGraphTx diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeMvMap.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeMvMap.scala index 8ab9f325..a2eec93d 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeMvMap.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeMvMap.scala @@ -29,7 +29,7 @@ class SuperNodeMvMap[HK, C[_, _], K, V]( .toMap ) - private lazy val counterByHotKey = new MvMap[HK, Counter]("counterBySuperNode", store) + private lazy val counterByHotKey = new MvMap[HK, Counter]("counterByHotKey", store) private def collectReadHotKey(k: HK): Counter = counterByHotKey.adjust(k)(_.fold(Counter(1, 1, 0, 0)) { case Counter(writeOps, readOps, added, removed) => diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala index d8d80e51..03e7bf9c 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala @@ -11,14 +11,14 @@ object Implicits { implicit val valueByBoxCodec: MultiMapCodec[java.util.Map, BoxId, Value] = ValueByBoxCodec implicit val blockInfoCodec: ValueCodec[BlockInfo] = BlockInfoCodec implicit val counterCodec: ValueCodec[Counter] = CounterCodec - implicit val addressCodec: ValueCodec[Address] = AddressCodec + implicit val addressCodec: ValueCodec[ErgoTreeHex] = ErgoTreeHexCodec - implicit val superNodeAddressCodec: HotKeyCodec[Address] = new HotKeyCodec[Address] { - import Address.unwrapped - def serialize(key: Address): String = key.unwrapped + implicit val superNodeErgoTreeCodec: HotKeyCodec[ErgoTreeHex] = new HotKeyCodec[ErgoTreeHex] { + import ErgoTreeHex.unwrapped + def serialize(key: ErgoTreeHex): String = key.unwrapped // do not call Address.fromStringUnsafe as it has been already validated in BlockBuilder - def deserialize(key: String): Address = key.asInstanceOf[Address] + def deserialize(key: String): ErgoTreeHex = key.asInstanceOf[ErgoTreeHex] } } diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala index d2e789da..a599d378 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala @@ -38,51 +38,51 @@ import scala.util.{Failure, Random, Success, Try} case class MvStorage( store: MVStore, - utxosByAddress: MultiMvMap[Address, java.util.Map, BoxId, Value], - addressByUtxo: MapLike[BoxId, Address], + utxosByErgoTreeHex: MultiMvMap[ErgoTreeHex, java.util.Map, BoxId, Value], + ergoTreeHexByUtxo: MapLike[BoxId, ErgoTreeHex], blockIdsByHeight: MapLike[Height, java.util.Set[BlockId]], blockById: MapLike[BlockId, BlockInfo] ) extends Storage with LazyLogging { def clear(): Try[Unit] = - utxosByAddress.clear() + utxosByErgoTreeHex.clear() def close(): Try[Unit] = Try(store.close()) def rollbackTo(version: Revision): Try[Unit] = Try(store.rollbackTo(version)) - private def removeInputBoxesByAddress(address: Address, inputBoxes: ArraySeq[BoxId]): Try[_] = - addressByUtxo.removeAllOrFail(inputBoxes).flatMap { _ => - utxosByAddress.removeAllOrFail(address, inputBoxes, inputBoxes.size) { existingBoxIds => + private def removeInputBoxesByErgoTree(ergoTree: ErgoTreeHex, inputBoxes: ArraySeq[BoxId]): Try[_] = + ergoTreeHexByUtxo.removeAllOrFail(inputBoxes).flatMap { _ => + utxosByErgoTreeHex.removeAllOrFail(ergoTree, inputBoxes, inputBoxes.size) { existingBoxIds => inputBoxes.foreach(existingBoxIds.remove) Option(existingBoxIds).collect { case m if !m.isEmpty => m } } } - private def persistUtxos(address: Address, boxes: Iterable[OutputRecord]): Try[_] = - addressByUtxo.putAllNewOrFail(boxes.iterator.map(b => b.boxId -> b.address)).flatMap { _ => - utxosByAddress.adjustAndForget(address, boxes.iterator.map(b => b.boxId -> b.value), boxes.size) + private def persistUtxos(ergoTreeHex: ErgoTreeHex, boxes: Iterable[OutputRecord]): Try[_] = + ergoTreeHexByUtxo.putAllNewOrFail(boxes.iterator.map(b => b.boxId -> b.ergoTree)).flatMap { _ => + utxosByErgoTreeHex.adjustAndForget(ergoTreeHex, boxes.iterator.map(b => b.boxId -> b.value), boxes.size) } def persistNewBlock(b: BlockWithInputs): Try[BlockWithInputs] = { val outputExceptionOpt = b.outputRecords - .groupBy(_.address) - .map { case (address, boxes) => - persistUtxos(address, boxes) + .groupBy(_.ergoTree) + .map { case (ergoTree, boxes) => + persistUtxos(ergoTree, boxes) } .collectFirst { case f @ Failure(_) => f } val inputExceptionOpt = b.inputRecords - .groupBy(_.address) + .groupBy(_.ergoTree) .view .mapValues(_.collect { case InputRecord(_, boxId, _, _) if boxId != Emission.inputBox && boxId != Foundation.box => boxId }) - .map { case (address, inputIds) => - removeInputBoxesByAddress(address, inputIds) + .map { case (ergoTree, inputIds) => + removeInputBoxesByErgoTree(ergoTree, inputIds) } .collectFirst { case f @ Failure(_) => f } @@ -103,15 +103,15 @@ case class MvStorage( } } - def writeReport: Try[_] = utxosByAddress.writeReport + def writeReport: Try[_] = utxosByErgoTreeHex.writeReport def getCompactReport: String = { val height = getLastHeight.getOrElse(0) - val MultiMapSize(superNodeSize, superNodeTotalSize, commonSize) = utxosByAddress.size + val MultiMapSize(superNodeSize, superNodeTotalSize, commonSize) = utxosByErgoTreeHex.size val nonEmptyAddressCount = superNodeSize + commonSize val progress = s"storage height: $height, " + - s"utxo count: ${addressByUtxo.size}, " + + s"utxo count: ${ergoTreeHexByUtxo.size}, " + s"supernode-utxo-count : $superNodeTotalSize, " + s"non-empty-address count: $nonEmptyAddressCount, " @@ -137,14 +137,14 @@ case class MvStorage( .map(_.asScala.flatMap(blockId => blockById.get(blockId).map(blockId -> _)).toMap) .getOrElse(Map.empty) - def getUtxosByAddress(address: Address): Option[java.util.Map[BoxId, Value]] = - utxosByAddress.getAll(address) + def getUtxosByErgoTreeHex(ergoTreeHex: ErgoTreeHex): Option[java.util.Map[BoxId, Value]] = + utxosByErgoTreeHex.getAll(ergoTreeHex) - def getUtxoValueByAddress(address: Address, utxo: BoxId): Option[Value] = - utxosByAddress.get(address, utxo: BoxId) + def getUtxoValueByErgoTreeHex(ergoTreeHex: ErgoTreeHex, utxo: BoxId): Option[Value] = + utxosByErgoTreeHex.get(ergoTreeHex, utxo: BoxId) def isEmpty: Boolean = - utxosByAddress.isEmpty && addressByUtxo.isEmpty && blockIdsByHeight.isEmpty && blockById.isEmpty + utxosByErgoTreeHex.isEmpty && ergoTreeHexByUtxo.isEmpty && blockIdsByHeight.isEmpty && blockById.isEmpty def getLastHeight: Option[Height] = blockIdsByHeight.lastKey @@ -160,7 +160,7 @@ case class MvStorage( def getBlockById(blockId: BlockId): Option[BlockInfo] = blockById.get(blockId) - def getAddressByUtxo(boxId: BoxId): Option[Address] = addressByUtxo.get(boxId) + def getErgoTreeHexByUtxo(boxId: BoxId): Option[ErgoTreeHex] = ergoTreeHexByUtxo.get(boxId) def findMissingHeights: TreeSet[Height] = { val lastHeight = getLastHeight @@ -219,11 +219,11 @@ object MvStorage extends LazyLogging { store.setRetentionTime(3600 * 1000 * 24 * 7) MvStorage( store, - new MultiMvMap[Address, java.util.Map, BoxId, Value]( - new MvMap[Address, java.util.Map[BoxId, Value]]("utxosByAddress", store), - SuperNodeMvMap[Address, java.util.Map, BoxId, Value](store, superNodeFile) + new MultiMvMap[ErgoTreeHex, java.util.Map, BoxId, Value]( + new MvMap[ErgoTreeHex, java.util.Map[BoxId, Value]]("utxosByErgoTreeHex", store), + SuperNodeMvMap[ErgoTreeHex, java.util.Map, BoxId, Value](store, superNodeFile) ), - new MvMap[BoxId, Address]("addressByUtxo", store), + new MvMap[BoxId, ErgoTreeHex]("ergoTreeHexByUtxo", store), new MvMap[Height, java.util.Set[BlockId]]("blockIdsByHeight", store), new MvMap[BlockId, BlockInfo]("blockById", store) ) diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockIdsCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockIdsCodec.scala index a5b72a22..225b0be2 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockIdsCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockIdsCodec.scala @@ -8,7 +8,7 @@ import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, M import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.ValueCodec -import org.ergoplatform.uexplorer.{Address, BlockId, Height} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BlockId, Height} import java.nio.ByteBuffer import java.util diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockInfoCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockInfoCodec.scala index 2f0114c5..c881e6f7 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockInfoCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/BlockInfoCodec.scala @@ -8,7 +8,6 @@ import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, M import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.ValueCodec -import org.ergoplatform.uexplorer.{Address, Height} import java.nio.ByteBuffer import java.util diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala index 12dbe4b0..0a5fb105 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala @@ -9,7 +9,6 @@ import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.SuperNodeCollector.Counter import org.ergoplatform.uexplorer.mvstore.ValueCodec -import org.ergoplatform.uexplorer.{Address, Height} import java.nio.ByteBuffer import java.util diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/AddressCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ErgoTreeHexCodec.scala similarity index 62% rename from modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/AddressCodec.scala rename to modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ErgoTreeHexCodec.scala index fe82ff61..b866ca0c 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/AddressCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ErgoTreeHexCodec.scala @@ -8,15 +8,15 @@ import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, M import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.ValueCodec -import org.ergoplatform.uexplorer.{Address, Height} +import org.ergoplatform.uexplorer.{ErgoTreeHex, Height} import java.nio.ByteBuffer import java.util import scala.util.Try -object AddressCodec extends ValueCodec[Address] { - override def readAll(bytes: Array[Byte]): Address = - new String(bytes).asInstanceOf[Address] // do not call Address.fromStringUnsafe as it has been already validated +object ErgoTreeHexCodec extends ValueCodec[ErgoTreeHex] { + override def readAll(bytes: Array[Byte]): ErgoTreeHex = + new String(bytes).asInstanceOf[ErgoTreeHex] // do not call Address.fromStringUnsafe as it has been already validated - override def writeAll(obj: Address): Array[Byte] = obj.asInstanceOf[String].getBytes("UTF-8") + override def writeAll(obj: ErgoTreeHex): Array[Byte] = obj.asInstanceOf[String].getBytes("UTF-8") } diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/KryoSerialization.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/KryoSerialization.scala index 8041b463..1ff7e0c9 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/KryoSerialization.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/KryoSerialization.scala @@ -10,7 +10,6 @@ import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.SuperNodeCollector.Counter import org.ergoplatform.uexplorer.mvstore.{MultiMapCodec, ValueCodec} import org.ergoplatform.uexplorer.storage.kryo.* -import org.ergoplatform.uexplorer.{Address, BlockId, BoxId, Height, Value} import java.nio.ByteBuffer import java.util @@ -29,7 +28,6 @@ object KryoSerialization { kryo.register(classOf[util.HashSet[_]], setSerializer) kryo.register(classOf[Counter]) kryo.register(classOf[BlockInfo]) - kryo.register(classOf[Address.Stats]) setSerializer.setAcceptsNull(false) mapSerializer.setKeyClass(classOf[String], kryo.getSerializer(classOf[String])) mapSerializer.setKeysCanBeNull(false) diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala index 476fbad9..525f32d3 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala @@ -8,7 +8,7 @@ import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, M import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.db.BlockInfo import org.ergoplatform.uexplorer.mvstore.* -import org.ergoplatform.uexplorer.{Address, BoxId, Height, Value} +import org.ergoplatform.uexplorer.{ErgoTreeHex, BoxId, Height, Value} import scala.language.unsafeNulls import java.nio.ByteBuffer