diff --git a/modules/backend/src/main/resources/h2-schema.sql b/modules/backend/src/main/resources/h2-schema.sql index ed40b3a6..d370edc9 100755 --- a/modules/backend/src/main/resources/h2-schema.sql +++ b/modules/backend/src/main/resources/h2-schema.sql @@ -39,9 +39,8 @@ create table if not exists ErgoTreeT8 ( create table if not exists Box ( boxId VARCHAR(64) NOT NULL PRIMARY KEY, - blockId VARCHAR(64) NOT NULL REFERENCES Block (blockId) ON DELETE CASCADE, txId VARCHAR(64) NOT NULL, - ergoTreeHash VARCHAR(64) NOT NULL REFERENCES ErgoTree (hash), + ergoTreeHash VARCHAR(64) NOT NULL REFERENCES ErgoTree (hash) ON DELETE CASCADE, ergoTreeT8Hash VARCHAR(64) REFERENCES ErgoTreeT8 (hash), ergValue BIGINT NOT NULL, r4 VARCHAR, @@ -54,16 +53,19 @@ create table if not exists Box ( create table if not exists Asset ( tokenId VARCHAR(64) NOT NULL PRIMARY KEY, - blockId VARCHAR(64) NOT NULL REFERENCES Block (blockId) ON DELETE CASCADE, - boxId VARCHAR(64) NOT NULL REFERENCES Box (boxId), + blockId VARCHAR(64) NOT NULL REFERENCES Block (blockId) ON DELETE CASCADE +); + +create table if not exists Asset2Box ( + tokenId VARCHAR(64) NOT NULL REFERENCES Asset (tokenId) ON DELETE CASCADE, + boxId VARCHAR(64) NOT NULL REFERENCES Box (boxId) ON DELETE CASCADE, amount BIGINT NOT NULL ); create table if not exists Utxo ( boxId VARCHAR(64) NOT NULL PRIMARY KEY REFERENCES Box (boxId), - blockId VARCHAR(64) NOT NULL REFERENCES Block (blockId) ON DELETE CASCADE, txId VARCHAR(64) NOT NULL, - ergoTreeHash VARCHAR(64) NOT NULL REFERENCES ErgoTree (hash), + ergoTreeHash VARCHAR(64) NOT NULL REFERENCES ErgoTree (hash) ON DELETE CASCADE, ergoTreeT8Hash VARCHAR(64) REFERENCES ErgoTreeT8 (hash), ergValue BIGINT NOT NULL, r4 VARCHAR, diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/H2Backend.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/H2Backend.scala index e1eab874..89958f46 100644 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/H2Backend.scala +++ b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/H2Backend.scala @@ -24,7 +24,7 @@ object H2Backend extends Backend { Server .serve((BlockRoutes() ++ BoxRoutes()).withDefaultErrorResponse) .fork - .provideSomeLayer(Server.defaultWithPort(8080)) + .provideSomeLayer(Server.defaultWithPort(8088)) override def isEmpty: Task[Boolean] = ??? diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/InmemoryRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/InmemoryRepo.scala deleted file mode 100644 index 74f2ae44..00000000 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/InmemoryRepo.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.ergoplatform.uexplorer.backend - -import org.ergoplatform.uexplorer.BlockId -import org.ergoplatform.uexplorer.Const.Protocol -import org.ergoplatform.uexplorer.backend.blocks.BlockRepo -import org.ergoplatform.uexplorer.backend.boxes.BoxRepo -import org.ergoplatform.uexplorer.db.{Block, LinkedBlock} -import zio.* - -import scala.collection.mutable - -case class InmemoryRepo(blockRepo: BlockRepo, boxRepo: BoxRepo) extends Repo: - override def isEmpty: Task[Boolean] = - for - blockRepoEmpty <- blockRepo.isEmpty - boxRepoEmpty <- boxRepo.isEmpty - yield blockRepoEmpty && boxRepoEmpty - - override def removeBlocks(blockIds: Set[BlockId]): Task[Unit] = blockRepo.delete(blockIds).unit - - override def writeBlock(b: LinkedBlock)(preTx: Task[Any], postTx: Task[Any]): Task[BlockId] = { - val ergoTrees = b.outputRecords.byErgoTree.keys - val ergoTreeT8s = b.outputRecords.byErgoTreeT8.keys - val utxos = b.outputRecords.byErgoTree.values.flatten - for - _ <- preTx - blockId <- blockRepo.insert(b.block) - _ <- boxRepo.insertUtxos(ergoTrees, ergoTreeT8s, b.outputRecords.assets, utxos) - _ <- boxRepo.deleteUtxos(b.b.transactions.transactions.flatMap(_.inputs.map(_.boxId))) - _ <- postTx - yield blockId - } - -object InmemoryRepo { - def layer: ZLayer[BlockRepo with BoxRepo, Nothing, InmemoryRepo] = - ZLayer.fromFunction(InmemoryRepo.apply _) - -} diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/PersistentRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/PersistentRepo.scala index ca7ac129..a11c8f76 100644 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/PersistentRepo.scala +++ b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/PersistentRepo.scala @@ -1,16 +1,12 @@ package org.ergoplatform.uexplorer.backend import io.getquill.* -import io.getquill.context.ZioJdbc.DataSourceLayer -import io.getquill.jdbczio.Quill import org.ergoplatform.uexplorer.* -import org.ergoplatform.uexplorer.backend.Codecs -import org.ergoplatform.uexplorer.backend.blocks.{BlockRepo, PersistentBlockRepo} -import org.ergoplatform.uexplorer.backend.boxes.{BoxRepo, PersistentBoxRepo} +import org.ergoplatform.uexplorer.backend.blocks.BlockRepo +import org.ergoplatform.uexplorer.backend.boxes.* import org.ergoplatform.uexplorer.db.* import zio.* -import java.util.UUID import javax.sql.DataSource case class PersistentRepo(ds: DataSource, blockRepo: BlockRepo, boxRepo: BoxRepo) extends Repo with Codecs: @@ -43,12 +39,18 @@ case class PersistentRepo(ds: DataSource, blockRepo: BlockRepo, boxRepo: BoxRepo val ergoTrees = outputs.byErgoTree.keys val ergoTreeT8s = outputs.byErgoTreeT8.keys val utxos = outputs.byErgoTree.values.flatten + val assetsToBox = + outputs.tokensByUtxo.flatMap { case (box, amountByToken) => + amountByToken.map { case (token, amount) => Asset2Box(token, box, amount) } + } + + val assets = outputs.utxosByTokenId.keySet.map(tokenId => Asset(tokenId, block.blockId)) ctx .transaction { for _ <- preTx blockId <- blockRepo.insert(block) - _ <- boxRepo.insertUtxos(ergoTrees, ergoTreeT8s, outputs.assets, utxos) + _ <- boxRepo.insertUtxos(ergoTrees, ergoTreeT8s, assetsToBox, assets, utxos) _ <- boxRepo.deleteUtxos(inputIds) _ <- postTx yield blockId diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/blocks/InmemoryBlockRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/blocks/InmemoryBlockRepo.scala deleted file mode 100644 index 20fafcc5..00000000 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/blocks/InmemoryBlockRepo.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.ergoplatform.uexplorer.backend.blocks - -import org.ergoplatform.uexplorer.BlockId -import org.ergoplatform.uexplorer.Const.Protocol -import org.ergoplatform.uexplorer.db.Block -import zio.* - -import scala.collection.mutable - -case class InmemoryBlockRepo(map: Ref[Map[BlockId, Block]]) extends BlockRepo: - - override def insert(block: Block): UIO[BlockId] = - map.update(_ + (block.blockId -> block)).as(block.blockId) - - override def lookup(headerId: BlockId): UIO[Option[Block]] = - map.get.map(_.get(headerId)) - - override def lookupBlocks(ids: Set[BlockId]): UIO[List[Block]] = - map.get.map(_.values.filter(b => ids.contains(b.blockId)).toList) - - override def isEmpty: Task[Boolean] = map.get.map(_.isEmpty) - - override def delete(blockId: BlockId): Task[Long] = map.update(_ - blockId).as(1) - - override def delete(blockIds: Set[BlockId]): Task[Long] = map.update(_ -- blockIds).as(1) - -object InmemoryBlockRepo { - def layer: ZLayer[Any, Nothing, InmemoryBlockRepo] = - ZLayer.fromZIO( - Ref.make(Map.empty).map(new InmemoryBlockRepo(_)) - ) - - def layerWithBlocks(blocks: List[Block]): ZLayer[Any, Nothing, InmemoryBlockRepo] = - ZLayer.fromZIO( - Ref.make(blocks.map(b => b.blockId -> b).toMap).map(new InmemoryBlockRepo(_)) - ) -} diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRepo.scala index 4c2010e9..e5a2b3ac 100644 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRepo.scala +++ b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRepo.scala @@ -1,25 +1,26 @@ package org.ergoplatform.uexplorer.backend.boxes import org.ergoplatform.uexplorer.db.* -import org.ergoplatform.uexplorer.{BlockId, BoxId, ErgoTreeHash, ErgoTreeT8Hash, TokenId} +import org.ergoplatform.uexplorer.{BoxId, ErgoTreeHash, ErgoTreeT8Hash, TokenId} import zio.* trait BoxRepo: def insertUtxos( - ergoTrees: Iterable[ErgoTree], - ergoTreeT8s: Iterable[ErgoTreeT8], - assets: List[Asset], - utxos: Iterable[Utxo] + ergoTrees: Iterable[ErgoTree], + ergoTreeT8s: Iterable[ErgoTreeT8], + assetsToBox: Iterable[Asset2Box], + assets: Iterable[Asset], + utxos: Iterable[Utxo] ): Task[Iterable[BoxId]] def deleteUtxo(boxId: BoxId): Task[Long] def deleteUtxos(boxId: Iterable[BoxId]): Task[Long] - def lookupUnspentAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] + def lookupUnspentAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset2Box]] - def lookupAnyAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] + def lookupAnyAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset2Box]] def lookupBox(boxId: BoxId): Task[Option[Box]] @@ -51,12 +52,13 @@ trait BoxRepo: object BoxRepo: def insertUtxos( - ergoTrees: Iterable[ErgoTree], - ergoTreeT8s: Iterable[ErgoTreeT8], - assets: List[Asset], - utxos: Iterable[Utxo] + ergoTrees: Iterable[ErgoTree], + ergoTreeT8s: Iterable[ErgoTreeT8], + assetsToBox: Iterable[Asset2Box], + assets: Iterable[Asset], + utxos: Iterable[Utxo] ): ZIO[BoxRepo, Throwable, Iterable[BoxId]] = - ZIO.serviceWithZIO[BoxRepo](_.insertUtxos(ergoTrees, ergoTreeT8s, assets, utxos)) + ZIO.serviceWithZIO[BoxRepo](_.insertUtxos(ergoTrees, ergoTreeT8s, assetsToBox, assets, utxos)) def deleteUtxo(boxId: BoxId): ZIO[BoxRepo, Throwable, Long] = ZIO.serviceWithZIO[BoxRepo](_.deleteUtxo(boxId)) diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxService.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxService.scala index 948585c8..416183e5 100644 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxService.scala +++ b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/BoxService.scala @@ -1,6 +1,6 @@ package org.ergoplatform.uexplorer.backend.boxes -import org.ergoplatform.uexplorer.db.{Asset, Box, Utxo} +import org.ergoplatform.uexplorer.db.{Asset, Asset2Box, Box, Utxo} import org.ergoplatform.uexplorer.parser.ErgoTreeParser import org.ergoplatform.uexplorer.{Address, BoxId, CoreConf, ErgoTreeHash, ErgoTreeHex, ErgoTreeT8Hash, ErgoTreeT8Hex, TokenId} import zio.http.QueryParams @@ -9,14 +9,14 @@ import zio.{Task, ZIO, ZLayer} case class BoxService(boxRepo: BoxRepo, coreConf: CoreConf) { import BoxService.allColumns - def getUnspentAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset]] = + def getUnspentAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset2Box]] = for tId <- ZIO.attempt(TokenId.fromStringUnsafe(tokenId)) indexFilter = params.map.view.mapValues(_.head).toMap assets <- boxRepo.lookupUnspentAssetsByTokenId(tId, allColumns, indexFilter) yield assets - def getSpentAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset]] = + def getSpentAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset2Box]] = for tId <- ZIO.attempt(TokenId.fromStringUnsafe(tokenId)) indexFilter = params.map.view.mapValues(_.head).toMap @@ -24,7 +24,7 @@ case class BoxService(boxRepo: BoxRepo, coreConf: CoreConf) { assets <- boxRepo.lookupAnyAssetsByTokenId(tId, allColumns, indexFilter) yield assets.filter(a => !utxoIds.contains(a.boxId)) - def getAnyAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset]] = + def getAnyAssetsByTokenId(tokenId: String, params: QueryParams): Task[Iterable[Asset2Box]] = for tId <- ZIO.attempt(TokenId.fromStringUnsafe(tokenId)) indexFilter = params.map.view.mapValues(_.head).toMap @@ -198,13 +198,13 @@ object BoxService { def layer: ZLayer[BoxRepo with CoreConf, Nothing, BoxService] = ZLayer.fromFunction(BoxService.apply _) - def getUnspentAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset]] = + def getUnspentAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset2Box]] = ZIO.serviceWithZIO[BoxService](_.getUnspentAssetsByTokenId(tokenId, params)) - def getSpentAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset]] = + def getSpentAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset2Box]] = ZIO.serviceWithZIO[BoxService](_.getSpentAssetsByTokenId(tokenId, params)) - def getAnyAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset]] = + def getAnyAssetsByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Asset2Box]] = ZIO.serviceWithZIO[BoxService](_.getAnyAssetsByTokenId(tokenId, params)) def getUnspentBoxesByTokenId(tokenId: String, params: QueryParams): ZIO[BoxService, Throwable, Iterable[Utxo]] = diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/InmemoryBoxRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/InmemoryBoxRepo.scala deleted file mode 100644 index 08426fa0..00000000 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/InmemoryBoxRepo.scala +++ /dev/null @@ -1,164 +0,0 @@ -package org.ergoplatform.uexplorer.backend.boxes - -import org.ergoplatform.uexplorer.backend.boxes.InmemoryBoxRepo.matchCaseClassWith -import org.ergoplatform.uexplorer.db.{Asset, Box, ErgoTree, ErgoTreeT8, Utxo} -import org.ergoplatform.uexplorer.{BoxId, ErgoTreeHash, ErgoTreeT8Hash, TokenId} -import zio.* - -case class BoxRepoState( - utxos: Map[BoxId, Utxo], - boxes: Map[BoxId, Box], - ets: Map[ErgoTreeHash, ErgoTree], - etT8s: Map[ErgoTreeT8Hash, ErgoTreeT8], - assets: List[Asset] -) - -object BoxRepoState { - def empty: BoxRepoState = BoxRepoState(Map.empty, Map.empty, Map.empty, Map.empty, List.empty) -} - -case class InmemoryBoxRepo(state: Ref[BoxRepoState]) extends BoxRepo: - override def insertUtxos( - ergoTrees: Iterable[ErgoTree], - ergoTreeT8s: Iterable[ErgoTreeT8], - assets: List[Asset], - utxos: Iterable[Utxo] - ): Task[Iterable[BoxId]] = - state.modify { state => - utxos.map(_.boxId) -> state.copy( - utxos = state.utxos ++ utxos.map(b => b.boxId -> b), - boxes = state.boxes ++ utxos.map(b => b.boxId -> b.toBox), - ets = state.ets ++ ergoTrees.map(et => et.hash -> et), - etT8s = state.etT8s ++ ergoTreeT8s.map(etT8 => etT8.hash -> etT8), - assets = state.assets ++ assets - ) - } - - override def deleteUtxo(boxId: BoxId): Task[Long] = state.modify(s => 0L -> s.copy(s.utxos.removed(boxId))) - - override def deleteUtxos(boxIds: Iterable[BoxId]): Task[Long] = - state.modify(s => 0L -> s.copy(s.utxos.removedAll(boxIds))) - - override def lookupUnspentAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] = - state.get.map { state => - val utxos = state.utxos.toSet - state.assets.filter { asset => - val matchingUtxos = utxos.filter(_._2.boxId == asset.boxId) - matchingUtxos.nonEmpty && matchCaseClassWith(matchingUtxos.asInstanceOf[Set[AnyRef]] + asset, columns, filter) - } - } - - override def lookupAnyAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] = - state.get.map { state => - val boxes = state.boxes.toSet - state.assets.filter { asset => - val matchingBoxes = boxes.filter(_._2.boxId == asset.boxId) - matchingBoxes.nonEmpty && matchCaseClassWith(matchingBoxes.asInstanceOf[Set[AnyRef]] + asset, columns, filter) - } - } - - override def lookupUtxosByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Utxo]] = - state.get.map { state => - val assets = state.assets.toSet - state.utxos.filter { case (_, utxo) => - val matchingAssets = assets.filter(_.boxId == utxo.boxId) - matchingAssets.nonEmpty && matchCaseClassWith(matchingAssets.asInstanceOf[Set[AnyRef]] + utxo, columns, filter) - }.values - } - - override def lookupBoxesByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Box]] = - state.get.map(state => - val assets = state.assets.toSet - state.boxes.filter { case (_, box) => - val matchingAssets = assets.filter(_.boxId == box.boxId) - matchingAssets.nonEmpty && matchCaseClassWith(matchingAssets.asInstanceOf[Set[AnyRef]] + box, columns, filter) - }.values - ) - - def lookupUtxoIdsByTokenId(tokenId: TokenId): Task[Set[BoxId]] = - state.get.map { state => - state.assets.collect { - case asset if state.utxos.contains(asset.boxId) && asset.tokenId == tokenId => - asset.boxId - }.toSet - } - - override def lookupBox(boxId: BoxId): Task[Option[Box]] = state.get.map(_.boxes.get(boxId)) - - override def lookupUtxo(boxId: BoxId): Task[Option[Utxo]] = state.get.map(_.utxos.get(boxId)) - - override def lookupBoxes(boxes: Set[BoxId]): Task[List[Box]] = - state.get.map(_.boxes.filter(t => boxes.contains(t._1)).valuesIterator.toList) - - override def lookupUtxos(utxos: Set[BoxId]): Task[List[Utxo]] = - state.get.map(_.utxos.filter(t => utxos.contains(t._1)).valuesIterator.toList) - - override def lookupBoxesByHash(etHash: ErgoTreeHash, columns: List[String], filter: Map[String, Any]): Task[Iterable[Box]] = - state.get.map(state => - state.boxes.collect { - case (_, box) if box.ergoTreeHash == etHash && matchCaseClassWith(List(box), columns, filter) => - box - } - ) - - override def lookupUtxosByHash(etHash: ErgoTreeHash, columns: List[String], filter: Map[String, Any]): Task[Iterable[Utxo]] = - state.get.map(state => - state.utxos.collect { - case (_, utxo) if utxo.ergoTreeHash == etHash && matchCaseClassWith(List(utxo), columns, filter) => - utxo - } - ) - - override def lookupBoxesByT8Hash(etT8Hash: ErgoTreeT8Hash, columns: List[String], filter: Map[String, Any]): Task[Iterable[Box]] = - state.get.map(state => - state.boxes.collect { - case (_, box) if box.ergoTreeT8Hash.contains(etT8Hash) && matchCaseClassWith(List(box), columns, filter) => - box - } - ) - - override def lookupUtxosByT8Hash(etT8Hash: ErgoTreeT8Hash, columns: List[String], filter: Map[String, Any]): Task[Iterable[Utxo]] = - state.get.map(state => - state.utxos.collect { - case (_, utxo) if utxo.ergoTreeT8Hash.contains(etT8Hash) => utxo - } - ) - - override def lookupUtxoIdsByHash(etHash: ErgoTreeHash): Task[Set[BoxId]] = - state.get.map(state => - state.utxos.collect { - case (_, utxo) if utxo.ergoTreeHash == etHash => utxo.boxId - }.toSet - ) - - override def lookupUtxoIdsByT8Hash(etT8Hash: ErgoTreeT8Hash): Task[Set[BoxId]] = - state.get.map(state => - state.utxos.collect { - case (_, utxo) if utxo.ergoTreeT8Hash.contains(etT8Hash) => utxo.boxId - }.toSet - ) - - override def isEmpty: Task[Boolean] = - state.get.map(s => s.boxes.isEmpty && s.utxos.isEmpty && s.ets.isEmpty && s.etT8s.isEmpty) - -object InmemoryBoxRepo { - - def matchCaseClassWith(caseClasses: Iterable[AnyRef], columns: List[String], filter: Map[String, Any]): Boolean = { - val ccFields: Map[String, Any] = - caseClasses - .foldLeft(Map.empty[String, Any]) { case (acc, cc) => - acc ++ cc.getClass.getDeclaredFields - .tapEach(_.setAccessible(true)) - .foldLeft(Map.empty[String, Any])((a, f) => a + (f.getName -> f.get(cc))) - } - filter.removedAll(columns).forall { case (k, v) => - ccFields.get(k).contains(v) - } - } - - def layer: ZLayer[Any, Nothing, InmemoryBoxRepo] = - ZLayer.fromZIO( - Ref.make(BoxRepoState.empty).map(new InmemoryBoxRepo(_)) - ) - -} diff --git a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/PersistentBoxRepo.scala b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/PersistentBoxRepo.scala index 5164a377..d74f267f 100644 --- a/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/PersistentBoxRepo.scala +++ b/modules/backend/src/main/scala/org/ergoplatform/uexplorer/backend/boxes/PersistentBoxRepo.scala @@ -15,6 +15,10 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: private val dsLayer = ZLayer.succeed(ds) + // TODO https://github.com/zio/zio-protoquill/issues/298 + inline def onConflictIgnoreForBatchQueries[T](inline entity: Insert[T]): Insert[T] = + sql"$entity ON CONFLICT DO NOTHING".as[Insert[T]] + private def insertUtxosQuery(utxos: Iterable[Utxo]) = quote { liftQuery(utxos).foreach(utxo => query[Utxo].insertValue(utxo)) @@ -24,38 +28,44 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: liftQuery(boxes).foreach(box => query[Box].insertValue(box)) } private def insertErgoTreesQuery(ergoTrees: Iterable[ErgoTree]) = - quote { // todo https://github.com/zio/zio-protoquill/issues/298 - liftQuery(ergoTrees).foreach(ergoTree => query[ErgoTree].insertValue(ergoTree).onConflictIgnore) + quote { + liftQuery(ergoTrees).foreach(ergoTree => onConflictIgnoreForBatchQueries(query[ErgoTree].insertValue(ergoTree))) } private def insertErgoTreeT8sQuery(ergoTreeT8s: Iterable[ErgoTreeT8]) = - quote { // todo https://github.com/zio/zio-protoquill/issues/298 - liftQuery(ergoTreeT8s).foreach(ergoTreeT8 => query[ErgoTreeT8].insertValue(ergoTreeT8).onConflictIgnore) - } - private def insertErgoTreeQuery(ergoTree: ErgoTree) = quote { - query[ErgoTree].insertValue(lift(ergoTree)).onConflictIgnore + liftQuery(ergoTreeT8s).foreach(ergoTreeT8 => onConflictIgnoreForBatchQueries(query[ErgoTreeT8].insertValue(ergoTreeT8))) } - private def insertErgoTreeT8Query(ergoTreeT8: ErgoTreeT8) = + private def insertAssetsToBox(assetsToBox: Iterable[Asset2Box]) = quote { - query[ErgoTreeT8].insertValue(lift(ergoTreeT8)).onConflictIgnore + liftQuery(assetsToBox).foreach(assetToBox => query[Asset2Box].insertValue(assetToBox)) } - private def insertAssets(assets: List[Asset]) = + private def insertAssets(assets: Iterable[Asset]) = quote { - liftQuery(assets).foreach(asset => query[Asset].insertValue(asset)) + liftQuery(assets).foreach(asset => onConflictIgnoreForBatchQueries(query[Asset].insertValue(asset))) } override def insertUtxos( ergoTrees: Iterable[ErgoTree], ergoTreeT8s: Iterable[ErgoTreeT8], - assets: List[Asset], + assetsToBox: Iterable[Asset2Box], + assets: Iterable[Asset], utxos: Iterable[Utxo] ): Task[Iterable[BoxId]] = (for { - _ <- ZIO.foreachDiscard(ergoTrees)(et => ctx.run(insertErgoTreeQuery(et))) // todo 298 - _ <- ZIO.foreachDiscard(ergoTreeT8s)(etT8 => ctx.run(insertErgoTreeT8Query(etT8))) + _ <- ZIO.collectAllParDiscard( + List( + ctx.run(insertErgoTreesQuery(ergoTrees)), + ctx.run(insertErgoTreeT8sQuery(ergoTreeT8s)), + ctx.run(insertAssets(assets)) + ) + ) _ <- ctx.run(insertBoxesQuery(utxos.map(_.toBox))) - _ <- ctx.run(insertUtxosQuery(utxos)) - _ <- ctx.run(insertAssets(assets)) + _ <- ZIO.collectAllParDiscard( + List( + ctx.run(insertAssetsToBox(assetsToBox)), + ctx.run(insertUtxosQuery(utxos)) + ) + ) } yield utxos.map(_.boxId)).provide(dsLayer) override def deleteUtxo(boxId: BoxId): Task[Long] = @@ -76,30 +86,30 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: } .provide(dsLayer) - def lookupUnspentAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] = + def lookupUnspentAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset2Box]] = ctx .run { quote { query[Utxo] - .join(query[Asset]) + .join(query[Asset2Box]) .on((a, utxo) => a.boxId == utxo.boxId) .filter((_, a) => a.tokenId == lift(tokenId)) - .map((_, a) => Asset(a.tokenId, a.blockId, a.boxId, a.amount)) + .map((_, a) => Asset2Box(a.tokenId, a.boxId, a.amount)) .filterByKeys(filter) .filterColumns(columns) } } .provide(dsLayer) - def lookupAnyAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset]] = + def lookupAnyAssetsByTokenId(tokenId: TokenId, columns: List[String], filter: Map[String, Any]): Task[Iterable[Asset2Box]] = ctx .run { quote { query[Box] - .join(query[Asset]) + .join(query[Asset2Box]) .on((a, box) => a.boxId == box.boxId) .filter((_, a) => a.tokenId == lift(tokenId)) - .map((_, a) => Asset(a.tokenId, a.blockId, a.boxId, a.amount)) + .map((_, a) => Asset2Box(a.tokenId, a.boxId, a.amount)) .filterByKeys(filter) .filterColumns(columns) } @@ -152,11 +162,11 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: ctx .run { quote { - query[Asset] + query[Asset2Box] .join(query[Utxo]) .on((a, utxo) => utxo.boxId == a.boxId) .filter((a, _) => a.tokenId == lift(tokenId)) - .map((_, b) => Utxo(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Utxo(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } @@ -167,11 +177,11 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: ctx .run { quote { - query[Asset] + query[Asset2Box] .join(query[Box]) .on((a, b) => b.boxId == a.boxId) .filter((a, _) => a.tokenId == lift(tokenId)) - .map((_, b) => Box(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Box(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } @@ -182,7 +192,7 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: ctx .run { quote { - query[Asset] + query[Asset2Box] .join(query[Utxo]) .on((a, b) => b.boxId == a.boxId) .filter((a, _) => a.tokenId == lift(tokenId)) @@ -200,7 +210,7 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: .join(query[Box]) .on((et, box) => et.hash == box.ergoTreeHash) .filter((et, _) => et.hash == lift(etHash)) - .map((_, b) => Box(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Box(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } @@ -215,7 +225,7 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: .join(query[Utxo]) .on((et, utxo) => et.hash == utxo.ergoTreeHash) .filter((et, _) => et.hash == lift(etHash)) - .map((_, b) => Utxo(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Utxo(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } @@ -242,7 +252,7 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: .join(query[Box]) .on((et, box) => box.ergoTreeT8Hash.contains(et.hash)) .filter((et, _) => et.hash == lift(etT8Hash)) - .map((_, b) => Box(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Box(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } @@ -257,7 +267,7 @@ case class PersistentBoxRepo(ds: DataSource) extends BoxRepo with Codecs: .join(query[Utxo]) .on((et, box) => box.ergoTreeT8Hash.contains(et.hash)) .filter((et, _) => et.hash == lift(etT8Hash)) - .map((_, b) => Utxo(b.boxId, b.blockId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) + .map((_, b) => Utxo(b.boxId, b.txId, b.ergoTreeHash, b.ergoTreeT8Hash, b.ergValue, b.r4, b.r5, b.r6, b.r7, b.r8, b.r9)) .filterByKeys(filter) .filterColumns(columns) } diff --git a/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/RouteSpec.scala b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/RouteSpec.scala new file mode 100644 index 00000000..c69b57c7 --- /dev/null +++ b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/RouteSpec.scala @@ -0,0 +1,29 @@ +package org.ergoplatform.uexplorer.backend + +import org.ergoplatform.uexplorer.{CoreConf, NetworkPrefix} +import org.ergoplatform.uexplorer.backend.blocks.{BlockRoutesSpec, PersistentBlockRepo} +import org.ergoplatform.uexplorer.backend.boxes.{BoxRoutesSpec, PersistentBoxRepo} +import org.ergoplatform.uexplorer.http.Rest +import zio.ZIO +import zio.test.{TestAspect, ZIOSpecDefault} + +object RouteSpec extends ZIOSpecDefault with BlockRoutesSpec with BoxRoutesSpec { + + implicit private val ps: CoreConf = CoreConf(NetworkPrefix.fromStringUnsafe("0")) + + def spec = suite("RouteSpec")( + blockRoutesSpec, + boxRoutesSpec + ) @@ TestAspect.beforeAll( + (for + repo <- ZIO.service[Repo] + blocks <- Rest.chain.forHeights(1 to 10) + _ <- ZIO.collectAllDiscard(blocks.map(b => repo.writeBlock(b)(ZIO.unit, ZIO.unit))) + yield ()).provide( + H2Backend.layer, + PersistentBlockRepo.layer, + PersistentBoxRepo.layer, + PersistentRepo.layer + ) + ) +} diff --git a/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/blocks/BlockRoutesSpec.scala b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/blocks/BlockRoutesSpec.scala index 0df5db67..096f2bb2 100644 --- a/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/blocks/BlockRoutesSpec.scala +++ b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/blocks/BlockRoutesSpec.scala @@ -10,34 +10,42 @@ import zio.test.* import eu.timepit.refined.auto.autoUnwrap import zio.json.interop.refined.* import org.ergoplatform.uexplorer.BlockId.unwrapped +import org.ergoplatform.uexplorer.backend.boxes.PersistentBoxRepo +import org.ergoplatform.uexplorer.{CoreConf, NetworkPrefix} +import org.ergoplatform.uexplorer.backend.{H2Backend, PersistentRepo, Repo} import org.ergoplatform.uexplorer.db.Block +import org.ergoplatform.uexplorer.http.Rest import java.util.concurrent.TimeUnit -object BlockRoutesSpec extends ZIOSpecDefault { +trait BlockRoutesSpec extends ZIOSpec[TestEnvironment] { - private val app = BlockRoutes() + private val blockRoutes = BlockRoutes() - def spec = suite("BlockRoutesSpec")( + def blockRoutesSpec = suite("BlockRoutesSpec")( test("get block by id") { - val path = Root / "blocks" / Protocol.genesisBlockId.unwrapped + val path = Root / "blocks" / Protocol.firstBlockId.unwrapped val req = Request.get(url = URL(path)) for { - expectedBody <- app.runZIO(req).flatMap(_.body.asString) - } yield assertTrue(new Block().toJson == expectedBody) + expectedBody <- blockRoutes.runZIO(req).flatMap(_.body.asString) + expectedBlock <- ZIO.fromEither(expectedBody.fromJson[Block]) + } yield assertTrue(1 == expectedBlock.height) }.provide( - InmemoryBlockRepo.layerWithBlocks(List(new Block())) + H2Backend.layer, + PersistentBlockRepo.layer ), test("get blocks by ids") { val path = Root / "blocks" - val req = Request.post(url = URL(path), body = Body.fromString(List(Protocol.genesisBlockId).toJson)) + val req = Request.post(url = URL(path), body = Body.fromString(List(Protocol.firstBlockId).toJson)) for { - expectedBody <- app.runZIO(req).flatMap(_.body.asString) - } yield assertTrue(List(new Block()).toJson == expectedBody) + expectedBody <- blockRoutes.runZIO(req).flatMap(_.body.asString) + expectedBlocks <- ZIO.fromEither(expectedBody.fromJson[List[Block]]) + } yield assertTrue(List(1) == expectedBlocks.map(_.height)) }.provide( - InmemoryBlockRepo.layerWithBlocks(List(new Block())) + H2Backend.layer, + PersistentBlockRepo.layer ) ) } diff --git a/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRoutesSpec.scala b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRoutesSpec.scala index bb8133bd..8f7f8ef4 100644 --- a/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRoutesSpec.scala +++ b/modules/backend/src/test/scala/org/ergoplatform/uexplorer/backend/boxes/BoxRoutesSpec.scala @@ -15,13 +15,12 @@ import zio.test.* import scala.collection.immutable.Set -object BoxRoutesSpec extends ZIOSpecDefault { +trait BoxRoutesSpec extends ZIOSpec[TestEnvironment] { - private val app = BoxRoutes() + private val boxRoutes = BoxRoutes() private val indexFilter: Map[String, Chunk[String]] = BoxService.indexWhiteList.map(key => key -> Chunk("")).toMap - implicit private val ps: CoreConf = CoreConf(NetworkPrefix.fromStringUnsafe("0")) - def spec = suite("BoxRoutesSpec")( + def boxRoutesSpec = suite("BoxRoutesSpec")( test("get spent/unspent/any box(es) by id") { val unspentBoxGet = Request.get(URL(Root / "boxes" / "unspent" / Protocol.firstBlockRewardBox.unwrapped)) val missingUnspentBoxGet = Request.get(URL(Root / "boxes" / "unspent" / Emission.outputBox.unwrapped)) @@ -46,14 +45,14 @@ object BoxRoutesSpec extends ZIOSpecDefault { ) for { - unspentBox <- app.runZIO(unspentBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Utxo]))) - unspentBoxes <- app.runZIO(unspentBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Utxo]]))) - spentBox <- app.runZIO(spentBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Box]))) - anyBox <- app.runZIO(anyBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Box]))) - spentBoxes <- app.runZIO(spentBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Box]]))) - anyBoxes <- app.runZIO(anyBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Box]]))) - missingSpentBoxStatus <- app.runZIO(missingSpentBoxGet).map(_.status) - missingUnspentBoxStatus <- app.runZIO(missingUnspentBoxGet).map(_.status) + unspentBox <- boxRoutes.runZIO(unspentBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Utxo]))) + unspentBoxes <- boxRoutes.runZIO(unspentBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Utxo]]))) + spentBox <- boxRoutes.runZIO(spentBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Box]))) + anyBox <- boxRoutes.runZIO(anyBoxGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Box]))) + spentBoxes <- boxRoutes.runZIO(spentBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Box]]))) + anyBoxes <- boxRoutes.runZIO(anyBoxPost).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[Set[Box]]))) + missingSpentBoxStatus <- boxRoutes.runZIO(missingSpentBoxGet).map(_.status) + missingUnspentBoxStatus <- boxRoutes.runZIO(missingUnspentBoxGet).map(_.status) } yield assertTrue( unspentBox.boxId == Protocol.firstBlockRewardBox, unspentBoxes.map(_.boxId) == Set(Protocol.firstBlockRewardBox), @@ -80,12 +79,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyBoxIdsByAddressGet = Request.get(URL(Root / "box-ids" / "any" / "by-address" / Emission.address)) for { - spentBoxesByAddress <- app.runZIO(spentBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentBoxesByAddress <- app.runZIO(unspentBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyBoxesByAddress <- app.runZIO(anyBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentBoxIdsByAddress <- app.runZIO(spentBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentBoxIdsByAddress <- app.runZIO(unspentBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyBoxIdsByAddress <- app.runZIO(anyBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentBoxesByAddress <- boxRoutes.runZIO(spentBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentBoxesByAddress <- boxRoutes.runZIO(unspentBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyBoxesByAddress <- boxRoutes.runZIO(anyBoxesByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentBoxIdsByAddress <- boxRoutes.runZIO(spentBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentBoxIdsByAddress <- boxRoutes.runZIO(unspentBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyBoxIdsByAddress <- boxRoutes.runZIO(anyBoxIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentBoxesByAddress.map(_.boxId).intersect(spentBoxesByAddress.map(_.boxId)).isEmpty, unspentBoxesByAddress.map(_.boxId).nonEmpty, @@ -112,12 +111,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyBoxIdsByErgoTreeGet = Request.get(URL(Root / "box-ids" / "any" / "contracts" / "by-ergo-tree" / Emission.ergoTreeHex)) for { - spentBoxesByErgoTree <- app.runZIO(spentBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentBoxesByErgoTree <- app.runZIO(unspentBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyBoxesByErgoTree <- app.runZIO(anyBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentBoxIdsByErgoTree <- app.runZIO(spentBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentBoxIdsByErgoTree <- app.runZIO(unspentBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyBoxIdsByErgoTree <- app.runZIO(anyBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentBoxesByErgoTree <- boxRoutes.runZIO(spentBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentBoxesByErgoTree <- boxRoutes.runZIO(unspentBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyBoxesByErgoTree <- boxRoutes.runZIO(anyBoxesByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentBoxIdsByErgoTree <- boxRoutes.runZIO(spentBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentBoxIdsByErgoTree <- boxRoutes.runZIO(unspentBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyBoxIdsByErgoTree <- boxRoutes.runZIO(anyBoxIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentBoxesByErgoTree.map(_.boxId).intersect(spentBoxesByErgoTree.map(_.boxId)).isEmpty, unspentBoxesByErgoTree.map(_.boxId).nonEmpty, @@ -143,12 +142,13 @@ object BoxRoutesSpec extends ZIOSpecDefault { val unspentBoxIdsByErgoTreeT8Get = Request.get(URL(Root / "box-ids" / "unspent" / "templates" / "by-ergo-tree" / Emission.ergoTreeHex)) val anyBoxIdsByErgoTreeT8Get = Request.get(URL(Root / "box-ids" / "any" / "templates" / "by-ergo-tree" / Emission.ergoTreeHex)) for { - spentBoxesByErgoTreeT8 <- app.runZIO(spentBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentBoxesByErgoTreeT8 <- app.runZIO(unspentBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyBoxesByErgoTreeT8 <- app.runZIO(anyBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentBoxIdsByErgoTreeT8 <- app.runZIO(spentBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentBoxIdsByErgoTreeT8 <- app.runZIO(unspentBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyBoxIdsByErgoTreeT8 <- app.runZIO(anyBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentBoxesByErgoTreeT8 <- boxRoutes.runZIO(spentBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentBoxesByErgoTreeT8 <- boxRoutes.runZIO(unspentBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyBoxesByErgoTreeT8 <- boxRoutes.runZIO(anyBoxesByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentBoxIdsByErgoTreeT8 <- boxRoutes.runZIO(spentBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentBoxIdsByErgoTreeT8 <- + boxRoutes.runZIO(unspentBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyBoxIdsByErgoTreeT8 <- boxRoutes.runZIO(anyBoxIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( // hard to get any real template when we have just first 4000 blocks that have no templates unspentBoxesByErgoTreeT8.map(_.boxId).intersect(spentBoxesByErgoTreeT8.map(_.boxId)).isEmpty, @@ -176,12 +176,15 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyBoxIdsByErgoTreeHashGet = Request.get(URL(Root / "box-ids" / "any" / "contracts" / "by-ergo-tree-hash" / Emission.ergoTreeHash)) for { - spentBoxesByErgoTreeHash <- app.runZIO(spentBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentBoxesByErgoTreeHash <- app.runZIO(unspentBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyBoxesByErgoTreeHash <- app.runZIO(anyBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentBoxIdsByErgoTreeHash <- app.runZIO(spentBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentBoxIdsByErgoTreeHash <- app.runZIO(unspentBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyBoxIdsByErgoTreeHash <- app.runZIO(anyBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentBoxesByErgoTreeHash <- boxRoutes.runZIO(spentBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentBoxesByErgoTreeHash <- + boxRoutes.runZIO(unspentBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyBoxesByErgoTreeHash <- boxRoutes.runZIO(anyBoxesByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentBoxIdsByErgoTreeHash <- + boxRoutes.runZIO(spentBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentBoxIdsByErgoTreeHash <- + boxRoutes.runZIO(unspentBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyBoxIdsByErgoTreeHash <- boxRoutes.runZIO(anyBoxIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentBoxesByErgoTreeHash.map(_.boxId).intersect(spentBoxesByErgoTreeHash.map(_.boxId)).isEmpty, unspentBoxesByErgoTreeHash.map(_.boxId).nonEmpty, @@ -208,12 +211,13 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyIdsByErgoTreeHashT8Get = Request.get(URL(Root / "box-ids" / "any" / "templates" / "by-ergo-tree-hash" / Emission.ergoTreeHash)) for { - spentByErgoTreeT8Hash <- app.runZIO(spentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByErgoTreeT8Hash <- app.runZIO(unspentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByErgoTreeT8Hash <- app.runZIO(anyByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByErgoTreeT8Hash <- app.runZIO(spentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByErgoTreeT8Hash <- app.runZIO(unspentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByErgoTreeT8Hash <- app.runZIO(anyIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByErgoTreeT8Hash <- boxRoutes.runZIO(spentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByErgoTreeT8Hash <- boxRoutes.runZIO(unspentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByErgoTreeT8Hash <- boxRoutes.runZIO(anyByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByErgoTreeT8Hash <- boxRoutes.runZIO(spentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByErgoTreeT8Hash <- + boxRoutes.runZIO(unspentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByErgoTreeT8Hash <- boxRoutes.runZIO(anyIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByErgoTreeT8Hash.map(_.boxId).intersect(spentByErgoTreeT8Hash.map(_.boxId)).isEmpty, unspentByErgoTreeT8Hash.map(_.boxId).isEmpty, @@ -240,12 +244,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyIdsByAddressGet = Request.get(URL(Root / "box-ids" / "any" / "by-address" / Emission.address).withQueryParams(indexFilter)) for { - spentByAddress <- app.runZIO(spentByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByAddress <- app.runZIO(unspentByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByAddress <- app.runZIO(anyByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByAddress <- app.runZIO(spentIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByAddress <- app.runZIO(unspentIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByAddress <- app.runZIO(anyIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByAddress <- boxRoutes.runZIO(spentByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByAddress <- boxRoutes.runZIO(unspentByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByAddress <- boxRoutes.runZIO(anyByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByAddress <- boxRoutes.runZIO(spentIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByAddress <- boxRoutes.runZIO(unspentIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByAddress <- boxRoutes.runZIO(anyIdsByAddressGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByAddress.map(_.boxId).intersect(spentByAddress.map(_.boxId)).isEmpty, unspentByAddress.map(_.boxId).isEmpty, @@ -274,12 +278,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyIdsByErgoTreeGet = Request.get(URL(Root / "boxes" / "any" / "contracts" / "by-ergo-tree" / Emission.ergoTreeHex).withQueryParams(indexFilter)) for { - spentByErgoTree <- app.runZIO(spentByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByErgoTree <- app.runZIO(unspentByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByErgoTree <- app.runZIO(anyByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByErgoTree <- app.runZIO(spentIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByErgoTree <- app.runZIO(unspentIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByErgoTree <- app.runZIO(anyIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByErgoTree <- boxRoutes.runZIO(spentByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByErgoTree <- boxRoutes.runZIO(unspentByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByErgoTree <- boxRoutes.runZIO(anyByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByErgoTree <- boxRoutes.runZIO(spentIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByErgoTree <- boxRoutes.runZIO(unspentIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByErgoTree <- boxRoutes.runZIO(anyIdsByErgoTreeGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByErgoTree.map(_.boxId).intersect(spentByErgoTree.map(_.boxId)).isEmpty, unspentByErgoTree.map(_.boxId).isEmpty, @@ -310,12 +314,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyIdsByErgoTreeT8Get = Request.get(URL(Root / "box-ids" / "any" / "templates" / "by-ergo-tree" / Emission.ergoTreeHex).withQueryParams(indexFilter)) for { - spentByErgoTreeT8 <- app.runZIO(spentByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByErgoTreeT8 <- app.runZIO(unspentByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByErgoTreeT8 <- app.runZIO(anyByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByErgoTreeT8 <- app.runZIO(spentIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByErgoTreeT8 <- app.runZIO(unspentIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByErgoTreeT8 <- app.runZIO(anyIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByErgoTreeT8 <- boxRoutes.runZIO(spentByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByErgoTreeT8 <- boxRoutes.runZIO(unspentByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByErgoTreeT8 <- boxRoutes.runZIO(anyByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByErgoTreeT8 <- boxRoutes.runZIO(spentIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByErgoTreeT8 <- boxRoutes.runZIO(unspentIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByErgoTreeT8 <- boxRoutes.runZIO(anyIdsByErgoTreeT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByErgoTreeT8.map(_.boxId).intersect(spentByErgoTreeT8.map(_.boxId)).isEmpty, unspentByErgoTreeT8.map(_.boxId).isEmpty, @@ -348,12 +352,12 @@ object BoxRoutesSpec extends ZIOSpecDefault { Request.get(URL(Root / "box-ids" / "any" / "contracts" / "by-ergo-tree-hash" / Emission.ergoTreeHash).withQueryParams(indexFilter)) for { - spentByErgoTreeHash <- app.runZIO(spentByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByErgoTreeHash <- app.runZIO(unspentByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByErgoTreeHash <- app.runZIO(anyByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByErgoTreeHash <- app.runZIO(spentIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByErgoTreeHash <- app.runZIO(unspentIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByErgoTreeHash <- app.runZIO(anyIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByErgoTreeHash <- boxRoutes.runZIO(spentByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByErgoTreeHash <- boxRoutes.runZIO(unspentByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByErgoTreeHash <- boxRoutes.runZIO(anyByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByErgoTreeHash <- boxRoutes.runZIO(spentIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByErgoTreeHash <- boxRoutes.runZIO(unspentIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByErgoTreeHash <- boxRoutes.runZIO(anyIdsByErgoTreeHashGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByErgoTreeHash.map(_.boxId).intersect(spentByErgoTreeHash.map(_.boxId)).isEmpty, unspentByErgoTreeHash.map(_.boxId).isEmpty, @@ -386,12 +390,13 @@ object BoxRoutesSpec extends ZIOSpecDefault { Request.get(URL(Root / "box-ids" / "any" / "templates" / "by-ergo-tree-hash" / Emission.ergoTreeHash).withQueryParams(indexFilter)) for { - spentByErgoTreeT8Hash <- app.runZIO(spentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - unspentByErgoTreeT8Hash <- app.runZIO(unspentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) - anyByErgoTreeT8Hash <- app.runZIO(anyByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) - spentIdsByErgoTreeT8Hash <- app.runZIO(spentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - unspentIdsByErgoTreeT8Hash <- app.runZIO(unspentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyIdsByErgoTreeT8Hash <- app.runZIO(anyIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentByErgoTreeT8Hash <- boxRoutes.runZIO(spentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + unspentByErgoTreeT8Hash <- boxRoutes.runZIO(unspentByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Utxo]]))) + anyByErgoTreeT8Hash <- boxRoutes.runZIO(anyByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Box]]))) + spentIdsByErgoTreeT8Hash <- boxRoutes.runZIO(spentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentIdsByErgoTreeT8Hash <- + boxRoutes.runZIO(unspentIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyIdsByErgoTreeT8Hash <- boxRoutes.runZIO(anyIdsByErgoTreeHashT8Get).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentByErgoTreeT8Hash.map(_.boxId).intersect(spentByErgoTreeT8Hash.map(_.boxId)).isEmpty, unspentByErgoTreeT8Hash.map(_.boxId).isEmpty, @@ -421,15 +426,15 @@ object BoxRoutesSpec extends ZIOSpecDefault { val anyBoxIdsByTokenIdGet = Request.get(URL(Root / "box-ids" / "any" / "by-token-id" / Protocol.firstBlockRewardBox.unwrapped)) for { - unspentAssets <- app.runZIO(unspentAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - spentAssets <- app.runZIO(spentAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - anyAssets <- app.runZIO(anyAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - unspentBoxes <- app.runZIO(unspentBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - unspentBoxIds <- app.runZIO(unspentBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - spentBoxes <- app.runZIO(spentBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - spentBoxIds <- app.runZIO(spentBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) - anyBoxes <- app.runZIO(anyBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) - anyBoxIds <- app.runZIO(anyBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + unspentAssets <- boxRoutes.runZIO(unspentAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + spentAssets <- boxRoutes.runZIO(spentAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + anyAssets <- boxRoutes.runZIO(anyAssetsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + unspentBoxes <- boxRoutes.runZIO(unspentBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + unspentBoxIds <- boxRoutes.runZIO(unspentBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + spentBoxes <- boxRoutes.runZIO(spentBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + spentBoxIds <- boxRoutes.runZIO(spentBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) + anyBoxes <- boxRoutes.runZIO(anyBoxesByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[Asset]]))) + anyBoxIds <- boxRoutes.runZIO(anyBoxIdsByTokenIdGet).flatMap(_.body.asString.flatMap(x => ZIO.fromEither(x.fromJson[List[BoxId]]))) } yield assertTrue( unspentAssets.isEmpty, spentAssets.isEmpty, @@ -447,16 +452,5 @@ object BoxRoutesSpec extends ZIOSpecDefault { BoxService.layer, CoreConf.layer ) - ) @@ TestAspect.beforeAll( - (for - repo <- ZIO.service[Repo] - blocks <- Rest.chain.forHeights(1 to 10) - _ <- ZIO.collectAllDiscard(blocks.map(b => repo.writeBlock(b)(ZIO.unit, ZIO.unit))) - yield ()).provide( - H2Backend.layer, - PersistentBlockRepo.layer, - PersistentBoxRepo.layer, - PersistentRepo.layer - ) ) } diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/StreamScheduler.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/StreamScheduler.scala index ccfd10c8..aa293942 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/StreamScheduler.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/StreamScheduler.scala @@ -27,14 +27,16 @@ case class StreamScheduler( def periodicSync: Task[MemPoolStateChanges] = for { - chainSyncResult <- streamExecutor.indexNewBlocks - stateChanges <- mempoolSyncer.syncMempool(chainSyncResult.storage) - _ <- pluginManager.executePlugins( - chainSyncResult.storage, - stateChanges, - chainSyncResult.graphTraversalSource, - chainSyncResult.lastBlock - ) + chainSyncResult <- streamExecutor.indexNewBlocks.logError("Syncing chain failed") + stateChanges <- mempoolSyncer.syncMempool(chainSyncResult.storage).logError("Syncing mempool failed") + _ <- pluginManager + .executePlugins( + chainSyncResult.storage, + stateChanges, + chainSyncResult.graphTraversalSource, + chainSyncResult.lastBlock + ) + .logError("Plugin execution failed") } yield stateChanges def validateAndSchedule( diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockWriter.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockWriter.scala index c0d1642d..fe40d0f4 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockWriter.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/chain/BlockWriter.scala @@ -28,7 +28,7 @@ case class BlockWriter( chainIndexerConf: ChainIndexerConf ) { - implicit private val ps: CoreConf = chainIndexerConf.core + implicit private val ps: CoreConf = chainIndexerConf.core implicit private val enc: ErgoAddressEncoder = ps.addressEncoder private def hasParentAndIsChained(fork: List[LinkedBlock]): Boolean = @@ -49,7 +49,7 @@ case class BlockWriter( ) } else { for { - _ <- ZIO.log(s"Adding fork from height ${winningFork.head.block.height} until ${winningFork.last.block.height}") + _ <- ZIO.log(s"Adding fork from height ${winningFork.head.block.height} until ${winningFork.last.block.height}") preForkVersion <- ZIO.attempt(storage.getBlockById(winningFork.head.b.header.id).map(_.revision).get) loosingFork = winningFork.flatMap(b => storage.getBlocksByHeight(b.block.height).filter(_._1 != b.b.header.id)).toMap _ <- ZIO.attempt(storage.rollbackTo(preForkVersion)) @@ -112,6 +112,8 @@ case class BlockWriter( *> storage.removeInputBoxesByErgoTree(b.b.transactions.transactions), storage.persistErgoTreeT8ByUtxo(b.outputRecords) *> storage.removeInputBoxesByErgoTreeT8(b.b.transactions.transactions), + storage.persistUtxosByTokenId(b.outputRecords.utxosByTokenId), + storage.persistTokensByUtxo(b.outputRecords.tokensByUtxo), storage.insertNewBlock(b.b.header.id, b.block, storage.getCurrentRevision) ) ), diff --git a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/tool/DownloadBlocksAsLinesFromNodeToFile.scala b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/tool/DownloadBlocksAsLinesFromNodeToFile.scala index 3b7c98f3..6fb26059 100644 --- a/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/tool/DownloadBlocksAsLinesFromNodeToFile.scala +++ b/modules/chain-indexer/src/main/scala/org/ergoplatform/uexplorer/indexer/tool/DownloadBlocksAsLinesFromNodeToFile.scala @@ -16,7 +16,7 @@ import zio.stream.* object DownloadBlocksAsLinesFromNodeToFile extends ZIOAppDefault { - private val targetFile = Paths.get(java.lang.System.getProperty("user.home"), ".ergo-uexplorer", "ergo-chain.lines.gz") + private val targetFile = Paths.get(java.lang.System.getProperty("user.home"), ".ergo-uexplorer", "ergo-chain.100k.gz") targetFile.toFile.getParentFile.mkdirs() def run = @@ -26,6 +26,7 @@ object DownloadBlocksAsLinesFromNodeToFile extends ZIOAppDefault { _ <- ZIO.log(s"Initiating download") result <- blockReader .blockIdSource(1) + .take(100000) .mapZIO(blockHttpClient.getBlockForIdAsString) .mapConcat(line => (line + java.lang.System.lineSeparator()).getBytes) .via(ZPipeline.gzip()) diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala index a65fa92a..048e25e9 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/Storage.scala @@ -1,14 +1,14 @@ package org.ergoplatform.uexplorer import org.ergoplatform.uexplorer.chain.ChainTip -import org.ergoplatform.uexplorer.db.{Block, OutputRecords} +import org.ergoplatform.uexplorer.db.{Asset, Block, LinkedBlock, OutputRecords} import org.ergoplatform.uexplorer.node.ApiTransaction import org.ergoplatform.uexplorer.{BlockId, BoxId, ErgoTreeHex, Height, Value} import zio.Task import java.nio.file.Path import scala.collection.compat.immutable.ArraySeq -import scala.collection.concurrent +import scala.collection.{concurrent, mutable} import scala.collection.immutable.{ArraySeq, TreeSet} import scala.util.Try @@ -39,6 +39,8 @@ trait ReadableStorage { trait WritableStorage extends ReadableStorage { + def removeDuplicates(b: LinkedBlock): LinkedBlock + def writeReportAndCompact(blocksIndexed: Int): Task[Unit] def commit(): Revision @@ -53,6 +55,10 @@ trait WritableStorage extends ReadableStorage { def persistErgoTreeT8ByUtxo(outputRecords: OutputRecords): Task[_] + def persistTokensByUtxo(tokensByUtxo: mutable.Map[BoxId, mutable.Map[TokenId, Amount]]): Task[_] + + def persistUtxosByTokenId(utxosByTokenId: mutable.Map[TokenId, mutable.Set[BoxId]]): Task[_] + def compact(indexing: Boolean): Task[Unit] def insertNewBlock( diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/chain/BlockProcessor.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/chain/BlockProcessor.scala index 43c85eda..bcab8c4a 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/chain/BlockProcessor.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/chain/BlockProcessor.scala @@ -1,8 +1,6 @@ package org.ergoplatform.uexplorer.chain -import org.ergoplatform.ErgoAddressEncoder import org.ergoplatform.uexplorer.CoreConf -import org.ergoplatform.uexplorer.chain.ChainLinker import org.ergoplatform.uexplorer.db.* import org.ergoplatform.uexplorer.node.ApiFullBlock import zio.* diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/LightBlockModel.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/LightBlockModel.scala index 790ca0fe..8a5fe222 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/LightBlockModel.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/LightBlockModel.scala @@ -12,7 +12,6 @@ import scala.collection.mutable case class Box( boxId: BoxId, - blockId: BlockId, txId: TxId, ergoTreeHash: ErgoTreeHash, ergoTreeT8Hash: Option[ErgoTreeT8Hash], @@ -32,7 +31,6 @@ object Box { case class Utxo( boxId: BoxId, - blockId: BlockId, txId: TxId, ergoTreeHash: ErgoTreeHash, ergoTreeT8Hash: Option[ErgoTreeT8Hash], @@ -47,7 +45,6 @@ case class Utxo( def toBox: Box = Box( boxId, - blockId, txId, ergoTreeHash, ergoTreeT8Hash, @@ -69,17 +66,23 @@ object Utxo { case class ErgoTree(hash: ErgoTreeHash, blockId: BlockId, hex: ErgoTreeHex) case class ErgoTreeT8(hash: ErgoTreeT8Hash, blockId: BlockId, hex: ErgoTreeT8Hex) -case class Asset(tokenId: TokenId, blockId: BlockId, boxId: BoxId, amount: Amount) - +case class Asset(tokenId: TokenId, blockId: BlockId) object Asset { implicit val encoder: JsonEncoder[Asset] = DeriveJsonEncoder.gen[Asset] implicit val decoder: JsonDecoder[Asset] = DeriveJsonDecoder.gen[Asset] } +case class Asset2Box(tokenId: TokenId, boxId: BoxId, amount: Amount) +object Asset2Box { + implicit val encoder: JsonEncoder[Asset2Box] = DeriveJsonEncoder.gen[Asset2Box] + implicit val decoder: JsonDecoder[Asset2Box] = DeriveJsonDecoder.gen[Asset2Box] +} + case class OutputRecords( byErgoTree: mutable.Map[ErgoTree, mutable.Set[Utxo]], byErgoTreeT8: mutable.Map[ErgoTreeT8, mutable.Set[Utxo]], - assets: List[Asset] + utxosByTokenId: mutable.Map[TokenId, mutable.Set[BoxId]], + tokensByUtxo: mutable.Map[BoxId, mutable.Map[TokenId, Amount]] ) final case class Block( @@ -139,4 +142,5 @@ final case class Block( object Block { implicit val encoder: JsonEncoder[Block] = DeriveJsonEncoder.gen[Block] + implicit val decoder: JsonDecoder[Block] = DeriveJsonDecoder.gen[Block] } diff --git a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala index 9fffb7db..b221807e 100644 --- a/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala +++ b/modules/explorer-core/src/main/scala/org/ergoplatform/uexplorer/db/OutputBuilder.scala @@ -12,18 +12,22 @@ object OutputBuilder { private def getOutputRecords(block: BlockWithReward)(implicit enc: ErgoAddressEncoder): Task[OutputRecords] = ZIO.attempt { - val byErgoTree = mutable.Map.empty[ErgoTree, mutable.Set[Utxo]] - val byErgoTreeT8 = mutable.Map.empty[ErgoTreeT8, mutable.Set[Utxo]] - val assets = List.newBuilder[Asset] + val byErgoTree = mutable.Map.empty[ErgoTree, mutable.Set[Utxo]] + val byErgoTreeT8 = mutable.Map.empty[ErgoTreeT8, mutable.Set[Utxo]] + val utxosByTokenId = mutable.Map.empty[TokenId, mutable.Set[BoxId]] + val tokensByUtxo = mutable.Map.empty[BoxId, mutable.Map[TokenId, Amount]] + block.b.transactions.transactions.foreach { tx => tx.outputs.foreach { o => val (ergoTreeHash, ergoTreeT8Opt) = ErgoTreeParser.ergoTreeHex2T8(o.ergoTree).get val additionalRegisters = o.additionalRegisters.view.mapValues(hex => RegistersParser.parseAny(hex).serializedValue).toMap - assets.addAll(o.assets.map(asset => Asset(asset.tokenId, block.b.header.id, o.boxId, asset.amount))) + o.assets.foreach { asset => + adjustMultiSet(utxosByTokenId, asset.tokenId, o.boxId) + adjustMultiMap(tokensByUtxo, o.boxId, asset.tokenId, asset.amount) + } val utxo = Utxo( o.boxId, - block.b.header.id, tx.id, ergoTreeHash, ergoTreeT8Opt.map(_._2), @@ -41,11 +45,14 @@ object OutputBuilder { } } } - OutputRecords(byErgoTree, byErgoTreeT8, assets.result()) + OutputRecords(byErgoTree, byErgoTreeT8, utxosByTokenId, tokensByUtxo) } - private def adjustMultiSet[ET, B](m: mutable.Map[ET, mutable.Set[B]], et: ET, boxId: B) = - m.adjust(et)(_.fold(mutable.Set(boxId))(_.addOne(boxId))) + private def adjustMultiSet[ET, K](m: mutable.Map[ET, mutable.Set[K]], et: ET, k: K) = + m.adjust(et)(_.fold(mutable.Set(k))(_.addOne(k))) + + private def adjustMultiMap[ET, K, V](m: mutable.Map[ET, mutable.Map[K, V]], et: ET, k: K, v: V) = + m.adjust(et)(_.fold(mutable.Map(k -> v))(_.addOne(k -> v))) def apply(block: BlockWithReward)(implicit enc: ErgoAddressEncoder): Task[BlockWithOutputs] = getOutputRecords(block).map(outputRecords => block.toBlockWithOutput(outputRecords)) diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeCollector.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeCollector.scala index eb6d297a..9686175e 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeCollector.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/SuperNodeCollector.scala @@ -1,37 +1,29 @@ package org.ergoplatform.uexplorer.mvstore -import org.h2.mvstore.MVStore - -import java.io.{BufferedInputStream, File, FileInputStream, FileWriter} -import java.nio.file.{Path, Paths} -import java.util.concurrent.ConcurrentHashMap +import java.io.BufferedInputStream import java.util.zip.GZIPInputStream -import scala.collection.concurrent import scala.io.Source import scala.jdk.CollectionConverters.* -import scala.util.{Random, Success, Try} class SuperNodeCollector[HK: HotKeyCodec](id: String) { private val hotKeyCodec: HotKeyCodec[HK] = implicitly[HotKeyCodec[HK]] private val stringifiedHotKeys: Map[HK, String] = - Source - .fromInputStream( - new GZIPInputStream( - new BufferedInputStream( - Thread - .currentThread() - .getContextClassLoader - .getResourceAsStream(s"hot-keys-$id.csv.gz") - ) - ) - ) - .getLines() - .map(_.trim) - .filterNot(_.isEmpty) - .toSet - .map(k => hotKeyCodec.deserialize(k) -> k) - .toMap + Option( + Thread + .currentThread() + .getContextClassLoader + .getResourceAsStream(s"hot-keys-$id.csv.gz") + ).map { resource => + Source + .fromInputStream(new GZIPInputStream(new BufferedInputStream(resource))) + .getLines() + .map(_.trim) + .filterNot(_.isEmpty) + .toSet + .map(k => hotKeyCodec.deserialize(k) -> k) + .toMap + }.getOrElse(Map.empty) def getExistingStringifiedHotKeys(mvStoreMapNames: Set[String]): Map[HK, String] = stringifiedHotKeys.filter(e => mvStoreMapNames.contains(e._2)) diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multimap/MultiMvMap.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multimap/MultiMvMap.scala index 82535d67..fa637af2 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multimap/MultiMvMap.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multimap/MultiMvMap.scala @@ -26,7 +26,7 @@ case class MultiMvMap[PK, C[_, _], K, V]( private val commonMap: MapLike[PK, C[K, V]] = new MvMap[PK, C[K, V]](id) private val superNodeMap: SuperNodeMvMap[PK, C, K, V] = SuperNodeMvMap[PK, C, K, V](id) - def clearEmptySuperNodes(): Task[Unit] = superNodeMap.clearEmptySuperNodes() + def clearEmptySuperNodes: Task[Unit] = superNodeMap.clearEmptySuperNodes() def getReport: (Path, Vector[(String, SuperNodeCounter)]) = ergoHomeDir.resolve(s"hot-keys-$id-$randomNumberPerRun.csv") -> superNodeMap.getReport diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiMvSet.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiMvSet.scala index e01715d2..74395f66 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiMvSet.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiMvSet.scala @@ -28,6 +28,13 @@ case class MultiMvSet[K, C[_], V]( def isEmpty: Boolean = superNodeMap.isEmpty && commonMap.isEmpty + def get(k: K): Option[C[V]] = + superNodeMap + .get(k) + .orElse(commonMap.get(k)) + + def contains(k: K): Boolean = superNodeMap.contains(k) || commonMap.containsKey(k) + def size: MultiColSize = MultiColSize(superNodeMap.size, superNodeMap.totalSize, commonMap.size) def clearEmptySuperNodes: Task[Unit] = diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiSetLike.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiSetLike.scala index caaf6262..45abd3bd 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiSetLike.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/MultiSetLike.scala @@ -1,6 +1,5 @@ package org.ergoplatform.uexplorer.mvstore.multiset - import org.ergoplatform.uexplorer.mvstore.MultiColSize import scala.util.Try @@ -9,6 +8,10 @@ trait MultiSetLike[K, C[_], V] { def isEmpty: Boolean + def get(k: K): Option[C[V]] + + def contains(k: K): Boolean + def size: MultiColSize def removeSubsetOrFail(k: K, values: IterableOnce[V], size: Int)(f: C[V] => Option[C[V]]): Try[Unit] diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeMvSet.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeMvSet.scala index a186cbc5..3e806c42 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeMvSet.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeMvSet.scala @@ -11,7 +11,7 @@ import java.nio.file.Path import java.util.Map.Entry import java.util.concurrent.ConcurrentHashMap import java.util.stream.Collectors -import scala.collection.concurrent +import scala.collection.{concurrent, mutable} import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} @@ -25,15 +25,13 @@ class SuperNodeMvSet[HK, C[_], V]( private lazy val counterByHotKey = new MvMap[HK, SuperNodeCounter](s"$id-counter") private def collectReadHotKey(k: HK): SuperNodeCounter = - counterByHotKey.adjust(k)(_.fold(SuperNodeCounter(1, 1, 0, 0)) { - case SuperNodeCounter(writeOps, readOps, added, removed) => - SuperNodeCounter(writeOps + 1, readOps + 1, added, removed) + counterByHotKey.adjust(k)(_.fold(SuperNodeCounter(1, 1, 0, 0)) { case SuperNodeCounter(writeOps, readOps, added, removed) => + SuperNodeCounter(writeOps + 1, readOps + 1, added, removed) }) private def collectInsertedHotKey(k: HK, size: Int): SuperNodeCounter = - counterByHotKey.adjust(k)(_.fold(SuperNodeCounter(1, 0, size, 0)) { - case SuperNodeCounter(writeOps, readOps, added, removed) => - SuperNodeCounter(writeOps + 1, readOps, added + size, removed) + counterByHotKey.adjust(k)(_.fold(SuperNodeCounter(1, 0, size, 0)) { case SuperNodeCounter(writeOps, readOps, added, removed) => + SuperNodeCounter(writeOps + 1, readOps, added + size, removed) }) private def collectRemovedHotKey(k: HK, size: Int): Option[SuperNodeCounter] = @@ -122,6 +120,29 @@ class SuperNodeMvSet[HK, C[_], V]( def isEmpty: Boolean = existingMapsByHotKey.forall(_._2.isEmpty) + def get(hotKey: HK): Option[C[V]] = + superNodeCollector + .getHotKeyString(hotKey) + .flatMap { _ => + existingMapsByHotKey.get(hotKey) + } + .map(codec.readAll) + + def contains(hotKey: HK): Boolean = + superNodeCollector + .getHotKeyString(hotKey) + .exists { _ => + existingMapsByHotKey.contains(hotKey) + } + + def contains(hotKey: HK, v: V): Option[Boolean] = + superNodeCollector + .getHotKeyString(hotKey) + .flatMap { _ => + existingMapsByHotKey.get(hotKey) + } + .map(codec.contains(v, _)) + def size: Int = existingMapsByHotKey.size def totalSize: Int = existingMapsByHotKey.iterator.map(_._2.size()).sum diff --git a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeSetLike.scala b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeSetLike.scala index 0b6ee089..ab690eb6 100644 --- a/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeSetLike.scala +++ b/modules/mvstore/src/main/scala/org/ergoplatform/uexplorer/mvstore/multiset/SuperNodeSetLike.scala @@ -8,6 +8,12 @@ trait SuperNodeSetLike[K, C[_], V] { def removeAllOrFail(hotKey: K, values: IterableOnce[V], size: Int): Option[Try[Unit]] + def get(hotKey: K): Option[C[V]] + + def contains(hotKey: K): Boolean + + def contains(hotKey: K, v: V): Option[Boolean] + def isEmpty: Boolean def size: Int diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala index 1389f3f6..d9602f58 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/Implicits.scala @@ -1,14 +1,15 @@ package org.ergoplatform.uexplorer.storage import org.ergoplatform.uexplorer.db.Block -import org.ergoplatform.uexplorer.* +import org.ergoplatform.uexplorer.{BoxId, *} import org.ergoplatform.uexplorer.mvstore.* import org.ergoplatform.uexplorer.mvstore.multimap.MultiMapCodec import org.ergoplatform.uexplorer.mvstore.multiset.MultiSetCodec import org.ergoplatform.uexplorer.storage.kryo.* object Implicits { - implicit def valueByBoxCodec[V]: MultiMapCodec[java.util.Map, BoxId, V] = new ValueByBoxCodec[V] + implicit def valueByBoxCodec[V]: MultiMapCodec[java.util.Map, BoxId, V] = new HashMultiMapCodec[BoxId, V] + implicit def amountByTokenCodec[V]: MultiMapCodec[java.util.Map, TokenId, V] = new HashMultiMapCodec[TokenId, V] implicit val blockIdsCodec: ValueCodec[java.util.Set[BlockId]] = BlockIdsCodec implicit val boxCodec: MultiSetCodec[java.util.Set, BoxId] = BoxCodec @@ -16,7 +17,12 @@ object Implicits { implicit val counterCodec: ValueCodec[SuperNodeCounter] = CounterCodec implicit val addressCodec: ValueCodec[ErgoTreeHex] = ErgoTreeHexCodec - implicit val hexCodec: HotKeyCodec[HexString] = new HotKeyCodec[HexString] { + implicit val hotBoxCodec: HotKeyCodec[BoxId] = new HotKeyCodec[BoxId] { + def serialize(key: BoxId): String = key.unwrapped + def deserialize(key: String): BoxId = BoxId(key) + } + + implicit val hotHexCodec: HotKeyCodec[HexString] = new HotKeyCodec[HexString] { import org.ergoplatform.uexplorer.HexString.unwrapped def serialize(key: HexString): String = key.unwrapped diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala index a6a6dc01..dca22a79 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/MvStorage.scala @@ -1,47 +1,32 @@ package org.ergoplatform.uexplorer.storage -import com.esotericsoftware.kryo.Kryo -import com.esotericsoftware.kryo.io.{ByteBufferInput, ByteBufferOutput, Input, Output} -import com.esotericsoftware.kryo.serializers.MapSerializer -import com.esotericsoftware.kryo.util.Pool -import org.apache.tinkerpop.shaded.kryo.pool.KryoPool import org.ergoplatform.uexplorer.* -import org.ergoplatform.uexplorer.Const.Protocol.{Emission, FeeContract, Foundation} import org.ergoplatform.uexplorer.chain.ChainTip import org.ergoplatform.uexplorer.db.* import org.ergoplatform.uexplorer.mvstore.* +import org.ergoplatform.uexplorer.mvstore.multimap.MultiMvMap import org.ergoplatform.uexplorer.mvstore.multiset.MultiMvSet -import org.ergoplatform.uexplorer.node.{ApiFullBlock, ApiTransaction} +import org.ergoplatform.uexplorer.node.ApiTransaction import org.ergoplatform.uexplorer.storage.Implicits.* -import org.ergoplatform.uexplorer.storage.MvStorage.* -import org.h2.mvstore.{MVMap, MVStore} +import org.h2.mvstore.MVStore import zio.* -import java.io.{BufferedInputStream, File} -import java.nio.ByteBuffer -import java.nio.file.{CopyOption, Files, Path, Paths} + +import java.io.File +import java.nio.file.Path import java.util -import java.util.Map.Entry -import java.util.concurrent.{ConcurrentHashMap, ConcurrentSkipListMap} -import java.util.stream.Collectors -import java.util.zip.GZIPInputStream -import scala.collection.compat.immutable.ArraySeq -import scala.collection.immutable.{ArraySeq, TreeMap, TreeSet} -import scala.collection.mutable.ListBuffer -import scala.collection.{concurrent, mutable} -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import scala.concurrent.duration.FiniteDuration -import scala.io.Source +import scala.collection.immutable.{ArraySeq, TreeSet} +import scala.collection.mutable import scala.jdk.CollectionConverters.* -import scala.util.{Failure, Random, Success, Try} case class MvStorage( - utxosByErgoTreeHex: MultiMvSet[ErgoTreeHex, java.util.Set, BoxId], - utxosByErgoTreeT8Hex: MultiMvSet[ErgoTreeT8Hex, java.util.Set, BoxId], + utxosByErgoTreeHex: MultiMvMap[ErgoTreeHex, util.Map, BoxId, Value], + utxosByErgoTreeT8Hex: MultiMvSet[ErgoTreeT8Hex, util.Set, BoxId], ergoTreeHexByUtxo: MapLike[BoxId, ErgoTreeHex], ergoTreeT8HexByUtxo: MapLike[BoxId, ErgoTreeT8Hex], - blockIdsByHeight: MapLike[Height, java.util.Set[BlockId]], - blockById: MapLike[BlockId, Block] + blockIdsByHeight: MapLike[Height, util.Set[BlockId]], + blockById: MapLike[BlockId, Block], + utxosByTokenId: MultiMvSet[TokenId, util.Set, BoxId], + tokensByUtxo: MultiMvMap[BoxId, util.Map, TokenId, Amount] )(implicit val store: MVStore, mvStoreConf: MvStoreConf) extends WritableStorage { @@ -51,6 +36,9 @@ case class MvStorage( utxosByErgoTreeT8Hex.getReport ) + def removeDuplicates(b: LinkedBlock): LinkedBlock = ??? + /// b.outputRecords.utxosByTokenId.filter { case (tokenId, _) => utxosByTokenId.contains(tokenId) } + def getChainTip: Task[ChainTip] = { val chainTip = ChainTip( @@ -137,7 +125,7 @@ case class MvStorage( } .groupBy(_._1) .foreach { case (et, inputBoxes) => - utxosByErgoTreeHex.removeSubsetOrFail(et, inputBoxes.iterator.map(_._2), inputBoxes.size) { existingBoxIds => + utxosByErgoTreeHex.removeAllOrFail(et, inputBoxes.iterator.map(_._2), inputBoxes.size) { existingBoxIds => inputBoxes.iterator.foreach(t => existingBoxIds.remove(t._2)) Option(existingBoxIds).collect { case m if !m.isEmpty => m } } @@ -173,11 +161,12 @@ case class MvStorage( ergoTreeHexByUtxo .putAllNewOrFail(boxes.iterator.map(b => b.boxId -> ergoTreeHex.hex)) .flatMap { _ => - utxosByErgoTreeHex.adjustAndForget(ergoTreeHex.hex, boxes.iterator.map(_.boxId), boxes.size) + utxosByErgoTreeHex.adjustAndForget(ergoTreeHex.hex, boxes.iterator.map(b => b.boxId -> b.ergValue), boxes.size) } .get } } + def persistErgoTreeT8ByUtxo(outputRecords: OutputRecords): Task[_] = ZIO.attempt { outputRecords.byErgoTreeT8 .foreach { case (ergoTreeT8, boxes) => @@ -190,6 +179,18 @@ case class MvStorage( } } + def persistTokensByUtxo(assets: mutable.Map[BoxId, mutable.Map[TokenId, Amount]]): Task[_] = ZIO.attempt { + assets.foreach { case (boxId, ammountByTokenId) => + tokensByUtxo.adjustAndForget(boxId, ammountByTokenId.iterator, ammountByTokenId.size) + } + } + + def persistUtxosByTokenId(assets: mutable.Map[TokenId, mutable.Set[BoxId]]): Task[_] = ZIO.attempt { + assets.foreach { case (tokenId, boxIds) => + utxosByTokenId.adjustAndForget(tokenId, boxIds.iterator, boxIds.size) + } + } + def insertNewBlock( blockId: BlockId, block: Block, @@ -256,7 +257,6 @@ case class MvStorage( } object MvStorage { - import scala.concurrent.duration.* private val VersionsToKeep = 10 @@ -282,12 +282,14 @@ object MvStorage { .tap(store => ZIO.log(s"Opened mvstore at version ${store.getCurrentVersion}")) .map { implicit store => MvStorage( - multiset.MultiMvSet[ErgoTreeHex, util.Set, BoxId]("utxosByErgoTreeHex"), - multiset.MultiMvSet[ErgoTreeT8Hex, util.Set, BoxId]("utxosByErgoTreeT8Hex"), + MultiMvMap[ErgoTreeHex, util.Map, BoxId, Value]("utxosByErgoTreeHex"), + MultiMvSet[ErgoTreeT8Hex, util.Set, BoxId]("utxosByErgoTreeT8Hex"), MvMap[BoxId, ErgoTreeHex]("ergoTreeHexByUtxo"), MvMap[BoxId, ErgoTreeT8Hex]("ergoTreeT8HexByUtxo"), MvMap[Height, util.Set[BlockId]]("blockIdsByHeight"), - MvMap[BlockId, Block]("blockById") + MvMap[BlockId, Block]("blockById"), + MultiMvSet[TokenId, util.Set, BoxId]("utxosByTokenId"), + MultiMvMap[BoxId, util.Map, TokenId, Amount]("tokensByUtxo") )(store, mvStoreConf.get) } } { storage => diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala index a91cb54f..746c6552 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/CounterCodec.scala @@ -1,16 +1,9 @@ package org.ergoplatform.uexplorer.storage.kryo -import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{ByteBufferOutput, Input} -import com.esotericsoftware.kryo.serializers.DefaultSerializers.CollectionsSingletonSetSerializer -import com.esotericsoftware.kryo.serializers.ImmutableCollectionsSerializers.JdkImmutableSetSerializer -import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, MapSerializer} -import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.mvstore.{SuperNodeCounter, ValueCodec} import java.nio.ByteBuffer -import java.util -import scala.util.Try object CounterCodec extends ValueCodec[SuperNodeCounter] { override def readAll(bytes: Array[Byte]): SuperNodeCounter = { diff --git a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/HashMultiMapCodec.scala similarity index 51% rename from modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala rename to modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/HashMultiMapCodec.scala index 961e5d43..1f489bbb 100644 --- a/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/ValueByBoxCodec.scala +++ b/modules/storage/src/main/scala/org/ergoplatform/uexplorer/storage/kryo/HashMultiMapCodec.scala @@ -1,48 +1,41 @@ package org.ergoplatform.uexplorer.storage.kryo -import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{ByteBufferOutput, Input} -import com.esotericsoftware.kryo.serializers.DefaultSerializers.CollectionsSingletonSetSerializer -import com.esotericsoftware.kryo.serializers.ImmutableCollectionsSerializers.JdkImmutableSetSerializer -import com.esotericsoftware.kryo.serializers.{ImmutableCollectionsSerializers, MapSerializer} -import com.esotericsoftware.kryo.util.Pool import org.ergoplatform.uexplorer.mvstore.* import org.ergoplatform.uexplorer.mvstore.multimap.MultiMapCodec -import org.ergoplatform.uexplorer.{BoxId, ErgoTreeHex, Height} import java.nio.ByteBuffer import java.util import scala.jdk.CollectionConverters.* import scala.language.unsafeNulls -import scala.util.Try -class ValueByBoxCodec[V] extends MultiMapCodec[java.util.Map, BoxId, V] { +class HashMultiMapCodec[K, V] extends MultiMapCodec[java.util.Map, K, V] { - override def readOne(key: BoxId, valueByBoxId: java.util.Map[BoxId, V]): Option[V] = + override def readOne(key: K, valueByBoxId: java.util.Map[K, V]): Option[V] = Option(valueByBoxId.get(key)) - override def readAll(bytes: Array[Byte]): java.util.Map[BoxId, V] = { + override def readAll(bytes: Array[Byte]): java.util.Map[K, V] = { val input = new Input(bytes) val kryo = KryoSerialization.pool.obtain() - try kryo.readObject(input, classOf[util.HashMap[BoxId, V]]) + try kryo.readObject(input, classOf[util.HashMap[K, V]]) finally { KryoSerialization.pool.free(kryo) input.close() } } - override def readPartially(only: IterableOnce[BoxId])( - existingOpt: Option[java.util.Map[BoxId, V]] - ): Option[java.util.Map[BoxId, V]] = + override def readPartially(only: IterableOnce[K])( + existingOpt: Option[java.util.Map[K, V]] + ): Option[java.util.Map[K, V]] = existingOpt.map { existingMap => - val partialResult = new util.HashMap[BoxId, V]() + val partialResult = new util.HashMap[K, V]() only.iterator.foreach { k => partialResult.put(k, existingMap.get(k)) } partialResult } - override def writeAll(valueByBoxId: java.util.Map[BoxId, V]): Array[Byte] = { + override def writeAll(valueByBoxId: java.util.Map[K, V]): Array[Byte] = { val buffer = ByteBuffer.allocate((valueByBoxId.size() * 72) + 512) val output = new ByteBufferOutput(buffer) val kryo = KryoSerialization.pool.obtain() @@ -54,9 +47,9 @@ class ValueByBoxCodec[V] extends MultiMapCodec[java.util.Map, BoxId, V] { buffer.array() } - override def append(newValueByBoxId: IterableOnce[(BoxId, V)])( - existingOpt: Option[java.util.Map[BoxId, V]] - ): (Appended, java.util.Map[BoxId, V]) = + override def append(newValueByBoxId: IterableOnce[(K, V)])( + existingOpt: Option[java.util.Map[K, V]] + ): (Appended, java.util.Map[K, V]) = existingOpt.fold(true -> javaMapOf(newValueByBoxId)) { existingMap => newValueByBoxId.iterator.forall { e => val replaced: V | Null = existingMap.put(e._1, e._2)