diff --git a/Circles.Index.Common/BlockEvent.cs b/Circles.Index.Common/BlockEvent.cs deleted file mode 100644 index 28ccfdc..0000000 --- a/Circles.Index.Common/BlockEvent.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace Circles.Index.Common; - -public record BlockEvent(long BlockNumber, long Timestamp, string BlockHash); \ No newline at end of file diff --git a/Circles.Index.Common/DatabaseSchema.cs b/Circles.Index.Common/DatabaseSchema.cs index a9f3858..fdb5b07 100644 --- a/Circles.Index.Common/DatabaseSchema.cs +++ b/Circles.Index.Common/DatabaseSchema.cs @@ -4,16 +4,19 @@ namespace Circles.Index.Common; public class DatabaseSchema : IDatabaseSchema { - public IDictionary Tables { get; } = new Dictionary - { + public ISchemaPropertyMap SchemaPropertyMap { get; } = new SchemaPropertyMap(); + public IEventDtoTableMap EventDtoTableMap { get; } = new EventDtoTableMap(); + + public IDictionary<(string Namespace, string Table), EventSchema> Tables { get; } = + new Dictionary<(string Namespace, string Table), EventSchema> { - "Block", - // Hash256 must be 32 bytes and was 0 bytes - new EventSchema("Block", new Hash256(new byte[32]), [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("BlockHash", ValueTypes.String, true) - ]) - } - }; + { + ("System", "Block"), + new EventSchema("System", "Block", new Hash256(new byte[32]), [ + new("blockNumber", ValueTypes.Int, false), + new("timestamp", ValueTypes.Int, true), + new("blockHash", ValueTypes.String, false) + ]) + } + }; } \ No newline at end of file diff --git a/Circles.Index.Common/Equals.cs b/Circles.Index.Common/Equals.cs index 16a2eb5..719faf7 100644 --- a/Circles.Index.Common/Equals.cs +++ b/Circles.Index.Common/Equals.cs @@ -2,10 +2,10 @@ namespace Circles.Index.Common; public class Equals : QueryPredicate { - internal Equals(IDatabase database, string table, string column, object? value) + internal Equals(IDatabase database, (string Namespace, string Table) table, string column, object? value) : base(database, table, column, value) { } - public override string ToSql() => $"{Field} = {ParameterName}"; + public override string ToSql() => $"\"{Field}\" = {ParameterName}"; } \ No newline at end of file diff --git a/Circles.Index.Common/EventSchema.cs b/Circles.Index.Common/EventSchema.cs index 38849ce..cae4ecc 100644 --- a/Circles.Index.Common/EventSchema.cs +++ b/Circles.Index.Common/EventSchema.cs @@ -5,8 +5,9 @@ namespace Circles.Index.Common; public record EventFieldSchema(string Column, ValueTypes Type, bool IsIndexed); -public class EventSchema(string table, Hash256 topic, List columns) +public class EventSchema(string @namespace, string table, Hash256 topic, List columns) { + public string Namespace { get; } = @namespace; public Hash256 Topic { get; } = topic; public string Table { get; } = table; public List Columns { get; } = columns; @@ -21,13 +22,14 @@ public class EventSchema(string table, Hash256 topic, List col /// ); /// ``` /// + /// /// /// /// Thrown when the event definition is invalid (according to this parser ;). /// /// Doesn't support all Solidity types yet (most notably arrays). Please handle events with these types manually. /// - public static EventSchema FromSolidity(string solidityEventDefinition) + public static EventSchema FromSolidity(string @namespace, string solidityEventDefinition) { var trimmedDefinition = solidityEventDefinition.Trim(); const string prefix = "event "; @@ -55,14 +57,22 @@ public static EventSchema FromSolidity(string solidityEventDefinition) .Substring(openParenthesisIndex + 1, closeParenthesisIndex - openParenthesisIndex - 1).Trim(); var columnDefinitions = parameters.Split(','); - var columns = new List(); + var columns = new List + { + new("blockNumber", ValueTypes.Int, false), + new("timestamp", ValueTypes.Int, true), + new("transactionIndex", ValueTypes.Int, false), + new("logIndex", ValueTypes.Int, false), + new("transactionHash", ValueTypes.String, true) + }; - StringBuilder sb = new StringBuilder(); - sb.Append(eventName); - sb.Append("("); + StringBuilder eventTopic = new StringBuilder(); + eventTopic.Append(eventName); + eventTopic.Append('('); - foreach (var columnDefinition in columnDefinitions) + for (int i = 0; i < columnDefinitions.Length; i++) { + var columnDefinition = columnDefinitions[i]; var parts = columnDefinition.Trim().Split(' '); if (parts.Length < 2) { @@ -70,34 +80,41 @@ public static EventSchema FromSolidity(string solidityEventDefinition) $"Invalid column definition '${columnDefinition}'. Must contain a type and a name."); } + if (i > 0) + { + eventTopic.Append(','); + } + if (parts.Length == 2) { var type = MapSolidityType(parts[0].Trim()); + eventTopic.Append(parts[0].Trim()); + var columnName = parts[1].Trim(); columns.Add(new EventFieldSchema(columnName, type, false)); - sb.Append($"{type} {columnName}"); } if (parts.Length == 3) { - var isIndexed = parts[0].Trim() == "indexed"; + var type = MapSolidityType(parts[0].Trim()); + eventTopic.Append(parts[0].Trim()); + + var isIndexed = parts[1].Trim() == "indexed"; if (!isIndexed) { throw new ArgumentException( $"Invalid column definition '${columnDefinition}'."); } - var type = MapSolidityType(parts[1].Trim()); var columnName = parts[2].Trim(); - columns.Add(new EventFieldSchema(columnName, type, false)); - sb.Append($"{type} {columnName}"); + columns.Add(new EventFieldSchema(columnName, type, isIndexed)); } } - sb.Append(")"); + eventTopic.Append(')'); - Hash256 topic = Keccak.Compute(sb.ToString()); - return new EventSchema(eventName, topic, columns); + Hash256 topic = Keccak.Compute(eventTopic.ToString()); + return new EventSchema(@namespace, eventName, topic, columns); } private static ValueTypes MapSolidityType(string type) @@ -112,8 +129,9 @@ private static ValueTypes MapSolidityType(string type) "uint128" => ValueTypes.BigInt, "uint256" => ValueTypes.BigInt, "string" => ValueTypes.String, + "bool" => ValueTypes.Boolean, _ => throw new ArgumentException( - $"'${type}' is not supported. Please handle this event manually.") + $"'{type}' is not supported. Please handle this event manually.") }; } } \ No newline at end of file diff --git a/Circles.Index.Common/GreaterThan.cs b/Circles.Index.Common/GreaterThan.cs index 08603ee..0c06d5d 100644 --- a/Circles.Index.Common/GreaterThan.cs +++ b/Circles.Index.Common/GreaterThan.cs @@ -2,10 +2,10 @@ namespace Circles.Index.Common; public class GreaterThan : QueryPredicate { - internal GreaterThan(IDatabase database, string table, string column, object value) + internal GreaterThan(IDatabase database, (string Namespace, string Table) table, string column, object value) : base(database, table, column, value) { } - public override string ToSql() => $"{Field} > {ParameterName}"; + public override string ToSql() => $"\"{Field}\" > {ParameterName}"; } \ No newline at end of file diff --git a/Circles.Index.Common/GreaterThanOrEqual.cs b/Circles.Index.Common/GreaterThanOrEqual.cs index cbb4d65..db82662 100644 --- a/Circles.Index.Common/GreaterThanOrEqual.cs +++ b/Circles.Index.Common/GreaterThanOrEqual.cs @@ -2,10 +2,10 @@ namespace Circles.Index.Common; public class GreaterThanOrEqual : QueryPredicate { - internal GreaterThanOrEqual(IDatabase database, string table, string column, object value) + internal GreaterThanOrEqual(IDatabase database, (string Namespace, string Table) table, string column, object value) : base(database, table, column, value) { } - public override string ToSql() => $"{Field} >= {ParameterName}"; + public override string ToSql() => $"\"{Field}\" >= {ParameterName}"; } \ No newline at end of file diff --git a/Circles.Index.Common/IDatabase.cs b/Circles.Index.Common/IDatabase.cs index 1d9b930..dc987e5 100644 --- a/Circles.Index.Common/IDatabase.cs +++ b/Circles.Index.Common/IDatabase.cs @@ -9,7 +9,7 @@ public interface IDatabase void Migrate(); Task DeleteFromBlockOnwards(long reorgAt); - Task WriteBatch(string table, IEnumerable data, ISchemaPropertyMap propertyMap); + Task WriteBatch(string @namespace, string table, IEnumerable data, ISchemaPropertyMap propertyMap); long? LatestBlock(); long? FirstGap(); IEnumerable<(long BlockNumber, Hash256 BlockHash)> LastPersistedBlocks(int count); diff --git a/Circles.Index.Common/IDatabaseSchema.cs b/Circles.Index.Common/IDatabaseSchema.cs index 5a6b019..9232cb9 100644 --- a/Circles.Index.Common/IDatabaseSchema.cs +++ b/Circles.Index.Common/IDatabaseSchema.cs @@ -2,5 +2,9 @@ namespace Circles.Index.Common; public interface IDatabaseSchema { - public IDictionary Tables { get; } + public ISchemaPropertyMap SchemaPropertyMap { get; } + + public IEventDtoTableMap EventDtoTableMap { get; } + + public IDictionary<(string Namespace, string Table), EventSchema> Tables { get; } } \ No newline at end of file diff --git a/Circles.Index.Common/LessThan.cs b/Circles.Index.Common/LessThan.cs index ffb4987..7d3832e 100644 --- a/Circles.Index.Common/LessThan.cs +++ b/Circles.Index.Common/LessThan.cs @@ -2,10 +2,10 @@ namespace Circles.Index.Common; public class LessThan : QueryPredicate { - internal LessThan(IDatabase database, string table, string column, object value) + internal LessThan(IDatabase database, (string Namespace, string Table) table, string column, object value) : base(database, table, column, value) { } - public override string ToSql() => $"{Field} < {ParameterName}"; + public override string ToSql() => $"\"{Field}\" < {ParameterName}"; } \ No newline at end of file diff --git a/Circles.Index.Common/Query.cs b/Circles.Index.Common/Query.cs index cd1c590..eb67513 100644 --- a/Circles.Index.Common/Query.cs +++ b/Circles.Index.Common/Query.cs @@ -19,20 +19,21 @@ private static IDatabase GetDatabase() return _database; } - public static Equals Equals(string table, string column, object? value) => + public static Equals Equals((string Namespace, string Table) table, string column, object? value) => new(GetDatabase(), table, column, value); - public static GreaterThan GreaterThan(string table, string column, object value) => + public static GreaterThan GreaterThan((string Namespace, string Table) table, string column, object value) => new(GetDatabase(), table, column, value); - public static GreaterThanOrEqual GreaterThanOrEqual(string table, string column, object value) => + public static GreaterThanOrEqual GreaterThanOrEqual((string Namespace, string Table) table, string column, + object value) => new(GetDatabase(), table, column, value); - public static LessThan LessThan(string table, string column, object value) => + public static LessThan LessThan((string Namespace, string Table) table, string column, object value) => new(GetDatabase(), table, column, value); public static LogicalAnd And(params IQuery[] subElements) => new(subElements); public static LogicalOr Or(params IQuery[] subElements) => new(subElements); - public static Select Select(string table, IEnumerable columns) => new(table, columns); + public static Select Select((string Namespace, string Table) table, IEnumerable columns) => new(table, columns); } \ No newline at end of file diff --git a/Circles.Index.Common/QueryPredicate.cs b/Circles.Index.Common/QueryPredicate.cs index ff44c23..e71e57a 100644 --- a/Circles.Index.Common/QueryPredicate.cs +++ b/Circles.Index.Common/QueryPredicate.cs @@ -6,13 +6,13 @@ namespace Circles.Index.Common; public abstract class QueryPredicate : IQuery { protected readonly string Field; - public readonly string Table; + public readonly (string Namespace, string Table) Table; public readonly string Column; protected readonly string ParameterName; public readonly object Value; protected readonly IDatabase Database; - internal QueryPredicate(IDatabase database, string table, string column, object value) + internal QueryPredicate(IDatabase database, (string Namespace, string Table) table, string column, object value) { Database = database; Table = table; diff --git a/Circles.Index.Common/SchemaPropertyMap.cs b/Circles.Index.Common/SchemaPropertyMap.cs index 84c5572..9e6d6ae 100644 --- a/Circles.Index.Common/SchemaPropertyMap.cs +++ b/Circles.Index.Common/SchemaPropertyMap.cs @@ -2,7 +2,10 @@ namespace Circles.Index.Common; public class CompositeDatabaseSchema : IDatabaseSchema { - public IDictionary Tables { get; } + public ISchemaPropertyMap SchemaPropertyMap { get; } + public IEventDtoTableMap EventDtoTableMap { get; } + + public IDictionary<(string Namespace, string Table), EventSchema> Tables { get; } public CompositeDatabaseSchema(IDatabaseSchema[] components) { @@ -12,19 +15,24 @@ public CompositeDatabaseSchema(IDatabaseSchema[] components) kvp => kvp.Key , kvp => kvp.Value ); + + SchemaPropertyMap = new CompositeSchemaPropertyMap(components.Select(o => o.SchemaPropertyMap).ToArray()); + EventDtoTableMap = new CompositeEventDtoTableMap(components.Select(o => o.EventDtoTableMap).ToArray()); } } public interface ISchemaPropertyMap { - Dictionary>> Map { get; } + Dictionary<(string Namespace, string Table), Dictionary>> Map { get; } + + public void Add((string Namespace, string Table) table, Dictionary> map); } public class SchemaPropertyMap : ISchemaPropertyMap { - public Dictionary>> Map { get; } = new(); + public Dictionary<(string Namespace, string Table), Dictionary>> Map { get; } = new(); - public void Add(string table, Dictionary> map) + public void Add((string Namespace, string Table) table, Dictionary> map) { Map[table] = map.ToDictionary( pair => pair.Key, @@ -35,9 +43,13 @@ public void Add(string table, Dictionary> public class CompositeSchemaPropertyMap : ISchemaPropertyMap { - public Dictionary>> Map { get; } + public Dictionary<(string Namespace, string Table), Dictionary>> Map { get; } + public void Add((string Namespace, string Table) table, Dictionary> map) + { + throw new NotImplementedException(); + } - public CompositeSchemaPropertyMap(SchemaPropertyMap[] components) + public CompositeSchemaPropertyMap(ISchemaPropertyMap[] components) { Map = components .SelectMany(c => c.Map) @@ -50,14 +62,17 @@ public CompositeSchemaPropertyMap(SchemaPropertyMap[] components) public interface IEventDtoTableMap { - Dictionary Map { get; } + Dictionary Map { get; } + + public void Add((string Namespace, string Table) table) + where TEvent : IIndexEvent; } public class EventDtoTableMap : IEventDtoTableMap { - public Dictionary Map { get; } = new(); + public Dictionary Map { get; } = new(); - public void Add(string table) + public void Add((string Namespace, string Table) table) where TEvent : IIndexEvent { Map[typeof(TEvent)] = table; @@ -66,9 +81,14 @@ public void Add(string table) public class CompositeEventDtoTableMap : IEventDtoTableMap { - public Dictionary Map { get; } + public Dictionary Map { get; } + + public void Add((string Namespace, string Table) table) where TEvent : IIndexEvent + { + throw new NotImplementedException(); + } - public CompositeEventDtoTableMap(EventDtoTableMap[] components) + public CompositeEventDtoTableMap(IEventDtoTableMap[] components) { Map = components .SelectMany(c => c.Map) diff --git a/Circles.Index.Common/Select.cs b/Circles.Index.Common/Select.cs index c6b0f09..7d1638c 100644 --- a/Circles.Index.Common/Select.cs +++ b/Circles.Index.Common/Select.cs @@ -5,14 +5,14 @@ namespace Circles.Index.Common; public class Select : IQuery { - public readonly string Table; - private readonly string _table; + public readonly (string Namespace, string Table) Table; + private readonly (string Namespace, string Table) _table; public readonly IEnumerable Columns; private readonly List _databaseFields; public readonly List Conditions; public readonly List<(string Column, SortOrder Order)> OrderBy = new(); - public Select(string table, IEnumerable columns) + public Select((string Namespace, string Table) table, IEnumerable columns) { var columnsArray = columns.ToArray(); Table = table; @@ -30,7 +30,8 @@ public Select Where(IQuery condition) public string ToSql() { - var sql = new StringBuilder($"SELECT {string.Join(", ", _databaseFields)} FROM {_table}"); + var sql = new StringBuilder( + $"SELECT {string.Join(", ", _databaseFields.Select(o => $"\"{o}\""))} FROM \"{_table}\""); if (Conditions.Any()) { sql.Append(" WHERE "); @@ -40,7 +41,7 @@ public string ToSql() if (OrderBy.Any()) { sql.Append(" ORDER BY "); - sql.Append(string.Join(", ", OrderBy.Select(o => $"{o.Column} {o.Order}"))); + sql.Append(string.Join(", ", OrderBy.Select(o => $"\"{o.Column}\" \"{o.Order}\""))); } return sql.ToString(); diff --git a/Circles.Index.Common/Sink.cs b/Circles.Index.Common/Sink.cs index c6ce375..7230e8c 100644 --- a/Circles.Index.Common/Sink.cs +++ b/Circles.Index.Common/Sink.cs @@ -1,3 +1,5 @@ +using System.Collections.Concurrent; + namespace Circles.Index.Common; public class Sink @@ -12,10 +14,14 @@ public class Sink public readonly IDatabase Database; + private readonly ConcurrentDictionary _addedEventCounts = new(); + private readonly ConcurrentDictionary _importedEventCounts = new(); + public Sink(IDatabase database, ISchemaPropertyMap schemaPropertyMap, - IEventDtoTableMap eventDtoTableMap, int batchSize = 100000) + IEventDtoTableMap eventDtoTableMap, int batchSize = 10000) { Database = database; + _batchSize = batchSize; _schemaPropertyMap = schemaPropertyMap; _eventDtoTableMap = eventDtoTableMap; @@ -33,6 +39,11 @@ private async Task PerformAddEvent(object indexEvent) { _insertBuffer.Add(indexEvent); + _addedEventCounts.AddOrUpdate( + indexEvent.GetType(), + 1, + (_, count) => count + 1); + if (_insertBuffer.Length >= _batchSize) { await Flush(); @@ -48,33 +59,60 @@ private async Task PerformFlush() { var snapshot = _insertBuffer.TakeSnapshot(); - Dictionary> eventsByTable = new(); + Dictionary> eventsByType = new(); foreach (var indexEvent in snapshot) { - if (!_eventDtoTableMap.Map.TryGetValue(indexEvent.GetType(), out var table)) + if (!eventsByType.TryGetValue(indexEvent.GetType(), out var typeEvents)) { - continue; + typeEvents = new List(); + eventsByType[indexEvent.GetType()] = typeEvents; } - if (!eventsByTable.TryGetValue(table, out var tableEvents)) + typeEvents.Add(indexEvent); + } + + IEnumerable tasks = eventsByType.Select(o => + { + if (!_eventDtoTableMap.Map.TryGetValue(o.Key, out var tableId)) { - tableEvents = new List(); - eventsByTable[table] = tableEvents; + // TODO: Use proper logger + Console.WriteLine($"Warning: No table mapping for {o.Key}"); + return Task.CompletedTask; } - tableEvents.Add(indexEvent); - } + var task = Database.WriteBatch(tableId.Namespace, tableId.Table, o.Value, _schemaPropertyMap); + + return task.ContinueWith(p => + { + if (p.Exception != null) + { + throw p.Exception; + } + + _importedEventCounts.AddOrUpdate( + o.Key, + o.Value.Count, + (_, count) => count + o.Value.Count); + }); + }); - List tasks = new(); - foreach (var tableEvents in eventsByTable) + await Task.WhenAll(tasks); + + + // Log event counts + var sw = new StringWriter(); + await sw.WriteLineAsync("Sink stats:"); + foreach (var (eventType, count) in _addedEventCounts) { - var table = tableEvents.Key; - var events = tableEvents.Value; + await sw.WriteLineAsync($" * Added {count} {eventType.Name} events"); + } - tasks.Add(Database.WriteBatch(table, events, _schemaPropertyMap)); + foreach (var (eventType, count) in _importedEventCounts) + { + await sw.WriteLineAsync($" * Imported {count} {eventType.Name} events"); } - await Task.WhenAll(tasks); + Console.WriteLine(sw.ToString()); } } \ No newline at end of file diff --git a/Circles.Index.Common/ValueTypes.cs b/Circles.Index.Common/ValueTypes.cs index a8a1f71..f26f086 100644 --- a/Circles.Index.Common/ValueTypes.cs +++ b/Circles.Index.Common/ValueTypes.cs @@ -2,9 +2,9 @@ namespace Circles.Index.Common; public enum ValueTypes { - Address = 0, - Int = 1, - BigInt = 2, + Boolean = 0, + Address = 1, + Int = 2, + BigInt = 3, String = 4, - Boolean = 6 } \ No newline at end of file diff --git a/Circles.Index.Postgres/PostgresDb.cs b/Circles.Index.Postgres/PostgresDb.cs index 954217e..16e234c 100644 --- a/Circles.Index.Postgres/PostgresDb.cs +++ b/Circles.Index.Postgres/PostgresDb.cs @@ -14,18 +14,69 @@ public class PostgresDb(string connectionString, IDatabaseSchema schema, ILogger { public IDatabaseSchema Schema { get; } = schema; + private bool HasPrimaryKey(NpgsqlConnection connection, EventSchema table) + { + var checkPkSql = $@" + SELECT 1 + FROM pg_constraint + WHERE conrelid = '""{table.Namespace}_{table.Table}""'::regclass + AND contype = 'p';"; + + using var command = connection.CreateCommand(); + command.CommandText = checkPkSql; + return command.ExecuteScalar() != null; + } + public void Migrate() { - foreach (var table in Schema.Tables) + using var connection = new NpgsqlConnection(connectionString); + connection.Open(); + + using var transaction = connection.BeginTransaction(); + try + { + StringBuilder ddlSql = new StringBuilder(); + foreach (var table in Schema.Tables) + { + var ddl = GetDdl(table.Value); + ddlSql.AppendLine(ddl); + } + + ExecuteNonQuery(connection, ddlSql.ToString()); + + StringBuilder primaryKeyDdl = new StringBuilder(); + foreach (var table in Schema.Tables) + { + if (!HasPrimaryKey(connection, table.Value)) + { + if (table.Value is { Namespace: "System", Table: "Block" }) + { + primaryKeyDdl.AppendLine( + $"ALTER TABLE \"{table.Value.Namespace}_{table.Value.Table}\" ADD PRIMARY KEY (\"blockNumber\");"); + } + else + { + primaryKeyDdl.AppendLine( + $"ALTER TABLE \"{table.Value.Namespace}_{table.Value.Table}\" ADD PRIMARY KEY (\"blockNumber\", \"transactionIndex\", \"logIndex\");"); + } + } + } + + ExecuteNonQuery(connection, primaryKeyDdl.ToString()); + } + catch (Exception) { - var ddl = GetDdl(table.Value); - ExecuteNonQuery(ddl); + transaction.Rollback(); + throw; } + + transaction.Commit(); } - public async Task WriteBatch(string table, IEnumerable data, ISchemaPropertyMap propertyMap) + public async Task WriteBatch(string @namespace, string table, IEnumerable data, + ISchemaPropertyMap propertyMap) { - var tableSchema = Schema.Tables[table]; + var tableSchema = Schema.Tables[(@namespace, table)]; var columnTypes = tableSchema.Columns.ToDictionary(o => o.Column, o => o.Type); var columnList = string.Join(", ", columnTypes.Select(o => $"\"{o.Key}\"")); @@ -33,7 +84,7 @@ public async Task WriteBatch(string table, IEnumerable data, ISchemaProp connection.Open(); await using var writer = await connection.BeginBinaryImportAsync( - $"COPY \"{table}\" ({columnList}) FROM STDIN (FORMAT BINARY)" + $"COPY \"{tableSchema.Namespace}_{tableSchema.Table}\" ({columnList}) FROM STDIN (FORMAT BINARY)" ); foreach (var indexEvent in data) @@ -41,7 +92,7 @@ public async Task WriteBatch(string table, IEnumerable data, ISchemaProp await writer.StartRowAsync(); foreach (var column in tableSchema.Columns) { - var value = propertyMap.Map[table][column.Column](indexEvent); + var value = propertyMap.Map[(@namespace, table)][column.Column](indexEvent); await writer.WriteAsync(value, GetNpgsqlDbType(column.Type)); } } @@ -52,7 +103,7 @@ public async Task WriteBatch(string table, IEnumerable data, ISchemaProp private string GetDdl(EventSchema @event) { StringBuilder ddlSql = new StringBuilder(); - ddlSql.AppendLine($"CREATE TABLE IF NOT EXISTS \"{@event.Table}\" ("); + ddlSql.AppendLine($"CREATE TABLE IF NOT EXISTS \"{@event.Namespace}_{@event.Table}\" ("); List columnDefinitions = new List(); List primaryKeyColumns = new List(); @@ -91,9 +142,9 @@ column is foreach (var column in indexedColumns) { - string indexName = $"idx_{@event.Table}_{column.Column}"; + string indexName = $"idx_{@event.Namespace}_{@event.Table}_{column.Column}"; ddlSql.AppendLine( - $"CREATE INDEX IF NOT EXISTS \"{indexName}\" ON \"{@event.Table}\" (\"{column.Column}\");"); + $"CREATE INDEX IF NOT EXISTS \"{indexName}\" ON \"{@event.Namespace}_{@event.Table}\" (\"{column.Column}\");"); } return ddlSql.ToString(); @@ -125,18 +176,11 @@ private NpgsqlDbType GetNpgsqlDbType(ValueTypes type) }; } - private void ExecuteNonQuery(string command, IDbDataParameter[]? parameters = null) + private void ExecuteNonQuery(NpgsqlConnection connection, string command, IDbDataParameter[]? parameters = null) { - using var connection = new NpgsqlConnection(connectionString); - connection.Open(); - using var cmd = connection.CreateCommand(); cmd.CommandText = command; cmd.Parameters.AddRange(parameters ?? []); - - Console.WriteLine($"Executing: {command}"); - Console.WriteLine( - $" with parameters: {string.Join(", ", cmd.Parameters.Select(o => o.Value?.ToString() ?? ""))}"); cmd.ExecuteNonQuery(); } @@ -147,7 +191,7 @@ private void ExecuteNonQuery(string command, IDbDataParameter[]? parameters = nu NpgsqlCommand cmd = connection.CreateCommand(); cmd.CommandText = $@" - SELECT MAX(""BlockNumber"") as block_number FROM ""{"Block"}"" + SELECT MAX(""blockNumber"") as block_number FROM ""{"System_Block"}"" "; object? result = cmd.ExecuteScalar(); @@ -166,14 +210,14 @@ SELECT MAX(""BlockNumber"") as block_number FROM ""{"Block"}"" NpgsqlCommand cmd = connection.CreateCommand(); cmd.CommandText = $@" - SELECT (prev.""BlockNumber"" + 1) AS gap_start + SELECT (prev.""blockNumber"" + 1) AS gap_start FROM ( - SELECT ""BlockNumber"", LEAD(""BlockNumber"") OVER (ORDER BY ""BlockNumber"") AS next_block_number + SELECT ""blockNumber"", LEAD(""blockNumber"") OVER (ORDER BY ""blockNumber"") AS next_block_number FROM ( - SELECT ""BlockNumber"" FROM ""{"Block"}"" ORDER BY ""BlockNumber"" DESC LIMIT 500000 + SELECT ""blockNumber"" FROM ""{"System_Block"}"" ORDER BY ""blockNumber"" DESC LIMIT 500000 ) AS sub ) AS prev - WHERE prev.next_block_number - prev.""BlockNumber"" > 1 + WHERE prev.next_block_number - prev.""blockNumber"" > 1 ORDER BY gap_start LIMIT 1; "; @@ -194,9 +238,9 @@ ORDER BY gap_start NpgsqlCommand cmd = connection.CreateCommand(); cmd.CommandText = $@" - SELECT ""BlockNumber"", ""BlockHash"" - FROM {"Block"} - ORDER BY ""BlockNumber"" DESC + SELECT ""blockNumber"", ""blockHash"" + FROM ""{"System_Block"}"" + ORDER BY ""blockNumber"" DESC LIMIT {count} "; @@ -241,10 +285,11 @@ public async Task DeleteFromBlockOnwards(long reorgAt) await using var transaction = await connection.BeginTransactionAsync(); try { - foreach (var tableName in Schema.Tables.Keys) + foreach (var table in Schema.Tables.Values) { await using var command = connection.CreateCommand(); - command.CommandText = $"DELETE FROM \"{tableName}\" WHERE \"{"BlockNumber"}\" >= @reorgAt;"; + command.CommandText = + $"DELETE FROM \"{table.Namespace}_{table.Table}\" WHERE \"{"blockNumber"}\" >= @reorgAt;"; command.Parameters.AddWithValue("@reorgAt", reorgAt); command.Transaction = transaction; command.ExecuteNonQuery(); diff --git a/Circles.Index.V1/DatabaseSchema.cs b/Circles.Index.V1/DatabaseSchema.cs index c739dd3..0fe058d 100644 --- a/Circles.Index.V1/DatabaseSchema.cs +++ b/Circles.Index.V1/DatabaseSchema.cs @@ -1,116 +1,122 @@ using System.Numerics; using Circles.Index.Common; -using Nethermind.Core.Crypto; namespace Circles.Index.V1; public class DatabaseSchema : IDatabaseSchema { - public SchemaPropertyMap SchemaPropertyMap { get; } = new(); + public ISchemaPropertyMap SchemaPropertyMap { get; } = new SchemaPropertyMap(); - public EventDtoTableMap EventDtoTableMap { get; } = new(); + public IEventDtoTableMap EventDtoTableMap { get; } = new EventDtoTableMap(); - public IDictionary Tables { get; } = new Dictionary - { - { - "CrcV1HubTransfer", - new EventSchema("CrcV1HubTransfer", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("FromAddress", ValueTypes.Address, true), - new("ToAddress", ValueTypes.Address, true), - new("Amount", ValueTypes.BigInt, false) - ]) - }, - { - "CrcV1Signup", - new EventSchema("CrcV1Signup", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("CirclesAddress", ValueTypes.Address, true), - new("TokenAddress", ValueTypes.Address, true) - ]) - }, + public static readonly EventSchema HubTransfer = EventSchema.FromSolidity("CrcV1", + "event HubTransfer(address indexed from, address indexed to, uint256 amount)"); + + public static readonly EventSchema Signup = EventSchema.FromSolidity("CrcV1", + "event Signup(address indexed user, address indexed token)"); + + public static readonly EventSchema OrganizationSignup = EventSchema.FromSolidity("CrcV1", + "event OrganizationSignup(address indexed organization)"); + + public static readonly EventSchema Trust = EventSchema.FromSolidity("CrcV1", + "event Trust(address indexed canSendTo, address indexed user, uint256 limit)"); + + public static readonly EventSchema Transfer = EventSchema.FromSolidity("CrcV1", + "event Transfer(address indexed from, address indexed to, uint256 amount)"); + + public IDictionary<(string Namespace, string Table), EventSchema> Tables { get; } = + new Dictionary<(string Namespace, string Table), EventSchema> { - "CrcV1Trust", - new EventSchema("CrcV1Trust", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("UserAddress", ValueTypes.Address, true), - new("CanSendToAddress", ValueTypes.Address, true), - new("Limit", ValueTypes.Int, false) - ]) - } - }; + { + ("CrcV1", "HubTransfer"), + HubTransfer + }, + { + ("CrcV1", "Signup"), + Signup + }, + { + ("CrcV1", "OrganizationSignup"), + OrganizationSignup + }, + { + ("CrcV1", "Trust"), + Trust + }, + { + ("CrcV1", "Transfer"), + Transfer + } + }; public DatabaseSchema() { - EventDtoTableMap.Add("CrcV1Signup"); - SchemaPropertyMap.Add("CrcV1Signup", - new Dictionary> + EventDtoTableMap.Add(("CrcV1", "Signup")); + SchemaPropertyMap.Add(("CrcV1", "Signup"), + new Dictionary> + { + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "user", e => e.User }, + { "token", e => e.Token } + }); + + EventDtoTableMap.Add(("CrcV1", "OrganizationSignup")); + SchemaPropertyMap.Add(("CrcV1", "OrganizationSignup"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "CirclesAddress", e => e.CirclesAddress }, - { "TokenAddress", e => e.TokenAddress ?? string.Empty } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "organization", e => e.Organization } }); - EventDtoTableMap.Add("CrcV1Trust"); - SchemaPropertyMap.Add("CrcV1Trust", - new Dictionary> + EventDtoTableMap.Add(("CrcV1", "Trust")); + SchemaPropertyMap.Add(("CrcV1", "Trust"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "UserAddress", e => e.UserAddress }, - { "CanSendToAddress", e => e.CanSendToAddress }, - { "Limit", e => e.Limit } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "canSendTo", e => e.CanSendTo }, + { "user", e => e.User }, + { "limit", e => e.Limit } }); - EventDtoTableMap.Add("CrcV1HubTransfer"); - SchemaPropertyMap.Add("CrcV1HubTransfer", - new Dictionary> + EventDtoTableMap.Add(("CrcV1", "HubTransfer")); + SchemaPropertyMap.Add(("CrcV1", "HubTransfer"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "FromAddress", e => e.FromAddress }, - { "ToAddress", e => e.ToAddress }, - { "Amount", e => (BigInteger)e.Amount } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "from", e => e.From }, + { "to", e => e.To }, + { "amount", e => (BigInteger)e.Amount } }); - EventDtoTableMap.Add("CrcV1HubTransfer"); - SchemaPropertyMap.Add("Erc20Transfer", - new Dictionary> + EventDtoTableMap.Add(("CrcV1", "Transfer")); + SchemaPropertyMap.Add(("CrcV1", "Transfer"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "TokenAddress", e => e.TokenAddress }, - { "FromAddress", e => e.From }, - { "ToAddress", e => e.To }, - { "Amount", e => (BigInteger)e.Value } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "tokenAddress", e => e.TokenAddress }, + { "from", e => e.From }, + { "to", e => e.To }, + { "amount", e => (BigInteger)e.Value } }); } } \ No newline at end of file diff --git a/Circles.Index.V1/Events.cs b/Circles.Index.V1/Events.cs index b366f4a..cedb959 100644 --- a/Circles.Index.V1/Events.cs +++ b/Circles.Index.V1/Events.cs @@ -3,7 +3,50 @@ namespace Circles.Index.V1; -public record CirclesSignupData(long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, string CirclesAddress, string? TokenAddress) : IIndexEvent; -public record CirclesTrustData(long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, string UserAddress, string CanSendToAddress, int Limit) : IIndexEvent; -public record CirclesHubTransferData(long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, string FromAddress, string ToAddress, UInt256 Amount) : IIndexEvent; -public record Erc20TransferData(long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, string TokenAddress, string From, string To, UInt256 Value) : IIndexEvent; \ No newline at end of file +public record Signup( + long BlockNumber, + long Timestamp, + int TransactionIndex, + int LogIndex, + string TransactionHash, + string User, + string Token) : IIndexEvent; + +public record OrganizationSignup( + long BlockNumber, + long Timestamp, + int TransactionIndex, + int LogIndex, + string TransactionHash, + string Organization) : IIndexEvent; + +public record Trust( + long BlockNumber, + long Timestamp, + int TransactionIndex, + int LogIndex, + string TransactionHash, + string User, + string CanSendTo, + int Limit) : IIndexEvent; + +public record HubTransfer( + long BlockNumber, + long Timestamp, + int TransactionIndex, + int LogIndex, + string TransactionHash, + string From, + string To, + UInt256 Amount) : IIndexEvent; + +public record Transfer( + long BlockNumber, + long Timestamp, + int TransactionIndex, + int LogIndex, + string TransactionHash, + string TokenAddress, + string From, + string To, + UInt256 Value) : IIndexEvent; \ No newline at end of file diff --git a/Circles.Index.V1/LogParser.cs b/Circles.Index.V1/LogParser.cs index eebda05..6cc8d24 100644 --- a/Circles.Index.V1/LogParser.cs +++ b/Circles.Index.V1/LogParser.cs @@ -2,7 +2,6 @@ using System.Globalization; using Circles.Index.Common; using Nethermind.Core; -using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; @@ -10,14 +9,8 @@ namespace Circles.Index.V1; public class LogParser(Address v1HubAddress) : ILogParser { - public static Hash256 CrcHubTransferEventTopic { get; } = Keccak.Compute("HubTransfer(address,address,uint256)"); - public static Hash256 CrcTrustEventTopic { get; } = Keccak.Compute("Trust(address,address,uint256)"); - public static Hash256 CrcSignupEventTopic { get; } = Keccak.Compute("Signup(address,address)"); - public static Hash256 CrcOrganisationSignupEventTopic { get; } = Keccak.Compute("OrganisationSignup(address)"); - public static Hash256 Erc20TransferTopic { get; } = Keccak.Compute("Transfer(address,address,uint256)"); - public static readonly ConcurrentDictionary CirclesTokenAddresses = new(); - + public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) { List events = new(); @@ -27,7 +20,7 @@ public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntr } var topic = log.Topics[0]; - if (topic == Erc20TransferTopic && + if (topic == DatabaseSchema.Transfer.Topic && CirclesTokenAddresses.ContainsKey(log.LoggersAddress)) { events.Add(Erc20Transfer(block, receipt, log, logIndex)); @@ -35,7 +28,7 @@ public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntr if (log.LoggersAddress == v1HubAddress) { - if (topic == CrcSignupEventTopic) + if (topic == DatabaseSchema.Signup.Topic) { var signupEvents = CrcSignup(block, receipt, log, logIndex); foreach (var signupEvent in signupEvents) @@ -44,21 +37,21 @@ public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntr } } - if (topic == CrcHubTransferEventTopic) + if (topic == DatabaseSchema.OrganizationSignup.Topic) { - events.Add(CrcHubTransfer(block, receipt, log, logIndex)); + events.Add(CrcOrgSignup(block, receipt, log, logIndex)); } - if (topic == CrcTrustEventTopic) + if (topic == DatabaseSchema.HubTransfer.Topic) { - events.Add(CrcTrust(block, receipt, log, logIndex)); + events.Add(CrcHubTransfer(block, receipt, log, logIndex)); } - if (topic == CrcOrganisationSignupEventTopic) + if (topic == DatabaseSchema.Trust.Topic) { - events.Add(CrcOrgSignup(block, receipt, log, logIndex)); + events.Add(CrcTrust(block, receipt, log, logIndex)); } - }; + } return events; } @@ -69,7 +62,7 @@ private IIndexEvent Erc20Transfer(Block block, TxReceipt receipt, LogEntry log, string to = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); UInt256 value = new(log.Data, true); - return new Erc20TransferData( + return new Transfer( receipt.BlockNumber , (long)block.Timestamp , receipt.Index @@ -83,16 +76,15 @@ private IIndexEvent Erc20Transfer(Block block, TxReceipt receipt, LogEntry log, private IIndexEvent CrcOrgSignup(Block block, TxReceipt receipt, LogEntry log, int logIndex) { - string user = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); + string org = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - return new CirclesSignupData( + return new OrganizationSignup( receipt.BlockNumber , (long)block.Timestamp , receipt.Index , logIndex , receipt.TxHash!.ToString() - , user - , null); + , org); } private IIndexEvent CrcTrust(Block block, TxReceipt receipt, LogEntry log, int logIndex) @@ -101,7 +93,7 @@ private IIndexEvent CrcTrust(Block block, TxReceipt receipt, LogEntry log, int l string canSendTo = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); int limit = new UInt256(log.Data, true).ToInt32(CultureInfo.InvariantCulture); - return new CirclesTrustData( + return new Trust( receipt.BlockNumber , (long)block.Timestamp , receipt.Index @@ -118,7 +110,7 @@ private IIndexEvent CrcHubTransfer(Block block, TxReceipt receipt, LogEntry log, string to = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); UInt256 amount = new(log.Data, true); - return new CirclesHubTransferData( + return new HubTransfer( receipt.BlockNumber , (long)block.Timestamp , receipt.Index @@ -134,7 +126,7 @@ private IEnumerable CrcSignup(Block block, TxReceipt receipt, LogEn string user = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); Address tokenAddress = new Address(log.Data.Slice(12)); - IIndexEvent signupEvent = new CirclesSignupData( + IIndexEvent signupEvent = new Signup( receipt.BlockNumber , (long)block.Timestamp , receipt.Index @@ -142,30 +134,31 @@ private IEnumerable CrcSignup(Block block, TxReceipt receipt, LogEn , receipt.TxHash!.ToString() , user , tokenAddress.ToString(true, false)); - + CirclesTokenAddresses.TryAdd(tokenAddress, null); IIndexEvent? signupBonusEvent = null; - + // Every signup comes together with an Erc20 transfer (the signup bonus). // Since the signup event is emitted after the transfer, the token wasn't known yet when we encountered the transfer. // Look for the transfer again and process it. - foreach (var repeatedLogEntry in receipt.Logs!) + for (int i = 0; i < receipt.Logs!.Length; i++) { + var repeatedLogEntry = receipt.Logs[i]; if (repeatedLogEntry.LoggersAddress != tokenAddress) { continue; } - if (repeatedLogEntry.Topics[0] == Erc20TransferTopic) + if (repeatedLogEntry.Topics[0] == DatabaseSchema.Transfer.Topic) { - signupBonusEvent = Erc20Transfer(block, receipt, repeatedLogEntry, logIndex); + signupBonusEvent = Erc20Transfer(block, receipt, repeatedLogEntry, i); break; } } - + return signupBonusEvent == null - ? new[] {signupEvent} - : new[] {signupEvent, signupBonusEvent}; + ? new[] { signupEvent } + : new[] { signupEvent, signupBonusEvent }; } } \ No newline at end of file diff --git a/Circles.Index.V2/DatabaseSchema.cs b/Circles.Index.V2/DatabaseSchema.cs index 8400e92..9a58071 100644 --- a/Circles.Index.V2/DatabaseSchema.cs +++ b/Circles.Index.V2/DatabaseSchema.cs @@ -3,371 +3,302 @@ namespace Circles.Index.V2; +/* + + hub/ + Hub.sol: + event PersonalMint(address indexed human, uint256 amount, uint256 startPeriod, uint256 endPeriod); + event RegisterHuman(address indexed avatar); + event RegisterGroup(address indexed group, address indexed mint, address indexed treasury, string indexed name, string indexed symbol); + event InviteHuman(address indexed inviter, address indexed invited); + event RegisterOrganization(address indexed organization, string name); + event Stopped(address indexed avatar); + event Trust(address indexed truster, address indexed trustee, uint256 expiryTime); + event URI(string value, uint256 indexed id); + event TransferSingle(address indexed operator, address indexed from, address indexed to, uint256 id, uint256 value); + event ApprovalForAll(address indexed account, address indexed operator, bool approved); + + Manual events: + event TransferBatch(address indexed operator, address indexed from, address indexed to, uint256[] ids, uint256[] values); + + lift/ + DemurrageCircles.sol: + event Deposit(address indexed account, uint256 amount, uint256 inflationaryAmount); + event Withdraw(address indexed account, uint256 amount, uint256 inflationaryAmount); + + InflationaryCircles.sol: + event Deposit(address indexed account, uint256 amount, uint256 demurragedAmount); + event Withdraw(address indexed account, uint256 amount, uint256 demurragedAmount); + + names/ + NameRegistry.sol: + event RegisterShortName(address indexed avatar, uint72 shortName, uint256 nonce); + event UpdateMetadataDigest(address indexed avatar, bytes32 metadataDigest); + + proxy/ + ProxyFactory.sol: + event ProxyCreation(address proxy, address masterCopy); + */ public class DatabaseSchema : IDatabaseSchema { - public SchemaPropertyMap SchemaPropertyMap { get; } = new(); + public ISchemaPropertyMap SchemaPropertyMap { get; } = new SchemaPropertyMap(); - public EventDtoTableMap EventDtoTableMap { get; } = new(); + public IEventDtoTableMap EventDtoTableMap { get; } = new EventDtoTableMap(); - public IDictionary Tables { get; } = new Dictionary - { - { - "CrcV2ConvertInflation", - new EventSchema("CrcV2ConvertInflation", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("InflationValue", ValueTypes.BigInt, false), - new("DemurrageValue", ValueTypes.BigInt, false), - new("Day", ValueTypes.BigInt, false) - ]) - }, - { - "CrcV2InviteHuman", - new EventSchema("CrcV2InviteHuman", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("InviterAddress", ValueTypes.Address, true), - new("InviteeAddress", ValueTypes.Address, true) - ]) - }, - { - "CrcV2PersonalMint", - new EventSchema("CrcV2PersonalMint", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("HumanAddress", ValueTypes.Address, true), - new("Amount", ValueTypes.BigInt, false), - new("StartPeriod", ValueTypes.BigInt, false), - new("EndPeriod", ValueTypes.BigInt, false) - ]) - }, - { - "CrcV2RegisterGroup", - new EventSchema("CrcV2RegisterGroup", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("GroupAddress", ValueTypes.Address, true), - new("MintPolicy", ValueTypes.Address, true), - new("Treasury", ValueTypes.Address, true), - new("GroupName", ValueTypes.String, true), - new("GroupSymbol", ValueTypes.String, true) - ]) - }, - { - "CrcV2RegisterHuman", - new EventSchema("CrcV2RegisterHuman", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("HumanAddress", ValueTypes.Address, true) - ]) - }, - { - "CrcV2RegisterOrganization", - new EventSchema("CrcV2RegisterOrganization", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("OrganizationAddress", ValueTypes.Address, true), - new("OrganizationName", ValueTypes.String, true) - ]) - }, - { - "CrcV2Stopped", - new EventSchema("CrcV2Stopped", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("Address", ValueTypes.Address, true) - ]) - }, - { - "CrcV2Trust", - new EventSchema("CrcV2Trust", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("TrusterAddress", ValueTypes.Address, true), - new("TrusteeAddress", ValueTypes.Address, true), - new("ExpiryTime", ValueTypes.BigInt, false) - ]) - }, - - - // Existing: - { - "Erc20Transfer", - new EventSchema("Erc20Transfer", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("TokenAddress", ValueTypes.Address, true), - new("FromAddress", ValueTypes.Address, true), - new("ToAddress", ValueTypes.Address, true), - new("Amount", ValueTypes.BigInt, false) - ]) - }, - { - "Erc1155ApprovalForAll", - new EventSchema("Erc1155ApprovalForAll", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("Owner", ValueTypes.Address, true), - new("Operator", ValueTypes.Address, true), - new("Approved", ValueTypes.Boolean, true) - ]) - }, - { - "Erc1155TransferBatch", - new EventSchema("Erc1155TransferBatch", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("BatchIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("OperatorAddress", ValueTypes.Address, true), - new("FromAddress", ValueTypes.Address, true), - new("ToAddress", ValueTypes.Address, true), - new("TokenId", ValueTypes.BigInt, true), - new("Value", ValueTypes.BigInt, false) - ]) - }, - { - "Erc1155TransferSingle", - new EventSchema("Erc1155TransferSingle", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("OperatorAddress", ValueTypes.Address, true), - new("FromAddress", ValueTypes.Address, true), - new("ToAddress", ValueTypes.Address, true), - new("TokenId", ValueTypes.BigInt, true), - new("Value", ValueTypes.BigInt, false) - ]) - }, + public static readonly EventSchema InviteHuman = + EventSchema.FromSolidity("CrcV2", "event InviteHuman(address indexed inviter, address indexed invited);"); + + public static readonly EventSchema PersonalMint = EventSchema.FromSolidity("CrcV2", + "event PersonalMint(address indexed human, uint256 amount, uint256 startPeriod, uint256 endPeriod)"); + + public static readonly EventSchema RegisterGroup = EventSchema.FromSolidity("CrcV2", + "event RegisterGroup(address indexed group, address indexed mint, address indexed treasury, string indexed name, string indexed symbol)"); + + public static readonly EventSchema RegisterHuman = + EventSchema.FromSolidity("CrcV2", "event RegisterHuman(address indexed avatar)"); + + public static readonly EventSchema RegisterOrganization = + EventSchema.FromSolidity("CrcV2", + "event RegisterOrganization(address indexed organization, string indexed name)"); + + public static readonly EventSchema Stopped = + EventSchema.FromSolidity("CrcV2", "event Stopped(address indexed avatar)"); + + public static readonly EventSchema Trust = + EventSchema.FromSolidity("CrcV2", + "event Trust(address indexed truster, address indexed trustee, uint256 expiryTime)"); + + public static readonly EventSchema TransferSingle = EventSchema.FromSolidity( + "CrcV2", + "event TransferSingle(address indexed operator, address indexed from, address indexed to, uint256 indexed id, uint256 value)"); + + public static readonly EventSchema URI = + EventSchema.FromSolidity("CrcV2", "event URI(string value, uint256 indexed id)"); + + public static readonly EventSchema ApprovalForAll = + EventSchema.FromSolidity( + "CrcV2", "event ApprovalForAll(address indexed account, address indexed operator, bool approved)"); + + public static readonly EventSchema TransferBatch = new("CrcV2", "TransferBatch", + Keccak.Compute("TransferBatch(address,address,address,uint256[],uint256[])"), + [ + new("blockNumber", ValueTypes.Int, true), + new("timestamp", ValueTypes.Int, true), + new("transactionIndex", ValueTypes.Int, true), + new("logIndex", ValueTypes.Int, true), + new("batchIndex", ValueTypes.Int, true), + new("transactionHash", ValueTypes.String, true), + new("operatorAddress", ValueTypes.Address, true), + new("fromAddress", ValueTypes.Address, true), + new("toAddress", ValueTypes.Address, true), + new("id", ValueTypes.BigInt, true), + new("value", ValueTypes.BigInt, false) + ]); + + + public IDictionary<(string Namespace, string Table), EventSchema> Tables { get; } = + new Dictionary<(string Namespace, string Table), EventSchema> { - "Erc1155Uri", - new EventSchema("Erc1155Uri", new Hash256(new byte[32]), - [ - new("BlockNumber", ValueTypes.Int, true), - new("Timestamp", ValueTypes.Int, true), - new("TransactionIndex", ValueTypes.Int, true), - new("LogIndex", ValueTypes.Int, true), - new("TransactionHash", ValueTypes.String, true), - new("TokenId", ValueTypes.BigInt, true), - new("Uri", ValueTypes.BigInt, false) - ]) - } - }; + { + ("CrcV2", "InviteHuman"), + InviteHuman + }, + { + ("CrcV2", "PersonalMint"), + PersonalMint + }, + { + ("CrcV2", "RegisterGroup"), + RegisterGroup + }, + { + ("CrcV2", "RegisterHuman"), + RegisterHuman + }, + { + ("CrcV2", "RegisterOrganization"), + RegisterOrganization + }, + { + ("CrcV2", "Stopped"), + Stopped + }, + { + ("CrcV2", "Trust"), + Trust + }, + { + ("CrcV2", "TransferSingle"), + TransferSingle + }, + { + ("CrcV2", "URI"), + URI + }, + { + ("CrcV2", "ApprovalForAll"), + ApprovalForAll + }, + { + ("CrcV2", "TransferBatch"), + TransferBatch + } + }; public DatabaseSchema() { - EventDtoTableMap.Add("CrcV2ConvertInflation"); - SchemaPropertyMap.Add("CrcV2ConvertInflation", - new Dictionary> - { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "InflationValue", e => e.InflationValue }, - { "DemurrageValue", e => e.DemurrageValue }, - { "Day", e => e.Day } - }); - - EventDtoTableMap.Add("CrcV2InviteHuman"); - SchemaPropertyMap.Add("CrcV2InviteHuman", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "InviteHuman")); + SchemaPropertyMap.Add(("CrcV2", "InviteHuman"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "InviterAddress", e => e.InviterAddress }, - { "InviteeAddress", e => e.InviteeAddress } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "inviter", e => e.Inviter }, + { "invited", e => e.Invited } }); - EventDtoTableMap.Add("CrcV2PersonalMint"); - SchemaPropertyMap.Add("CrcV2PersonalMint", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "PersonalMint")); + SchemaPropertyMap.Add(("CrcV2", "PersonalMint"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "HumanAddress", e => e.HumanAddress }, - { "Amount", e => e.Amount }, - { "StartPeriod", e => e.StartPeriod }, - { "EndPeriod", e => e.EndPeriod } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "human", e => e.Human }, + { "amount", e => e.Amount }, + { "startPeriod", e => e.StartPeriod }, + { "endPeriod", e => e.EndPeriod } }); - EventDtoTableMap.Add("CrcV2RegisterGroup"); - SchemaPropertyMap.Add("CrcV2RegisterGroup", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "RegisterGroup")); + SchemaPropertyMap.Add(("CrcV2", "RegisterGroup"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "GroupAddress", e => e.GroupAddress }, - { "MintPolicy", e => e.MintPolicy }, - { "Treasury", e => e.Treasury }, - { "GroupName", e => e.GroupName }, - { "GroupSymbol", e => e.GroupSymbol } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "group", e => e.Group }, + { "mint", e => e.Mint }, + { "treasury", e => e.Treasury }, + { "name", e => e.Name }, + { "symbol", e => e.Symbol } }); - EventDtoTableMap.Add("CrcV2RegisterHuman"); - SchemaPropertyMap.Add("CrcV2RegisterHuman", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "RegisterHuman")); + SchemaPropertyMap.Add(("CrcV2", "RegisterHuman"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "HumanAddress", e => e.HumanAddress } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "avatar", e => e.Avatar } }); - EventDtoTableMap.Add("CrcV2RegisterOrganization"); - SchemaPropertyMap.Add("CrcV2RegisterOrganization", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "RegisterOrganization")); + SchemaPropertyMap.Add(("CrcV2", "RegisterOrganization"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "OrganizationAddress", e => e.OrganizationAddress }, - { "OrganizationName", e => e.OrganizationName } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "organization", e => e.Organization }, + { "name", e => e.Name } }); - EventDtoTableMap.Add("CrcV2Stopped"); - SchemaPropertyMap.Add("CrcV2Stopped", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "Stopped")); + SchemaPropertyMap.Add(("CrcV2", "Stopped"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "Address", e => e.Address } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "avatar", e => e.Avatar } }); - EventDtoTableMap.Add("CrcV2Trust"); - SchemaPropertyMap.Add("CrcV2Trust", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "Trust")); + SchemaPropertyMap.Add(("CrcV2", "Trust"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "TrusterAddress", e => e.TrusterAddress }, - { "TrusteeAddress", e => e.TrusteeAddress }, - { "ExpiryTime", e => e.ExpiryTime } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "truster", e => e.Truster }, + { "trustee", e => e.Trustee }, + { "expiryTime", e => e.ExpiryTime } }); - EventDtoTableMap.Add("Erc1155ApprovalForAll"); - SchemaPropertyMap.Add("Erc1155ApprovalForAll", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "ApprovalForAll")); + SchemaPropertyMap.Add(("CrcV2", "ApprovalForAll"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "Owner", e => e.Owner }, - { "Operator", e => e.Operator }, - { "Approved", e => e.Approved } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "account", e => e.Account }, + { "operator", e => e.Operator }, + { "approved", e => e.Approved } }); - EventDtoTableMap.Add("Erc1155TransferSingle"); - SchemaPropertyMap.Add("Erc1155TransferSingle", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "TransferSingle")); + SchemaPropertyMap.Add(("CrcV2", "TransferSingle"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "OperatorAddress", e => e.OperatorAddress }, - { "FromAddress", e => e.FromAddress }, - { "ToAddress", e => e.ToAddress }, - { "TokenId", e => e.TokenId }, - { "Value", e => e.Value } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "operator", e => e.Operator }, + { "from", e => e.From }, + { "to", e => e.To }, + { "id", e => e.Id }, + { "value", e => e.Value } }); - EventDtoTableMap.Add("Erc1155TransferBatch"); - SchemaPropertyMap.Add("Erc1155TransferBatch", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "TransferBatch")); + SchemaPropertyMap.Add(("CrcV2", "TransferBatch"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "OperatorAddress", e => e.OperatorAddress }, - { "FromAddress", e => e.FromAddress }, - { "ToAddress", e => e.ToAddress }, - { "TokenId", e => e.TokenId }, - { "Value", e => e.Value } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "operator", e => e.Operator }, + { "from", e => e.From }, + { "to", e => e.To }, + { "id", e => e.Id }, + { "value", e => e.Value } }); - EventDtoTableMap.Add("Erc1155Uri"); - SchemaPropertyMap.Add("Erc1155Uri", - new Dictionary> + EventDtoTableMap.Add(("CrcV2", "URI")); + SchemaPropertyMap.Add(("CrcV2", "URI"), + new Dictionary> { - { "BlockNumber", e => e.BlockNumber }, - { "Timestamp", e => e.Timestamp }, - { "TransactionIndex", e => e.TransactionIndex }, - { "LogIndex", e => e.LogIndex }, - { "TransactionHash", e => e.TransactionHash }, - { "TokenId", e => e.TokenId }, - { "Uri", e => e.Uri } + { "blockNumber", e => e.BlockNumber }, + { "timestamp", e => e.Timestamp }, + { "transactionIndex", e => e.TransactionIndex }, + { "logIndex", e => e.LogIndex }, + { "transactionHash", e => e.TransactionHash }, + { "value", e => e.Value }, + { "id", e => e.Id } }); } } \ No newline at end of file diff --git a/Circles.Index.V2/Events.cs b/Circles.Index.V2/Events.cs index 69f5ba8..88db6ea 100644 --- a/Circles.Index.V2/Events.cs +++ b/Circles.Index.V2/Events.cs @@ -3,123 +3,113 @@ namespace Circles.Index.V2; -public record CrcV2RegisterOrganizationData( +public record RegisterOrganization( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string OrganizationAddress, - string OrganizationName) : IIndexEvent; + string Organization, + string Name) : IIndexEvent; -public record CrcV2RegisterGroupData( +public record RegisterGroup( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string GroupAddress, - string MintPolicy, + string Group, + string Mint, string Treasury, - string GroupName, - string GroupSymbol) : IIndexEvent; + string Name, + string Symbol) : IIndexEvent; -public record CrcV2RegisterHumanData( +public record RegisterHuman( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string HumanAddress) : IIndexEvent; + string Avatar) : IIndexEvent; -public record CrcV2PersonalMintData( +public record PersonalMint( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string HumanAddress, + string Human, UInt256 Amount, UInt256 StartPeriod, UInt256 EndPeriod) : IIndexEvent; -public record CrcV2InviteHumanData( +public record InviteHuman( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string InviterAddress, - string InviteeAddress) : IIndexEvent; + string Inviter, + string Invited) : IIndexEvent; -public record CrcV2ConvertInflationData( +public record Trust( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - UInt256 InflationValue, - UInt256 DemurrageValue, - ulong Day) : IIndexEvent; - -public record CrcV2TrustData( - long BlockNumber, - long Timestamp, - int TransactionIndex, - int LogIndex, - string TransactionHash, - string TrusterAddress, - string TrusteeAddress, + string Truster, + string Trustee, UInt256 ExpiryTime) : IIndexEvent; -public record CrcV2StoppedData( +public record Stopped( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string Address) : IIndexEvent; + string Avatar) : IIndexEvent; -public record Erc1155ApprovalForAllData( +public record ApprovalForAll( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string Owner, + string Account, string Operator, bool Approved) : IIndexEvent; -public record Erc1155TransferSingleData( +public record TransferSingle( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - string OperatorAddress, - string FromAddress, - string ToAddress, - UInt256 TokenId, + string Operator, + string From, + string To, + UInt256 Id, UInt256 Value) : IIndexEvent; -public record Erc1155TransferBatchData( +public record TransferBatch( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, int BatchIndex, - string OperatorAddress, - string FromAddress, - string ToAddress, - UInt256 TokenId, + string Operator, + string From, + string To, + UInt256 Id, UInt256 Value) : IIndexEvent; -public record Erc1155UriData( +public record URI( long BlockNumber, long Timestamp, int TransactionIndex, int LogIndex, string TransactionHash, - UInt256 TokenId, - string Uri) : IIndexEvent; \ No newline at end of file + UInt256 Id, + string Value) : IIndexEvent; \ No newline at end of file diff --git a/Circles.Index.V2/LogParser.cs b/Circles.Index.V2/LogParser.cs index beb065b..f918566 100644 --- a/Circles.Index.V2/LogParser.cs +++ b/Circles.Index.V2/LogParser.cs @@ -1,9 +1,7 @@ -using System.Globalization; using System.Numerics; using System.Text; using Circles.Index.Common; using Nethermind.Core; -using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; @@ -11,34 +9,6 @@ namespace Circles.Index.V2; public class LogParser(Address v2HubAddress) : ILogParser { - public static Hash256 CrcV2RegisterHumanTopic { get; } = Keccak.Compute("RegisterHuman(address)"); - public static Hash256 CrcV2InviteHumanTopic { get; } = Keccak.Compute("InviteHuman(address,address)"); - - public static Hash256 CrcV2RegisterOrganizationTopic { get; } = - Keccak.Compute("RegisterOrganization(address,string)"); - - public static Hash256 CrcV2RegisterGroupTopic { get; } = - Keccak.Compute("RegisterGroup(address,address,address,string,string)"); - - public static Hash256 CrcV2TrustTopic { get; } = Keccak.Compute("Trust(address,address,uint256)"); - public static Hash256 CrcV2StoppedTopic { get; } = Keccak.Compute("Stopped(address)"); - - public static Hash256 CrcV2PersonalMintTopic { get; } = - Keccak.Compute("PersonalMint(address,uint256,uint256,uint256)"); - - public static Hash256 CrcV2ConvertInflationTopic { get; } = - Keccak.Compute("ConvertInflation(uint256,uint256,uint64"); - - // All ERC1155 events - public static Hash256 Erc1155TransferSingleTopic { get; } = - Keccak.Compute("TransferSingle(address,address,address,uint256,uint256)"); - - public static Hash256 Erc1155TransferBatchTopic { get; } = - Keccak.Compute("TransferBatch(address,address,address,uint256[],uint256[])"); - - public static Hash256 Erc1155ApprovalForAllTopic { get; } = Keccak.Compute("ApprovalForAll(address,address,bool)"); - public static Hash256 Erc1155UriTopic { get; } = Keccak.Compute("URI(uint256,string)"); - public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) { if (log.Topics.Length == 0) @@ -52,73 +22,75 @@ public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntr yield break; } - if (topic == CrcV2StoppedTopic) + if (topic == DatabaseSchema.Stopped.Topic) { yield return CrcV2Stopped(block, receipt, log, logIndex); } - if (topic == CrcV2TrustTopic) + if (topic == DatabaseSchema.Trust.Topic) { yield return CrcV2Trust(block, receipt, log, logIndex); } - if (topic == CrcV2ConvertInflationTopic) + if (topic == DatabaseSchema.InviteHuman.Topic) { - yield return CrcV2ConvertInflation(block, receipt, log, logIndex); + yield return CrcV2InviteHuman(block, receipt, log, logIndex); } - if (topic == CrcV2InviteHumanTopic) + if (topic == DatabaseSchema.PersonalMint.Topic) { - yield return CrcV2InviteHuman(block, receipt, log, logIndex); + yield return CrcV2PersonalMint(block, receipt, log, logIndex); } - if (topic == CrcV2PersonalMintTopic) + if (topic == DatabaseSchema.RegisterHuman.Topic) { - yield return CrcV2PersonalMint(block, receipt, log, logIndex); + yield return CrcV2RegisterHuman(block, receipt, log, logIndex); } - if (topic == CrcV2RegisterHumanTopic) { yield return CrcV2RegisterHuman(block, receipt, log, logIndex); } - if (topic == CrcV2RegisterGroupTopic) + if (topic == DatabaseSchema.RegisterGroup.Topic) { yield return CrcV2RegisterGroup(block, receipt, log, logIndex); } - if (topic == CrcV2RegisterOrganizationTopic) + if (topic == DatabaseSchema.RegisterOrganization.Topic) { yield return CrcV2RegisterOrganization(block, receipt, log, logIndex); } - if (topic == Erc1155TransferBatchTopic) + if (topic == DatabaseSchema.TransferBatch.Topic) { - yield return Erc1155TransferBatch(block, receipt, log, logIndex); + foreach (var batchEvent in Erc1155TransferBatch(block, receipt, log, logIndex)) + { + yield return batchEvent; + } } - if (topic == Erc1155TransferSingleTopic) + if (topic == DatabaseSchema.TransferSingle.Topic) { yield return Erc1155TransferSingle(block, receipt, log, logIndex); } - if (topic == Erc1155ApprovalForAllTopic) + if (topic == DatabaseSchema.ApprovalForAll.Topic) { yield return Erc1155ApprovalForAll(block, receipt, log, logIndex); } - if (topic == Erc1155UriTopic) + if (topic == DatabaseSchema.URI.Topic) { yield return Erc1155Uri(block, receipt, log, logIndex); } } - private Erc1155UriData Erc1155Uri(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private URI Erc1155Uri(Block block, TxReceipt receipt, LogEntry log, int logIndex) { var tokenId = new UInt256(log.Topics[1].Bytes, true); var uri = Encoding.UTF8.GetString(log.Data); - return new Erc1155UriData( + return new URI( block.Number, (long)block.Timestamp, receipt.Index, @@ -128,28 +100,28 @@ private Erc1155UriData Erc1155Uri(Block block, TxReceipt receipt, LogEntry log, uri); } - private Erc1155ApprovalForAllData Erc1155ApprovalForAll(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private ApprovalForAll Erc1155ApprovalForAll(Block block, TxReceipt receipt, LogEntry log, int logIndex) { throw new NotImplementedException(); } - private Erc1155TransferSingleData Erc1155TransferSingle(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private TransferSingle Erc1155TransferSingle(Block block, TxReceipt receipt, LogEntry log, int logIndex) { throw new NotImplementedException(); } - private Erc1155TransferBatchData Erc1155TransferBatch(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private IEnumerable Erc1155TransferBatch(Block block, TxReceipt receipt, LogEntry log, int logIndex) { throw new NotImplementedException(); } - private CrcV2RegisterOrganizationData CrcV2RegisterOrganization(Block block, TxReceipt receipt, LogEntry log, + private RegisterOrganization CrcV2RegisterOrganization(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string orgAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); string orgName = Encoding.UTF8.GetString(log.Data); - return new CrcV2RegisterOrganizationData( + return new RegisterOrganization( block.Number, (long)block.Timestamp, receipt.Index, @@ -159,7 +131,7 @@ private CrcV2RegisterOrganizationData CrcV2RegisterOrganization(Block block, TxR orgName); } - private CrcV2RegisterGroupData CrcV2RegisterGroup(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private RegisterGroup CrcV2RegisterGroup(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string groupAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); string mintPolicy = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); @@ -173,7 +145,7 @@ private CrcV2RegisterGroupData CrcV2RegisterGroup(Block block, TxReceipt receipt int symbolLength = (int)new BigInteger(log.Data.Slice(symbolOffset, 32).ToArray()); string groupSymbol = Encoding.UTF8.GetString(log.Data.Slice(symbolOffset + 32, symbolLength)); - return new CrcV2RegisterGroupData( + return new RegisterGroup( block.Number, (long)block.Timestamp, receipt.Index, @@ -187,11 +159,11 @@ private CrcV2RegisterGroupData CrcV2RegisterGroup(Block block, TxReceipt receipt } - private CrcV2RegisterHumanData CrcV2RegisterHuman(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private RegisterHuman CrcV2RegisterHuman(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string humanAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - return new CrcV2RegisterHumanData( + return new RegisterHuman( block.Number, (long)block.Timestamp, receipt.Index, @@ -200,14 +172,14 @@ private CrcV2RegisterHumanData CrcV2RegisterHuman(Block block, TxReceipt receipt humanAddress); } - private CrcV2PersonalMintData CrcV2PersonalMint(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private PersonalMint CrcV2PersonalMint(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string toAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); UInt256 amount = new UInt256(log.Data.Slice(0, 32), true); UInt256 startPeriod = new UInt256(log.Data.Slice(32, 32), true); UInt256 endPeriod = new UInt256(log.Data.Slice(64), true); - return new CrcV2PersonalMintData( + return new PersonalMint( block.Number, (long)block.Timestamp, receipt.Index, @@ -219,14 +191,14 @@ private CrcV2PersonalMintData CrcV2PersonalMint(Block block, TxReceipt receipt, endPeriod); } - private CrcV2InviteHumanData CrcV2InviteHuman(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private InviteHuman CrcV2InviteHuman(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string inviterAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); string inviteeAddress = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - return new CrcV2InviteHumanData( + return new InviteHuman( block.Number, (long)block.Timestamp, receipt.Index, @@ -236,31 +208,14 @@ private CrcV2InviteHumanData CrcV2InviteHuman(Block block, TxReceipt receipt, Lo inviteeAddress); } - private CrcV2ConvertInflationData CrcV2ConvertInflation(Block block, TxReceipt receipt, LogEntry log, int logIndex) - { - UInt256 inflationValue = new UInt256(log.Data.Slice(0, 32), true); - UInt256 demurrageValue = new UInt256(log.Data.Slice(32, 32), true); - ulong day = new UInt256(log.Data.Slice(64), true).ToUInt64(CultureInfo.InvariantCulture); - - return new CrcV2ConvertInflationData( - block.Number, - (long)block.Timestamp, - receipt.Index, - logIndex, - receipt.TxHash.ToString(), - inflationValue, - demurrageValue, - day); - } - - private CrcV2TrustData CrcV2Trust(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private Trust CrcV2Trust(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string userAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); string canSendToAddress = "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); UInt256 limit = new UInt256(log.Data, true); - return new CrcV2TrustData( + return new Trust( block.Number, (long)block.Timestamp, receipt.Index, @@ -271,11 +226,11 @@ private CrcV2TrustData CrcV2Trust(Block block, TxReceipt receipt, LogEntry log, limit); } - private CrcV2StoppedData CrcV2Stopped(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private Stopped CrcV2Stopped(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string address = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - return new CrcV2StoppedData( + return new Stopped( block.Number, (long)block.Timestamp, receipt.Index, diff --git a/Circles.Index/Indexer/BlockIndexer.cs b/Circles.Index/Indexer/BlockIndexer.cs index 516308e..7fdc200 100644 --- a/Circles.Index/Indexer/BlockIndexer.cs +++ b/Circles.Index/Indexer/BlockIndexer.cs @@ -78,7 +78,7 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) , _receiptFinder.Get(block))); TransformBlock receiptsSource = new( block => findReceipts.Call(block!) - , CreateOptions(cancellationToken)); + , CreateOptions(cancellationToken, Environment.ProcessorCount, Environment.ProcessorCount)); blockSource.LinkTo(receiptsSource, b => b != null); @@ -109,7 +109,7 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) receiptsSource.LinkTo(parser); ActionBlock<(BlockWithReceipts, IEnumerable)> sink = new(Sink, - CreateOptions(cancellationToken, 100000, 1)); + CreateOptions(cancellationToken, 50000, 1)); parser.LinkTo(sink); return blockSource; @@ -177,13 +177,13 @@ private async Task PerformFlushBlocks() var blocks = _blockBuffer.TakeSnapshot(); var map = new SchemaPropertyMap(); - map.Add("Block", new Dictionary>() + map.Add(("System", "Block"), new Dictionary> { - { "BlockNumber", o => o.Number }, - { "Timestamp", o => (long)o.Timestamp }, - { "Hash", o => o.Hash!.ToString() } + { "blockNumber", o => o.Number }, + { "timestamp", o => (long)o.Timestamp }, + { "blockHash", o => o.Hash!.ToString() } }); - await _sink.Database.WriteBatch("Block", blocks, map); + await _sink.Database.WriteBatch("System", "Block", blocks, map); } } \ No newline at end of file diff --git a/Circles.Index/Indexer/Context.cs b/Circles.Index/Indexer/Context.cs index d1b6790..b62533f 100644 --- a/Circles.Index/Indexer/Context.cs +++ b/Circles.Index/Indexer/Context.cs @@ -8,4 +8,6 @@ public record Context( INethermindApi NethermindApi, ILogger Logger, Settings Settings, - IDatabase Database); \ No newline at end of file + IDatabase Database, + ILogParser[] LogParsers, + Sink Sink); \ No newline at end of file diff --git a/Circles.Index/Indexer/StateMachine.cs b/Circles.Index/Indexer/StateMachine.cs index e10b3d4..f0b30be 100644 --- a/Circles.Index/Indexer/StateMachine.cs +++ b/Circles.Index/Indexer/StateMachine.cs @@ -1,4 +1,3 @@ -using Circles.Index.Common; using Circles.Index.Data; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; @@ -190,37 +189,17 @@ private async Task Sync() { context.Logger.Info("Starting syncing process."); - var v1Schema = new V1.DatabaseSchema(); - var v2Schema = new V2.DatabaseSchema(); - - IEventDtoTableMap compositeEventDtoTableMap = new CompositeEventDtoTableMap([ - v1Schema.EventDtoTableMap, v2Schema.EventDtoTableMap - ]); - - ISchemaPropertyMap compositeSchemaPropertyMap = new CompositeSchemaPropertyMap([ - v1Schema.SchemaPropertyMap, v2Schema.SchemaPropertyMap - ]); - - Sink sink = new Sink(context.Database, compositeSchemaPropertyMap, - compositeEventDtoTableMap); - - ILogParser[] parsers = - [ - new V1.LogParser(context.Settings.CirclesV1HubAddress), - new V2.LogParser(context.Settings.CirclesV2HubAddress) - ]; - try { ImportFlow flow = new ImportFlow(blockTree , receiptFinder - , parsers - , sink); + , context.LogParsers + , context.Sink); IAsyncEnumerable blocksToSync = GetBlocksToSync(); Range importedBlockRange = await flow.Run(blocksToSync, cancellationToken); - await sink.Flush(); + await context.Sink.Flush(); await flow.FlushBlocks(); if (importedBlockRange is { Min: long.MaxValue, Max: long.MinValue }) diff --git a/Circles.Index/Plugin.cs b/Circles.Index/Plugin.cs index 80b7725..0991ec9 100644 --- a/Circles.Index/Plugin.cs +++ b/Circles.Index/Plugin.cs @@ -31,10 +31,11 @@ public Task Init(INethermindApi nethermindApi) IDatabaseSchema common = new Common.DatabaseSchema(); IDatabaseSchema v1 = new V1.DatabaseSchema(); IDatabaseSchema v2 = new V2.DatabaseSchema(); + IDatabaseSchema databaseSchema = new CompositeDatabaseSchema([common, v1, v2]); ILogger baseLogger = nethermindApi.LogManager.GetClassLogger(); - ILogger pluginLogger = new LoggerWithPrefix(Name, baseLogger); + ILogger pluginLogger = new LoggerWithPrefix($"{Name}: ", baseLogger); Settings settings = new(); pluginLogger.Info("Index Db connection string: " + settings.IndexDbConnectionString); @@ -43,9 +44,26 @@ public Task Init(INethermindApi nethermindApi) pluginLogger.Info("Start index from: " + settings.StartBlock); IDatabase database = new PostgresDb(settings.IndexDbConnectionString, databaseSchema, pluginLogger); - _indexerContext = new Context(nethermindApi, pluginLogger, settings, database); + + Sink sink = new Sink( + database, + new CompositeSchemaPropertyMap([ + v1.SchemaPropertyMap, v2.SchemaPropertyMap + ]), + new CompositeEventDtoTableMap([ + v1.EventDtoTableMap, v2.EventDtoTableMap + ])); + + ILogParser[] logParsers = + [ + new V1.LogParser(settings.CirclesV1HubAddress), + new V2.LogParser(settings.CirclesV2HubAddress) + ]; + + _indexerContext = new Context(nethermindApi, pluginLogger, settings, database, logParsers, sink); _indexerContext.Database.Migrate(); + Query.Initialize(_indexerContext.Database); Run(nethermindApi); @@ -137,7 +155,7 @@ public Task InitNetworkProtocol() public async Task InitRpcModules() { - await Task.Delay(1000); + await Task.Delay(5000); if (_indexerContext?.NethermindApi == null) { @@ -152,8 +170,6 @@ public async Task InitRpcModules() CirclesRpcModule circlesRpcModule = new(_indexerContext); _indexerContext.NethermindApi.ForRpc.GetFromApi.RpcModuleProvider?.Register( new SingletonModulePool(circlesRpcModule)); - - return; } public ValueTask DisposeAsync() diff --git a/Circles.Index/Rpc/CirclesRpcModule.cs b/Circles.Index/Rpc/CirclesRpcModule.cs index 6be8f80..9800dd9 100644 --- a/Circles.Index/Rpc/CirclesRpcModule.cs +++ b/Circles.Index/Rpc/CirclesRpcModule.cs @@ -22,6 +22,7 @@ public CirclesRpcModule(Context indexerContext) { ILogger baseLogger = indexerContext.NethermindApi.LogManager.GetClassLogger(); _pluginLogger = new LoggerWithPrefix("Circles.Index.Rpc:", baseLogger); + _indexerContext = indexerContext; } public async Task> circles_getTotalBalance(Address address) @@ -53,7 +54,7 @@ public ResultWrapper> circles_query(CirclesQuery query) throw new InvalidOperationException("Table is null"); } - var select = Query.Select(query.Table, + var select = Query.Select((query.Namespace, query.Table), query.Columns ?? throw new InvalidOperationException("Columns are null")); if (query.Conditions.Count != 0) @@ -98,14 +99,14 @@ public ResultWrapper circles_computeTransfer(string from, string to, str private IEnumerable
TokenAddressesForAccount(Address circlesAccount) { var select = Query.Select( - "Erc20Transfer" + ("CrcV1", "Transfer") , new[] { "TokenAddress" }) .Where( Query.Equals( - "Erc20Transfer" + ("CrcV1", "Transfer") , "ToAddress" , circlesAccount.ToString(true, false))); @@ -180,7 +181,7 @@ private UInt256 TotalBalance(IEthRpcModule rpcModule, Address address) return totalBalance; } - private IQuery BuildCondition(string table, QueryExpression queryExpression) + private IQuery BuildCondition((string Namespace, string Table) table, QueryExpression queryExpression) { if (queryExpression.Type == "Equals") { diff --git a/Circles.Index/Rpc/ICirclesRpcModule.cs b/Circles.Index/Rpc/ICirclesRpcModule.cs index 854700b..b5c21ea 100644 --- a/Circles.Index/Rpc/ICirclesRpcModule.cs +++ b/Circles.Index/Rpc/ICirclesRpcModule.cs @@ -24,6 +24,7 @@ public CirclesTokenBalance(Address token, string balance) public class CirclesQuery { + public string? Namespace { get; set; } public string? Table { get; set; } public string[]? Columns { get; set; } public List Conditions { get; set; } = new();