Ethereum eventdata

core/build.gradle

@@ -74,6 +74,8 @@ dependencies {
         exclude group: "com.github.spotbugs"
     }
 
+    implementation 'com.squareup.okhttp3:okhttp:4.11.0'
+
     // --- Test Compile ---
     testImplementation group: "junit", name: "junit", version: junit_version
     testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version  // BSD 3-clause

core/src/main/java/org/polypheny/db/adapter/Adapter.java

@@ -81,6 +81,8 @@ public abstract class Adapter {
     @Getter
     private final String adapterName;
 
+    public boolean canCache = false;
+
 
     @Target(ElementType.TYPE)
     @Retention(RetentionPolicy.RUNTIME)

core/src/main/java/org/polypheny/db/adapter/DataSource.java

@@ -23,9 +23,12 @@
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.pf4j.ExtensionPoint;
+import org.polypheny.db.catalog.Catalog.Collation;
 import org.polypheny.db.catalog.entity.CatalogAdapter.AdapterType;
 import org.polypheny.db.catalog.entity.CatalogGraphDatabase;
 import org.polypheny.db.catalog.entity.CatalogGraphPlacement;
+import org.polypheny.db.ddl.DdlManager.ColumnTypeInformation;
+import org.polypheny.db.ddl.DdlManager.FieldInformation;
 import org.polypheny.db.prepare.Context;
 import org.polypheny.db.type.PolyType;
 
@@ -93,6 +96,16 @@ public String getDisplayType() {
             return typeStr;
         }
 
+
+        public ColumnTypeInformation toColumnTypeInformation() {
+            return new ColumnTypeInformation( type, collectionsType, length, scale, dimension, cardinality, nullable );
+
+        }
+
+        public FieldInformation toFieldInformation(){
+            return new FieldInformation( name, toColumnTypeInformation(), PolyType.STRING_TYPES.contains( type ) ? Collation.getDefaultCollation() : null, null, physicalPosition );
+        }
+
     }
 
 

core/src/main/java/org/polypheny/db/catalog/Catalog.java

@@ -98,6 +98,7 @@
 
 public abstract class Catalog implements ExtensionPoint {
 
+    public static final String HIDDEN_PREFIX = "__hidden__";
     public static Adapter defaultStore;
     public static Adapter defaultSource;
     public static int defaultUserId = 0;
@@ -490,9 +491,11 @@ protected final boolean isValidIdentifier( final String str ) {
      * @param ownerId The if of the owner
      * @param entityType The table type
      * @param modifiable Whether the content of the table can be modified
+     * @param cached
+     * @param hidden
      * @return The id of the inserted table
      */
-    public abstract long addTable( String name, long namespaceId, int ownerId, EntityType entityType, boolean modifiable );
+    public abstract long addTable( String name, long namespaceId, int ownerId, EntityType entityType, boolean modifiable, boolean cached, boolean hidden );
 
 
     /**

core/src/main/java/org/polypheny/db/catalog/entity/CatalogTable.java

@@ -53,6 +53,8 @@ public class CatalogTable implements CatalogObject, Comparable<CatalogTable> {
 
     @Getter
     public final ImmutableList<Long> connectedViews;
+    public final boolean cached;
+    public final boolean hidden;
 
 
     public CatalogTable(
@@ -66,7 +68,9 @@ public CatalogTable(
             final Long primaryKey,
             @NonNull final ImmutableList<Integer> dataPlacements,
             boolean modifiable,
-            PartitionProperty partitionProperty ) {
+            PartitionProperty partitionProperty,
+            boolean cached,
+            boolean hidden ) {
         this.id = id;
         this.name = name;
         this.fieldIds = fieldIds;
@@ -76,6 +80,8 @@ public CatalogTable(
         this.entityType = type;
         this.primaryKey = primaryKey;
         this.modifiable = modifiable;
+        this.cached = cached;
+        this.hidden = hidden;
 
         this.partitionProperty = partitionProperty;
         this.connectedViews = ImmutableList.of();
@@ -100,7 +106,9 @@ public CatalogTable(
             @NonNull final ImmutableList<Integer> dataPlacements,
             boolean modifiable,
             PartitionProperty partitionProperty,
-            ImmutableList<Long> connectedViews ) {
+            ImmutableList<Long> connectedViews,
+            boolean cached,
+            boolean hidden ) {
         this.id = id;
         this.name = name;
         this.fieldIds = fieldIds;
@@ -116,6 +124,8 @@ public CatalogTable(
         this.connectedViews = connectedViews;
 
         this.dataPlacements = ImmutableList.copyOf( dataPlacements );
+        this.cached = cached;
+        this.hidden = hidden;
 
         if ( type == EntityType.ENTITY && !modifiable ) {
             throw new RuntimeException( "Tables of table type TABLE must be modifiable!" );
@@ -220,7 +230,7 @@ public CatalogTable getRenamed( String newName ) {
                 dataPlacements,
                 modifiable,
                 partitionProperty,
-                connectedViews );
+                connectedViews, false, false );
     }
 
 
@@ -237,7 +247,9 @@ public CatalogTable getConnectedViews( ImmutableList<Long> newConnectedViews ) {
                 dataPlacements,
                 modifiable,
                 partitionProperty,
-                newConnectedViews );
+                newConnectedViews,
+                cached,
+                hidden );
     }
 
 
@@ -254,7 +266,9 @@ public CatalogTable getTableWithColumns( ImmutableList<Long> newColumnIds ) {
                 dataPlacements,
                 modifiable,
                 partitionProperty,
-                connectedViews );
+                connectedViews,
+                cached,
+                hidden );
     }
 
 

core/src/main/java/org/polypheny/db/catalog/entity/CatalogView.java

@@ -65,7 +65,7 @@ public CatalogView(
             ImmutableMap<Long, ImmutableList<Long>> underlyingTables,
             String language ) {
         super( id, name, columnIds, schemaId, databaseId, ownerId, entityType, primaryKey, dataPlacements,
-                modifiable, partitionProperty, connectedViews );
+                modifiable, partitionProperty, connectedViews, false, false );
         this.query = query;
         this.algCollation = algCollation;
         this.underlyingTables = underlyingTables;

core/src/main/java/org/polypheny/db/ddl/DdlManager.java

@@ -449,9 +449,11 @@ public static DdlManager getInstance() {
      * @param ifNotExists whether to silently ignore if the table already exists
      * @param stores list of data stores on which to create a full placement for this table
      * @param placementType which placement type should be used for the initial placements
+     * @param cached
      * @param statement the used statement
+     * @param hidden
      */
-    public abstract void createTable( long schemaId, String tableName, List<FieldInformation> columns, List<ConstraintInformation> constraints, boolean ifNotExists, List<DataStore> stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException, ColumnNotExistsException, UnknownPartitionTypeException, UnknownColumnException, PartitionGroupNamesNotUniqueException;
+    public abstract void createTable( long schemaId, String tableName, List<FieldInformation> columns, List<ConstraintInformation> constraints, boolean ifNotExists, List<DataStore> stores, PlacementType placementType, boolean cached, Statement statement, boolean hidden ) throws EntityAlreadyExistsException, ColumnNotExistsException, UnknownPartitionTypeException, UnknownColumnException, PartitionGroupNamesNotUniqueException;
 
     /**
      * Create a new view

core/src/main/java/org/polypheny/db/tools/AlgBuilder.java

@@ -34,17 +34,60 @@
 package org.polypheny.db.tools;
 
 
+import static org.polypheny.db.util.Static.RESOURCE;
+
 import com.google.common.base.Preconditions;
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import lombok.Getter;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.linq4j.function.Experimental;
 import org.bson.BsonValue;
-import org.polypheny.db.algebra.*;
+import org.polypheny.db.algebra.AlgCollation;
+import org.polypheny.db.algebra.AlgCollations;
+import org.polypheny.db.algebra.AlgDistribution;
+import org.polypheny.db.algebra.AlgFieldCollation;
+import org.polypheny.db.algebra.AlgNode;
 import org.polypheny.db.algebra.constant.Kind;
 import org.polypheny.db.algebra.constant.SemiJoinType;
-import org.polypheny.db.algebra.core.*;
+import org.polypheny.db.algebra.core.Aggregate;
+import org.polypheny.db.algebra.core.AggregateCall;
+import org.polypheny.db.algebra.core.AlgFactories;
 import org.polypheny.db.algebra.core.AlgFactories.ScanFactory;
+import org.polypheny.db.algebra.core.CorrelationId;
+import org.polypheny.db.algebra.core.Filter;
+import org.polypheny.db.algebra.core.Intersect;
+import org.polypheny.db.algebra.core.Join;
+import org.polypheny.db.algebra.core.JoinAlgType;
+import org.polypheny.db.algebra.core.Match;
+import org.polypheny.db.algebra.core.Minus;
+import org.polypheny.db.algebra.core.Modify.Operation;
+import org.polypheny.db.algebra.core.Project;
+import org.polypheny.db.algebra.core.Scan;
+import org.polypheny.db.algebra.core.SemiJoin;
+import org.polypheny.db.algebra.core.Sort;
+import org.polypheny.db.algebra.core.Union;
+import org.polypheny.db.algebra.core.Values;
 import org.polypheny.db.algebra.fun.AggFunction;
 import org.polypheny.db.algebra.logical.document.LogicalDocumentProject;
 import org.polypheny.db.algebra.logical.document.LogicalDocumentScan;
@@ -53,6 +96,7 @@
 import org.polypheny.db.algebra.logical.lpg.LogicalLpgProject;
 import org.polypheny.db.algebra.logical.lpg.LogicalLpgScan;
 import org.polypheny.db.algebra.logical.relational.LogicalFilter;
+import org.polypheny.db.algebra.logical.relational.LogicalModify;
 import org.polypheny.db.algebra.logical.relational.LogicalProject;
 import org.polypheny.db.algebra.metadata.AlgMetadataQuery;
 import org.polypheny.db.algebra.operators.OperatorName;
@@ -63,26 +107,46 @@
 import org.polypheny.db.languages.OperatorRegistry;
 import org.polypheny.db.languages.QueryLanguage;
 import org.polypheny.db.nodes.Operator;
-import org.polypheny.db.plan.*;
-import org.polypheny.db.rex.*;
+import org.polypheny.db.plan.AlgOptCluster;
+import org.polypheny.db.plan.AlgOptPredicateList;
+import org.polypheny.db.plan.AlgOptSchema;
+import org.polypheny.db.plan.AlgOptTable;
+import org.polypheny.db.plan.AlgOptUtil;
+import org.polypheny.db.plan.Context;
+import org.polypheny.db.plan.Contexts;
+import org.polypheny.db.prepare.Prepare.CatalogReader;
+import org.polypheny.db.rex.RexBuilder;
+import org.polypheny.db.rex.RexCall;
+import org.polypheny.db.rex.RexCorrelVariable;
+import org.polypheny.db.rex.RexExecutor;
+import org.polypheny.db.rex.RexInputRef;
+import org.polypheny.db.rex.RexLiteral;
+import org.polypheny.db.rex.RexNode;
+import org.polypheny.db.rex.RexShuttle;
+import org.polypheny.db.rex.RexSimplify;
+import org.polypheny.db.rex.RexUtil;
 import org.polypheny.db.runtime.Hook;
 import org.polypheny.db.runtime.PolyCollections.PolyDictionary;
 import org.polypheny.db.schema.ModelTrait;
 import org.polypheny.db.schema.SchemaPlus;
 import org.polypheny.db.schema.graph.PolyNode;
 import org.polypheny.db.transaction.Statement;
 import org.polypheny.db.type.PolyType;
-import org.polypheny.db.util.*;
+import org.polypheny.db.util.DateString;
+import org.polypheny.db.util.Holder;
+import org.polypheny.db.util.ImmutableBitSet;
+import org.polypheny.db.util.ImmutableIntList;
+import org.polypheny.db.util.ImmutableNullableList;
+import org.polypheny.db.util.Litmus;
+import org.polypheny.db.util.NlsString;
+import org.polypheny.db.util.Pair;
+import org.polypheny.db.util.TimeString;
+import org.polypheny.db.util.TimestampString;
+import org.polypheny.db.util.Util;
+import org.polypheny.db.util.ValidatorUtil;
 import org.polypheny.db.util.mapping.Mapping;
 import org.polypheny.db.util.mapping.Mappings;
 
-import javax.annotation.Nonnull;
-import java.math.BigDecimal;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.polypheny.db.util.Static.RESOURCE;
-
 
 /**
  * Builder for relational expressions.
@@ -2501,6 +2565,13 @@ public void clear() {
     }
 
 
+    public AlgBuilder insert( AlgOptTable table ) {
+        LogicalModify modify = LogicalModify.create( table, (CatalogReader) algOptSchema, stack.pop().alg, Operation.INSERT, null, null, false );
+        stack.add( new Frame( modify ) );
+        return this;
+    }
+
+
     /**
      * Information necessary to create a call to an aggregate function.
      *

core/src/main/java/org/polypheny/db/util/BsonUtil.java

@@ -20,6 +20,7 @@
 import java.io.InputStream;
 import java.io.PushbackInputStream;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
@@ -354,6 +355,8 @@ private static BsonValue handleBigInt( Object obj ) {
 
         if ( obj instanceof Long ) {
             return new BsonInt64( (Long) obj );
+        } else if ( obj instanceof BigInteger ) {
+            return new BsonInt64(( (BigInteger) obj ).longValue());
         } else {
             return new BsonInt64( (Integer) obj );
         }

core/src/test/java/org/polypheny/db/catalog/MockCatalog.java

@@ -348,7 +348,7 @@ public CatalogTable getTable( long databaseId, String schemaName, String tableNa
 
 
     @Override
-    public long addTable( String name, long namespaceId, int ownerId, EntityType entityType, boolean modifiable ) {
+    public long addTable( String name, long namespaceId, int ownerId, EntityType entityType, boolean modifiable, boolean cached, boolean hidden ) {
         throw new NotImplementedException();
     }
 

dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java

@@ -243,7 +243,8 @@ public void addAdapter( String uniqueName, String adapterName, AdapterType adapt
                 exportedColumns = ((DataSource) adapter).getExportedColumns();
             } catch ( Exception e ) {
                 AdapterManager.getInstance().removeAdapter( adapter.getAdapterId() );
-                throw new RuntimeException( "Could not deploy adapter", e );
+                // throw new RuntimeException( "Could not deploy adapter", e );
+                throw new RuntimeException( "Could not deploy adapter: " + e.getMessage(), e );
             }
             // Create table, columns etc.
             for ( Map.Entry<String, List<ExportedColumn>> entry : exportedColumns.entrySet() ) {
@@ -257,7 +258,7 @@ public void addAdapter( String uniqueName, String adapterName, AdapterType adapt
                     tableName += i;
                 }
 
-                long tableId = catalog.addTable( tableName, 1, 1, EntityType.SOURCE, !((DataSource) adapter).isDataReadOnly() );
+                long tableId = catalog.addTable( tableName, 1, 1, EntityType.SOURCE, !((DataSource) adapter).isDataReadOnly(), adapter.canCache, false );
                 List<Long> primaryKeyColIds = new ArrayList<>();
                 int colPos = 1;
                 String physicalSchemaName = null;
@@ -2171,7 +2172,7 @@ private List<Long> getUnderlyingColumns( AlgNode algNode, AlgDataType fieldList
 
 
     @Override
-    public void createTable( long schemaId, String name, List<FieldInformation> fields, List<ConstraintInformation> constraints, boolean ifNotExists, List<DataStore> stores, PlacementType placementType, Statement statement ) throws EntityAlreadyExistsException {
+    public void createTable( long schemaId, String name, List<FieldInformation> fields, List<ConstraintInformation> constraints, boolean ifNotExists, List<DataStore> stores, PlacementType placementType, boolean cached, Statement statement, boolean hidden ) throws EntityAlreadyExistsException {
         name = adjustNameIfNeeded( name, schemaId );
 
         try {
@@ -2209,7 +2210,9 @@ public void createTable( long schemaId, String name, List<FieldInformation> fiel
                     schemaId,
                     statement.getPrepareContext().getCurrentUserId(),
                     EntityType.ENTITY,
-                    true );
+                    true,
+                    cached,
+                    hidden );
 
             // Initially create DataPlacement containers on every store the table should be placed.
             stores.forEach( store -> catalog.addDataPlacement( store.getAdapterId(), tableId ) );

dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java

@@ -638,7 +638,7 @@ public void copyPartitionData( Transaction transaction, CatalogAdapter store, Ca
         CatalogPrimaryKey primaryKey = Catalog.getInstance().getPrimaryKey( sourceTable.primaryKey );
 
         // Check Lists
-        List<CatalogColumnPlacement> targetColumnPlacements = new LinkedList<>();
+        List<CatalogColumnPlacement> targetColumnPlacements = new ArrayList<>();
         for ( CatalogColumn catalogColumn : columns ) {
             targetColumnPlacements.add( Catalog.getInstance().getColumnPlacement( store.id, catalogColumn.id ) );
         }