From 9ebef7d57c53d12f887d36199fa721bf96bd3b2c Mon Sep 17 00:00:00 2001 From: Eslam Ashraf Date: Sun, 23 Jun 2024 18:58:59 +0300 Subject: [PATCH 01/24] add presntation type in model --- plugins/cypher-language/build.gradle | 2 ++ .../db/notebooks/model/response/NotebookCellModel.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/plugins/cypher-language/build.gradle b/plugins/cypher-language/build.gradle index 3950c9186d..759c6d4ec9 100644 --- a/plugins/cypher-language/build.gradle +++ b/plugins/cypher-language/build.gradle @@ -98,6 +98,8 @@ jar { attributes "Copyright": "The Polypheny Project (polypheny.org)" attributes "Version": "$project.version" } + duplicatesStrategy(DuplicatesStrategy.EXCLUDE) + } java { withJavadocJar() diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java index eff0cb22a5..d52663e657 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java @@ -30,6 +30,10 @@ public class NotebookCellModel { @SerializedName("cell_type") private String cellType; + @Getter + @SerializedName("cell_present") + private String presentType; + @Getter private String id; @@ -54,6 +58,7 @@ public NotebookCellModel( String code ) { public NotebookCellModel( String code, List outputs ) { this.cellType = "code"; + this.presentType="skip"; this.id = UUID.randomUUID().toString(); this.source = code; this.metadata = new CellMetadataModel(); From 5f0e3f713f22f5344942affd013021e9f1f58125 Mon Sep 17 00:00:00 2001 From: Eslam Ashraf Date: Sun, 23 Jun 2024 19:13:02 +0300 Subject: [PATCH 02/24] add showOutput flag in notebook model --- .../db/notebooks/model/response/NotebookCellModel.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java index d52663e657..3d092c60fb 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/response/NotebookCellModel.java @@ -50,6 +50,9 @@ public class NotebookCellModel { @Getter private List outputs; + @Getter + private boolean showOutput; + public NotebookCellModel( String code ) { this( code, new LinkedList<>() ); @@ -59,6 +62,7 @@ public NotebookCellModel( String code ) { public NotebookCellModel( String code, List outputs ) { this.cellType = "code"; this.presentType="skip"; + this.showOutput=true; this.id = UUID.randomUUID().toString(); this.source = code; this.metadata = new CellMetadataModel(); From 1a6a259cd5011cac78f18cb3a0b09c7f0b641ed8 Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 24 Jun 2024 14:48:16 +0200 Subject: [PATCH 03/24] safety in statistic to prevent random race-conditions --- .../polypheny/db/algebra/convert/ConverterImpl.java | 5 ++++- .../statistics/NumericalStatisticColumn.java | 10 ++++++++-- .../db/monitoring/statistics/QueryResult.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/convert/ConverterImpl.java b/core/src/main/java/org/polypheny/db/algebra/convert/ConverterImpl.java index 4201565fb8..6f8a9f1e94 100644 --- a/core/src/main/java/org/polypheny/db/algebra/convert/ConverterImpl.java +++ b/core/src/main/java/org/polypheny/db/algebra/convert/ConverterImpl.java @@ -70,7 +70,10 @@ protected ConverterImpl( AlgCluster cluster, AlgTraitDef traitDef, AlgTraitSe @Override public AlgOptCost computeSelfCost( AlgPlanner planner, AlgMetadataQuery mq ) { - double dRows = mq.getTupleCount( getInput() ); + Double dRows = mq.getTupleCount( getInput() ); + if ( dRows == null ) { + dRows = Double.MAX_VALUE; + } double dIo = 0; return planner.getCostFactory().makeCost( dRows, dRows, dIo ); } diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java index 0386442dbe..ba073737b9 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java @@ -19,7 +19,12 @@ import com.google.gson.annotations.Expose; import java.util.List; +import java.util.Set; +import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArraySet; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -44,8 +49,8 @@ public class NumericalStatisticColumn extends StatisticColumn { @Setter private PolyNumber max; - private final TreeSet minCache = new TreeSet<>(); - private final TreeSet maxCache = new TreeSet<>(); + private final SortedSet minCache = new ConcurrentSkipListSet<>(); + private final SortedSet maxCache = new ConcurrentSkipListSet<>(); public NumericalStatisticColumn( QueryResult column ) { @@ -72,6 +77,7 @@ public void insert( PolyValue val ) { if ( !uniqueValues.isEmpty() ) { uniqueValues.add( val ); } + minCache.add( val.asNumber() ); maxCache.add( val.asNumber() ); } diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/QueryResult.java b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/QueryResult.java index 09ad9ef519..1f07321718 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/QueryResult.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/QueryResult.java @@ -28,7 +28,7 @@ */ @Getter @Data -class QueryResult { +public class QueryResult { private final Entity entity; private final LogicalColumn column; From 40a1ae52e74c7ddadbb8780d3902f2d46c48fa10 Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 27 Jun 2024 15:56:10 +0200 Subject: [PATCH 04/24] fixed notebooks queries --- .../db/algebra/operators/OperatorName.java | 2 +- .../db/notebooks/model/JupyterKernel.java | 50 ++++++++++--------- .../model/language/IPythonKernelLanguage.java | 5 +- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java b/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java index 80fefb1cfb..78b038a31b 100644 --- a/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java +++ b/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java @@ -1456,4 +1456,4 @@ public enum OperatorName { MQL_LT, MQL_LTE ); -} \ No newline at end of file +} diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java index f977894e4a..73bcf96381 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java @@ -16,6 +16,8 @@ package org.polypheny.db.notebooks.model; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; @@ -23,6 +25,7 @@ import java.io.IOException; import java.net.URI; import java.net.http.WebSocket; +import java.net.http.WebSocket.Builder; import java.net.http.WebSocket.Listener; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -39,11 +42,13 @@ import org.eclipse.jetty.websocket.api.Session; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; +import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.notebooks.model.language.JupyterKernelLanguage; import org.polypheny.db.notebooks.model.language.JupyterKernelLanguage.JupyterQueryPart; import org.polypheny.db.notebooks.model.language.JupyterLanguageFactory; import org.polypheny.db.processing.QueryContext; +import org.polypheny.db.webui.HttpServer; import org.polypheny.db.webui.crud.LanguageCrud; import org.polypheny.db.webui.models.requests.QueryRequest; import org.polypheny.db.webui.models.results.Result; @@ -64,12 +69,13 @@ public class JupyterKernel { private final WebSocket webSocket; private final Set subscribers = new HashSet<>(); private final Gson gson = new Gson(); - private final Gson resultSetGson; private final JsonObject statusMsg; private boolean isRestarting = false; private final Map activePolyCells = new ConcurrentHashMap<>(); private final JupyterSessionManager jsm = JupyterSessionManager.getInstance(); + private final ObjectMapper mapper = new ObjectMapper(); + /** * Create a new JupyterKernel and open a websocket connection to the corresponding kernel in the jupyter server. @@ -79,7 +85,7 @@ public class JupyterKernel { * @param builder the Websocket.Builder used to open the websocket * @param host the address of the Docker container running the jupyter server */ - public JupyterKernel( String kernelId, String name, WebSocket.Builder builder, String host ) { + public JupyterKernel( String kernelId, String name, Builder builder, String host ) { this.kernelId = kernelId; this.name = name; this.clientId = UUID.randomUUID().toString(); @@ -97,9 +103,6 @@ public JupyterKernel( String kernelId, String name, WebSocket.Builder builder, S content.addProperty( "execution_state", "starting" ); this.statusMsg.add( "content", content ); sendInitCode(); - - GsonBuilder gsonBuilder = new GsonBuilder(); - resultSetGson = gsonBuilder.create(); } @@ -215,9 +218,13 @@ public void execute( String code, String uuid ) { * @param expandParams whether to expand variables specified in the form ${varname} */ public void executePolyCell( - String query, String uuid, String language, String namespace, - String variable, boolean expandParams ) { - if ( query.strip().isEmpty() ) { + String query, + String uuid, + String language, + String namespace, + String variable, + boolean expandParams ) { + if ( query.isBlank() ) { execute( "", uuid ); // properly terminates the request return; } @@ -257,7 +264,11 @@ private String anyQuery( String query, String language, String namespace ) { .origin( "Notebooks" ) .transactionManager( jsm.getTransactionManager() ) .build(), queryRequest ); - return resultSetGson.toJson( results ); + try { + return mapper.writeValueAsString( results ); + } catch ( JsonProcessingException e ) { + throw new GenericRuntimeException( e ); + } } @@ -384,14 +395,14 @@ public int getSubscriberCount() { } - private class WebSocketClient implements WebSocket.Listener { + private class WebSocketClient implements Listener { private final StringBuilder textBuilder = new StringBuilder(); @Override public void onOpen( WebSocket webSocket ) { - WebSocket.Listener.super.onOpen( webSocket ); + Listener.super.onOpen( webSocket ); } @@ -402,7 +413,7 @@ public CompletionStage onText( WebSocket webSocket, CharSequence data, boolea handleText( textBuilder.toString() ); textBuilder.setLength( 0 ); } - return WebSocket.Listener.super.onText( webSocket, data, last ); + return Listener.super.onText( webSocket, data, last ); } @@ -425,18 +436,9 @@ public void onError( WebSocket webSocket, Throwable error ) { /** - * Represents a query cell that is currently being executed. - */ - private static class ActivePolyCell { - - @Getter - private final String language, namespace; - - - public ActivePolyCell( String language, String namespace ) { - this.language = language; - this.namespace = namespace; - } + * Represents a query cell that is currently being executed. + */ + private record ActivePolyCell( @Getter String language, @Getter String namespace ) { } diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java index 565fccf991..574c637033 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java @@ -16,6 +16,7 @@ package org.polypheny.db.notebooks.model.language; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -34,7 +35,7 @@ public List getInitCode() { @Override public List transformToQuery( String query, String language, String namespace, String varName, boolean expandParams ) { - List queries = new LinkedList<>(); + List queries = new ArrayList<>(); String cleanQuery = query.strip(); // -i: use stdin, -j: output json, -t: query is template @@ -54,7 +55,7 @@ public List getExportedInitCode() { @Override public List exportedQuery( String query, String language, String namespace, String varName, boolean expandParams ) { - List code = new LinkedList<>(); + List code = new ArrayList<>(); String cleanQuery = query.strip(); code.add( (expandParams ? "%%poly -t " : "%%poly ") + language + " " + namespace + "\n" + cleanQuery ); code.add( varName + " = _" ); From 297d0e81b3522790ff91af44fc8aab3bf0646bc7 Mon Sep 17 00:00:00 2001 From: datomo Date: Fri, 28 Jun 2024 21:36:15 +0200 Subject: [PATCH 05/24] adjusting of locking mechanism --- .../org/polypheny/db/algebra/AlgNode.java | 9 +++ ...EnumerableConditionalExecuteFalseRule.java | 10 ++- .../entity/logical/LogicalForeignKey.java | 1 - .../processing/LogicalAlgAnalyzeShuttle.java | 2 +- .../org/polypheny/db/transaction/PUID.java | 7 +- .../polypheny/db/transaction/Transaction.java | 8 +- .../org/polypheny/db/ddl/DdlManagerImpl.java | 8 +- .../db/processing/AbstractQueryProcessor.java | 28 +++++-- .../processing/ConstraintEnforceAttacher.java | 2 +- .../org/polypheny/db/transaction/Lock.java | 8 +- .../db/transaction/TransactionImpl.java | 31 ++++---- .../polypheny/db/adapter/jdbc/JdbcSchema.java | 76 +------------------ .../jdbc/sources/AbstractJdbcSource.java | 21 +++-- .../jdbc/stores/AbstractJdbcStore.java | 23 ++++-- .../db/prisminterface/ClientManager.java | 2 - .../PIPreparedIndexedStatement.java | 2 +- .../statements/PIPreparedStatement.java | 7 +- .../java/org/polypheny/db/webui/Crud.java | 7 -- 18 files changed, 105 insertions(+), 147 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/AlgNode.java b/core/src/main/java/org/polypheny/db/algebra/AlgNode.java index 08293b170c..0f7449dcbf 100644 --- a/core/src/main/java/org/polypheny/db/algebra/AlgNode.java +++ b/core/src/main/java/org/polypheny/db/algebra/AlgNode.java @@ -37,6 +37,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.jetbrains.annotations.Nullable; import org.polypheny.db.algebra.core.Correlate; import org.polypheny.db.algebra.core.CorrelationId; @@ -392,6 +394,13 @@ default boolean containsEntity() { return getInputs().stream().anyMatch( AlgNode::containsEntity ); } + default Set getEntities(){ + if ( getEntity() != null ) { + return Set.of(getEntity()); + } + return getInputs().stream().map( AlgNode::getEntities ).reduce( (a,b) -> Stream.concat(a.stream(), b.stream()).collect( Collectors.toSet()) ).orElse( Set.of() ); + } + /** * Context of an algebra expression, for purposes of checking validity. */ diff --git a/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java b/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java index e566f50296..3581161bd8 100644 --- a/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java +++ b/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java @@ -25,6 +25,7 @@ import org.polypheny.db.algebra.core.common.ConditionalExecute.Condition; import org.polypheny.db.algebra.enumerable.EnumerableConvention; import org.polypheny.db.algebra.logical.common.LogicalConditionalExecute; +import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.plan.Convention; @@ -38,11 +39,14 @@ public EnumerableConditionalExecuteFalseRule() { } - @SneakyThrows @Override public AlgNode convert( AlgNode alg ) { ConditionalExecute ce = (ConditionalExecute) alg; - throw ce.getExceptionClass().getConstructor( String.class ).newInstance( ce.getExceptionMessage() ); + try { + throw ce.getExceptionClass().getConstructor( String.class ).newInstance( ce.getExceptionMessage() ); + } catch ( Exception e ) { + throw new GenericRuntimeException( e ); + } } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java index 692a4ec2de..89811943cd 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java @@ -152,7 +152,6 @@ public static class LogicalForeignKeyField implements PolyObject { private final String foreignKeyFieldName; - @SneakyThrows @Override public PolyValue[] getParameterArray() { return Catalog.snapshot() diff --git a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java index 28f5d785a9..b8ab8df0ff 100644 --- a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java +++ b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java @@ -85,7 +85,7 @@ public class LogicalAlgAnalyzeShuttle extends AlgShuttleImpl { public Set hashBasis = new HashSet<>(); - public Map availableColumns = new LinkedHashMap<>(); // column id -> schemaName.tableName.ColumnName + public Map availableColumns = new HashMap<>(); // column id -> schemaName.tableName.ColumnName public Map availableColumnsWithTable = new HashMap<>(); // columnId -> tableId diff --git a/core/src/main/java/org/polypheny/db/transaction/PUID.java b/core/src/main/java/org/polypheny/db/transaction/PUID.java index 634df05c4b..2a2a3a85ad 100644 --- a/core/src/main/java/org/polypheny/db/transaction/PUID.java +++ b/core/src/main/java/org/polypheny/db/transaction/PUID.java @@ -17,9 +17,11 @@ package org.polypheny.db.transaction; +import java.io.Serial; import java.io.Serializable; import java.util.UUID; import lombok.EqualsAndHashCode; +import lombok.Getter; /** @@ -28,6 +30,7 @@ @EqualsAndHashCode(doNotUseGetters = true) public class PUID implements Serializable { + @Serial private static final long serialVersionUID = 1L; public static final PUID EMPTY_PUID = new PUID( 0L, 0L ); @@ -112,6 +115,7 @@ public String toString() { } + @Getter public enum Type { OTHER( (byte) 0x00 ), RANDOM( (byte) 0xff ), @@ -139,9 +143,6 @@ public static Type extractType( final PUID puid ) { } - public byte getIndicator() { - return indicator; - } } diff --git a/core/src/main/java/org/polypheny/db/transaction/Transaction.java b/core/src/main/java/org/polypheny/db/transaction/Transaction.java index 73fc015925..e10227d2c1 100644 --- a/core/src/main/java/org/polypheny/db/transaction/Transaction.java +++ b/core/src/main/java/org/polypheny/db/transaction/Transaction.java @@ -48,7 +48,7 @@ public interface Transaction { void registerInvolvedAdapter( Adapter adapter ); - List> getInvolvedAdapters(); + Set> getInvolvedAdapters(); Snapshot getSnapshot(); @@ -68,8 +68,6 @@ public interface Transaction { LogicalNamespace getDefaultNamespace(); - void addChangedTable( String qualifiedTableName ); - String getOrigin(); MultimediaFlavor getFlavor(); @@ -82,7 +80,9 @@ public interface Transaction { boolean getUseCache(); - Set getLogicalTables(); + void addUsedTable(LogicalTable table); + + void removeUsedTable(LogicalTable table); void setAcceptsOutdated( boolean acceptsOutdated ); diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 611e4ede96..b368520c95 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -545,7 +545,7 @@ public void createForeignKey( LogicalTable table, LogicalTable refTable, List columnNames, Stat } } } - statement.getTransaction().getLogicalTables().add( table ); + statement.getTransaction().addUsedTable( table ); } @@ -846,7 +846,7 @@ public void createUniqueConstraint( LogicalTable table, List columnNames columnIds.add( logicalColumn.id ); } catalog.getLogicalRel( table.namespaceId ).addUniqueConstraint( table.id, constraintName, columnIds ); - statement.getTransaction().getLogicalTables().add( table ); + statement.getTransaction().addUsedTable( table ); } @@ -2959,7 +2959,7 @@ public void dropTable( LogicalTable table, Statement statement ) { prepareMonitoring( statement, Kind.DROP_TABLE, table ); // ON_COMMIT constraint needs no longer to be enforced if entity does no longer exist - statement.getTransaction().getLogicalTables().remove( table ); + statement.getTransaction().removeUsedTable( table ); // Reset plan cache implementation cache & routing cache statement.getQueryProcessor().resetCaches(); diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index 36c53b6c6a..425edc5aa6 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -21,7 +21,6 @@ import com.google.common.collect.Lists; import java.lang.reflect.Type; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -78,8 +77,10 @@ import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.entity.Entity; +import org.polypheny.db.catalog.entity.allocation.AllocationEntity; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; import org.polypheny.db.catalog.entity.logical.LogicalTable; +import org.polypheny.db.catalog.entity.physical.PhysicalEntity; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.catalog.logistic.DataModel; import org.polypheny.db.config.RuntimeConfig; @@ -536,10 +537,12 @@ private List routePlans( AlgRoot indexLookupRoot, LogicalQueryInformation if ( plans == null ) { plans = route( indexLookupRoot, statement, logicalQueryInformation ).stream().map( p -> new Plan().proposedRoutingPlan( p ) ).toList(); } + return plans; } + private void flattenPlans( List plans ) { for ( Plan plan : plans ) { AlgRoot routedRoot = plan.proposedRoutingPlan().getRoutedRoot(); @@ -564,7 +567,7 @@ private void acquireLock( boolean isAnalyze, AlgRoot logicalRoot, Map> idAccessMap = new ArrayList<>(); + List> idAccessMap = new ArrayList<>(); // Get locks for individual entities EntityAccessMap accessMap = new EntityAccessMap( logicalRoot.alg, accessedPartitions ); // Get a shared global schema lock (only DDLs acquire an exclusive global schema lock) @@ -1185,8 +1188,8 @@ private LogicalQueryInformation analyzeQueryAndPrepareMonitoring( Statement stat logicalRoot.alg.accept( analyzer ); // Get partitions of logical information - Map> partitionValueFilterPerScan = analyzer.getPartitionValueFilterPerScan(); - Map> accessedPartitions = this.getAccessedPartitionsPerScan( logicalRoot.alg, partitionValueFilterPerScan ); + + Map> accessedPartitions = extractPartitions( logicalRoot.alg.getEntities() ); // Build queryClass from query-name and partitions. String queryHash = analyzer.getQueryName() + accessedPartitions; @@ -1209,6 +1212,21 @@ private LogicalQueryInformation analyzeQueryAndPrepareMonitoring( Statement stat } + private Map> extractPartitions( Set entities ) { + Map> map = new HashMap<>(); + for ( Entity entity : entities ) { + if ( entity.isLogical() ) { + map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).add( entity.id ); + }else if ( entity.isAllocation() ){ + map.computeIfAbsent( ((AllocationEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getLogicalId() ); + }else if( entity.isPhysical()){ + map.computeIfAbsent( ((PhysicalEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((PhysicalEntity)entity).getLogicalId() ); + } + } + return map; + } + + /** * Traverses all TablesScans used during execution and identifies for the corresponding table all * associated partitions that needs to be accessed, on the basis of the provided partitionValues identified in a LogicalFilter @@ -1490,7 +1508,7 @@ public void unlock( Statement statement ) { @Override public void lock( Statement statement ) { try { - LockManager.INSTANCE.lock( Collections.singletonList( Pair.of( LockManager.GLOBAL_LOCK, LockMode.SHARED ) ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( List.of( Pair.of( LockManager.GLOBAL_LOCK, LockMode.SHARED ) ), (TransactionImpl) statement.getTransaction() ); } catch ( DeadlockException e ) { throw new GenericRuntimeException( "DeadLock while locking to reevaluate statistics", e ); } diff --git a/dbms/src/main/java/org/polypheny/db/processing/ConstraintEnforceAttacher.java b/dbms/src/main/java/org/polypheny/db/processing/ConstraintEnforceAttacher.java index 4d1cb22afb..d1e8a58ca5 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/ConstraintEnforceAttacher.java +++ b/dbms/src/main/java/org/polypheny/db/processing/ConstraintEnforceAttacher.java @@ -131,7 +131,7 @@ public static void attachOnCommitConstraints( AlgNode node, Statement statement throw new GenericRuntimeException( "The tree did no conform, while generating the constraint enforcement query!" ); } - statement.getTransaction().getLogicalTables().add( modify.entity.unwrap( LogicalTable.class ).orElseThrow() ); + statement.getTransaction().addUsedTable( modify.entity.unwrap( LogicalTable.class ).orElseThrow() ); } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java index 97df60682b..b410f8c663 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java @@ -18,6 +18,7 @@ import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -28,7 +29,7 @@ // Based on code taken from https://github.com/dstibrany/LockManager public class Lock { - private final Set owners = new HashSet<>(); + private final Set owners = new CopyOnWriteArraySet<>(); private final ReentrantLock lock = new ReentrantLock( true ); private final Condition waiters = lock.newCondition(); private final WaitForGraph waitForGraph; @@ -112,11 +113,6 @@ LockMode getMode() { } - Set getOwners() { - return owners; - } - - private void acquireSLock( TransactionImpl txn ) throws InterruptedException { lock.lock(); try { diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java index 81921f3093..fbfb58ff15 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -94,13 +95,12 @@ public class TransactionImpl implements Transaction, Comparable { private final List statements = new ArrayList<>(); - private final List changedTables = new ArrayList<>(); @Getter private final Set logicalTables = new TreeSet<>(); @Getter - private final List> involvedAdapters = new CopyOnWriteArrayList<>(); + private final Set> involvedAdapters = new ConcurrentSkipListSet<>( (a,b) -> Math.toIntExact( a.adapterId - b.adapterId ) ); private final Set lockList = new HashSet<>(); private boolean useCache = true; @@ -146,9 +146,7 @@ public InformationManager getQueryAnalyzer() { @Override public void registerInvolvedAdapter( Adapter adapter ) { - if ( !involvedAdapters.contains( adapter ) ) { - involvedAdapters.add( adapter ); - } + involvedAdapters.add( adapter ); } @@ -282,17 +280,6 @@ public StatementImpl createStatement() { } - @Override - public void addChangedTable( String qualifiedTableName ) { - if ( !this.changedTables.contains( qualifiedTableName ) ) { - if ( log.isDebugEnabled() ) { - log.debug( "Add changed table: {}", qualifiedTableName ); - } - this.changedTables.add( qualifiedTableName ); - } - } - - @Override public int compareTo( @NonNull Object o ) { Transaction that = (Transaction) o; @@ -331,6 +318,18 @@ public boolean getUseCache() { } + @Override + public void addUsedTable( LogicalTable table ) { + this.logicalTables.add( table ); + } + + + @Override + public void removeUsedTable( LogicalTable table ) { + this.logicalTables.remove( table ); + } + + /** * Used to specify if a TX was started using freshness tolerance levels and * therefore allows the usage of outdated replicas. diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java index d808d04d73..97e371d3cd 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java @@ -74,31 +74,11 @@ public class JdbcSchema extends Namespace implements Expressible { @Getter private final JdbcConvention convention; - private final Map tableMap; - public final Adapter adapter; @Getter private final long id; - private JdbcSchema( - long id, - @NonNull ConnectionFactory connectionFactory, - @NonNull SqlDialect dialect, - JdbcConvention convention, - Map tableMap, - Adapter adapter ) { - super( id, adapter.getAdapterId() ); - this.id = id; - this.connectionFactory = connectionFactory; - this.dialect = dialect; - this.convention = convention; - this.tableMap = tableMap; - this.adapter = adapter; - - } - - /** * Creates a JDBC schema. * @@ -118,7 +98,6 @@ public JdbcSchema( this.dialect = dialect; convention.setJdbcSchema( this ); this.convention = convention; - this.tableMap = new HashMap<>(); this.adapter = adapter; } @@ -155,62 +134,9 @@ public ConnectionHandler getConnectionHandler( DataContext dataContext ) { } - protected Multimap getFunctions() { - // TODO: populate map from JDBC metadata - return ImmutableMultimap.of(); - } - - - public synchronized ImmutableMap getTableMap() { - return ImmutableMap.copyOf( tableMap ); - } - - - /** - * Given "INTEGER", returns BasicSqlType(INTEGER). - * Given "VARCHAR(10)", returns BasicSqlType(VARCHAR, 10). - * Given "NUMERIC(10, 2)", returns BasicSqlType(NUMERIC, 10, 2). - */ - private AlgDataType parseTypeString( AlgDataTypeFactory typeFactory, String typeString ) { - int precision = -1; - int scale = -1; - int open = typeString.indexOf( "(" ); - if ( open >= 0 ) { - int close = typeString.indexOf( ")", open ); - if ( close >= 0 ) { - String rest = typeString.substring( open + 1, close ); - typeString = typeString.substring( 0, open ); - int comma = rest.indexOf( "," ); - if ( comma >= 0 ) { - precision = Integer.parseInt( rest.substring( 0, comma ) ); - scale = Integer.parseInt( rest.substring( comma ) ); - } else { - precision = Integer.parseInt( rest ); - } - } - } - try { - final PolyType typeName = PolyType.valueOf( typeString ); - return typeName.allowsPrecScale( true, true ) - ? typeFactory.createPolyType( typeName, precision, scale ) - : typeName.allowsPrecScale( true, false ) - ? typeFactory.createPolyType( typeName, precision ) - : typeFactory.createPolyType( typeName ); - } catch ( IllegalArgumentException e ) { - return typeFactory.createTypeWithNullability( typeFactory.createPolyType( PolyType.ANY ), true ); - } - } - - - protected Map getTypes() { - // TODO: populate map from JDBC metadata - return ImmutableMap.of(); - } - - @Override public Expression asExpression() { - return this.adapter.getNamespaceAsExpression( id ); //todo change + return this.adapter.getNamespaceAsExpression( id ); } } diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java index 7040220735..3d4320b38a 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java @@ -159,11 +159,14 @@ protected void executeUpdate( StringBuilder builder, Context context ) { } - @SneakyThrows @Override public boolean prepare( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - return connectionFactory.getConnectionHandler( xid ).prepare(); + try { + return connectionFactory.getConnectionHandler( xid ).prepare(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to prepare (Unique name: {}, XID: {})! Returning true.", getUniqueName(), xid ); return true; @@ -171,22 +174,28 @@ public boolean prepare( PolyXid xid ) { } - @SneakyThrows @Override public void commit( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - connectionFactory.getConnectionHandler( xid ).commit(); + try { + connectionFactory.getConnectionHandler( xid ).commit(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to commit (Unique name: {}, XID: {})!", getUniqueName(), xid ); } } - @SneakyThrows @Override public void rollback( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - connectionFactory.getConnectionHandler( xid ).rollback(); + try { + connectionFactory.getConnectionHandler( xid ).rollback(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to rollback (Unique name: {}, XID: {})!", getUniqueName(), xid ); } diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java index 9cb8b65d67..2e4eec76ab 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.pf4j.ExtensionPoint; @@ -35,6 +34,7 @@ import org.polypheny.db.adapter.jdbc.JdbcTable; import org.polypheny.db.adapter.jdbc.JdbcUtils; import org.polypheny.db.adapter.jdbc.connection.ConnectionFactory; +import org.polypheny.db.adapter.jdbc.connection.ConnectionHandlerException; import org.polypheny.db.catalog.catalogs.RelAdapterCatalog; import org.polypheny.db.catalog.entity.allocation.AllocationCollection; import org.polypheny.db.catalog.entity.allocation.AllocationGraph; @@ -446,11 +446,14 @@ public void restoreCollection( AllocationCollection alloc, List } - @SneakyThrows @Override public boolean prepare( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - return connectionFactory.getConnectionHandler( xid ).prepare(); + try { + return connectionFactory.getConnectionHandler( xid ).prepare(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to prepare (Uniquename: {}, XID: {})! Returning true.", getUniqueName(), xid ); return true; @@ -458,22 +461,28 @@ public boolean prepare( PolyXid xid ) { } - @SneakyThrows @Override public void commit( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - connectionFactory.getConnectionHandler( xid ).commit(); + try { + connectionFactory.getConnectionHandler( xid ).commit(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to commit (Uniquename: {}, XID: {})!", getUniqueName(), xid ); } } - @SneakyThrows @Override public void rollback( PolyXid xid ) { if ( connectionFactory.hasConnectionHandler( xid ) ) { - connectionFactory.getConnectionHandler( xid ).rollback(); + try { + connectionFactory.getConnectionHandler( xid ).rollback(); + } catch ( ConnectionHandlerException e ) { + throw new GenericRuntimeException( e ); + } } else { log.warn( "There is no connection to rollback (Uniquename: {}, XID: {})!", getUniqueName(), xid ); } diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java index 93b3956a6e..8f300c7e22 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java @@ -94,8 +94,6 @@ String registerConnection( ConnectionRequest connectionRequest, Transport t ) th log.trace( "User {} tries to establish connection via prism interface.", uuid ); } final LogicalUser user = getUser( connectionRequest, t ); - Transaction transaction = transactionManager.startTransaction( user.id, false, "prism-interface" ); - transaction.commit(); LogicalNamespace namespace = getNamespaceOrDefault( connectionRequest ); boolean isAutocommit = getAutocommitOrDefault( connectionRequest ); PIClient client = new PIClient( diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java index 613b870133..da38a2f3df 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java @@ -71,7 +71,7 @@ public List executeBatch( List> valuesBatch ) { } StatementProcessor.implement( this ); updateCounts.add( StatementProcessor.executeAndGetResult( this ).getScalar() ); - //} + return updateCounts; } diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java index 5ad99896c5..402a5db79b 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java @@ -17,6 +17,7 @@ package org.polypheny.db.prisminterface.statements; import java.util.List; +import lombok.Setter; import org.jetbrains.annotations.NotNull; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; import org.polypheny.db.languages.QueryLanguage; @@ -24,16 +25,12 @@ import org.polypheny.db.prisminterface.statementProcessing.StatementProcessor; import org.polypheny.prism.ParameterMeta; +@Setter public abstract class PIPreparedStatement extends PIStatement implements Signaturizable { protected List parameterMetas; - public void setParameterMetas( List parameterMetas ) { - this.parameterMetas = parameterMetas; - } - - public List getParameterMetas() { if ( parameterMetas == null ) { StatementProcessor.prepare( this ); diff --git a/webui/src/main/java/org/polypheny/db/webui/Crud.java b/webui/src/main/java/org/polypheny/db/webui/Crud.java index 01ea338088..7e1f1b0e73 100644 --- a/webui/src/main/java/org/polypheny/db/webui/Crud.java +++ b/webui/src/main/java/org/polypheny/db/webui/Crud.java @@ -863,7 +863,6 @@ private String computeWherePK( final LogicalTable table, final Map Date: Mon, 1 Jul 2024 11:47:05 +0200 Subject: [PATCH 06/24] removed non-thread safe locking, replaced with conservative locking --- .../java/org/polypheny/db/PolyphenyDb.java | 2 +- .../db/processing/AbstractQueryProcessor.java | 15 +- .../db/transaction/EntityAccessMap.java | 4 + .../org/polypheny/db/transaction/Lock.java | 2 + .../polypheny/db/transaction/LockManager.java | 131 ++++++------------ .../db/transaction/TransactionImpl.java | 22 +-- .../db/view/MaterializedViewManagerImpl.java | 12 +- .../statistics/StatisticQueryProcessor.java | 2 +- .../org/polypheny/db/cql/CqlProcessor.java | 4 +- .../polypheny/db/cypher/CypherProcessor.java | 4 +- .../db/adapter/jdbc/ResultSetEnumerable.java | 2 +- .../db/adapter/jdbc/package-info.java | 22 --- .../polypheny/db/languages/MqlProcessor.java | 4 +- .../org/polypheny/db/sql/SqlProcessor.java | 4 +- 14 files changed, 64 insertions(+), 166 deletions(-) delete mode 100644 plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/package-info.java diff --git a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java index 88148bf8e3..a331f93ceb 100644 --- a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java +++ b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java @@ -131,7 +131,7 @@ public class PolyphenyDb { public boolean daemonMode = false; @Option(name = { "-defaultStore" }, description = "Type of default storeId") - public String defaultStoreName = "hsqldb"; + public String defaultStoreName = "monetdb"; @Option(name = { "-defaultSource" }, description = "Type of default source") public static String defaultSourceName = "csv"; diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index 425edc5aa6..5f35132e57 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -139,7 +139,6 @@ import org.polypheny.db.tools.Programs; import org.polypheny.db.tools.RoutedAlgBuilder; import org.polypheny.db.transaction.EntityAccessMap; -import org.polypheny.db.transaction.EntityAccessMap.EntityIdentifier; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.transaction.LockManager; import org.polypheny.db.transaction.Statement; @@ -312,6 +311,7 @@ private ProposedImplementations prepareQueries( AlgRoot logicalRoot, AlgDataType if ( isAnalyze ) { statement.getProcessingDuration().start( "Locking" ); } + if ( lock ) { this.acquireLock( isAnalyze, logicalRoot, logicalQueryInformation.getAccessedPartitions() ); } @@ -557,24 +557,17 @@ private void flattenPlans( List plans ) { private void acquireLock( boolean isAnalyze, AlgRoot logicalRoot, Map> accessedPartitions ) { - // TODO @HENNLO Check if this is this is necessary to pass the partitions explicitly. - // This currently only works for queries. Since DMLs are evaluated during routing. - // This SHOULD be adjusted - if ( accessedPartitions.isEmpty() ) { // TODO: Does this happen for create table? // Do not acquire any locks if nothing is accessed return; } // Locking try { - List> idAccessMap = new ArrayList<>(); // Get locks for individual entities EntityAccessMap accessMap = new EntityAccessMap( logicalRoot.alg, accessedPartitions ); // Get a shared global schema lock (only DDLs acquire an exclusive global schema lock) - idAccessMap.add( Pair.of( LockManager.GLOBAL_LOCK, LockMode.SHARED ) ); - idAccessMap.addAll( accessMap.getAccessedEntityPair() ); - LockManager.INSTANCE.lock( idAccessMap, (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( accessMap.getNeededLock(), (TransactionImpl) statement.getTransaction() ); } catch ( DeadlockException e ) { throw new GenericRuntimeException( e ); } @@ -1497,7 +1490,7 @@ private CachedProposedRoutingPlan selectCachedPlan( List> getAccessedEntityPair() { return accessLockMap.entrySet(); } + public LockMode getNeededLock() { + return accessLockMap.values().stream().anyMatch( l -> l == LockMode.EXCLUSIVE ) ? LockMode.EXCLUSIVE : LockMode.SHARED; + } + /** * Determines whether an entity is accessed at all. diff --git a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java index b410f8c663..e1690a49bd 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java @@ -22,6 +22,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import lombok.Getter; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.transaction.Transaction.AccessMode; @@ -29,6 +30,7 @@ // Based on code taken from https://github.com/dstibrany/LockManager public class Lock { + @Getter private final Set owners = new CopyOnWriteArraySet<>(); private final ReentrantLock lock = new ReentrantLock( true ); private final Condition waiters = lock.newCondition(); diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 4cf24cc8c3..1fcd284397 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -17,139 +17,88 @@ package org.polypheny.db.transaction; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; +import java.util.HashSet; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import lombok.Getter; +import com.google.common.base.Stopwatch; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.polypheny.db.transaction.EntityAccessMap.EntityIdentifier; -import org.polypheny.db.transaction.EntityAccessMap.EntityIdentifier.NamespaceLevel; +import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.util.DeadlockException; +import javax.transaction.xa.Xid; - -// Based on code taken from https://github.com/dstibrany/LockManager @Slf4j public class LockManager { public static final LockManager INSTANCE = new LockManager(); - public static final EntityIdentifier GLOBAL_LOCK = new EntityIdentifier( -1L, -1L, NamespaceLevel.ENTITY_LEVEL ); // For locking whole schema - private final ConcurrentHashMap lockTable; - @Getter - private final WaitForGraph waitForGraph; + private boolean isExclusive = false; + private final Set owners = new HashSet<>(); private LockManager() { - lockTable = new ConcurrentHashMap<>(); - waitForGraph = new WaitForGraph(); } - public void lock( @NonNull Collection> idAccessMap, @NonNull TransactionImpl transaction ) throws DeadlockException { + public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws DeadlockException { // Decide on which locking approach to focus - if ( transaction.acceptsOutdated() ) { - handleSecondaryLocks( idAccessMap, transaction ); - } else { - handlePrimaryLocks( idAccessMap, transaction ); + Stopwatch watch = Stopwatch.createStarted(); + while ( !handleSimpleLock(mode, transaction )){ + if( watch.elapsed().getSeconds() > 200 ){ + throw new DeadlockException( new GenericRuntimeException( "Could not get lock after retry" ) ); + } + try { + Thread.sleep( 5 ); + } catch ( InterruptedException e ) { + // ignored + } } } - /** - * Used in traditional transactional workload to lck all entities that will eagerly receive any update - */ - private void handlePrimaryLocks( @NonNull Collection> idAccessMap, @NonNull TransactionImpl transaction ) throws DeadlockException { - Iterator> iter = idAccessMap.stream().sorted( ( a, b ) -> Math.toIntExact( a.getKey().entityId - b.getKey().entityId ) ).iterator(); - Entry pair; - while ( iter.hasNext() ) { - pair = iter.next(); - lockTable.putIfAbsent( pair.getKey(), new Lock( waitForGraph ) ); + private synchronized boolean handleSimpleLock( @NonNull LockMode mode, TransactionImpl transaction ) { + if ( mode == LockMode.EXCLUSIVE ) { + // get w + if ( owners.isEmpty() || (owners.size() == 1 && owners.contains( transaction.getXid() ))) { - Lock lock = lockTable.get( pair.getKey() ); + isExclusive = true; + owners.add( transaction.getXid() ); + return true; + } - try { - if ( hasLock( transaction, pair.getKey() ) && pair.getValue() == lock.getMode() ) { - continue; - } else if ( pair.getValue() == Lock.LockMode.SHARED && hasLock( transaction, pair.getKey() ) && lock.getMode() == Lock.LockMode.EXCLUSIVE ) { - continue; - } else if ( pair.getValue() == Lock.LockMode.EXCLUSIVE && hasLock( transaction, pair.getKey() ) && lock.getMode() == Lock.LockMode.SHARED ) { - lock.upgrade( transaction ); - } else { - lock.acquire( transaction, pair.getValue() ); - } - } catch ( InterruptedException e ) { - removeTransaction( transaction ); - throw new DeadlockException( e ); + } else { + // get r + if ( !isExclusive ) { + owners.add( transaction.getXid() ); + return true; } - transaction.addLock( lock ); } + return false; } - /** - * Used in freshness related workload to lock all entities that will lazily receive updates (considered secondaries) - */ - private void handleSecondaryLocks( @NonNull Collection> idAccessMap, @NonNull TransactionImpl transaction ) throws DeadlockException { - // Try locking secondaries first. - // If this cannot be fulfilled by data distribution fallback and try to acquire a regular primary lock - // TODO @HENNLO Check if this decision should even be made here or somewhere else - // This is mainly relevant for Queries on secondaries/outdated nodes. - // In theory, we already know for each query which partitions are going to be accessed. - // The FreshnessManager could therefore already be invoked prior to Routing to decide if the Freshness can be - // guaranteed or if we need to fall back to primary locking mechanisms. - } + public synchronized void unlock( @NonNull TransactionImpl transaction ) { - - public void unlock( @NonNull Collection ids, @NonNull TransactionImpl transaction ) { - Iterator iter = ids.iterator(); - EntityIdentifier entityIdentifier; - while ( iter.hasNext() ) { - entityIdentifier = iter.next(); - Lock lock = lockTable.get( entityIdentifier ); - if ( lock != null ) { - lock.release( transaction ); - } - transaction.removeLock( lock ); + if ( !owners.contains( transaction.getXid()) ) { + log.debug( "Transaction is no owner" ); + return; } - } + if ( isExclusive ) { + isExclusive = false; - public void removeTransaction( @NonNull TransactionImpl transaction ) { - Set txnLockList = transaction.getLocks(); - for ( Lock lock : txnLockList ) { - lock.release( transaction ); } - } + owners.remove( transaction.getXid() ); - public boolean hasLock( @NonNull TransactionImpl transaction, @NonNull EntityAccessMap.EntityIdentifier entityIdentifier ) { - Set locks = transaction.getLocks(); - if ( locks == null ) { - return false; - } - for ( Lock txnLock : locks ) { - if ( txnLock == lockTable.get( entityIdentifier ) ) { - return true; - } - } - return false; } - Lock.LockMode getLockMode( @NonNull EntityAccessMap.EntityIdentifier entityIdentifier ) { - return lockTable.get( entityIdentifier ).getMode(); + public void removeTransaction( @NonNull TransactionImpl transaction ) { + unlock( transaction ); } - public Map getLocks() { - return Map.copyOf( lockTable ); - } - } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java index fbfb58ff15..00fdefec97 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java @@ -102,7 +102,6 @@ public class TransactionImpl implements Transaction, Comparable { @Getter private final Set> involvedAdapters = new ConcurrentSkipListSet<>( (a,b) -> Math.toIntExact( a.adapterId - b.adapterId ) ); - private final Set lockList = new HashSet<>(); private boolean useCache = true; private boolean acceptsOutdated = false; @@ -233,6 +232,7 @@ public void rollback() throws TransactionException { return; } try { + log.warn( "rolling back transaction" ); // Rollback changes to the adapters for ( Adapter adapter : involvedAdapters ) { adapter.rollback( xid ); @@ -386,26 +386,6 @@ public void updateAccessMode( AccessMode accessModeCandidate ) { } - // - // For locking - // - - - Set getLocks() { - return lockList; - } - - - void addLock( Lock lock ) { - lockList.add( lock ); - } - - - void removeLock( Lock lock ) { - lockList.remove( lock ); - } - - void abort() { Thread.currentThread().interrupt(); } diff --git a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java index 5dba2a8a62..4dcfc5d2ef 100644 --- a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java @@ -296,15 +296,7 @@ public void prepareToUpdate( long materializedId ) { return; } - // Get locks for individual tables - EntityAccessMap access = new EntityAccessMap( optionalEntity.get().getDefinition(), new HashMap<>() ); - Collection> idAccesses = new ArrayList<>( access.getAccessedEntityPair() ); - // if we don't lock exclusively for the target here we can produce deadlocks on underlying stores, - // as we could end up with two concurrent shared locks waiting for an exclusive lock on the same entity or each other's entities - catalog.getSnapshot().alloc().getFromLogical( materializedId ) - .forEach( allocation -> idAccesses.add( Pair.of( new EntityIdentifier( entity.id, allocation.id, NamespaceLevel.ENTITY_LEVEL ), LockMode.EXCLUSIVE ) ) ); - - LockManager.INSTANCE.lock( idAccesses, (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( LockMode.EXCLUSIVE, (TransactionImpl) statement.getTransaction() ); } catch ( DeadlockException e ) { throw new GenericRuntimeException( "DeadLock while locking for materialized view update", e ); } @@ -411,7 +403,7 @@ public void commitTransaction( Transaction transaction ) { } } finally { // Release lock - LockManager.INSTANCE.unlock( Collections.singletonList( LockManager.GLOBAL_LOCK ), (TransactionImpl) transaction ); + LockManager.INSTANCE.unlock( (TransactionImpl) transaction ); } } diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/StatisticQueryProcessor.java b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/StatisticQueryProcessor.java index 12cfd0f549..aed0948f32 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/StatisticQueryProcessor.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/StatisticQueryProcessor.java @@ -42,10 +42,10 @@ import org.polypheny.db.type.entity.PolyValue; +@Getter @Slf4j public class StatisticQueryProcessor { - @Getter private final TransactionManager transactionManager; diff --git a/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java b/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java index f15b32ecf1..13172c9b4c 100644 --- a/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java +++ b/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java @@ -72,13 +72,13 @@ public AlgRoot translate( Statement statement, ParsedQueryContext context ) { @Override public void unlock( Statement statement ) { - LockManager.INSTANCE.unlock( Collections.singletonList( LockManager.GLOBAL_LOCK ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.unlock( (TransactionImpl) statement.getTransaction() ); } @Override protected void lock( Statement statement ) throws DeadlockException { - LockManager.INSTANCE.lock( Collections.singletonList( Pair.of( LockManager.GLOBAL_LOCK, LockMode.EXCLUSIVE ) ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( LockMode.EXCLUSIVE, (TransactionImpl) statement.getTransaction() ); } diff --git a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/CypherProcessor.java b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/CypherProcessor.java index c7355c61b5..5cc45e1b2a 100644 --- a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/CypherProcessor.java +++ b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/CypherProcessor.java @@ -129,13 +129,13 @@ public AlgRoot translate( Statement statement, ParsedQueryContext context ) { @Override public void unlock( Statement statement ) { - LockManager.INSTANCE.unlock( List.of( LockManager.GLOBAL_LOCK ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.unlock( (TransactionImpl) statement.getTransaction() ); } @Override protected void lock( Statement statement ) throws DeadlockException { - LockManager.INSTANCE.lock( List.of( Pair.of( LockManager.GLOBAL_LOCK, LockMode.EXCLUSIVE ) ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( LockMode.EXCLUSIVE, (TransactionImpl) statement.getTransaction() ); } diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java index 1fac032f1c..e70918760a 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java @@ -78,7 +78,7 @@ public class ResultSetEnumerable extends AbstractEnumerable { // timestamp do factor in the timezones, which means that 10:00 is 9:00 with - // an one hour shift, as we lose this timezone information on retrieval + // a one hour shift, as we lose this timezone information on retrieval // therefore we use the offset if needed public final static int OFFSET = Calendar.getInstance().getTimeZone().getRawOffset(); diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/package-info.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/package-info.java deleted file mode 100644 index 136edf75a8..0000000000 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Query provider based on a JDBC data source. - */ - -package org.polypheny.db.adapter.jdbc; - diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java index 3c27c392ff..9a7cd7ad0e 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java @@ -150,13 +150,13 @@ public AlgRoot translate( Statement statement, ParsedQueryContext context ) { @Override public void unlock( Statement statement ) { - LockManager.INSTANCE.unlock( Collections.singletonList( LockManager.GLOBAL_LOCK ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.unlock( (TransactionImpl) statement.getTransaction() ); } @Override public void lock( Statement statement ) throws DeadlockException { - LockManager.INSTANCE.lock( Collections.singletonList( Pair.of( LockManager.GLOBAL_LOCK, LockMode.EXCLUSIVE ) ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( LockMode.EXCLUSIVE, (TransactionImpl) statement.getTransaction() ); } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlProcessor.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlProcessor.java index 36ba04116e..5daabccfc1 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlProcessor.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlProcessor.java @@ -224,13 +224,13 @@ public AlgRoot translate( Statement statement, ParsedQueryContext context ) { @Override public void unlock( Statement statement ) { - LockManager.INSTANCE.unlock( List.of( LockManager.GLOBAL_LOCK ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.unlock( (TransactionImpl) statement.getTransaction() ); } @Override public void lock( Statement statement ) throws DeadlockException { - LockManager.INSTANCE.lock( List.of( Pair.of( LockManager.GLOBAL_LOCK, LockMode.EXCLUSIVE ) ), (TransactionImpl) statement.getTransaction() ); + LockManager.INSTANCE.lock( LockMode.EXCLUSIVE, (TransactionImpl) statement.getTransaction() ); } From 6162f1826594ce8494f9d94b4675cca1c81a1801 Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 1 Jul 2024 12:58:09 +0200 Subject: [PATCH 07/24] added missing case, where xLock already is held by tx --- .../main/java/org/polypheny/db/transaction/LockManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 1fcd284397..8025aab9ab 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -44,7 +44,7 @@ public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws D // Decide on which locking approach to focus Stopwatch watch = Stopwatch.createStarted(); while ( !handleSimpleLock(mode, transaction )){ - if( watch.elapsed().getSeconds() > 200 ){ + if( watch.elapsed().getSeconds() > 10 ){ throw new DeadlockException( new GenericRuntimeException( "Could not get lock after retry" ) ); } try { @@ -68,7 +68,7 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti } else { // get r - if ( !isExclusive ) { + if ( !isExclusive || owners.contains( transaction.getXid()) ) { owners.add( transaction.getXid() ); return true; } @@ -88,7 +88,6 @@ public synchronized void unlock( @NonNull TransactionImpl transaction ) { if ( isExclusive ) { isExclusive = false; - } owners.remove( transaction.getXid() ); From 8ab90921b4eb2ccd2764134614a7db726b7bb83d Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 1 Jul 2024 13:19:04 +0200 Subject: [PATCH 08/24] reverted to default store --- dbms/src/main/java/org/polypheny/db/PolyphenyDb.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java index a331f93ceb..88148bf8e3 100644 --- a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java +++ b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java @@ -131,7 +131,7 @@ public class PolyphenyDb { public boolean daemonMode = false; @Option(name = { "-defaultStore" }, description = "Type of default storeId") - public String defaultStoreName = "monetdb"; + public String defaultStoreName = "hsqldb"; @Option(name = { "-defaultSource" }, description = "Type of default source") public static String defaultSourceName = "csv"; From 1b3c0930a673fc7490ba121b002b103da680ccc3 Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 1 Jul 2024 14:40:05 +0200 Subject: [PATCH 09/24] fixed extraction of partitions --- .../java/org/polypheny/db/languages/LanguageManager.java | 1 - .../java/org/polypheny/db/routing/ColumnDistribution.java | 4 +--- .../polypheny/db/processing/AbstractQueryProcessor.java | 8 ++++---- .../org/polypheny/db/routing/routers/SimpleRouter.java | 2 +- .../org/polypheny/db/misc/HorizontalPartitioningTest.java | 1 + 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/languages/LanguageManager.java b/core/src/main/java/org/polypheny/db/languages/LanguageManager.java index 48b70e3c20..9a253a020f 100644 --- a/core/src/main/java/org/polypheny/db/languages/LanguageManager.java +++ b/core/src/main/java/org/polypheny/db/languages/LanguageManager.java @@ -241,7 +241,6 @@ private static void cancelTransaction( @Nullable Transaction transaction ) { public List anyQuery( QueryContext context ) { List prepared = anyPrepareQuery( context, context.getTransactions().get( context.getTransactions().size() - 1 ) ); - List executedContexts = new ArrayList<>(); for ( ImplementationContext implementation : prepared ) { diff --git a/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java b/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java index cbcb34a3c2..a21169afc0 100644 --- a/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java +++ b/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java @@ -128,9 +128,7 @@ private Map> getPlacementColumns() { private List getPossiblePartitions() { - List partitions = new ArrayList<>(); - snapshot.alloc().getPartitions().stream().filter( p -> p.logicalEntityId == logicalEntityId && sourcePartitions.contains( p.id ) ).forEach( partitions::add ); - return partitions; + return snapshot.alloc().getPartitions().stream().filter( p -> p.logicalEntityId == logicalEntityId && sourcePartitions.contains( p.id ) ).toList(); } diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index 5f35132e57..daf545a996 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -1209,11 +1209,11 @@ private Map> extractPartitions( Set entities ) { Map> map = new HashMap<>(); for ( Entity entity : entities ) { if ( entity.isLogical() ) { - map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).add( entity.id ); + map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).addAll( Catalog.snapshot().alloc().getPartitionsFromLogical( entity.getId()).stream().map( p -> p.id ).toList() ); }else if ( entity.isAllocation() ){ - map.computeIfAbsent( ((AllocationEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getLogicalId() ); - }else if( entity.isPhysical()){ - map.computeIfAbsent( ((PhysicalEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((PhysicalEntity)entity).getLogicalId() ); + map.computeIfAbsent( ((AllocationEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getPartitionId() ); + }else if( entity.isPhysical() ){ + map.computeIfAbsent( ((PhysicalEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((PhysicalEntity)entity).getAllocationId() ); } } return map; diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java index 52ea4aca64..5ddaa11c3f 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java @@ -78,7 +78,7 @@ protected List handleHorizontalPartitioning( AlgNode node, Log // Only one builder available //builders.get( 0 ).addPhysicalInfo( placementDistribution ); - ColumnDistribution columnDistribution = new ColumnDistribution( table.id, table.getColumnIds(), partitionIds, partitionIds, List.of(), context.getSnapshot() ); + ColumnDistribution columnDistribution = new ColumnDistribution( table.id, table.getColumnIds(), partitionIds, partitionIds, List.of(), Catalog.snapshot() ); context.fieldDistribution = columnDistribution; builders.get( 0 ).push( super.buildJoinedScan( columnDistribution, context ) ); diff --git a/dbms/src/test/java/org/polypheny/db/misc/HorizontalPartitioningTest.java b/dbms/src/test/java/org/polypheny/db/misc/HorizontalPartitioningTest.java index 2ea77e57b6..86261531aa 100644 --- a/dbms/src/test/java/org/polypheny/db/misc/HorizontalPartitioningTest.java +++ b/dbms/src/test/java/org/polypheny/db/misc/HorizontalPartitioningTest.java @@ -300,6 +300,7 @@ public void dataMigrationTest() throws SQLException { statement.executeUpdate( "INSERT INTO hashpartition VALUES (1, 3, 'hans')" ); statement.executeUpdate( "INSERT INTO hashpartition VALUES (2, 7, 'bob')" ); + TestHelper.checkResultSet( statement.executeQuery( "SELECT * FROM hashpartition ORDER BY tprimary" ), ImmutableList.of( From 2cb3fdbf7cc9b072a24fe0e2a0c75595ee3a051d Mon Sep 17 00:00:00 2001 From: datomo Date: Tue, 2 Jul 2024 11:09:00 +0200 Subject: [PATCH 10/24] finally fixed deadlock scenario --- .../org/polypheny/db/config/RuntimeConfig.java | 5 +++++ .../org/polypheny/db/transaction/LockManager.java | 13 +++++++------ .../org/polypheny/db/prisminterface/MqlTest.java | 1 - .../statementProcessing/StatementProcessor.java | 14 +++++++++++++- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java index 012731831b..902b8bf87d 100644 --- a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java +++ b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java @@ -505,6 +505,11 @@ public enum RuntimeConfig { "runtime/serialization", "How big the buffersize for catalog objects should be.", 200000, + ConfigType.INTEGER ), + LOCKING_MAX_TIMEOUT_SECONDS( + "runtime/maxTimeout", + "How long a transactions should wait for a lock until it is aborted", + 10, ConfigType.INTEGER ); diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 8025aab9ab..973b94a178 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -17,15 +17,16 @@ package org.polypheny.db.transaction; +import com.google.common.base.Stopwatch; import java.util.HashSet; import java.util.Set; -import com.google.common.base.Stopwatch; +import javax.transaction.xa.Xid; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; +import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.util.DeadlockException; -import javax.transaction.xa.Xid; @Slf4j public class LockManager { @@ -44,7 +45,7 @@ public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws D // Decide on which locking approach to focus Stopwatch watch = Stopwatch.createStarted(); while ( !handleSimpleLock(mode, transaction )){ - if( watch.elapsed().getSeconds() > 10 ){ + if ( watch.elapsed().getSeconds() > RuntimeConfig.LOCKING_MAX_TIMEOUT_SECONDS.getInteger() ) { throw new DeadlockException( new GenericRuntimeException( "Could not get lock after retry" ) ); } try { @@ -60,7 +61,7 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti if ( mode == LockMode.EXCLUSIVE ) { // get w if ( owners.isEmpty() || (owners.size() == 1 && owners.contains( transaction.getXid() ))) { - + log.debug( "x lock {}", transaction.getXid() ); isExclusive = true; owners.add( transaction.getXid() ); return true; @@ -69,6 +70,7 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti } else { // get r if ( !isExclusive || owners.contains( transaction.getXid()) ) { + log.debug( "r lock {}", transaction.getXid() ); owners.add( transaction.getXid() ); return true; } @@ -80,7 +82,6 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti public synchronized void unlock( @NonNull TransactionImpl transaction ) { - if ( !owners.contains( transaction.getXid()) ) { log.debug( "Transaction is no owner" ); return; @@ -89,7 +90,7 @@ public synchronized void unlock( @NonNull TransactionImpl transaction ) { if ( isExclusive ) { isExclusive = false; } - + log.debug( "release {}", transaction.getXid() ); owners.remove( transaction.getXid() ); } diff --git a/dbms/src/test/java/org/polypheny/db/prisminterface/MqlTest.java b/dbms/src/test/java/org/polypheny/db/prisminterface/MqlTest.java index ec6b2f0b99..65e87560cc 100644 --- a/dbms/src/test/java/org/polypheny/db/prisminterface/MqlTest.java +++ b/dbms/src/test/java/org/polypheny/db/prisminterface/MqlTest.java @@ -64,7 +64,6 @@ public void simpleMqlTest() throws SQLException { Result result = polyStatement.execute( "mqltest", MQL_LANGUAGE_NAME, TEST_QUERY ); assertEquals( ResultType.DOCUMENT, result.getResultType() ); } ); - } } diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statementProcessing/StatementProcessor.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statementProcessing/StatementProcessor.java index fe48b6f7cc..820b1ec018 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statementProcessing/StatementProcessor.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statementProcessing/StatementProcessor.java @@ -34,6 +34,7 @@ import org.polypheny.db.processing.QueryContext; import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; +import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.util.Pair; import org.polypheny.prism.Frame; import org.polypheny.prism.StatementResult; @@ -66,7 +67,18 @@ public static void implement( PIStatement piStatement ) { .build(); List implementations = LanguageManager.getINSTANCE().anyPrepareQuery( context, statement ); if ( implementations.stream().anyMatch( i -> i.getQuery().isAutoGenerated() ) ) { - throw new GenericRuntimeException( "This query relies on an auto-generated subquery, which is not supported by this interface." ); + // we remove additional transaction which where opened for auto generation todo adjust if interface supports it + implementations.stream() + .filter( i -> !i.getStatement().getTransaction().getXid().equals( piStatement.getStatement().getTransaction().getXid() ) ) + .forEach( i -> { + try { + i.getStatement().getTransaction().rollback(); + } catch ( TransactionException e ) { + throw new GenericRuntimeException( e ); + } + } ); + + throw new GenericRuntimeException( "This query relies on an auto-generated subquery, which is not supported by this interface. Transaction was rolled backed." ); } if ( implementations.get( 0 ).getImplementation() == null ) { From 31999057a5d0b9a9418fdcdeeae6e98b3339f84c Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 08:34:22 +0200 Subject: [PATCH 11/24] fixed wrong transaction times --- .../polypheny/db/processing/ImplementationContext.java | 9 ++++----- .../java/org/polypheny/db/webui/crud/LanguageCrud.java | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java b/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java index e85b9891df..06d173aed9 100644 --- a/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java +++ b/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java @@ -18,6 +18,7 @@ import java.util.Optional; +import com.google.common.base.Stopwatch; import lombok.EqualsAndHashCode; import lombok.Value; import lombok.experimental.NonFinal; @@ -51,14 +52,12 @@ public static ImplementationContext ofError( Throwable e, ParsedQueryContext par public ExecutedContext execute( Statement statement ) { - long time = System.nanoTime(); + Stopwatch stopwatch = Stopwatch.createStarted(); try { ResultIterator result = implementation.execute( statement, query.getBatch(), query.isAnalysed(), query.isAnalysed(), false ); - time = System.nanoTime() - time; - return new ExecutedContext( implementation, null, query, time, result, statement ); + return new ExecutedContext( implementation, null, query, stopwatch.elapsed().getNano(), result, statement ); } catch ( Throwable e ) { - time = System.nanoTime() - time; - return ExecutedContext.ofError( e, this, time ); + return ExecutedContext.ofError( e, this, (long) stopwatch.elapsed().getNano() ); } } diff --git a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java index 13778e2e75..87e28ba1fe 100644 --- a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java +++ b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java @@ -162,14 +162,13 @@ public static long getNamespaceIdOrDefault( String namespace ) { results.add( builder.apply( executedContext, request, executedContext.getStatement() ).build() ); } - commitAndFinish( executedContexts, transaction.getQueryAnalyzer(), results, executedContexts.stream().map( ExecutedContext::getExecutionTime ).reduce( Long::sum ).orElse( -1L ) ); + commitAndFinish( executedContexts, transaction.getQueryAnalyzer(), results, executedContexts.stream().map( ExecutedContext::getExecutionTime ).reduce( (a,b) -> a + b ).orElse( -1L ) ); return results; } public static void commitAndFinish( List executedContexts, InformationManager queryAnalyzer, List> results, long executionTime ) { - executionTime = System.nanoTime() - executionTime; String commitStatus = "Error on starting committing"; for ( Transaction transaction : executedContexts.stream().flatMap( c -> c.getQuery().getTransactions().stream() ).toList() ) { // this has a lot of unnecessary no-op commits atm From 8bca347e89a314b67284c925b62cfd4c9b96e01f Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 13:45:41 +0200 Subject: [PATCH 12/24] added more robust locking, with starving protection --- .../db/processing/DataMigratorImpl.java | 4 +- .../polypheny/db/transaction/LockManager.java | 103 ++++++++++++++++-- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java index 5d866a48e0..46c62e9d9e 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java @@ -440,8 +440,8 @@ public AlgRoot buildInsertStatement( Statement statement, List // this often leads to errors, and can be prevented by sorting List placements = to.stream().sorted( Comparator.comparingLong( p -> p.columnId ) ).toList(); - List columnNames = new LinkedList<>(); - List values = new LinkedList<>(); + List columnNames = new ArrayList<>(); + List values = new ArrayList<>(); for ( AllocationColumn ccp : placements ) { LogicalColumn logicalColumn = Catalog.getInstance().getSnapshot().rel().getColumn( ccp.columnId ).orElseThrow(); columnNames.add( ccp.getLogicalColumnName() ); diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 973b94a178..ca621aa43c 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -20,21 +20,36 @@ import com.google.common.base.Stopwatch; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; import javax.transaction.xa.Xid; +import com.google.common.base.Supplier; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.util.DeadlockException; +import org.polypheny.db.util.Pair; +import org.polypheny.db.util.Triple; @Slf4j -public class LockManager { +public class LockManager implements Runnable { public static final LockManager INSTANCE = new LockManager(); private boolean isExclusive = false; private final Set owners = new HashSet<>(); + private final ConcurrentLinkedQueue waiters = new ConcurrentLinkedQueue<>(); + private final ReentrantLock lock = new ReentrantLock(); + private final Condition condition = lock.newCondition(); private LockManager() { @@ -43,24 +58,74 @@ private LockManager() { public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws DeadlockException { // Decide on which locking approach to focus - Stopwatch watch = Stopwatch.createStarted(); - while ( !handleSimpleLock(mode, transaction )){ - if ( watch.elapsed().getSeconds() > RuntimeConfig.LOCKING_MAX_TIMEOUT_SECONDS.getInteger() ) { - throw new DeadlockException( new GenericRuntimeException( "Could not get lock after retry" ) ); + synchronized ( this ) { + if ( owners.isEmpty() ) { + handleLockOrThrow( mode, transaction ); + return; } + } + Thread thread = Thread.currentThread(); + + waiters.add( thread ); + + Stopwatch watch = Stopwatch.createStarted(); + // wait + while ( true ) { + lock.lock(); try { - Thread.sleep( 5 ); + while ( waiters.peek() != thread ) { + if ( watch.elapsed().getSeconds() > RuntimeConfig.LOCKING_MAX_TIMEOUT_SECONDS.getInteger() ) { + cleanup( thread ); + throw new DeadlockException( new GenericRuntimeException( "Could not acquire lock, after max timeout was reached" ) ); + } + condition.await(); + } } catch ( InterruptedException e ) { - // ignored + cleanup( thread ); + throw new GenericRuntimeException( e ); + } + lock.unlock(); + + synchronized ( this ) { + // try execute + if ( handleSimpleLock( mode, transaction ) ) { + // remove successful + waiters.poll(); + // signal + signalAll(); + + return; + } + + if ( owners.isEmpty() ) { + waiters.remove( thread ); + throw new GenericRuntimeException( "Could not acquire lock" ); + } } + // we wait until next signal + } + + } + + + private void cleanup( Thread thread ) { + waiters.remove( thread ); + lock.unlock(); + } + + + private void handleLockOrThrow( LockMode mode, @NotNull TransactionImpl transaction ) { + if ( !handleSimpleLock( mode, transaction ) ) { + throw new GenericRuntimeException( "Could not acquire lock, as single transaction" ); } } + private synchronized boolean handleSimpleLock( @NonNull LockMode mode, TransactionImpl transaction ) { if ( mode == LockMode.EXCLUSIVE ) { // get w - if ( owners.isEmpty() || (owners.size() == 1 && owners.contains( transaction.getXid() ))) { + if ( owners.isEmpty() || (owners.size() == 1 && owners.contains( transaction.getXid() )) ) { log.debug( "x lock {}", transaction.getXid() ); isExclusive = true; owners.add( transaction.getXid() ); @@ -69,7 +134,7 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti } else { // get r - if ( !isExclusive || owners.contains( transaction.getXid()) ) { + if ( !isExclusive || owners.contains( transaction.getXid() ) ) { log.debug( "r lock {}", transaction.getXid() ); owners.add( transaction.getXid() ); return true; @@ -80,9 +145,20 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti } + private void signalAll() { + lock.lock(); + try{ + synchronized ( condition ) { + condition.signalAll(); + } + }finally { + lock.unlock(); + } + } + public synchronized void unlock( @NonNull TransactionImpl transaction ) { - if ( !owners.contains( transaction.getXid()) ) { + if ( !owners.contains( transaction.getXid() ) ) { log.debug( "Transaction is no owner" ); return; } @@ -93,6 +169,8 @@ public synchronized void unlock( @NonNull TransactionImpl transaction ) { log.debug( "release {}", transaction.getXid() ); owners.remove( transaction.getXid() ); + // wake up waiters + signalAll(); } @@ -101,4 +179,9 @@ public void removeTransaction( @NonNull TransactionImpl transaction ) { } + @Override + public void run() { + + } + } From cfa8ddae063354955404bee65f485c86116c1d97 Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 14:55:14 +0200 Subject: [PATCH 13/24] fix for materialized views from derived values --- .../main/java/org/polypheny/db/ddl/DdlManagerImpl.java | 10 +++++----- .../org/polypheny/db/processing/DataMigratorImpl.java | 3 ++- .../polypheny/db/view/MaterializedViewManagerImpl.java | 1 - .../db/webui/models/catalog/FieldDefinition.java | 5 +++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index b368520c95..165e871563 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -1657,7 +1657,7 @@ public void createView( String viewName, long namespaceId, AlgNode algNode, AlgC AlgDataType fieldList = algNode.getTupleType(); - List columns = getColumnInformation( projectedColumns, fieldList ); + List columns = getViewColumnInformation( projectedColumns, fieldList ); Map> underlyingTables = new HashMap<>(); @@ -1753,7 +1753,7 @@ public void createMaterializedView( String viewName, long namespaceId, AlgRoot a ); // Creates a list with all columns, tableId is needed to create the primary key - List fields = getColumnInformation( projectedColumns, fieldList, true, view.id ); + List fields = getViewColumnInformation( projectedColumns, fieldList, true, view.id ); Map ids = new LinkedHashMap<>(); @@ -1953,12 +1953,12 @@ public void dropGraph( long graphId, boolean ifExists, Statement statement ) { } - private List getColumnInformation( List projectedColumns, AlgDataType fieldList ) { - return getColumnInformation( projectedColumns, fieldList, false, 0 ); + private List getViewColumnInformation( List projectedColumns, AlgDataType fieldList ) { + return getViewColumnInformation( projectedColumns, fieldList, false, 0 ); } - private List getColumnInformation( List projectedColumns, AlgDataType fieldList, boolean addPrimary, long tableId ) { + private List getViewColumnInformation( List projectedColumns, AlgDataType fieldList, boolean addPrimary, long tableId ) { List columns = new ArrayList<>(); int position = 1; diff --git a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java index 46c62e9d9e..8b10e00d09 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java @@ -91,6 +91,7 @@ import org.polypheny.db.type.entity.graph.PolyGraph; import org.polypheny.db.type.entity.numerical.PolyInteger; import org.polypheny.db.util.Pair; +import org.polypheny.db.webui.models.catalog.FieldDefinition; @Slf4j @@ -323,7 +324,7 @@ public void executeQuery( List selectedColumns, AlgRoot source for ( AllocationColumn column : selectedColumns ) { int i = 0; for ( AlgDataTypeField metaData : implementation.getTupleType().getFields() ) { - if ( metaData.getName().equalsIgnoreCase( column.getLogicalColumnName() ) ) { + if ( FieldDefinition.normalizeViewColumnName( metaData.getName() ).equalsIgnoreCase( column.getLogicalColumnName() ) ) { resultColMapping.put( column.getColumnId(), i ); } i++; diff --git a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java index 4dcfc5d2ef..85f3bf4694 100644 --- a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java @@ -336,7 +336,6 @@ public void addData( Transaction transaction, @Nullable List> store addMaterializedInfo( materializedView.id, materializedView.getMaterializedCriteria() ); } - /** * Deletes all the data from a materialized view and adds the newest data to the materialized view * diff --git a/webui/src/main/java/org/polypheny/db/webui/models/catalog/FieldDefinition.java b/webui/src/main/java/org/polypheny/db/webui/models/catalog/FieldDefinition.java index be47b3fc3f..94f9a07ea5 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/catalog/FieldDefinition.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/catalog/FieldDefinition.java @@ -50,4 +50,9 @@ public static FieldDefinition of( AlgDataTypeField field ) { return FieldDefinition.builder().name( field.getName() ).dataType( field.getType().getFullTypeString() ).build(); } + + public static String normalizeViewColumnName( String name ) { + return name.toLowerCase().replaceAll( "[^A-Za-z0-9]", "_" ); + } + } From 479d266f3068d9c6df5b9c734474f05377d6e56b Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 17:05:14 +0200 Subject: [PATCH 14/24] fixed some cases where lock already acquired --- .../polypheny/db/transaction/LockManager.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index ca621aa43c..8f6798b2d0 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -20,16 +20,10 @@ import com.google.common.base.Stopwatch; import java.util.HashSet; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import javax.transaction.xa.Xid; -import com.google.common.base.Supplier; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -37,8 +31,6 @@ import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.util.DeadlockException; -import org.polypheny.db.util.Pair; -import org.polypheny.db.util.Triple; @Slf4j public class LockManager implements Runnable { @@ -62,6 +54,10 @@ public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws D if ( owners.isEmpty() ) { handleLockOrThrow( mode, transaction ); return; + } else if ( owners.contains( transaction.getXid() ) && (mode == LockMode.SHARED || isExclusive) ) { + log.debug( "already locked {}", transaction.getXid() ); + // already have the required lock + return; } } Thread thread = Thread.currentThread(); @@ -78,6 +74,7 @@ public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws D cleanup( thread ); throw new DeadlockException( new GenericRuntimeException( "Could not acquire lock, after max timeout was reached" ) ); } + log.debug( "wait {} ", transaction.getXid() ); condition.await(); } } catch ( InterruptedException e ) { @@ -126,6 +123,11 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti if ( mode == LockMode.EXCLUSIVE ) { // get w if ( owners.isEmpty() || (owners.size() == 1 && owners.contains( transaction.getXid() )) ) { + if ( isExclusive ) { + log.debug( "lock already exclusive" ); + return true; + } + log.debug( "x lock {}", transaction.getXid() ); isExclusive = true; owners.add( transaction.getXid() ); From bfa7f6305cb0165f01e479989db25ce657b96f4f Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 17:57:29 +0200 Subject: [PATCH 15/24] fix for namespace syntax in mql --- .../java/org/polypheny/db/languages/MongoLanguagePlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MongoLanguagePlugin.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MongoLanguagePlugin.java index 56db38b764..d90388580c 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MongoLanguagePlugin.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MongoLanguagePlugin.java @@ -104,7 +104,7 @@ private static List anyQuerySplitter( QueryContext context ) if ( queryNode.getEntity() == null ) { continue; } - Optional collection = snapshot.getLogicalEntity( context.getNamespaceId(), queryNode.getEntity() ); + Optional collection = snapshot.getLogicalEntity( query.getNamespaceId(), queryNode.getEntity() ); if ( collection.isEmpty() && !created.contains( Pair.of( context.getNamespaceId(), queryNode.getEntity() ) ) ) { if ( queryNode instanceof MqlCreateCollection || queryNode instanceof MqlCreateView ) { // entity was created during this query From 08e0238711dcef4292ed46eeca3a0e5ae6b18213 Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 21:48:33 +0200 Subject: [PATCH 16/24] stop creation of document views --- .../db/languages/mql/MqlCreateView.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql/MqlCreateView.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql/MqlCreateView.java index 37871fff4a..e49b17e4bf 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql/MqlCreateView.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql/MqlCreateView.java @@ -22,13 +22,15 @@ import org.polypheny.db.algebra.AlgCollation; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.catalog.logistic.PlacementType; -import org.polypheny.db.ddl.DdlManager; import org.polypheny.db.languages.ParserPos; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.languages.mql.Mql.Type; import org.polypheny.db.nodes.ExecutableStatement; import org.polypheny.db.prepare.Context; +import org.polypheny.db.processing.Processor; +import org.polypheny.db.processing.QueryContext; import org.polypheny.db.processing.QueryContext.ParsedQueryContext; import org.polypheny.db.transaction.Statement; @@ -52,27 +54,25 @@ public MqlCreateView( ParserPos pos, String name, String namespace, String sourc public void execute( Context context, Statement statement, ParsedQueryContext parsedQueryContext ) { long database = parsedQueryContext.getQueryNode().orElseThrow().getNamespaceId(); - long schemaId = context.getSnapshot().getNamespace( database ).orElseThrow().id; + long namespaceId = context.getSnapshot().getNamespace( database ).orElseThrow().id; - AlgRoot algRoot = statement.getTransaction() - .getProcessor( QueryLanguage.from( "mongo" ) ) - .translate( statement, parsedQueryContext ); + QueryContext queryContext = QueryContext.builder() + .query( buildQuery() ) + .origin( "MQL create view" ) + .language( parsedQueryContext.getLanguage() ) + .namespaceId( parsedQueryContext.getNamespaceId() ) + .transactionManager( parsedQueryContext.getTransactionManager() ) + .transactions( parsedQueryContext.getTransactions() ) + .build(); + Processor processor = statement.getTransaction().getProcessor( QueryLanguage.from( "mongo" ) ); + + AlgRoot algRoot = processor.translate( statement, ParsedQueryContext.fromQuery( buildQuery(), processor.parse( buildQuery() ).get( 0 ), queryContext ) ); PlacementType placementType = PlacementType.AUTOMATIC; AlgNode algNode = algRoot.alg; AlgCollation algCollation = algRoot.collation; - DdlManager.getInstance().createView( - name, - schemaId, - algNode, - algCollation, - true, - statement, - placementType, - algRoot.alg.getTupleType().getFieldNames(), - buildQuery(), - QueryLanguage.from( "mongo" ) ); + throw new GenericRuntimeException( "Document views are currently not supported" ); } From 7c9756206094b36b017f7a0472bde97475c6cc23 Mon Sep 17 00:00:00 2001 From: datomo Date: Wed, 3 Jul 2024 22:53:31 +0200 Subject: [PATCH 17/24] adjusted waiting of threads --- .../java/org/polypheny/db/config/RuntimeConfig.java | 2 +- .../org/polypheny/db/transaction/LockManager.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java index 902b8bf87d..c97f7d24d2 100644 --- a/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java +++ b/core/src/main/java/org/polypheny/db/config/RuntimeConfig.java @@ -509,7 +509,7 @@ public enum RuntimeConfig { LOCKING_MAX_TIMEOUT_SECONDS( "runtime/maxTimeout", "How long a transactions should wait for a lock until it is aborted", - 10, + 30, ConfigType.INTEGER ); diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 8f6798b2d0..28e0fba29c 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import javax.transaction.xa.Xid; @@ -70,12 +71,12 @@ public void lock( LockMode mode, @NonNull TransactionImpl transaction ) throws D lock.lock(); try { while ( waiters.peek() != thread ) { - if ( watch.elapsed().getSeconds() > RuntimeConfig.LOCKING_MAX_TIMEOUT_SECONDS.getInteger() ) { + log.debug( "wait {} ", transaction.getXid() ); + boolean successful = condition.await( RuntimeConfig.LOCKING_MAX_TIMEOUT_SECONDS.getInteger(), TimeUnit.SECONDS ); + if ( !successful ) { cleanup( thread ); throw new DeadlockException( new GenericRuntimeException( "Could not acquire lock, after max timeout was reached" ) ); } - log.debug( "wait {} ", transaction.getXid() ); - condition.await(); } } catch ( InterruptedException e ) { cleanup( thread ); @@ -118,7 +119,6 @@ private void handleLockOrThrow( LockMode mode, @NotNull TransactionImpl transact } - private synchronized boolean handleSimpleLock( @NonNull LockMode mode, TransactionImpl transaction ) { if ( mode == LockMode.EXCLUSIVE ) { // get w @@ -149,11 +149,11 @@ private synchronized boolean handleSimpleLock( @NonNull LockMode mode, Transacti private void signalAll() { lock.lock(); - try{ + try { synchronized ( condition ) { condition.signalAll(); } - }finally { + } finally { lock.unlock(); } } From f2998e73a855d8922411b79173564b6154aebf6d Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 4 Jul 2024 09:55:47 +0200 Subject: [PATCH 18/24] fix for local and current time and date functions --- .../db/functions/TemporalFunctions.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/functions/TemporalFunctions.java b/core/src/main/java/org/polypheny/db/functions/TemporalFunctions.java index 6115e73038..a014c7d65a 100644 --- a/core/src/main/java/org/polypheny/db/functions/TemporalFunctions.java +++ b/core/src/main/java/org/polypheny/db/functions/TemporalFunctions.java @@ -17,9 +17,12 @@ package org.polypheny.db.functions; import java.sql.Timestamp; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; import java.util.TimeZone; import org.apache.calcite.linq4j.function.NonDeterministic; +import org.jetbrains.annotations.NotNull; import org.polypheny.db.adapter.DataContext; import org.polypheny.db.nodes.TimeUnitRange; import org.polypheny.db.type.entity.PolyInterval; @@ -38,6 +41,7 @@ public class TemporalFunctions { public static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + public static final String TIMEZONE = System.getProperty( "user.timezone" ); @SuppressWarnings("unused") @@ -478,7 +482,8 @@ public static Integer toIntOptional( java.util.Date v, TimeZone timeZone ) { @SuppressWarnings("unused") public static PolyTimestamp currentTimestamp( DataContext root ) { // Cast required for JDK 1.6. - return PolyTimestamp.of( (long) DataContext.Variable.CURRENT_TIMESTAMP.get( root ) ); + Date date = new Date(); + return PolyTimestamp.of( date.getTime() + timeZone( root ).getRawOffset() ); } @@ -508,17 +513,18 @@ public static PolyDate currentDate( DataContext root ) { if ( time < 0 ) { --date; } - return PolyDate.of( date ); + return PolyDate.ofDays( date ); } /** * SQL {@code LOCAL_TIMESTAMP} function. */ + @NotNull @NonDeterministic - public static long localTimestamp( DataContext root ) { - // Cast required for JDK 1.6. - return DataContext.Variable.LOCAL_TIMESTAMP.get( root ); + public static PolyTimestamp localTimestamp( DataContext root ) { + ZonedDateTime now = ZonedDateTime.now( ZoneId.of( TIMEZONE ) ); + return PolyTimestamp.of( (now.toEpochSecond() + now.getOffset().getTotalSeconds()) * DateTimeUtils.MILLIS_PER_SECOND ); } @@ -528,13 +534,14 @@ public static long localTimestamp( DataContext root ) { @SuppressWarnings("unused") @NonDeterministic public static PolyTime localTime( DataContext root ) { - return PolyTime.of( (int) (localTimestamp( root ) % DateTimeUtils.MILLIS_PER_DAY) ); + ZonedDateTime now = ZonedDateTime.now( ZoneId.of( TIMEZONE ) ); + return PolyTime.of( ((now.toEpochSecond() + now.getOffset().getTotalSeconds()) * DateTimeUtils.MILLIS_PER_SECOND) % DateTimeUtils.MILLIS_PER_DAY ); } @NonDeterministic public static TimeZone timeZone( DataContext root ) { - return DataContext.Variable.TIME_ZONE.get( root ); + return TimeZone.getTimeZone( TIMEZONE ); } From 5832df0c9c4029fac576e331964a56f06f000619 Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 4 Jul 2024 14:59:52 +0200 Subject: [PATCH 19/24] added enforcement of new constraints for during creation and during runtime --- .../common/LogicalConstraintEnforcer.java | 3 ++ .../catalogs/LogicalRelationalCatalog.java | 10 ++-- .../db/catalog/entity/LogicalConstraint.java | 11 ++++- .../impl/logical/RelationalCatalog.java | 25 +++++----- .../java/org/polypheny/db/ddl/DdlManager.java | 2 +- .../polypheny/db/transaction/Transaction.java | 13 +++++- .../org/polypheny/db/ddl/DdlManagerImpl.java | 28 +++++------ .../db/transaction/TransactionImpl.java | 46 ++++++++++++++++--- .../polypheny/db/webui/crud/LanguageCrud.java | 2 +- 9 files changed, 99 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java b/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java index 55ef4f83e9..5e68038b90 100644 --- a/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java +++ b/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java @@ -247,6 +247,9 @@ public static EnforcementInformation getControl( LogicalTable table, Statement s constraints.add( pkc ); } + // get new constraints + constraints.addAll( statement.getTransaction().getUsedConstraints( table.id ).stream().filter( c -> c.type != ConstraintType.FOREIGN ).toList() ); + AlgNode constrainedNode; // diff --git a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java index 9d87eb8042..0fdb8d59fc 100644 --- a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java @@ -37,6 +37,7 @@ import org.polypheny.db.catalog.logistic.ForeignKeyOption; import org.polypheny.db.catalog.logistic.IndexType; import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.transaction.Statement; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.entity.PolyValue; @@ -195,8 +196,9 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * * @param tableId The id of the table * @param columnIds The id of key which will be part of the primary keys + * @param statement The statement used to attach constraint enforcement on commit */ - void addPrimaryKey( long tableId, List columnIds ); + void addPrimaryKey( long tableId, List columnIds, Statement statement ); /** @@ -218,8 +220,9 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * @param tableId The id of the table * @param constraintName The name of the constraint * @param columnIds A list of column ids + * @param statement The statement to attach the constraint checks on commit */ - void addUniqueConstraint( long tableId, String constraintName, List columnIds ); + void addUniqueConstraint( long tableId, String constraintName, List columnIds, Statement statement ); /** @@ -229,8 +232,9 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * @param constraintName The name of the constraint * @param columnIds A list of column ids * @param type The type of the constraint + * @param statement */ - void addConstraint( long tableId, String constraintName, List columnIds, ConstraintType type ); + long addConstraint( long tableId, String constraintName, List columnIds, ConstraintType type, Statement statement ); /** * Deletes the specified primary key (including the entry in the key table). If there is an index on this key, make sure to delete it first. diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/LogicalConstraint.java b/core/src/main/java/org/polypheny/db/catalog/entity/LogicalConstraint.java index d9cb9c71a2..6b5aa59cd5 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/LogicalConstraint.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/LogicalConstraint.java @@ -23,13 +23,14 @@ import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.Value; +import org.jetbrains.annotations.NotNull; import org.polypheny.db.catalog.entity.logical.LogicalKey; import org.polypheny.db.catalog.logistic.ConstraintType; @EqualsAndHashCode @Value -public class LogicalConstraint implements Serializable { +public class LogicalConstraint implements Serializable, Comparable { @Serialize public long id; @@ -57,4 +58,12 @@ public LogicalConstraint( } + @Override + public int compareTo( @NotNull LogicalConstraint o ) { + if ( this == o ) { + return 0; + } + return Long.compare( id, o.id ); + } + } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java index b152e42ebd..e7e585cbe7 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java @@ -62,6 +62,7 @@ import org.polypheny.db.catalog.snapshot.Snapshot; import org.polypheny.db.catalog.util.CatalogEvent; import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.transaction.Statement; import org.polypheny.db.type.PolySerializable; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.entity.PolyValue; @@ -217,9 +218,8 @@ public void deleteTable( long tableId ) { @Override public void setPrimaryKey( long tableId, @Nullable Long keyId ) { - LogicalTable oldTable = tables.get( tableId ); // we temporarily can remove the primary, to clean-up old primaries before adding a new one - tables.put( tableId, oldTable.toBuilder().primaryKey( keyId ).build() ); + tables.computeIfPresent( tableId, ( k, oldTable ) -> oldTable.toBuilder().primaryKey( keyId ).build() ); if ( keyId != null ) { keys.put( keyId, new LogicalPrimaryKey( keys.get( keyId ) ) ); @@ -371,18 +371,16 @@ public void deleteDefaultValue( long columnId ) { @Override - public void addPrimaryKey( long tableId, List columnIds ) { + public void addPrimaryKey( long tableId, List columnIds, Statement statement ) { if ( columnIds.stream().anyMatch( id -> columns.get( id ).nullable ) ) { throw new GenericRuntimeException( "Primary key is not allowed to use nullable columns." ); } - // TODO: Check if the current values are unique - // Check if there is already a primary key defined for this table and if so, delete it. LogicalTable table = tables.get( tableId ); + if ( table.primaryKey != null ) { - // CatalogCombinedKey combinedKey = getCombinedKey( table.primaryKey ); if ( getKeyUniqueCount( table.primaryKey ) == 1 && isForeignKey( table.primaryKey ) ) { // This primary key is the only constraint for the uniqueness of this key. throw new GenericRuntimeException( "This key is referenced by at least one foreign key which requires this key to be unique. To drop this primary key, first drop the foreign keys or create a unique constraint." ); @@ -394,8 +392,8 @@ public void addPrimaryKey( long tableId, List columnIds ) { } long keyId = getOrAddKey( tableId, columnIds, EnforcementTime.ON_QUERY ); setPrimaryKey( tableId, keyId ); + change( CatalogEvent.PRIMARY_KEY_CREATED, tableId, keyId ); - tables.get( tableId ); } @@ -507,7 +505,7 @@ public void addForeignKey( long tableId, List columnIds, long referencesTa @Override - public void addUniqueConstraint( long tableId, String constraintName, List columnIds ) { + public void addUniqueConstraint( long tableId, String constraintName, List columnIds, Statement statement ) { long keyId = getOrAddKey( tableId, columnIds, EnforcementTime.ON_QUERY ); // Check if there is already a unique constraint List logicalConstraints = constraints.values().stream() @@ -516,20 +514,23 @@ public void addUniqueConstraint( long tableId, String constraintName, List if ( !logicalConstraints.isEmpty() ) { throw new GenericRuntimeException( "There is already a unique constraint!" ); } - addConstraint( tableId, constraintName, columnIds, ConstraintType.UNIQUE ); + long id = addConstraint( tableId, constraintName, columnIds, ConstraintType.UNIQUE, statement ); + statement.getTransaction().addNewConstraint( tableId, constraints.get( id ) ); } @Override - public void addConstraint( long tableId, String constraintName, List columnIds, ConstraintType type ) { + public long addConstraint( long tableId, String constraintName, List columnIds, ConstraintType type, Statement statement ) { long keyId = getOrAddKey( tableId, columnIds, EnforcementTime.ON_QUERY ); long id = idBuilder.getNewConstraintId(); + LogicalConstraint constraint = new LogicalConstraint( id, keyId, type, constraintName, Objects.requireNonNull( keys.get( keyId ) ) ); synchronized ( this ) { - constraints.put( id, new LogicalConstraint( id, keyId, type, constraintName, Objects.requireNonNull( keys.get( keyId ) ) ) ); + constraints.put( id, constraint ); change( CatalogEvent.CONSTRAINT_CREATED, null, id ); } - tables.get( tableId ); + statement.getTransaction().addNewConstraint( tableId, constraint ); + return id; } diff --git a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java index 2205918381..c4c8653198 100644 --- a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java +++ b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java @@ -476,7 +476,7 @@ public static DdlManager getInstance() { /** * Adds a new constraint to a table */ - public abstract void createConstraint( ConstraintInformation information, long namespaceId, List columnIds, long tableId ); + public abstract void createConstraint( ConstraintInformation information, long namespaceId, List columnIds, long tableId, Statement statement ); /** * Drop a NAMESPACE diff --git a/core/src/main/java/org/polypheny/db/transaction/Transaction.java b/core/src/main/java/org/polypheny/db/transaction/Transaction.java index e10227d2c1..b327aab06a 100644 --- a/core/src/main/java/org/polypheny/db/transaction/Transaction.java +++ b/core/src/main/java/org/polypheny/db/transaction/Transaction.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.polypheny.db.adapter.Adapter; import org.polypheny.db.adapter.java.JavaTypeFactory; +import org.polypheny.db.catalog.entity.LogicalConstraint; import org.polypheny.db.catalog.entity.LogicalUser; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; import org.polypheny.db.catalog.entity.logical.LogicalTable; @@ -80,9 +81,15 @@ public interface Transaction { boolean getUseCache(); - void addUsedTable(LogicalTable table); + void addUsedTable( LogicalTable table ); - void removeUsedTable(LogicalTable table); + void removeUsedTable( LogicalTable table ); + + void getNewEntityConstraints( long entity ); + + void addNewConstraint( long entityId, LogicalConstraint constraint ); + + void removeNewConstraint( long entityId, LogicalConstraint constraint ); void setAcceptsOutdated( boolean acceptsOutdated ); @@ -94,6 +101,8 @@ public interface Transaction { TransactionManager getTransactionManager(); + List getUsedConstraints( long id ); + /** * Flavor, how multimedia results should be returned from a store. */ diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 165e871563..ebe60fb402 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -543,7 +543,7 @@ public void createForeignKey( LogicalTable table, LogicalTable refTable, List columnNames, Stat dropConstraint( table, ConstraintType.PRIMARY.name() ); } - catalog.getLogicalRel( table.namespaceId ).addPrimaryKey( table.id, columnIds ); - catalog.getLogicalRel( table.namespaceId ).addConstraint( table.id, ConstraintType.PRIMARY.name(), columnIds, ConstraintType.PRIMARY ); + catalog.getLogicalRel( table.namespaceId ).addPrimaryKey( table.id, columnIds, statement ); + catalog.getLogicalRel( table.namespaceId ).addConstraint( table.id, ConstraintType.PRIMARY.name(), columnIds, ConstraintType.PRIMARY, statement ); // Add new column placements List placements = catalog.getSnapshot().alloc().getPlacementsFromLogical( table.id ); @@ -840,12 +840,12 @@ public void createUniqueConstraint( LogicalTable table, List columnNames checkModelLogic( table, null ); - List columnIds = new LinkedList<>(); + List columnIds = new ArrayList<>(); for ( String columnName : columnNames ) { LogicalColumn logicalColumn = catalog.getSnapshot().rel().getColumn( table.id, columnName ).orElseThrow(); columnIds.add( logicalColumn.id ); } - catalog.getLogicalRel( table.namespaceId ).addUniqueConstraint( table.id, constraintName, columnIds ); + catalog.getLogicalRel( table.namespaceId ).addUniqueConstraint( table.id, constraintName, columnIds, statement ); statement.getTransaction().addUsedTable( table ); } @@ -1763,8 +1763,8 @@ public void createMaterializedView( String viewName, long namespaceId, AlgRoot a // Sets previously created primary key long pkId = ids.get( fields.get( fields.size() - 1 ).name() ).id; - catalog.getLogicalRel( namespaceId ).addPrimaryKey( view.id, List.of( pkId ) ); - catalog.getLogicalRel( view.namespaceId ).addConstraint( view.id, ConstraintType.PRIMARY.name(), List.of( pkId ), ConstraintType.PRIMARY ); + catalog.getLogicalRel( namespaceId ).addPrimaryKey( view.id, List.of( pkId ), statement ); + catalog.getLogicalRel( view.namespaceId ).addConstraint( view.id, ConstraintType.PRIMARY.name(), List.of( pkId ), ConstraintType.PRIMARY, statement ); AllocationPartitionGroup group = catalog.getAllocRel( namespaceId ).addPartitionGroup( view.id, UNPARTITIONED, namespaceId, PartitionType.NONE, 1, false ); AllocationPartition partition = catalog.getAllocRel( namespaceId ).addPartition( view.id, namespaceId, group.id, null, false, PlacementType.AUTOMATIC, DataPlacementRole.UP_TO_DATE, null, PartitionType.NONE ); @@ -2077,7 +2077,7 @@ public void createTable( long namespaceId, String name, List f // create foreign keys later on for ( ConstraintInformation constraint : constraints.stream().filter( c -> c.getType() != ConstraintType.FOREIGN ).toList() ) { List columnIds = constraint.columnNames.stream().map( key -> ids.get( key ).id ).toList(); - createConstraint( constraint, namespaceId, columnIds, logical.id ); + createConstraint( constraint, namespaceId, columnIds, logical.id, statement ); if ( constraint.type == ConstraintType.PRIMARY ) { pkIds = columnIds; @@ -2106,7 +2106,7 @@ public void createTable( long namespaceId, String name, List f constraints.stream().filter( c -> c.getType() == ConstraintType.FOREIGN ).forEach( c -> { List columnIds = c.columnNames.stream().map( key -> ids.get( key ).id ).toList(); - createConstraint( c, namespaceId, columnIds, logical.id ); + createConstraint( c, namespaceId, columnIds, logical.id, statement ); } ); catalog.updateSnapshot(); @@ -2784,18 +2784,18 @@ private LogicalColumn addColumn( long namespaceId, String columnName, ColumnType @Override - public void createConstraint( ConstraintInformation information, long namespaceId, List columnIds, long tableId ) { + public void createConstraint( ConstraintInformation information, long namespaceId, List columnIds, long tableId, Statement statement ) { String constraintName = information.name; if ( constraintName == null ) { constraintName = NameGenerator.generateConstraintName(); } switch ( information.getType() ) { case UNIQUE: - catalog.getLogicalRel( namespaceId ).addUniqueConstraint( tableId, constraintName, columnIds ); + catalog.getLogicalRel( namespaceId ).addUniqueConstraint( tableId, constraintName, columnIds, statement ); break; case PRIMARY: - catalog.getLogicalRel( namespaceId ).addPrimaryKey( tableId, columnIds ); - catalog.getLogicalRel( namespaceId ).addConstraint( tableId, ConstraintType.PRIMARY.name(), columnIds, ConstraintType.PRIMARY ); + catalog.getLogicalRel( namespaceId ).addPrimaryKey( tableId, columnIds, statement ); + catalog.getLogicalRel( namespaceId ).addConstraint( tableId, ConstraintType.PRIMARY.name(), columnIds, ConstraintType.PRIMARY, statement ); break; case FOREIGN: String foreignKeyTable = information.foreignKeyTable; @@ -2810,7 +2810,7 @@ public void createConstraint( ConstraintInformation information, long namespaceI } long columnId = catalog.getSnapshot().rel().getColumn( foreignTableId, information.foreignKeyColumnName ).orElseThrow().id; catalog.getLogicalRel( namespaceId ).addForeignKey( tableId, columnIds, foreignTableId, List.of( columnId ), constraintName, ForeignKeyOption.NONE, ForeignKeyOption.NONE ); - catalog.getLogicalRel( namespaceId ).addConstraint( tableId, ConstraintType.FOREIGN.name(), columnIds, ConstraintType.FOREIGN ); + catalog.getLogicalRel( namespaceId ).addConstraint( tableId, ConstraintType.FOREIGN.name(), columnIds, ConstraintType.FOREIGN, statement ); break; } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java index 00fdefec97..09139f8f20 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java @@ -18,13 +18,13 @@ import java.util.ArrayList; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -40,6 +40,7 @@ import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.algebra.logical.common.LogicalConstraintEnforcer.EnforcementInformation; import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.entity.LogicalConstraint; import org.polypheny.db.catalog.entity.LogicalUser; import org.polypheny.db.catalog.entity.logical.LogicalKey.EnforcementTime; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; @@ -97,7 +98,10 @@ public class TransactionImpl implements Transaction, Comparable { @Getter - private final Set logicalTables = new TreeSet<>(); + private final Set usedTables = new TreeSet<>(); + + @Getter + private final Map> entityConstraints = new HashMap<>(); @Getter private final Set> involvedAdapters = new ConcurrentSkipListSet<>( (a,b) -> Math.toIntExact( a.adapterId - b.adapterId ) ); @@ -172,11 +176,11 @@ public void commit() throws TransactionException { } } - if ( !logicalTables.isEmpty() ) { + if ( !usedTables.isEmpty() ) { Statement statement = createStatement(); QueryProcessor processor = statement.getQueryProcessor(); List infos = ConstraintEnforceAttacher - .getConstraintAlg( logicalTables, statement, EnforcementTime.ON_COMMIT ); + .getConstraintAlg( usedTables, statement, EnforcementTime.ON_COMMIT ); List results = infos .stream() .map( s -> processor.prepareQuery( AlgRoot.of( s.control(), Kind.SELECT ), s.control().getCluster().getTypeFactory().builder().build(), false, true, false ) ).toList(); @@ -320,13 +324,34 @@ public boolean getUseCache() { @Override public void addUsedTable( LogicalTable table ) { - this.logicalTables.add( table ); + this.usedTables.add( table ); } @Override public void removeUsedTable( LogicalTable table ) { - this.logicalTables.remove( table ); + this.usedTables.remove( usedTables ); + } + + + @Override + public void getNewEntityConstraints( long entity ) { + this.entityConstraints.getOrDefault( entity, List.of() ); + } + + + @Override + public void addNewConstraint( long entityId, LogicalConstraint constraint ) { + this.entityConstraints.putIfAbsent( entityId, new ArrayList<>() ); + this.entityConstraints.get( entityId ).add( constraint ); + } + + + @Override + public void removeNewConstraint( long entityId, LogicalConstraint constraint ) { + List constraints = this.entityConstraints.get( entityId ); + constraints.remove( constraint ); + this.entityConstraints.put( entityId, constraints ); } @@ -386,6 +411,13 @@ public void updateAccessMode( AccessMode accessModeCandidate ) { } + + @Override + public List getUsedConstraints( long id ) { + return this.entityConstraints.getOrDefault( id, List.of() ); + } + + void abort() { Thread.currentThread().interrupt(); } diff --git a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java index 87e28ba1fe..30bb2032c2 100644 --- a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java +++ b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java @@ -162,7 +162,7 @@ public static long getNamespaceIdOrDefault( String namespace ) { results.add( builder.apply( executedContext, request, executedContext.getStatement() ).build() ); } - commitAndFinish( executedContexts, transaction.getQueryAnalyzer(), results, executedContexts.stream().map( ExecutedContext::getExecutionTime ).reduce( (a,b) -> a + b ).orElse( -1L ) ); + commitAndFinish( executedContexts, transaction.getQueryAnalyzer(), results, executedContexts.stream().map( ExecutedContext::getExecutionTime ).reduce( Long::sum ).orElse( -1L ) ); return results; } From c1fe14527cf29d82cffa3291a5d4c1ffc37283eb Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 4 Jul 2024 15:33:22 +0200 Subject: [PATCH 20/24] fixed incorrect remove of table for constraint --- .../main/java/org/polypheny/db/transaction/TransactionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java index 09139f8f20..71778118e9 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java @@ -330,7 +330,7 @@ public void addUsedTable( LogicalTable table ) { @Override public void removeUsedTable( LogicalTable table ) { - this.usedTables.remove( usedTables ); + this.usedTables.remove( table ); } From 16caf79606b7f9bf54b7fac53368f600be1dae84 Mon Sep 17 00:00:00 2001 From: datomo Date: Sun, 7 Jul 2024 12:38:20 +0200 Subject: [PATCH 21/24] reformating, optimized imports, adjusted headers --- .../main/java/org/polypheny/db/algebra/AlgNode.java | 6 +++--- .../common/EnumerableConditionalExecuteFalseRule.java | 1 - .../logical/common/LogicalConstraintEnforcer.java | 2 +- .../db/catalog/entity/logical/LogicalForeignKey.java | 3 +-- .../db/catalog/impl/logical/RelationalCatalog.java | 1 - .../db/processing/ImplementationContext.java | 2 +- .../db/processing/LogicalAlgAnalyzeShuttle.java | 1 - .../org/polypheny/db/routing/ColumnDistribution.java | 7 +++---- .../main/java/org/polypheny/db/transaction/PUID.java | 7 ++++++- .../db/processing/AbstractQueryProcessor.java | 11 +++++------ .../org/polypheny/db/processing/DataMigratorImpl.java | 3 +-- .../org/polypheny/db/transaction/EntityAccessMap.java | 1 + .../main/java/org/polypheny/db/transaction/Lock.java | 1 - .../org/polypheny/db/transaction/TransactionImpl.java | 2 +- .../db/view/MaterializedViewManagerImpl.java | 5 +---- .../statistics/NumericalStatisticColumn.java | 4 ---- .../main/java/org/polypheny/db/cql/CqlProcessor.java | 1 - .../org/polypheny/db/adapter/jdbc/JdbcSchema.java | 10 ---------- .../db/adapter/jdbc/sources/AbstractJdbcSource.java | 1 - .../java/org/polypheny/db/languages/MqlProcessor.java | 1 - .../polypheny/db/notebooks/model/JupyterKernel.java | 8 +++----- .../model/language/IPythonKernelLanguage.java | 3 +-- .../polypheny/db/prisminterface/ClientManager.java | 1 - 23 files changed, 28 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/org/polypheny/db/algebra/AlgNode.java b/core/src/main/java/org/polypheny/db/algebra/AlgNode.java index 0f7449dcbf..c384aeec25 100644 --- a/core/src/main/java/org/polypheny/db/algebra/AlgNode.java +++ b/core/src/main/java/org/polypheny/db/algebra/AlgNode.java @@ -394,11 +394,11 @@ default boolean containsEntity() { return getInputs().stream().anyMatch( AlgNode::containsEntity ); } - default Set getEntities(){ + default Set getEntities() { if ( getEntity() != null ) { - return Set.of(getEntity()); + return Set.of( getEntity() ); } - return getInputs().stream().map( AlgNode::getEntities ).reduce( (a,b) -> Stream.concat(a.stream(), b.stream()).collect( Collectors.toSet()) ).orElse( Set.of() ); + return getInputs().stream().map( AlgNode::getEntities ).reduce( ( a, b ) -> Stream.concat( a.stream(), b.stream() ).collect( Collectors.toSet() ) ).orElse( Set.of() ); } /** diff --git a/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java b/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java index 3581161bd8..da7ddf5410 100644 --- a/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java +++ b/core/src/main/java/org/polypheny/db/algebra/enumerable/common/EnumerableConditionalExecuteFalseRule.java @@ -17,7 +17,6 @@ package org.polypheny.db.algebra.enumerable.common; -import lombok.SneakyThrows; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.convert.ConverterRule; import org.polypheny.db.algebra.core.AlgFactories; diff --git a/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java b/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java index 5e68038b90..81d0987077 100644 --- a/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java +++ b/core/src/main/java/org/polypheny/db/algebra/logical/common/LogicalConstraintEnforcer.java @@ -400,7 +400,7 @@ public AlgNode accept( AlgShuttle shuttle ) { } - public record EnforcementInformation(AlgNode control, List> errorClasses, List errorMessages) { + public record EnforcementInformation( AlgNode control, List> errorClasses, List errorMessages ) { /** * {@link EnforcementInformation} holds all needed information regarding a constraint. diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java index 89811943cd..a4e1d14162 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalForeignKey.java @@ -26,7 +26,6 @@ import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.Value; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.entity.PolyObject; @@ -162,7 +161,7 @@ public PolyValue[] getParameterArray() { } - public record PrimitiveCatalogForeignKeyColumn(String pktableCat, String pktableSchem, String pktableName, String pkcolumnName, String fktableCat, String fktableSchem, String fktableName, String fkcolumnName, int keySeq, Integer updateRule, Integer deleteRule, String fkName, String pkName, Integer deferrability) { + public record PrimitiveCatalogForeignKeyColumn( String pktableCat, String pktableSchem, String pktableName, String pkcolumnName, String fktableCat, String fktableSchem, String fktableName, String fkcolumnName, int keySeq, Integer updateRule, Integer deleteRule, String fkName, String pkName, Integer deferrability ) { } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java index e7e585cbe7..013534d3e9 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java @@ -379,7 +379,6 @@ public void addPrimaryKey( long tableId, List columnIds, Statement stateme // Check if there is already a primary key defined for this table and if so, delete it. LogicalTable table = tables.get( tableId ); - if ( table.primaryKey != null ) { if ( getKeyUniqueCount( table.primaryKey ) == 1 && isForeignKey( table.primaryKey ) ) { // This primary key is the only constraint for the uniqueness of this key. diff --git a/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java b/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java index 06d173aed9..32c01bd57f 100644 --- a/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java +++ b/core/src/main/java/org/polypheny/db/processing/ImplementationContext.java @@ -17,8 +17,8 @@ package org.polypheny.db.processing; -import java.util.Optional; import com.google.common.base.Stopwatch; +import java.util.Optional; import lombok.EqualsAndHashCode; import lombok.Value; import lombok.experimental.NonFinal; diff --git a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java index b8ab8df0ff..243c543178 100644 --- a/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java +++ b/core/src/main/java/org/polypheny/db/processing/LogicalAlgAnalyzeShuttle.java @@ -20,7 +20,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java b/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java index a21169afc0..70220385a7 100644 --- a/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java +++ b/core/src/main/java/org/polypheny/db/routing/ColumnDistribution.java @@ -341,7 +341,7 @@ private List pick( ColumnDistribution columnDistribution, long } - public record PartialPartition(AllocationEntity entity, List columns) { + public record PartialPartition( AllocationEntity entity, List columns ) { } @@ -356,10 +356,9 @@ public record PartialPartition(AllocationEntity entity, List c * * * - * * @param partials */ - public record FullPartition(long id, List partials) implements FieldDistribution { + public record FullPartition( long id, List partials ) implements FieldDistribution { public boolean needsJoin() { return partials.size() > 1; @@ -383,7 +382,7 @@ public List getOrderedColumns() { } - public record RoutedDistribution(LogicalEntity entity, List partitions) { + public record RoutedDistribution( LogicalEntity entity, List partitions ) { public boolean needsUnion() { return partitions.size() > 1; diff --git a/core/src/main/java/org/polypheny/db/transaction/PUID.java b/core/src/main/java/org/polypheny/db/transaction/PUID.java index 2a2a3a85ad..d03323e4bd 100644 --- a/core/src/main/java/org/polypheny/db/transaction/PUID.java +++ b/core/src/main/java/org/polypheny/db/transaction/PUID.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 The Polypheny Project + * Copyright 2019-2024 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -161,6 +161,7 @@ private NodeId( final String nodeId ) { public static NodeId fromString( final String nodeId ) { return new NodeId( nodeId ); } + } @@ -179,6 +180,7 @@ private UserId( final byte[] userName ) { public static UserId fromString( final String userName ) { return new UserId( userName.getBytes() ); } + } @@ -197,6 +199,7 @@ private ConnectionId( final String connectionId ) { public static ConnectionId fromString( final String connectionId ) { return new ConnectionId( connectionId ); } + } @@ -225,5 +228,7 @@ public static StatementId fromString( final String statementId ) { public static StatementId fromInt( final int statementId ) { return new StatementId( statementId ); } + } + } diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index daf545a996..c739d724e2 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -542,7 +542,6 @@ private List routePlans( AlgRoot indexLookupRoot, LogicalQueryInformation } - private void flattenPlans( List plans ) { for ( Plan plan : plans ) { AlgRoot routedRoot = plan.proposedRoutingPlan().getRoutedRoot(); @@ -1209,11 +1208,11 @@ private Map> extractPartitions( Set entities ) { Map> map = new HashMap<>(); for ( Entity entity : entities ) { if ( entity.isLogical() ) { - map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).addAll( Catalog.snapshot().alloc().getPartitionsFromLogical( entity.getId()).stream().map( p -> p.id ).toList() ); - }else if ( entity.isAllocation() ){ - map.computeIfAbsent( ((AllocationEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getPartitionId() ); - }else if( entity.isPhysical() ){ - map.computeIfAbsent( ((PhysicalEntity)entity).getLogicalId(), k -> new ArrayList<>() ).add( ((PhysicalEntity)entity).getAllocationId() ); + map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).addAll( Catalog.snapshot().alloc().getPartitionsFromLogical( entity.getId() ).stream().map( p -> p.id ).toList() ); + } else if ( entity.isAllocation() ) { + map.computeIfAbsent( ((AllocationEntity) entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getPartitionId() ); + } else if ( entity.isPhysical() ) { + map.computeIfAbsent( ((PhysicalEntity) entity).getLogicalId(), k -> new ArrayList<>() ).add( ((PhysicalEntity) entity).getAllocationId() ); } } return map; diff --git a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java index 8b10e00d09..c21515e9c0 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java +++ b/dbms/src/main/java/org/polypheny/db/processing/DataMigratorImpl.java @@ -422,7 +422,6 @@ public AlgRoot buildDeleteStatement( Statement statement, List builder.project( values, columnNames ); } - AlgNode node = LogicalRelModify.create( allocation, builder.build(), Modify.Operation.DELETE, null, null, false ); return AlgRoot.of( node, Kind.DELETE ); @@ -659,7 +658,7 @@ private Source getSource( Transaction transaction, List sourceT } - private record Source(Statement sourceStatement, AlgRoot sourceAlg) { + private record Source( Statement sourceStatement, AlgRoot sourceAlg ) { } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/EntityAccessMap.java b/dbms/src/main/java/org/polypheny/db/transaction/EntityAccessMap.java index 9751ee67a1..c679ef1e28 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/EntityAccessMap.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/EntityAccessMap.java @@ -170,6 +170,7 @@ public Collection> getAccessedEntityPair() { return accessLockMap.entrySet(); } + public LockMode getNeededLock() { return accessLockMap.values().stream().anyMatch( l -> l == LockMode.EXCLUSIVE ) ? LockMode.EXCLUSIVE : LockMode.SHARED; } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java index e1690a49bd..a96458b819 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/Lock.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/Lock.java @@ -16,7 +16,6 @@ package org.polypheny.db.transaction; -import java.util.HashSet; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.locks.Condition; diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java index 71778118e9..8c0c345fae 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionImpl.java @@ -104,7 +104,7 @@ public class TransactionImpl implements Transaction, Comparable { private final Map> entityConstraints = new HashMap<>(); @Getter - private final Set> involvedAdapters = new ConcurrentSkipListSet<>( (a,b) -> Math.toIntExact( a.adapterId - b.adapterId ) ); + private final Set> involvedAdapters = new ConcurrentSkipListSet<>( ( a, b ) -> Math.toIntExact( a.adapterId - b.adapterId ) ); private boolean useCache = true; diff --git a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java index 85f3bf4694..1bb2ff6732 100644 --- a/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/view/MaterializedViewManagerImpl.java @@ -61,9 +61,6 @@ import org.polypheny.db.processing.DataMigrator; import org.polypheny.db.rex.RexBuilder; import org.polypheny.db.tools.AlgBuilder; -import org.polypheny.db.transaction.EntityAccessMap; -import org.polypheny.db.transaction.EntityAccessMap.EntityIdentifier; -import org.polypheny.db.transaction.EntityAccessMap.EntityIdentifier.NamespaceLevel; import org.polypheny.db.transaction.Lock.LockMode; import org.polypheny.db.transaction.LockManager; import org.polypheny.db.transaction.PolyXid; @@ -73,7 +70,6 @@ import org.polypheny.db.transaction.TransactionImpl; import org.polypheny.db.transaction.TransactionManager; import org.polypheny.db.util.DeadlockException; -import org.polypheny.db.util.Pair; import org.polypheny.db.util.background.BackgroundTask.TaskPriority; import org.polypheny.db.util.background.BackgroundTask.TaskSchedulingType; import org.polypheny.db.util.background.BackgroundTaskManager; @@ -336,6 +332,7 @@ public void addData( Transaction transaction, @Nullable List> store addMaterializedInfo( materializedView.id, materializedView.getMaterializedCriteria() ); } + /** * Deletes all the data from a materialized view and adds the newest data to the materialized view * diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java index ba073737b9..f8c1629b29 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/statistics/NumericalStatisticColumn.java @@ -19,12 +19,8 @@ import com.google.gson.annotations.Expose; import java.util.List; -import java.util.Set; import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CopyOnWriteArraySet; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; diff --git a/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java b/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java index 13172c9b4c..a7586387a7 100644 --- a/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java +++ b/plugins/cql-language/src/main/java/org/polypheny/db/cql/CqlProcessor.java @@ -17,7 +17,6 @@ package org.polypheny.db.cql; import java.util.Arrays; -import java.util.Collections; import java.util.List; import org.polypheny.db.adapter.java.JavaTypeFactory; import org.polypheny.db.algebra.AlgRoot; diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java index 97e371d3cd..e1f23fe96d 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcSchema.java @@ -33,11 +33,6 @@ package org.polypheny.db.adapter.jdbc; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimap; -import java.util.HashMap; -import java.util.Map; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -47,16 +42,11 @@ import org.polypheny.db.adapter.jdbc.connection.ConnectionFactory; import org.polypheny.db.adapter.jdbc.connection.ConnectionHandler; import org.polypheny.db.adapter.jdbc.connection.ConnectionHandlerException; -import org.polypheny.db.algebra.type.AlgDataType; -import org.polypheny.db.algebra.type.AlgDataTypeFactory; -import org.polypheny.db.algebra.type.AlgProtoDataType; import org.polypheny.db.catalog.entity.physical.PhysicalTable; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; -import org.polypheny.db.schema.Function; import org.polypheny.db.schema.Namespace; import org.polypheny.db.schema.types.Expressible; import org.polypheny.db.sql.language.SqlDialect; -import org.polypheny.db.type.PolyType; /** diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java index 3d4320b38a..47f361c263 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java @@ -25,7 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.apache.commons.dbcp2.BasicDataSource; diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java index 9a7cd7ad0e..befaccb09f 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/MqlProcessor.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java index 73bcf96381..c1450f31fd 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/JupyterKernel.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; import java.io.IOException; @@ -48,7 +47,6 @@ import org.polypheny.db.notebooks.model.language.JupyterKernelLanguage.JupyterQueryPart; import org.polypheny.db.notebooks.model.language.JupyterLanguageFactory; import org.polypheny.db.processing.QueryContext; -import org.polypheny.db.webui.HttpServer; import org.polypheny.db.webui.crud.LanguageCrud; import org.polypheny.db.webui.models.requests.QueryRequest; import org.polypheny.db.webui.models.results.Result; @@ -436,9 +434,9 @@ public void onError( WebSocket webSocket, Throwable error ) { /** - * Represents a query cell that is currently being executed. - */ - private record ActivePolyCell( @Getter String language, @Getter String namespace ) { + * Represents a query cell that is currently being executed. + */ + private record ActivePolyCell( @Getter String language, @Getter String namespace ) { } diff --git a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java index 574c637033..52414f882d 100644 --- a/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java +++ b/plugins/notebooks/src/main/java/org/polypheny/db/notebooks/model/language/IPythonKernelLanguage.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The Polypheny Project + * Copyright 2019-2024 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; /** diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java index 8f300c7e22..e4afa73021 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/ClientManager.java @@ -31,7 +31,6 @@ import org.polypheny.db.prisminterface.PIPlugin.PrismInterface; import org.polypheny.db.prisminterface.transport.Transport; import org.polypheny.db.prisminterface.utils.PropertyUtils; -import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.transaction.TransactionManager; import org.polypheny.prism.ConnectionRequest; From 13278ea40e12348f3351b978207f942939b3834c Mon Sep 17 00:00:00 2001 From: Martin Vahlensieck Date: Mon, 15 Jul 2024 09:51:02 +0200 Subject: [PATCH 22/24] Use Transaction instead of TransactionImpl --- .../db/processing/AbstractQueryProcessor.java | 6 +++--- .../java/org/polypheny/db/transaction/LockManager.java | 10 +++++----- .../polypheny/db/view/MaterializedViewManagerImpl.java | 4 ++-- .../main/java/org/polypheny/db/cql/CqlProcessor.java | 4 ++-- .../java/org/polypheny/db/cypher/CypherProcessor.java | 4 ++-- .../java/org/polypheny/db/languages/MqlProcessor.java | 4 ++-- .../main/java/org/polypheny/db/sql/SqlProcessor.java | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index c739d724e2..fce15ec4cc 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -566,7 +566,7 @@ private void acquireLock( boolean isAnalyze, AlgRoot logicalRoot, Map Date: Mon, 15 Jul 2024 10:01:58 +0200 Subject: [PATCH 23/24] Remove unused variable --- .../src/main/java/org/polypheny/db/transaction/LockManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index f886eb25f9..28a7e02585 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -17,7 +17,6 @@ package org.polypheny.db.transaction; -import com.google.common.base.Stopwatch; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; @@ -65,7 +64,6 @@ public void lock( LockMode mode, @NonNull Transaction transaction ) throws Deadl waiters.add( thread ); - Stopwatch watch = Stopwatch.createStarted(); // wait while ( true ) { lock.lock(); From 3e0c17e3bc7942516efcef4b47ac9e7a1800230e Mon Sep 17 00:00:00 2001 From: Martin Vahlensieck Date: Mon, 15 Jul 2024 10:02:25 +0200 Subject: [PATCH 24/24] Remove unused Runnable interface from LockManager --- .../java/org/polypheny/db/transaction/LockManager.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java index 28a7e02585..8d68fb580e 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/LockManager.java @@ -33,7 +33,7 @@ import org.polypheny.db.util.DeadlockException; @Slf4j -public class LockManager implements Runnable { +public class LockManager { public static final LockManager INSTANCE = new LockManager(); @@ -178,10 +178,4 @@ public void removeTransaction( @NonNull Transaction transaction ) { unlock( transaction ); } - - @Override - public void run() { - - } - }