Skip to content

Commit

Permalink
added some catching in tests to not block transactions
Browse files Browse the repository at this point in the history
  • Loading branch information
@datomo
datomo committed Dec 11, 2024
1 parent 4d33a73 commit 0fc9e93
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 87 deletions.
4 changes: 1 addition & 3 deletions core/src/main/java/org/polypheny/db/algebra/polyalg/PolyAlgMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ public PolyAlgMetadata addCosts( AlgOptCost nonCumulative, AlgOptCost cumulative
addMetadata( "tuplesCost", "Processed Tuples", nonCumulative.getRows(), cumulative.getRows(), false, "Most Tuples Processed" );
addMetadata( "cpuCost", "CPU Cost", nonCumulative.getCpu(), cumulative.getCpu(), false, "Most CPU" );
setOutConnection( "tupleCount", false );
//addMetadata( "ioCost", "IO Cost", nonCumulative.getIo(), cumulative.getIo(), false );
return this;
}

Expand Down Expand Up @@ -152,7 +151,7 @@ private ObjectNode findTableEntry( String key ) {

public static class GlobalStats {

private final double EPS = 0.0000001;
private final static double EPS = 0.0000001;
private final Map<String, Double> maxValues = new HashMap<>();
private final Map<String, Double> maxCumulativeValues = new HashMap<>();

Expand Down Expand Up @@ -182,7 +181,6 @@ public void updateMaxCosts( AlgOptCost nonCumulative, Double tupleCount ) {
update( "tupleCount", tupleCount );
update( "tuplesCost", nonCumulative.getRows() );
update( "cpuCost", nonCumulative.getCpu() );
//update( "ioCost", nonCumulative.getIo() );
}


Expand Down
186 changes: 102 additions & 84 deletions dbms/src/test/java/org/polypheny/db/polyalg/PolyAlgParsingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -72,6 +73,7 @@
import org.polypheny.db.type.entity.PolyValue;


@Slf4j
@SuppressWarnings("SqlNoDataSourceInspection")
public class PolyAlgParsingTest {

Expand Down Expand Up @@ -171,60 +173,65 @@ private static void testQueryRoundTrip( String query, QueryLanguage ql, String n
TransactionManager transactionManager = TransactionManagerImpl.getInstance();
Transaction transaction = transactionManager.startTransaction( Catalog.defaultUserId, ns, true, ORIGIN );

QueryContext qc = QueryContext.builder()
.query( query )
.language( ql )
.isAnalysed( true )
.usesCache( true )
.origin( ORIGIN )
.namespaceId( ns )
.batch( -1 )
.transactionManager( transactionManager )
.transactions( List.of( transaction ) ).build();
List<ExecutedContext> executedContexts = LanguageManager.getINSTANCE().anyQuery( qc );
String result = getResultAsString( executedContexts, ql.dataModel() );

String logical = null, allocation = null, physical = null;

int tries = 5;
try {
// plans are serialized in a separate thread, which might take some time
while ( UiRoutingPageUtil.runningTasks() > 0 && tries-- > 0 ) {
Thread.sleep( 2000 );
}
if( tries == 0 ) {
throw new RuntimeException( "Took too long to set all plans" );
}

for ( Information info : transaction.getQueryAnalyzer().getInformationArray() ) {
if ( info instanceof InformationPolyAlg polyInfo && polyInfo.getTextualPolyAlg() != null ) {
switch ( PlanType.valueOf( polyInfo.planType ) ) {
case LOGICAL -> logical = polyInfo.getTextualPolyAlg();
case ALLOCATION -> allocation = polyInfo.getTextualPolyAlg();
case PHYSICAL -> physical = polyInfo.getTextualPolyAlg();
QueryContext qc = QueryContext.builder()
.query( query )
.language( ql )
.isAnalysed( true )
.usesCache( true )
.origin( ORIGIN )
.namespaceId( ns )
.batch( -1 )
.transactionManager( transactionManager )
.transactions( List.of( transaction ) ).build();
List<ExecutedContext> executedContexts = LanguageManager.getINSTANCE().anyQuery( qc );
String result = getResultAsString( executedContexts, ql.dataModel() );

String logical = null, allocation = null, physical = null;

int tries = 5;
try {
// plans are serialized in a separate thread, which might take some time
while ( UiRoutingPageUtil.runningTasks() > 0 && tries-- > 0 ) {
Thread.sleep( 2000 );
}
if ( tries == 0 ) {
throw new RuntimeException( "Took too long to set all plans" );
}

for ( Information info : transaction.getQueryAnalyzer().getInformationArray() ) {
if ( info instanceof InformationPolyAlg polyInfo && polyInfo.getTextualPolyAlg() != null ) {
switch ( PlanType.valueOf( polyInfo.planType ) ) {
case LOGICAL -> logical = polyInfo.getTextualPolyAlg();
case ALLOCATION -> allocation = polyInfo.getTextualPolyAlg();
case PHYSICAL -> physical = polyInfo.getTextualPolyAlg();
}
}
}
}

assertNotNull( logical );
assertNotNull( allocation );
assertNotNull( physical ); // Physical is not yet tested further since it is only partially implemented
assertNotNull( logical );
assertNotNull( allocation );
assertNotNull( physical ); // Physical is not yet tested further since it is only partially implemented

} catch ( InterruptedException e ) {
throw new RuntimeException( e );
} finally {
transaction.commit(); // execute PolyAlg creates new transaction, as long as only DQLs are tested like this
}
if ( transactionManager.getNumberOfActiveTransactions() > 0 ) {
throw new RuntimeException();
}

} catch ( InterruptedException e ) {
// Check that parsing and executing again returns the same result
String resultFromLogical = executePolyAlg( logical, PlanType.LOGICAL, ql );
assertEquals( result, resultFromLogical, "Result from query does not match result when executing the logical plan." );
String resultFromAllocation = executePolyAlg( allocation, PlanType.ALLOCATION, ql );
assertEquals( result, resultFromAllocation, "Result from query does not match result when executing the allocation plan." );
} catch ( Exception e ) {
transaction.rollback( "Error during testing round trip: " + e.getMessage() );
throw new RuntimeException( e );
} finally {
transaction.commit(); // execute PolyAlg creates new transaction, as long as only DQLs are tested like this
}
if ( transactionManager.getNumberOfActiveTransactions() > 0 ) {
throw new RuntimeException();
}


// Check that parsing and executing again returns the same result
String resultFromLogical = executePolyAlg( logical, PlanType.LOGICAL, ql );
assertEquals( result, resultFromLogical, "Result from query does not match result when executing the logical plan." );
String resultFromAllocation = executePolyAlg( allocation, PlanType.ALLOCATION, ql );
assertEquals( result, resultFromAllocation, "Result from query does not match result when executing the allocation plan." );

}

Expand All @@ -240,28 +247,34 @@ private static String executePolyAlg( String polyAlg, PlanType planType, QueryLa
TransactionManager transactionManager = TransactionManagerImpl.getInstance();
Transaction transaction = transactionManager.startTransaction( Catalog.defaultUserId, Catalog.defaultNamespaceId, true, ORIGIN );
Statement statement = transaction.createStatement();
AlgRoot root = buildFromPolyAlg( polyAlg, planType, statement );
assertEqualAfterRoundtrip( polyAlg, root.alg );

QueryContext qc = QueryContext.builder()
.query( polyAlg )
.language( ql )
.isAnalysed( true )
.usesCache( true )
.origin( ORIGIN )
.batch( -1 )
.transactionManager( transactionManager )
.transactions( List.of( transaction ) )
.statement( statement ).build();
TranslatedQueryContext translated = TranslatedQueryContext.fromQuery( polyAlg, root, planType == PlanType.ALLOCATION, qc );
List<ExecutedContext> executedContexts = LanguageManager.getINSTANCE().anyQuery( translated );
String result = getResultAsString( executedContexts, ql.dataModel() );
try {
transaction.commit();
} catch ( TransactionException e ) {
AlgRoot root = buildFromPolyAlg( polyAlg, planType, statement );
assertEqualAfterRoundtrip( polyAlg, root.alg );

QueryContext qc = QueryContext.builder()
.query( polyAlg )
.language( ql )
.isAnalysed( true )
.usesCache( true )
.origin( ORIGIN )
.batch( -1 )
.transactionManager( transactionManager )
.transactions( List.of( transaction ) )
.statement( statement ).build();
TranslatedQueryContext translated = TranslatedQueryContext.fromQuery( polyAlg, root, planType == PlanType.ALLOCATION, qc );
List<ExecutedContext> executedContexts = LanguageManager.getINSTANCE().anyQuery( translated );
String result = getResultAsString( executedContexts, ql.dataModel() );
try {
transaction.commit();
} catch ( TransactionException e ) {
throw new RuntimeException( e );
}
return result;

} catch ( Exception e ) {
transaction.rollback( "Error during execution of polyAlg: " + e.getMessage() );
throw new RuntimeException( e );
}
return result;
}


Expand All @@ -270,29 +283,34 @@ private static String getResultAsString( List<ExecutedContext> executedContexts,
ExecutedContext context = executedContexts.get( 0 );

assertTrue( context.getException().isEmpty(), "Query resulted in an exception" );
@NotNull ResultIterator iter = context.getIterator();
String tupleType = context.getImplementation().tupleType.toString();
List<List<PolyValue>> rows = iter.getAllRowsAndClose();

StringBuilder sb = new StringBuilder( tupleType );

for ( List<PolyValue> row : rows ) {
sb.append( "\n" );
for ( PolyValue v : row ) {
String json = v == null ? "NULL" : v.toJson();
if ( json.contains( "\"id\"" ) ) {
try {
ObjectMapper objectMapper = new ObjectMapper();
JsonNode jsonNode = objectMapper.readTree( json );
((ObjectNode) jsonNode).remove( "id" );
json = objectMapper.writeValueAsString( jsonNode );
} catch ( JsonProcessingException ignored ) {
try {

@NotNull ResultIterator iter = context.getIterator();
String tupleType = context.getImplementation().tupleType.toString();
List<List<PolyValue>> rows = iter.getAllRowsAndClose();

StringBuilder sb = new StringBuilder( tupleType );

for ( List<PolyValue> row : rows ) {
sb.append( "\n" );
for ( PolyValue v : row ) {
String json = v == null ? "NULL" : v.toJson();
if ( json.contains( "\"id\"" ) ) {
try {
ObjectMapper objectMapper = new ObjectMapper();
JsonNode jsonNode = objectMapper.readTree( json );
((ObjectNode) jsonNode).remove( "id" );
json = objectMapper.writeValueAsString( jsonNode );
} catch ( JsonProcessingException ignored ) {
}
}
sb.append( json ).append( "," );
}
sb.append( json ).append( "," );
}
return sb.toString();
} catch ( Exception e ) {
throw new RuntimeException( e );
}
return sb.toString();
}


Expand Down

0 comments on commit 0fc9e93

Please sign in to comment.