Skip to content

Commit

Permalink
MODFQMMGR-230: Update mod-fqm-manager to support nested entity types
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvsharp
bvsharp committed Apr 23, 2024
1 parent c6803cc commit f37279f
Show file tree
Hide file tree
Showing 20 changed files with 1,082 additions and 129 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<properties>
<!-- runtime dependencies -->
<folio-spring-base.version>8.1.0</folio-spring-base.version>
<folio-query-tool-metadata.version>2.0.0</folio-query-tool-metadata.version>
<folio-query-tool-metadata.version>2.1.0-SNAPSHOT</folio-query-tool-metadata.version>
<mapstruct.version>1.5.2.Final</mapstruct.version>
<lib-fqm-query-processor.version>2.0.0</lib-fqm-query-processor.version>
<coffee-boots.version>4.0.0</coffee-boots.version>
Expand Down
84 changes: 84 additions & 0 deletions src/main/java/org/folio/fqm/repository/EntityTypeRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,32 @@
package org.folio.fqm.repository;

import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;

import org.folio.fqm.exception.EntityTypeNotFoundException;
import org.folio.querytool.domain.dto.BooleanType;
import org.folio.querytool.domain.dto.EntityTypeColumn;
import org.folio.querytool.domain.dto.EntityTypeSource;
import org.folio.querytool.domain.dto.EntityTypeSourceJoin;
import org.folio.querytool.domain.dto.ValueWithLabel;
import org.jooq.DSLContext;
import org.jooq.Condition;
import org.jooq.Field;

import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Repository;

import com.fasterxml.jackson.databind.ObjectMapper;

import org.folio.querytool.domain.dto.EntityType;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;

import static org.apache.commons.collections4.CollectionUtils.isEmpty;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.or;
Expand Down Expand Up @@ -110,6 +137,63 @@ public void replaceEntityTypeDefinitions(List<EntityType> entityTypes) {
});
}

public Optional<EntityType> getCompositeEntityTypeDefinition(UUID entityTypeId) {
log.info("Getting COMPOSITE definition for entity type ID: {}", entityTypeId);
Field<String> definitionField = field("definition", String.class);

EntityType entityType = jooqContext
.select(definitionField)
.from(table(TABLE_NAME))
.where(field(ID_FIELD_NAME).eq(entityTypeId))
.fetchOptional(definitionField)
.map(this::unmarshallEntityType)
.map(currentEntityType -> {
String customFieldsEntityTypeId = currentEntityType.getCustomFieldEntityTypeId();
if (customFieldsEntityTypeId != null) {
currentEntityType.getColumns().addAll(fetchColumnNamesForCustomFields(UUID.fromString(customFieldsEntityTypeId)));
}
return currentEntityType;
}).orElseThrow();

// Build entity type from sources and joins
// var source1 = entityType.getSources().get(0);
// buildCompositeEntityTypeDefinition(entityType);

return Optional.of(entityType);
}

private void buildCompositeEntityTypeDefinition(EntityType entityType) {
System.out.println("BUILD COMPOSITE DEFINITION");
List<EntityTypeSourceJoin> entityTypeSourceJoins = new ArrayList<>();
List<EntityTypeSource> sources = entityType.getSources();
entityType.setFromClause("BUILD COMPOSITE DEFINITION");
StringBuilder fromClause = new StringBuilder();
for (int i = 0; i < sources.size(); i++) {
EntityTypeSource source = sources.get(i);
System.out.println("Building source: " + source.getAlias());
// TODO: below if-block checks if source is the first in the array, since there is nothing to join if it is.
// But this needs to be done better
if (i == 0) { // TODO: think about replacing this with -- if (source.getJoin() == null)
fromClause.append(source.getAlias());
} else {
if (source.getType().equals("db")) {
System.out.println("Building db source");
EntityTypeSourceJoin join = source.getJoin();
String joinType = join.getType();
String joinCondition = join.getCondition();
String alias = source.getAlias();
fromClause.append(" " + joinType + " " + alias + " ON " + joinCondition);
} else if (source.getType().equals("entity-type")) {
System.out.println("Building entity-type source");
// get entity type definition
// get sources from that one recursively
}
}
}
String fromClauseString = fromClause.toString();
entityType.setFromClause(fromClauseString);
}

private List<EntityTypeColumn> fetchColumnNamesForCustomFields(UUID entityTypeId) {
log.info("Getting columns for entity type ID: {}", entityTypeId);
EntityType entityTypeDefinition = getEntityTypeDefinition(entityTypeId)
Expand Down
42 changes: 29 additions & 13 deletions src/main/java/org/folio/fqm/repository/IdStreamer.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package org.folio.fqm.repository;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.fqm.exception.EntityTypeNotFoundException;
import org.folio.fqm.model.IdsWithCancelCallback;
import org.folio.fqm.service.EntityTypeFlatteningService;
import org.folio.fqm.service.FqlToSqlConverterService;
import org.folio.fqm.utils.IdColumnUtils;
import org.folio.fqm.utils.StreamHelper;
import org.folio.fql.model.Fql;
import org.folio.querytool.domain.dto.EntityType;
import org.folio.querytool.domain.dto.EntityTypeDefaultSort;
import org.folio.querytool.domain.dto.EntityTypeSource;
import org.jooq.Condition;
import org.jooq.Cursor;
import org.jooq.DSLContext;
Expand All @@ -27,19 +30,20 @@
import java.util.stream.Stream;

import static org.apache.commons.lang3.ObjectUtils.isEmpty;
import static org.folio.fqm.repository.EntityTypeRepository.ID_FIELD_NAME;
import static org.folio.fqm.utils.IdColumnUtils.RESULT_ID_FIELD;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.table;

@Repository
@RequiredArgsConstructor(onConstructor = @__(@Autowired))
@Log4j2
public class IdStreamer {

@Qualifier("readerJooqContext") private final DSLContext jooqContext;
private final EntityTypeRepository entityTypeRepository;
@Qualifier("readerJooqContext")
private final DSLContext jooqContext;
private final QueryDetailsRepository queryDetailsRepository;
private final EntityTypeFlatteningService entityTypeFlatteningService;

/**
* Executes the given Fql Query and stream the result Ids back.
Expand All @@ -49,7 +53,9 @@ public int streamIdsInBatch(UUID entityTypeId,
Fql fql,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
EntityType entityType = getEntityType(entityTypeId);
EntityType entityType = entityTypeFlatteningService
.getFlattenedEntityType(entityTypeId, true)
.orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
Condition sqlWhereClause = FqlToSqlConverterService.getSqlCondition(fql.fqlCondition(), entityType);
return this.streamIdsInBatch(entityType, sortResults, sqlWhereClause, batchSize, idsConsumer);
}
Expand All @@ -63,12 +69,12 @@ public int streamIdsInBatch(UUID queryId,
Consumer<IdsWithCancelCallback> idsConsumer) {
UUID entityTypeId = queryDetailsRepository.getEntityTypeId(queryId);
EntityType entityType = getEntityType(entityTypeId);
Condition condition = field(ID_FIELD_NAME).in(
Condition condition = field("\"source1\".id").in(
select(RESULT_ID_FIELD)
.from(table("query_results"))
.where(field("query_id").eq(queryId))
);
return streamIdsInBatch(entityType, sortResults, condition, batchSize, idsConsumer);
return streamIdsInBatch(entityType, sortResults, condition, batchSize, idsConsumer); // TODO: fix this
}

public List<List<String>> getSortedIds(String derivedTableName,
Expand All @@ -91,19 +97,29 @@ public List<List<String>> getSortedIds(String derivedTableName,
}

private int streamIdsInBatch(EntityType entityType,
boolean sortResults,
Condition sqlWhereClause,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
boolean sortResults,
Condition sqlWhereClause,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
String finalJoinClause = entityTypeFlatteningService.getJoinClause(entityType);
Field<String[]> idValueGetter = IdColumnUtils.getResultIdValueGetter(entityType);
String finalWhereClause = sqlWhereClause.toString();
for (EntityTypeSource source : entityType.getSources()) {
String toReplace = ":" + source.getAlias();
String alias = "\"" + source.getAlias() + "\"";
finalWhereClause = finalWhereClause.replace(toReplace, alias);
}
System.out.println("Final where clause: " + finalWhereClause);
try (
// NEW WAY
Cursor<Record1<String[]>> idsCursor = jooqContext.dsl()
.select(field(idValueGetter))
.from(entityType.getFromClause())
.where(sqlWhereClause)
.from(finalJoinClause)
.where(finalWhereClause)
.orderBy(getSortFields(entityType, sortResults))
.fetchSize(batchSize)
.fetchLazy();

Stream<String[]> idStream = idsCursor
.stream()
.map(row -> row.getValue(idValueGetter));
Expand Down Expand Up @@ -136,7 +152,7 @@ private static SortField<Object> toSortField(EntityTypeDefaultSort entityTypeDef
}

private EntityType getEntityType(UUID entityTypeId) {
return entityTypeRepository.getEntityTypeDefinition(entityTypeId)
return entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, true)
.orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
}
}
38 changes: 30 additions & 8 deletions src/main/java/org/folio/fqm/repository/ResultSetRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.folio.fqm.repository;

import static org.folio.fqm.repository.EntityTypeRepository.ID_FIELD_NAME;
import static org.jooq.impl.DSL.field;

import java.sql.SQLException;
Expand All @@ -14,6 +13,7 @@
import org.folio.fql.model.Fql;
import org.folio.fqm.exception.FieldNotFoundException;
import org.folio.fqm.exception.EntityTypeNotFoundException;
import org.folio.fqm.service.EntityTypeFlatteningService;
import org.folio.fqm.service.FqlToSqlConverterService;
import org.folio.fqm.utils.IdColumnUtils;
import org.folio.fqm.utils.SqlFieldIdentificationUtils;
Expand All @@ -22,6 +22,7 @@

import org.apache.commons.collections4.CollectionUtils;
import org.folio.querytool.domain.dto.EntityTypeColumn;
import org.folio.querytool.domain.dto.EntityTypeSource;
import org.jooq.Condition;
import org.jooq.DSLContext;
import org.jooq.Field;
Expand All @@ -42,7 +43,7 @@
public class ResultSetRepository {

@Qualifier("readerJooqContext") private final DSLContext jooqContext;
private final EntityTypeRepository entityTypeRepository;
private final EntityTypeFlatteningService entityTypeFlatteningService;

public List<Map<String, Object>> getResultSet(UUID entityTypeId,
List<String> fields,
Expand Down Expand Up @@ -81,8 +82,12 @@ public List<Map<String, Object>> getResultSet(UUID entityTypeId,
whereClause = whereClause.and(field(idColumnValueGetter).in(idColumnValues));
}
}

String fromClause = entityTypeFlatteningService.getJoinClause(entityType);

// Have to get FROM clause
var result = jooqContext.select(fieldsToSelect)
.from(entityType.getFromClause())
.from(fromClause)
.where(whereClause)
.fetch();
return recordToMap(result);
Expand All @@ -104,11 +109,28 @@ public List<Map<String, Object>> getResultSet(UUID entityTypeId, Fql fql, List<S
}

Condition condition = FqlToSqlConverterService.getSqlCondition(fql.fqlCondition(), entityType);
// TODO: might want to put next 5 lines in its own method in EntityTypeFlatteningService
// also is it even necessary?
String finalWhereClause = condition.toString();
for (EntityTypeSource source : entityType.getSources()) {
String toReplace = ":" + source.getAlias();
String alias = "\"" + source.getAlias() + "\"";
finalWhereClause = finalWhereClause.replace(toReplace, alias);
}
var fieldsToSelect = getSqlFields(entityType, fields);
var sortCriteria = hasIdColumn(entityType) ? field(ID_FIELD_NAME) : DSL.noField();
String idFieldWithAlias = "";
String idColumnName = entityType
.getColumns()
.stream()
.filter(EntityTypeColumn::getIsIdColumn)
.findFirst()
.orElseThrow()
.getName();
var sortCriteria = hasIdColumn(entityType) ? field(idColumnName) : DSL.noField(); // TODO: new changes break sorting
String fromClause = entityTypeFlatteningService.getJoinClause(entityType);
var result = jooqContext.select(fieldsToSelect)
.from(entityType.getFromClause())
.where(condition)
.from(fromClause)
.where(finalWhereClause)
.and(afterIdCondition)
.orderBy(sortCriteria)
.limit(limit)
Expand All @@ -126,13 +148,13 @@ private List<Field<Object>> getSqlFields(EntityType entityType, List<String> fie
}

private EntityType getEntityType(UUID entityTypeId) {
return entityTypeRepository.getEntityTypeDefinition(entityTypeId)
return entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, true)
.orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
}

private boolean hasIdColumn(EntityType entityType) {
return entityType.getColumns().stream()
.anyMatch(col -> ID_FIELD_NAME.equals(col.getName()));
.anyMatch(col -> col.getIsIdColumn());
}

private List<Map<String, Object>> recordToMap(Result<Record> result) {
Expand Down
Loading

0 comments on commit f37279f

Please sign in to comment.