FINERACT-1910 Support Pagination, sorting and filtering for Datatable

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/PagedLocalRequest.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service;
+
+import java.util.Locale;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import org.apache.fineract.infrastructure.core.serialization.JsonParserHelper;
+
+@Getter
+@Setter
+@RequiredArgsConstructor
+public class PagedLocalRequest<T> extends PagedRequest<T> {
+
+    private String dateFormat;
+
+    private String dateTimeFormat;
+
+    private String locale;
+
+    public Locale getLocaleObject() {
+        return locale == null ? null : JsonParserHelper.localeFromString(locale);
+    }
+}

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/JdbcJavaType.java

@@ -30,14 +30,14 @@ public enum JdbcJavaType {
 
         @Override
         public Object toJdbcValueImpl(@NotNull DatabaseType dialect, Object value) {
-            return Boolean.TRUE.equals(value) ? 1 : 0;
+            return value == null ? null : (Boolean.TRUE.equals(value) ? 1 : 0);
         }
     },
     BOOLEAN(JavaType.BOOLEAN, new DialectType(JDBCType.BIT), new DialectType(JDBCType.BOOLEAN, null, "BOOL")) { //
 
         @Override
         public Object toJdbcValueImpl(@NotNull DatabaseType dialect, Object value) {
-            return dialect.isMySql() ? (Boolean.TRUE.equals(value) ? 1 : 0) : super.toJdbcValueImpl(dialect, value);
+            return (value != null && dialect.isMySql()) ? (Boolean.TRUE.equals(value) ? 1 : 0) : super.toJdbcValueImpl(dialect, value);
         }
     },
     SMALLINT(JavaType.SHORT, new DialectType(JDBCType.SMALLINT, true), new DialectType(JDBCType.SMALLINT, null, "INT2")), //

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/SqlOperator.java

@@ -47,7 +47,7 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public String formatPlaceholderImpl(String definition, String placeholder) {
+        public String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
             return format("%s %s CONCAT('%%', %s, '%%')", definition, getSymbol(), placeholder);
         }
     },
@@ -60,7 +60,7 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public String formatPlaceholderImpl(String definition, String placeholder) {
+        public String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
             return format("%s %s CONCAT('%%', %s, '%%')", definition, getSymbol(), placeholder);
         }
     },
@@ -74,7 +74,7 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public String formatPlaceholderImpl(String definition, String placeholder) {
+        public String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
             return format("%s %s %s AND %s", definition, getSymbol(), placeholder, placeholder);
         }
     },
@@ -88,7 +88,7 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public String formatPlaceholderImpl(String definition, String placeholder) {
+        public String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
             return format("%s %s %s AND %s", definition, getSymbol(), placeholder, placeholder);
         }
     },
@@ -102,8 +102,13 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public boolean isPlaceholderSupported() {
-            return false;
+        protected String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
+            return format("%s %s (%s)", definition, getSymbol(), placeholder + (", " + placeholder).repeat(paramCount - 1));
+        }
+
+        @Override
+        protected String formatNamedParamImpl(String definition, int paramCount, String namedParam) {
+            return format("%s %s (%s)", definition, getSymbol(), namedParam);
         }
     },
     NIN("NOT IN", -1) { //
@@ -116,8 +121,13 @@ public String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, Jdb
         }
 
         @Override
-        public boolean isPlaceholderSupported() {
-            return false;
+        protected String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
+            return format("%s %s (%s)", definition, getSymbol(), placeholder + (", " + placeholder).repeat(paramCount - 1));
+        }
+
+        @Override
+        protected String formatNamedParamImpl(String definition, int paramCount, String namedParam) {
+            return format("%s %s (%s)", definition, getSymbol(), namedParam);
         }
     },
     NULL("IS NULL", 0), //
@@ -160,38 +170,45 @@ protected String formatImpl(@NotNull DatabaseSpecificSQLGenerator sqlGenerator,
                 : format("%s %s %s", definition, symbol, sqlGenerator.formatValue(columnType, values[0]));
     }
 
-    public boolean isPlaceholderSupported() {
-        return true;
+    public String formatNamedParam(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, String definition, int paramCount, String alias) {
+        validateParamCount(paramCount);
+        if (paramCount > 1) {
+            throw new PlatformServiceUnavailableException("error.msg.database.operator.named.invalid",
+                    "Named parameter is not allowed on " + this);
+        }
+        return formatNamedParamImpl(sqlGenerator.alias(sqlGenerator.escape(definition), alias), paramCount, ":" + definition);
+    }
+
+    protected String formatNamedParamImpl(String definition, int paramCount, String namedParam) {
+        return formatPlaceholderImpl(definition, paramCount, namedParam);
     }
 
-    public String formatPlaceholder(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, String definition, String alias) {
-        return formatPlaceholder(sqlGenerator, definition, alias, "?");
+    public String formatPlaceholder(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, String definition, int paramCount, String alias) {
+        return formatPlaceholder(sqlGenerator, definition, paramCount, alias, "?");
     }
 
-    public String formatPlaceholder(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, String definition, String alias,
+    public String formatPlaceholder(@NotNull DatabaseSpecificSQLGenerator sqlGenerator, String definition, int paramCount, String alias,
             String placeholder) {
-        return formatPlaceholderImpl(sqlGenerator.alias(sqlGenerator.escape(definition), alias), placeholder);
+        validateParamCount(paramCount);
+        return formatPlaceholderImpl(sqlGenerator.alias(sqlGenerator.escape(definition), alias), paramCount, placeholder);
     }
 
-    protected String formatPlaceholderImpl(String definition, String placeholder) {
-        if (!isPlaceholderSupported()) {
-            throw new UnsupportedOperationException("Placeholder is not supported for this operator");
-        }
+    protected String formatPlaceholderImpl(String definition, int paramCount, String placeholder) {
         return paramCount == 0 ? format("%s %s", definition, symbol) : format("%s %s %s", definition, symbol, placeholder);
     }
 
     public void validateValues(String... values) {
-        if (values == null ? paramCount != 0 : (paramCount < 0 ? values.length < -paramCount : values.length != paramCount)) {
-            throw new PlatformServiceUnavailableException("error.msg.database.operator.invalid",
-                    "Number of parameters " + Arrays.toString(values) + " must be " + Math.abs(paramCount) + " on " + this);
-        }
+        validateParamCount(values == null ? 0 : values.length);
     }
 
     public void validateValues(List<?> values) {
-        int size = values == null ? 0 : values.size();
-        if (paramCount < 0 ? size < -paramCount : size != paramCount) {
+        validateParamCount(values == null ? 0 : values.size());
+    }
+
+    public void validateParamCount(int paramCount) {
+        if (this.paramCount < 0 ? paramCount < -this.paramCount : paramCount != this.paramCount) {
             throw new PlatformServiceUnavailableException("error.msg.database.operator.invalid",
-                    "Number of parameters " + size + " must be " + Math.abs(paramCount) + " on " + this);
+                    "Number of parameters " + paramCount + " must be " + Math.abs(this.paramCount) + " on " + this);
         }
     }
 

fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableSearchRequest.java

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.data;
+
+import java.io.Serializable;
+import java.util.List;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * Immutable data object representing datatable data.
+ */
+@Data
+@NoArgsConstructor
+public final class DatatableSearchRequest implements Serializable {
+
+    private List<ColumnFilter> columnFilters;
+
+    private List<String> resultColumns;
+
+    private String datatable;
+
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/DatatablesApiResource.java

@@ -52,11 +52,14 @@
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
 import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer;
+import org.apache.fineract.infrastructure.core.service.PagedLocalRequest;
 import org.apache.fineract.infrastructure.dataqueries.data.DatatableData;
 import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData;
 import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService;
 import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
 import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
+import org.apache.fineract.portfolio.search.data.AdvancedQueryData;
+import org.springframework.data.domain.Page;
 import org.springframework.stereotype.Component;
 
 @Path("/v1/datatables")
@@ -225,6 +228,19 @@ public String queryValues(@PathParam("datatable") @Parameter(description = "data
         return this.toApiJsonSerializer.serializePretty(ApiParameterHelper.prettyPrint(uriInfo.getQueryParameters()), result);
     }
 
+    @POST
+    @Path("{datatable}/query")
+    @Consumes({ MediaType.APPLICATION_JSON })
+    @Produces({ MediaType.APPLICATION_JSON })
+    @Operation(summary = "Query Data Table values", description = "Query values from a registered data table.")
+    @ApiResponses({
+            @ApiResponse(responseCode = "200", description = "OK", content = @Content(schema = @Schema(implementation = List.class))) })
+    public String advancedQuery(@PathParam("datatable") @Parameter(description = "datatable") final String datatable,
+            PagedLocalRequest<AdvancedQueryData> queryRequest, @Context final UriInfo uriInfo) {
+        final Page<JsonObject> result = this.readWriteNonCoreDataService.queryDataTableAdvanced(datatable, queryRequest);
+        return this.toApiJsonSerializer.serializePretty(ApiParameterHelper.prettyPrint(uriInfo.getQueryParameters()), result);
+    }
+
     @GET
     @Path("{datatable}/{apptableId}")
     @Consumes({ MediaType.APPLICATION_JSON })

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DataTableValidator.java

@@ -18,8 +18,16 @@
  */
 package org.apache.fineract.infrastructure.dataqueries.data;
 
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_COLUMN;
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_FILTERS;
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_OPERATOR;
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_QUERY;
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_RESULTCOLUMNS;
+import static org.apache.fineract.portfolio.search.SearchConstants.API_PARAM_TABLE;
+
 import com.google.gson.JsonElement;
 import com.google.gson.reflect.TypeToken;
+import jakarta.validation.constraints.NotNull;
 import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -29,9 +37,13 @@
 import java.util.Set;
 import org.apache.fineract.infrastructure.core.data.ApiParameterError;
 import org.apache.fineract.infrastructure.core.data.DataValidatorBuilder;
-import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException;
 import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper;
 import org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant;
+import org.apache.fineract.portfolio.search.data.AdvancedQueryData;
+import org.apache.fineract.portfolio.search.data.AdvancedQueryRequest;
+import org.apache.fineract.portfolio.search.data.ColumnFilterData;
+import org.apache.fineract.portfolio.search.data.FilterData;
+import org.apache.fineract.portfolio.search.data.TableQueryData;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -64,9 +76,56 @@ public void validateDataTableRegistration(final String json) {
             baseDataValidator.reset().parameter(DataTableApiConstant.categoryParamName).value(category).isOneOfTheseValues(objectArray);
         }
 
-        if (!dataValidationErrors.isEmpty()) {
-            throw new PlatformApiDataValidationException(dataValidationErrors);
+        baseDataValidator.throwValidationErrors();
+    }
+
+    public void validateTableSearch(@NotNull AdvancedQueryRequest queryRequest) {
+        final List<ApiParameterError> errors = new ArrayList<>();
+        final DataValidatorBuilder validator = new DataValidatorBuilder(errors).resource(DataTableApiConstant.DATATABLE_RESOURCE_NAME);
+        AdvancedQueryData baseQuery = queryRequest.getBaseQuery();
+        if (baseQuery != null) {
+            validateQueryData(baseQuery, validator);
         }
+        List<TableQueryData> datatableQueries = queryRequest.getDatatableQueries();
+        if (datatableQueries != null) {
+            for (TableQueryData datatableQuery : datatableQueries) {
+                validator.reset().parameter(API_PARAM_TABLE).value(datatableQuery.getTable()).notBlank();
+                AdvancedQueryData queryData = datatableQuery.getQuery();
+                validator.reset().parameter(API_PARAM_QUERY).value(queryData).notBlank();
+                if (queryData != null) {
+                    validateQueryData(queryData, validator);
+                }
+            }
+        }
+        validator.throwValidationErrors();
+    }
 
+    public void validateTableSearch(@NotNull AdvancedQueryData queryData) {
+        final DataValidatorBuilder validator = new DataValidatorBuilder(new ArrayList<>())
+                .resource(DataTableApiConstant.DATATABLE_RESOURCE_NAME);
+        validateQueryData(queryData, validator);
+        validator.throwValidationErrors();
+    }
+
+    private void validateQueryData(@NotNull AdvancedQueryData queryData, @NotNull DataValidatorBuilder validator) {
+        List<ColumnFilterData> columnFilters = queryData.getColumnFilters();
+        if (columnFilters != null) {
+            for (ColumnFilterData columnFilter : columnFilters) {
+                validator.reset().parameter(API_PARAM_COLUMN).value(columnFilter.getColumn()).notNull();
+                List<FilterData> filters = columnFilter.getFilters();
+                validator.reset().parameter(API_PARAM_FILTERS).value(filters == null ? null : filters.toArray()).notNull().arrayNotEmpty();
+                if (filters != null) {
+                    for (FilterData filter : filters) {
+                        validator.reset().parameter(API_PARAM_OPERATOR).value(filter.getOperator()).notNull();
+                    }
+                }
+            }
+            List<String> resultColumns = queryData.getResultColumns();
+            if (resultColumns != null) {
+                for (String resultColumn : resultColumns) {
+                    validator.reset().parameter(API_PARAM_RESULTCOLUMNS).value(resultColumn).notBlank();
+                }
+            }
+        }
     }
 }