Skip to content

Commit

Permalink
Merge branch 'master' into MODFQMMGR-496-fqm-query-async-results-get-…
Browse files Browse the repository at this point in the history
…protects-more-than-one-endpoint

# Conflicts:
#	descriptors/ModuleDescriptor-template.json
  • Loading branch information
vgema committed Oct 23, 2024
2 parents 284b21c + 7e9c0b9 commit f8a3196
Show file tree
Hide file tree
Showing 93 changed files with 3,864 additions and 2,543 deletions.
1 change: 1 addition & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# 2.0.x
- [MODFQMMGR-523](https://folio-org.atlassian.net/browse/MODFQMMGR-523) Upgrade `holdings-storage` to 8.0

## 2.0.0
- [MODFQMMGR-87](https://folio-org.atlassian.net/browse/MODFQMMGR-87) Move FQM-specific code from lib-fqm-query-processor
Expand Down
10 changes: 9 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ mvn clean install
| DB_PASSWORD | postgres | Postgres password |
| DB_DATABASE | postgres | Postgres database name |
| MAX_QUERY_SIZE | 1250000 | max result count per query |
| mod-fqm-manager.bypass-permissions | false | Disable all permission checks, for local development only |
| mod-fqm-manager.permissions-cache-timeout-seconds | 60 | Cache duration for user permissions |
| mod-fqm-manager.entity-type-cache-timeout-seconds | 3600 | Cache duration for entity type definitions |
| server.port | 8081 | Server port |
Expand All @@ -72,6 +71,15 @@ to mod-fqm-manager to ensure optimal performance and query handling. For larger
used, allocate at least 2 gigabytes. In extreme cases with extremely large responses and concurrent usage, 5 gigabytes of
heap space should be sufficient to maintain performance and stability.

### Local development only environment variables

> [!WARNING]
>
> These environment variables are intended for local development only and should not be used in production environments!
| Name | Default Value | Description |
|------------------------------------|---------------|-------------------------------|
| mod-fqm-manager.bypass-permissions | false | Disable all permission checks |

## Installing the module
Follow the guide of Deploying Modules sections of the [Okapi Guide](https://github.com/folio-org/okapi/blob/master/doc/guide.md#example-1-deploying-and-using-a-simple-module) and Reference, which describe the process in detail.
Expand Down
53 changes: 29 additions & 24 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"methods": ["POST"],
"pathPattern": "/_/tenant",
"permissionsRequired": [],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get","permissions.users.item.get"]
},
{
"methods": ["GET", "DELETE"],
Expand All @@ -28,24 +28,25 @@
"methods": ["GET"],
"pathPattern": "/entity-types/{entity-type-id}",
"permissionsRequired": ["fqm.entityTypes.item.get"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get"]
},
{
"methods": ["GET"],
"pathPattern": "/entity-types",
"permissionsRequired": ["fqm.entityTypes.collection.get"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get"]
},
{
"methods": ["GET"],
"pathPattern": "/entity-types/{entity-type-id}/columns/{column-name}/values",
"permissionsRequired": ["fqm.entityTypes.item.columnValues.get"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get","search.facets.collection.get", "configuration.entries.collection.get"]
},
{
"methods": ["POST"],
"pathPattern": "/entity-types/materialized-views/refresh",
"permissionsRequired": ["fqm.materializedViews.post"]
"permissionsRequired": ["fqm.materializedViews.post"],
"modulePermissions": ["configuration.entries.collection.get", "finance.exchange-rate.item.get"]
}
]
},
Expand All @@ -57,7 +58,7 @@
"methods": ["GET"],
"pathPattern": "/query",
"permissionsRequired": ["fqm.query.sync.get"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get"]
},
{
"methods": ["GET"],
Expand All @@ -75,13 +76,13 @@
"methods": ["GET"],
"pathPattern": "/fqm/version",
"permissionsRequired": ["fqm.version.get"],
"modulePermissions": ["perms.users.get"]
"modulePermissions": ["perms.users.get", "permissions.users.item.get"]
},
{
"methods": ["POST"],
"pathPattern": "/fqm/migrate",
"permissionsRequired": ["fqm.migrate.post"],
"modulePermissions": ["perms.users.get"]
"modulePermissions": ["perms.users.get", "permissions.users.item.get"]
},
{
"methods": ["POST"],
Expand All @@ -93,18 +94,18 @@
"methods": ["POST"],
"pathPattern": "/query",
"permissionsRequired": ["fqm.query.async.post"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get"]
},
{
"methods": ["POST"],
"pathPattern": "/query/purge",
"permissionsRequired": ["fqm.query.purge"]
"permissionsRequired": ["fqm.query.purge.post"]
},
{
"methods": ["DELETE"],
"pathPattern": "/query/{query-id}",
"permissionsRequired": ["fqm.query.async.delete"],
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get"]
"modulePermissions": ["perms.users.get", "user-tenants.collection.get", "consortia.user-tenants.collection.get", "permissions.users.item.get"]
}
]
},
Expand All @@ -119,7 +120,7 @@
"unit": "hour",
"delay": "1",
"modulePermissions": [
"fqm.query.purge"
"fqm.query.purge.post"
]
},
{
Expand All @@ -136,7 +137,8 @@
],
"permissionSets" : [
{
"permissionName": "fqm.query.purge",
"permissionName": "fqm.query.purge.post",
"replaces": ["fqm.query.purge"],
"displayName": "FQM - purge old queries",
"description": "Purge old queries",
"visible": true
Expand Down Expand Up @@ -222,11 +224,7 @@
"permissionName": "fqm.materializedViews.post",
"displayName": "FQM - Refresh materialized views",
"description": "Refresh FQM materialized views",
"visible": true,
"subPermissions": [
"configuration.entries.collection.get",
"finance.exchange-rate.item.get"
]
"visible": true
},
{
"permissionName": "fqm.query.all",
Expand All @@ -253,11 +251,15 @@
},
{
"id": "holdings-storage",
"version": "6.0 7.0"
"version": "6.0 7.0 8.0"
},
{
"id": "instance-storage",
"version": "10.0"
"version": "10.0 11.0"
},
{
"id": "instance-date-types",
"version": "1.0"
},
{
"id": "instance-types",
Expand All @@ -271,10 +273,6 @@
"id": "material-types",
"version": "2.2"
},
{
"id": "shelf-locations",
"version": "1.1"
},
{
"id": "location-units",
"version": "2.0"
Expand Down Expand Up @@ -386,6 +384,10 @@
{
"id": "voucher-storage.voucher-number",
"version": "1.1"
},
{
"id": "search",
"version": "1.3"
}
],
"optional": [
Expand Down Expand Up @@ -445,6 +447,9 @@
},
{ "name": "mod.fqm-manager.entity-type-cache-timeout-seconds",
"value": "300"
},
{ "name": "IS_EUREKA",
"value": "false"
}
]
}
Expand Down
Binary file modified entity-type-creator/bun.lockb
Binary file not shown.
42 changes: 21 additions & 21 deletions entity-type-creator/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,36 @@
"dependencies": {
"@codemirror/lang-java": "^6.0.1",
"@codemirror/lang-json": "^6.0.1",
"@codemirror/lang-sql": "^6.6.2",
"@emotion/cache": "^11.11.0",
"@emotion/react": "^11.11.4",
"@codemirror/lang-sql": "^6.8.0",
"@emotion/cache": "^11.13.1",
"@emotion/react": "^11.13.3",
"@emotion/server": "^11.11.0",
"@emotion/styled": "^11.11.5",
"@mui/icons-material": "^5.15.15",
"@mui/material": "^5.15.15",
"@mui/material-nextjs": "^5.15.11",
"@uiw/react-codemirror": "^4.21.25",
"@emotion/styled": "^11.13.0",
"@mui/icons-material": "^6.1.3",
"@mui/material": "^6.1.3",
"@mui/material-nextjs": "^6.1.3",
"@uiw/react-codemirror": "^4.23.5",
"change-case": "^5.4.4",
"dotenv": "^16.4.5",
"genson-js": "^0.0.8",
"json-schema-to-typescript": "^13.1.2",
"json-schema-to-typescript": "^15.0.2",
"json5": "^2.2.3",
"next": "^14.1.4",
"next": "^14.2.15",
"postgres": "^3.4.4",
"prettier": "^3.2.5",
"react": "^18",
"react-dom": "^18",
"socket.io": "^4.7.5",
"socket.io-client": "^4.7.5",
"sql-formatter": "^15.3.0",
"prettier": "^3.3.3",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"socket.io": "^4.8.0",
"socket.io-client": "^4.8.0",
"sql-formatter": "^15.4.3",
"uuid": "^9.0.1"
},
"devDependencies": {
"@types/bun": "^1.0.12",
"@types/node": "^20",
"@types/react": "^18",
"@types/react-dom": "^18",
"@types/bun": "^1.1.11",
"@types/node": "^20.16.11",
"@types/react": "^18.3.11",
"@types/react-dom": "^18.3.0",
"@types/uuid": "^9.0.8",
"typescript": "^5"
"typescript": "^5.6.3"
}
}
29 changes: 17 additions & 12 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

<properties>
<!-- runtime dependencies -->
<folio-spring-base.version>8.1.0</folio-spring-base.version>
<folio-spring-base.version>8.3.0-SNAPSHOT</folio-spring-base.version>
<folio-query-tool-metadata.version>2.1.0-SNAPSHOT</folio-query-tool-metadata.version>
<mapstruct.version>1.5.2.Final</mapstruct.version>
<lib-fqm-query-processor.version>2.1.0-SNAPSHOT</lib-fqm-query-processor.version>
Expand All @@ -32,13 +32,14 @@
<org.postgresql.version>42.5.4</org.postgresql.version>
<org.jooq.version>3.18.4</org.jooq.version>
<org.apache.commons.version>4.4</org.apache.commons.version>
<org.apache.commons-lang3.version>3.17.0</org.apache.commons-lang3.version>

<!-- test dependencies -->
<wiremock.version>3.2.0</wiremock.version>
<mockwebserver.version>4.11.0</mockwebserver.version>

<!-- plugins -->
<openapi-generator.version>6.2.1</openapi-generator.version>
<openapi-generator.version>7.4.0</openapi-generator.version>
<copy-rename-maven-plugin.version>1.0.1</copy-rename-maven-plugin.version>
<maven-release-plugin.version>3.0.0-M7</maven-release-plugin.version>
<lombok.version>1.18.30</lombok.version>
Expand Down Expand Up @@ -130,6 +131,12 @@
<version>${org.apache.commons.version}</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${org.apache.commons-lang3.version}</version>
</dependency>

<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
Expand Down Expand Up @@ -324,6 +331,14 @@
<supportingFilesToGenerate>ApiUtil.java</supportingFilesToGenerate>
<generateModelDocumentation>true</generateModelDocumentation>
<skipIfSpecIsUnchanged>true</skipIfSpecIsUnchanged>
<typeMappings>
<typeMapping>object+FqmMigrateRequest=FqmMigrateRequest</typeMapping>
<typeMapping>object+FqmMigrateResponse=FqmMigrateResponse</typeMapping>
</typeMappings>
<importMappings>
<importMapping>FqmMigrateRequest=org.folio.querytool.domain.dto.FqmMigrateRequest</importMapping>
<importMapping>FqmMigrateResponse=org.folio.querytool.domain.dto.FqmMigrateResponse</importMapping>
</importMappings>
<configOptions>
<java8>true</java8>
<dateLibrary>java</dateLibrary>
Expand All @@ -333,16 +348,6 @@
</configOptions>
</configuration>
</execution>
<execution>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<inputSpec>${mod-fqm-manager.yaml.file}</inputSpec>
<output>${project.build.directory}/docs/mod-fqm-manager</output>
<generatorName>html2</generatorName>
</configuration>
</execution>
</executions>
</plugin>

Expand Down
28 changes: 28 additions & 0 deletions src/main/java/org/folio/fqm/client/ModRolesKeycloakClient.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package org.folio.fqm.client;

import com.fasterxml.jackson.annotation.JsonProperty;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;

import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;

@Service
@FeignClient(name = "permissions")
public interface ModRolesKeycloakClient {
@GetMapping(value = "/users/{id}", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
UserPermissions getPermissionsUser(@RequestHeader("X-Okapi-Tenant") String tenant, @PathVariable UUID id);

record UserPermissions(@JsonProperty("permissions") List<String> permissionNames,
@JsonProperty("userId") UUID userId) {
public Set<String> getPermissionNames() {
return new HashSet<>(permissionNames);
}
}
}
14 changes: 13 additions & 1 deletion src/main/java/org/folio/fqm/client/SimpleHttpClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;

import java.util.Map;

Expand All @@ -19,6 +20,17 @@ public interface SimpleHttpClient {
* @param queryParams - a map of query parameters to pass to the API endpoint
* @return the body of the response (JSON)
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
@GetMapping(value = "/{path}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams);

/**
* Retrieve arbitrary data from a FOLIO API endpoint for the specified tenant.
*
* @param path - the path of the API endpoint
* @param queryParams - a map of query parameters to pass to the API endpoint
* @param tenant - FOLIO tenant from which to retrieve data
* @return the body of the response (JSON)
*/
@GetMapping(value = "/{path}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams, @RequestHeader("X-Okapi-Tenant") String tenant);
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public class V0POCMigration extends AbstractSimpleMigrationStrategy {
Map.entry("instance_source", "instance.source"),
Map.entry("instance_primary_contributor", "instance.instance_primary_contributor"),
Map.entry("instance_contributor_type_ids", "instance.contributors[*]->contributor_type_id"),
Map.entry("instance_contributor_type", "instance.contributors[*]->contributor_type_text"),
Map.entry("instance_contributor_type", "instance.contributors[*]->contributor_type"),
Map.entry("instance_contributor_type_name_ids", "instance.contributors[*]->contributor_name_type_id"),
Map.entry("instance_contributor_name_type", "instance.contributors[*]->name"),
Map.entry("instance_language", "instance.languages")
Expand Down
Loading

0 comments on commit f8a3196

Please sign in to comment.