Feat/cold data

packages/itmat-apis/src/trpc/dataProcedure.ts

@@ -198,7 +198,7 @@ export class DataRouter {
                 fieldIds: z.optional(z.array(z.string())),
                 useCache: z.optional(z.boolean()),
                 forceUpdate: z.optional(z.boolean()),
-                formatted: z.optional(z.string())
+                fromCold: z.optional(z.boolean())
             })).query(async (opts) => {
                 return await this.dataCore.getData(
                     opts.ctx.req.user,
@@ -207,7 +207,8 @@ export class DataRouter {
                     opts.input.versionId,
                     opts.input.aggregation,
                     opts.input.useCache,
-                    opts.input.forceUpdate
+                    opts.input.forceUpdate,
+                    opts.input.fromCold
                 );
             }),
             /**

packages/itmat-cores/src/coreFunc/dataCore.ts

@@ -748,7 +748,7 @@ export class DataCore {
      *
      * @return Partial<IData>[] - The list of objects of Partial<IData>
      */
-    public async getData(requester: IUserWithoutToken | undefined, studyId: string, selectedFieldIds?: string[], dataVersion?: string | null | Array<string | null>, aggregation?: Record<string, Array<{ operationName: enumDataTransformationOperation, params: Record<string, unknown> }>>, useCache?: boolean, forceUpdate?: boolean) {
+    public async getData(requester: IUserWithoutToken | undefined, studyId: string, selectedFieldIds?: string[], dataVersion?: string | null | Array<string | null>, aggregation?: Record<string, Array<{ operationName: enumDataTransformationOperation, params: Record<string, unknown> }>>, useCache?: boolean, forceUpdate?: boolean, fromCold?: boolean) {
         if (!requester) {
             throw new CoreError(
                 enumCoreErrors.NOT_LOGGED_IN,
@@ -811,7 +811,7 @@ export class DataCore {
                 return await getJsonFileContents(this.objStore, 'cache', hashedInfo[0].uri);
             } else {
                 // raw data by the permission
-                const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds);
+                const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds, fromCold);
                 // data transformation if aggregation is provided
                 const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(data as unknown as IDataTransformationClipArray, aggregation) : data;
                 // write to minio and cache collection
@@ -843,7 +843,7 @@ export class DataCore {
             }
         } else {
             // raw data by the permission
-            const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds);
+            const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds, fromCold);
             // data transformation if aggregation is provided
             const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(data as unknown as IDataTransformationClipArray, aggregation) : data;
             return transformed;
@@ -1275,12 +1275,14 @@ export class DataCore {
     }
 
 
-    public async getDataByRoles(requester: IUserWithoutToken, roles: IRole[], studyId: string, dataVersions: Array<string | null>, fieldIds?: string[]) {
+    public async getDataByRoles(requester: IUserWithoutToken, roles: IRole[], studyId: string, dataVersions: Array<string | null>, fieldIds?: string[], fromCold?: boolean) {
         const matchFilter: Filter<IData> = {
             studyId: studyId,
             dataVersion: { $in: dataVersions }
         };
 
+        const dbCollection = fromCold ? this.db.collections.colddata_collection : this.db.collections.data_collection;
+
         const roleArr: Filter<IData>[] = [];
         for (const role of roles) {
             const permissionArr: Filter<IData>[] = [];
@@ -1316,12 +1318,15 @@ export class DataCore {
         const queryField = async (fieldId: string) => {
             if (availableFieldIds.includes(fieldId) || availableFieldIds.some(el => new RegExp(el).test(fieldId))) {
                 const propertyFilter: Record<string, string> = {};
+                if (!availableFields[fieldId]) {
+                    return [];
+                }
                 if (availableFields[fieldId].properties) {
                     for (const property of availableFields[fieldId].properties) {
                         propertyFilter[`${property.name}`] = `$properties.${property.name}`;
                     }
                 }
-                const data = await this.db.collections.data_collection.aggregate<IData>([{
+                const data = await dbCollection.aggregate<IData>([{
                     $match: {
                         ...matchFilter,
                         fieldId: fieldId,

packages/itmat-interface/src/server/router.ts

@@ -100,7 +100,7 @@ export class Router {
         this.app.use(rateLimit({
             windowMs: 1 * 60 * 1000,
             max: async function (req) {
-                const minimumQPS = 1000;
+                const minimumQPS = 5000;
                 let qps = minimumQPS;
                 if (req.user) {
                     const userConfig = await db.collections.configs_collection.findOne({ type: enumConfigType.USERCONFIG, key: req.user.id });

packages/itmat-types/src/types/file.ts

@@ -29,6 +29,7 @@ export enum enumFileTypes {
     DOCX = 'DOCX',
     XLSX = 'XLSX',
     XLS = 'XLS',
+    MAT = 'MAT',
     // images
     JPG = 'JPG',
     JPEG = 'JPEG',

packages/itmat-ui-react/src/Fence.tsx

@@ -25,7 +25,7 @@ export const Fence: FunctionComponent = () => {
 
     const whoAmI = trpc.user.whoAmI.useQuery();
     const recoverSession = trpc.user.recoverSessionExpireTime.useQuery(undefined, {
-        refetchInterval: 60 * 60 * 1000
+        refetchInterval: 30 * 60 * 1000
     });
 
     const [component, setComponent] = useState<JSX.Element | null>(null);