Skip to content

Commit

Permalink
fix: Fix the key issues by introducing a standard pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
@wsy19961129
wsy19961129 committed Aug 21, 2024
1 parent 61c67af commit 9a8ec08
Show file tree
Hide file tree
Showing 17 changed files with 727 additions and 222 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@
"connect-timeout": "1.9.0",
"core-js": "^3.37.1",
"cors": "2.8.5",
"crypto-browserify": "3.12.0",
"csv-parse": "5.5.6",
"dayjs": "1.11.11",
"deepmerge": "4.3.1",
Expand Down
2 changes: 1 addition & 1 deletion packages/itmat-apis/src/graphql/resolvers/studyResolvers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ export class StudyResolvers {

async getDataRecords(_parent, { studyId, queryString, versionId }: { queryString: IQueryString, studyId: string, versionId: string | null | undefined }, context) {
try {
const result = (await this.dataCore.getData(context.req.user, studyId, queryString.data_requested, versionId))['raw'] as unknown as IData[] & { properties: { subjectId: string, visitId: string }, fieldId: string, value: string }[];
const result = (await this.dataCore.getData(context.req.user, studyId, queryString.data_requested, versionId)) as unknown as IData[] & { properties: { subjectId: string, visitId: string }, fieldId: string, value: string }[];
const groupedResult: IGroupedData = {};
for (let i = 0; i < result.length; i++) {
const { subjectId, visitId } = result[i].properties;
Expand Down
53 changes: 40 additions & 13 deletions packages/itmat-apis/src/trpc/userProcedure.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,13 +273,40 @@ export class UserRouter {
return await this.userCore.registerPubkey(opts.ctx.user, opts.input.pubkey, opts.input.signature, opts.input.associatedUserId);
}),
/**
* Issue an access token.
* Request an access token.
* @param username - The username of the user.
* @param pubkey - The public key.
*
* @return challenge
*/
requestAccessToken: this.baseProcedure.input(z.object({
username: z.string(),
pubkey: z.string()
})).mutation(async (opts) => {
return await this.userCore.requestAccessToken(opts.input.username, opts.input.pubkey);
}),
/**
* Get an access token.
* @param username - The username of the user.
* @param pubkey - The public key.
* @param signature - The signature of the public key.
* @param life - The life of the token.
* @return IAccessToken
*
* @return token
*/
getAccessToken: this.baseProcedure.input(z.object({
username: z.string(),
pubkey: z.string(),
signature: z.string()
})).mutation(async (opts) => {
return await this.userCore.getAccessToken(opts.input.username, opts.input.pubkey, opts.input.signature);
}),
/**
* Issue an access token.
*
* @param pubkey - The public key.
* @param signature - The signature of the public key.
* @param life - The life of the token.
* @return IAccessToken
*/
issueAccessToken: this.baseProcedure.input(z.object({
pubkey: z.string(),
signature: z.string(),
Expand All @@ -288,17 +315,17 @@ export class UserRouter {
return await this.userCore.issueAccessToken(opts.input.pubkey, opts.input.signature, opts.input.life);
}),
/**
* Delete a public key.
*
* @param keyId - The id of the public key.
* @param associatedUserId - The id of the user.
*/
* Delete a public key.
*
* @param keyId - The id of the public key.
* @param associatedUserId - The id of the user.
*/
deletePubkey: this.baseProcedure.input(z.object({
keyId: z.string(),
associatedUserId: z.string()
associatedUserId: z.string(),
keyId: z.string()
})).mutation(async (opts) => {
return await this.userCore.deletePubkey(opts.ctx.user, opts.input.keyId, opts.input.associatedUserId);
return await this.userCore.deletePubkey(opts.ctx.user, opts.input.associatedUserId, opts.input.keyId);
})
});
}
}
}
110 changes: 61 additions & 49 deletions packages/itmat-cores/src/coreFunc/dataCore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,16 +803,11 @@ export class DataCore {
return await getJsonFileContents(this.objStore, 'cache', hashedInfo[0].uri);
} else {
// raw data by the permission
const data = await this.getDataByRoles(roles, studyId, availableDataVersions, fieldIds);
// data versioning
const filteredData = this.dataTransformationCore.transformationAggregate(data, { raw: this.genVersioningAggregation((config.properties as IStudyConfig).defaultVersioningKeys, availableDataVersions.includes(null)) });
if (!Array.isArray(filteredData['raw']) || (filteredData['raw'].length > 0 && Array.isArray(filteredData['raw'][0]))) {
throw new Error('Input data must be of type IDataTransformationClipArray (A[]) and not A[][]');
}
const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds);
// data transformation if aggregation is provided
const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(filteredData['raw'] as IDataTransformationClipArray, aggregation) : filteredData;
const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(data as unknown as IDataTransformationClipArray, aggregation) : data;
// write to minio and cache collection
const info = await convertToBufferAndUpload(this.fileCore, requester, transformed);
const info = await convertToBufferAndUpload(this.fileCore, requester, { data: data });
const newHashInfo = {
id: uuid(),
keyHash: hash,
Expand Down Expand Up @@ -840,14 +835,9 @@ export class DataCore {
}
} else {
// raw data by the permission
const data = await this.getDataByRoles(roles, studyId, availableDataVersions, fieldIds);
// data versioning
const filteredData = this.dataTransformationCore.transformationAggregate(data, { raw: this.genVersioningAggregation((config.properties as IStudyConfig).defaultVersioningKeys, availableDataVersions.includes(null)) });
if (!Array.isArray(filteredData['raw']) || (filteredData['raw'].length > 0 && Array.isArray(filteredData['raw'][0]))) {
throw new Error('Input data must be of type IDataTransformationClipArray (A[]) and not A[][]');
}
const data = await this.getDataByRoles(requester, roles, studyId, availableDataVersions, fieldIds);
// data transformation if aggregation is provided
const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(filteredData['raw'] as IDataTransformationClipArray, aggregation) : filteredData;
const transformed = aggregation ? this.dataTransformationCore.transformationAggregate(data as unknown as IDataTransformationClipArray, aggregation) : data;
return transformed;
}
}
Expand Down Expand Up @@ -1006,18 +996,18 @@ export class DataCore {
if (fieldIds.length === 0) {
return [];
}
const fileDataRecords = (await this.getData(
const fileDataRecords: IData[] = (await this.getData(
requester,
studyId,
fieldIds,
availableDataVersions,
undefined,
false
))['raw'];
)) as unknown as IData[];
if (!Array.isArray(fileDataRecords)) {
return [];
}
const files = await this.db.collections.files_collection.find({ id: { $in: fileDataRecords.map(el => el.value) } }).toArray();
const files = await this.db.collections.files_collection.find({ id: { $in: fileDataRecords.map(el => String(el.value)) } }).toArray();
if (readable) {
const users = await this.db.collections.users_collection.find({}).toArray();
const edited = [...files];
Expand Down Expand Up @@ -1074,19 +1064,12 @@ export class DataCore {
}


public async getDataByRoles(roles: IRole[], studyId: string, dataVersions: Array<string | null>, fieldIds?: string[]) {
public async getDataByRoles(requester: IUserWithoutToken, roles: IRole[], studyId: string, dataVersions: Array<string | null>, fieldIds?: string[]) {
const matchFilter: Filter<IData> = {
studyId: studyId,
dataVersion: { $in: dataVersions }
};
if (fieldIds && fieldIds[0]) {
// we ask that for regular expressions, ^ and $ must be used
if (fieldIds[0][0] === '^' && fieldIds[0][fieldIds[0].length - 1] === '$') {
matchFilter.fieldId = { $in: fieldIds.map(el => new RegExp(el)) };
} else {
matchFilter.fieldId = { $in: fieldIds };
}
}

const roleArr: Filter<IData>[] = [];
for (const role of roles) {
const permissionArr: Filter<IData>[] = [];
Expand All @@ -1112,11 +1095,44 @@ export class DataCore {
}
roleArr.push({ $or: permissionArr });
}
const res = await this.db.collections.data_collection.aggregate([{
$match: { ...matchFilter }
}, {
$match: { $or: roleArr }
}], { allowDiskUse: true }).toArray();

// we need to query each field based on its properties
const availableFields = (await this.getStudyFields(requester, studyId, dataVersions)).reduce((a, c) => {
a[c.fieldId] = c;
return a;
}, {});
const availableFieldIds = Object.keys(availableFields);
const refactoredFieldIds = fieldIds ?? Object.keys(availableFields);
let res: IData[] = [];
for (const fieldId of refactoredFieldIds) {
if (availableFieldIds.includes(fieldId) || availableFieldIds.some(el => new RegExp(el).test(fieldId))) {
const propertyFilter = {};
if (availableFields[fieldId].properties) {
for (const property of availableFields[fieldId].properties) {
propertyFilter[`${property.name}`] = `$properties.${property.name}`;
}
}
const data = await this.db.collections.data_collection.aggregate<IData>([{
$match: { ...matchFilter, fieldId: fieldId }
}, {
$match: { $or: roleArr }
}, {
$sort: {
'life.createdTime': -1
}
}, {
$group: {
_id: {
...propertyFilter
},
latestDocument: { $first: '$$ROOT' }
}
}, {
$replaceRoot: { newRoot: '$latestDocument' }
}], { allowDiskUse: true }).toArray();
res = res.concat(data);
}
}
return res;
}

Expand Down Expand Up @@ -1320,22 +1336,18 @@ export class DataCore {
}

const generatedSummary = async () => {
const numberOfDataRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId });
const numberOfDataAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, value: { $ne: null } });
const numberOfDataDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, value: null });

const numberOfVersionedRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null } });
const numberOfVersionedAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null }, value: { $ne: null } });
const numberOfVersionedDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null }, value: null });

const numberOfUnversionedRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null });
const numberOfUnversionedAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null, value: { $ne: null } });
const numberOfUnversionedDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null, value: null });

const numberOfDataRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId }, { allowDiskUse: true });
const numberOfDataAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, value: { $ne: null } }, { allowDiskUse: true });
const numberOfDataDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, value: null }, { allowDiskUse: true });
const numberOfVersionedRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null } }, { allowDiskUse: true });
const numberOfVersionedAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null }, value: { $ne: null } }, { allowDiskUse: true });
const numberOfVersionedDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: { $ne: null }, value: null }, { allowDiskUse: true });
const numberOfUnversionedRecords: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null }, { allowDiskUse: true });
const numberOfUnversionedAdds: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null, value: { $ne: null } }, { allowDiskUse: true });
const numberOfUnversionedDeletes: number = await this.db.collections.data_collection.countDocuments({ studyId: studyId, dataVersion: null, value: null }, { allowDiskUse: true });
const numberOfFields: number = (await this.db.collections.field_dictionary_collection.distinct('fieldId', { studyId: studyId })).length;
const numberOfVersionedFields: number = (await this.db.collections.field_dictionary_collection.distinct('fieldId', { studyId: studyId, dataVersion: { $ne: null } })).length;
const numberOfUnversionedFields: number = (await this.db.collections.field_dictionary_collection.distinct('fieldId', { studyId: studyId, dataVersion: null })).length;

const dataByUploaders = await this.db.collections.data_collection.aggregate<{ userId: string, count: number }>([
{ $match: { studyId: studyId } },
{
Expand All @@ -1351,8 +1363,8 @@ export class DataCore {
count: 1
}
}
]).toArray();

], { allowDiskUse: true }).toArray();
console.log('dataByUploaders done');
const events = ['GET_DATA_RECORDS', 'GET_STUDY_FIELDS', 'GET_STUDY', 'data.getStudyFields',
'data.getStudyData', 'data.getStudyDataLatest', 'data.getFiles'
];
Expand All @@ -1379,8 +1391,8 @@ export class DataCore {
count: 1
}
}
]).toArray();

], { allowDiskUse: true }).toArray();
console.log('dataByUsers done');
return {
numberOfDataRecords: numberOfDataRecords,
numberOfDataAdds: numberOfDataAdds,
Expand Down
4 changes: 3 additions & 1 deletion packages/itmat-cores/src/coreFunc/domainCore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ export class DomainCore {
);
}

const obj: Filter<IDomain> = {};
const obj: Filter<IDomain> = {
'life.deletedTime': null
};
if (domainId) {
obj.id = domainId;
}
Expand Down
2 changes: 1 addition & 1 deletion packages/itmat-cores/src/coreFunc/logCore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export class LogCore {
}
let logs: ILog[];
if (indexRange) {
logs = await this.db.collections.log_collection.find(filters).skip(indexRange[0]).limit(indexRange[1] - indexRange[0]).toArray();
logs = await this.db.collections.log_collection.find(filters).sort({ 'life.createdTime': -1 }).skip(indexRange[0]).limit(indexRange[1] - indexRange[0]).toArray();
} else {
logs = await this.db.collections.log_collection.find(filters).toArray();
}
Expand Down
Loading

0 comments on commit 9a8ec08

Please sign in to comment.