Skip to content

Commit

Permalink
Merge pull request #9 from openfoodfacts/obsolete-handling
Browse files Browse the repository at this point in the history 
feat: Obsolete product handling
  • Loading branch information
@john-gom
john-gom authored Oct 12, 2023
2 parents cf19c21 + 72b0f2b commit 35b2a1f
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 27 deletions.
2 changes: 1 addition & 1 deletion .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ COMPOSE_PATH_SEPARATOR=;
TAG=latest
QUERY_PORT=127.0.0.1:5511
POSTGRES_HOST=localhost
POSTGRES_PORT=5512
POSTGRES_PORT=127.0.0.1:5512
POSTGRES_DB=query
POSTGRES_USER=productopener
POSTGRES_PASSWORD=productopener
Expand Down
121 changes: 107 additions & 14 deletions src/domain/services/query.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,36 @@ describe('count', () => {
});
it('should cope with no filters', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue, aminoValue, neucleotideValue } =
await createTestTags(app);
await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.count(null);
expect(response).toBeGreaterThan(2);
});
});

it('should be able to count obsolete products', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.count({
obsolete: 1,
origins_tags: originValue,
});
expect(response).toBe(1);
});
});

it('should be able to count not obsolete products', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.count({
obsolete: 0,
origins_tags: originValue,
});
expect(response).toBe(3);
});
});
});

describe('aggregate', () => {
Expand Down Expand Up @@ -129,6 +152,20 @@ describe('aggregate', () => {
});
});

it('should filter products when grouping by a product field', async () => {
await createTestingModule([DomainModule], async (app) => {
const { aminoValue, creatorValue } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.aggregate([
{ $match: { amino_acids_tags: aminoValue } },
{ $group: { _id: '$creator' } },
]);
const myTag = response.find((r) => r._id === creatorValue);
expect(myTag).toBeTruthy();
expect(parseInt(myTag.count)).toBe(1);
});
});

it('should be able to do not filtering', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue, aminoValue } = await createTestTags(app);
Expand Down Expand Up @@ -179,13 +216,26 @@ describe('aggregate', () => {
expect(parseInt(myTag.count)).toBe(1);
});
});

it('should be able to group obsolete products', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.aggregate([
{ $match: { obsolete: true } },
{ $group: { _id: '$origins_tags' } },
]);
const myTag = response.find((r) => r._id === originValue);
expect(myTag).toBeTruthy();
expect(parseInt(myTag.count)).toBe(1);
});
});
});

describe('select', () => {
it('should return matching products', async () =>{
it('should return matching products', async () => {
await createTestingModule([DomainModule], async (app) => {
const { originValue, aminoValue, neucleotideValue, product1, product2, product3 } =
await createTestTags(app);
const { aminoValue, product1 } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.select({
amino_acids_tags: aminoValue,
Expand All @@ -195,24 +245,43 @@ describe('select', () => {
expect(p1).toBeTruthy();
});
});

it('should return obsolete matching products', async () => {
await createTestingModule([DomainModule], async (app) => {
const { aminoValue, product4 } = await createTestTags(app);
const queryService = app.get(QueryService);
const response = await queryService.select({
amino_acids_tags: aminoValue,
obsolete: 'true',
});
expect(response).toHaveLength(1);
const p4 = response.find((r) => r.code === product4.code);
expect(p4).toBeTruthy();
});
});
});

async function createTestTags(app) {
const em = app.get(EntityManager);
// Create some dummy products with a specific tag
const product1 = em.create(Product, { code: randomCode() });
const product2 = em.create(Product, { code: randomCode() });
const product3 = em.create(Product, { code: randomCode() });

// Using origins and amino acids as they are smaller than most
const originValue = randomCode();
const aminoValue = randomCode();
const neucleotideValue = randomCode();
const creatorValue = randomCode();

// Create some dummy products with a specific tag
const product1 = em.create(Product, { code: randomCode() });
const product2 = em.create(Product, { code: randomCode(), creator: creatorValue });
const product3 = em.create(Product, { code: randomCode(), creator: creatorValue });
const product4 = em.create(Product, { code: randomCode(), obsolete: true });

// Matrix for testing
// Product | Origin | AminoAcid | Neucleotide
// Product1 | x | x | x
// Product2 | x | x |
// Product3 | x | | x
// Product | Origin | AminoAcid | Neucleotide | Obsolete | Creator
// Product1 | x | x | x | |
// Product2 | x | x | | | x
// Product3 | x | | x | | x
// Product4 | x | x | x | x |

em.create(ProductOriginsTag, {
product: product1,
Expand All @@ -226,6 +295,11 @@ async function createTestTags(app) {
product: product3,
value: originValue,
});
em.create(ProductOriginsTag, {
product: product4,
value: originValue,
obsolete: true,
});

em.create(ProductAminoAcidsTag, {
product: product1,
Expand All @@ -235,6 +309,11 @@ async function createTestTags(app) {
product: product2,
value: aminoValue,
});
em.create(ProductAminoAcidsTag, {
product: product4,
value: aminoValue,
obsolete: true,
});

em.create(ProductNucleotidesTag, {
product: product1,
Expand All @@ -244,7 +323,21 @@ async function createTestTags(app) {
product: product3,
value: neucleotideValue,
});
em.create(ProductNucleotidesTag, {
product: product4,
value: neucleotideValue,
obsolete: true,
});

await em.flush();
return { originValue, aminoValue, neucleotideValue, product1, product2, product3 };
return {
originValue,
aminoValue,
neucleotideValue,
creatorValue,
product1,
product2,
product3,
product4,
};
}
31 changes: 21 additions & 10 deletions src/domain/services/query.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ export class QueryService {
} else {
qb.select(`${column}`).distinct();
}
qb.where('not pt.obsolete');
qb.where(this.obsoleteWhere(match));

const whereLog = this.addMatches(match, qb);
const whereLog = this.addMatches(match, qb, entity);

if (count) {
qb = this.em.createQueryBuilder(qb, 'temp');
Expand Down Expand Up @@ -63,7 +63,8 @@ export class QueryService {
return results;
}

private addMatches(match: any, qb: QueryBuilder<object>, parentKey = 'pt.product_id') {
private addMatches(match: any, qb: QueryBuilder<object>, parentEntity) {
const parentId = parentEntity === Product ? 'id': 'product_id';
const whereLog = [];
for (const [matchTag, matchValue] of Object.entries(match)) {
let whereValue = matchValue;
Expand All @@ -76,7 +77,7 @@ export class QueryService {
const qbWhere = this.em
.createQueryBuilder(matchEntity, 'pt2')
.select('*')
.where(`pt2.product_id = ${parentKey} and pt2.${matchColumn} = ?`, [
.where(`pt2.product_id = pt.${parentId} and pt2.${matchColumn} = ?`, [
whereValue,
]);
qb.andWhere(`${not ? 'NOT ' : ''}EXISTS (${qbWhere.getKnexQuery()})`);
Expand All @@ -85,16 +86,23 @@ export class QueryService {
return whereLog;
}

obsoleteWhere(body: any) {
const obsolete = !!body?.obsolete;
delete body?.obsolete;
return `${obsolete ? '' : 'not '}pt.obsolete`;
}

async count(body: any) {
const start = Date.now();
this.logger.debug(body);

const obsoleteWhere = this.obsoleteWhere(body);
const tags = Object.keys(body ?? {});
const tag = tags?.[0];
const { entity, column } = this.getEntityAndColumn(tag);
const qb = this.em.createQueryBuilder(entity, 'pt');
qb.select(`count(*) count`);
qb.where('not pt.obsolete');
qb.where(obsoleteWhere);

let whereLog = [];
if (tag) {
Expand All @@ -106,7 +114,7 @@ export class QueryService {
}
qb.andWhere(`${not ? 'NOT ' : ''}pt.${column} = ?`, [matchValue]);
delete body[tag];
whereLog.push(...this.addMatches(body, qb));
whereLog.push(...this.addMatches(body, qb, entity));
}

this.logger.debug(qb.getFormattedQuery());
Expand All @@ -122,16 +130,19 @@ export class QueryService {
const start = Date.now();
this.logger.debug(body);

const tags = Object.keys(body);
const obsoleteWhere = this.obsoleteWhere(body);
let entity: EntityName<object> = Product;
const qb = this.em.createQueryBuilder(entity, 'p');
const qb = this.em.createQueryBuilder(entity, 'pt');
qb.select(`*`);
qb.where('not p.obsolete');
qb.where(obsoleteWhere);

const whereLog = this.addMatches(body, qb, 'p.id');
const whereLog = this.addMatches(body, qb, entity);

this.logger.debug(qb.getFormattedQuery());
const results = await qb.execute();
this.logger.log(
`Processed ${whereLog.join(' and ')} in ${Date.now() - start} ms. Selected ${results.length} records`,
);
return results;
}

Expand Down
2 changes: 1 addition & 1 deletion src/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ async function bootstrap() {
const app = await NestFactory.create(AppModule);
const migrator = app.get(MikroORM).getMigrator();
await migrator.up();
await app.listen(5510);
await app.listen(5510, '0.0.0.0');
}
bootstrap();
2 changes: 1 addition & 1 deletion src/mikro-orm.config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export default defineConfig({
user: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD,
host: process.env.POSTGRES_HOST,
port: parseInt(process.env.POSTGRES_PORT),
port: parseInt(process.env.POSTGRES_PORT.split(':').pop()),
schema: SCHEMA,
driverOptions: {
searchPath: [SCHEMA, 'public'],
Expand Down

0 comments on commit 35b2a1f

Please sign in to comment.