Skip to content

Commit

Permalink
Merge pull request #1512 from betagouv/feat/update_dataset
Browse files Browse the repository at this point in the history
Feat/update dataset
  • Loading branch information
jonathanfallon authored Oct 13, 2021
2 parents bced4d4 + 4e44b98 commit 147f774
Show file tree
Hide file tree
Showing 16 changed files with 1,290 additions and 1,261 deletions.
10 changes: 7 additions & 3 deletions api/db/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
"coverage": "exit 0"
},
"dependencies": {
"db-migrate": "^0.11.11",
"db-migrate-pg": "^1.2.2"
"@types/node": "^16.9.4",
"@types/sinon": "^10.0.3",
"concurrently": "^6.2.1",
"db-migrate": "^0.11.12",
"db-migrate-pg": "^1.2.2",
"ts-node": "^10.2.1"
}
}
}
3 changes: 2 additions & 1 deletion api/services/trip/src/ServiceProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,11 @@ import { ScopeToGroupMiddleware } from './middleware/ScopeToGroupMiddleware';
import { TripCacheWarmCron } from './cron/TripCacheWarmCron';
import { BuildExcelsExportAction } from './actions/BuildExcelExportAction';
import { ActiveCampaignExcelExportAction } from './actions/ActiveCampaignExcelExportAction';
import { DataGouvProvider } from './providers/DataGouvProvider';

@serviceProvider({
config,
providers: [TripRepositoryProvider, StatCacheRepositoryProvider, S3StorageProvider, CryptoProvider],
providers: [TripRepositoryProvider, StatCacheRepositoryProvider, S3StorageProvider, CryptoProvider, DataGouvProvider],
validator: [
listBinding,
searchCountBinding,
Expand Down
10 changes: 2 additions & 8 deletions api/services/trip/src/actions/BuildExcelExportAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ContextType, handler } from '@ilos/common';
import { Action } from '@ilos/core';
import { BucketName, S3StorageProvider } from '@pdc/provider-file';
import { internalOnlyMiddlewares } from '@pdc/provider-middleware';
import { getDefaultEndDate } from '../helpers/getDefaultDates';
import { ResultInterface as Campaign } from '../shared/policy/find.contract';
import { handlerConfig, ParamsInterface, ResultInterface } from '../shared/trip/excelExport.contract';
import { alias } from '../shared/trip/excelExport.schema';
Expand Down Expand Up @@ -68,7 +69,7 @@ export class BuildExcelsExportAction extends Action {

private castOrGetDefaultDates(params: ParamsInterface): { start_date: Date; end_date: Date } {
if (!params.query.date) {
const endDate: Date = this.endOfPreviousMonthDate();
const endDate: Date = getDefaultEndDate();
return { start_date: this.startOfPreviousMonthDate(endDate), end_date: endDate };
} else {
const start_date = new Date(params.query.date.start);
Expand All @@ -77,13 +78,6 @@ export class BuildExcelsExportAction extends Action {
}
}

private endOfPreviousMonthDate(): Date {
const endDate = new Date();
endDate.setDate(1);
endDate.setHours(0, 0, 0, -1);
return endDate;
}

private startOfPreviousMonthDate(endDate: Date): Date {
const startDate = new Date(endDate.valueOf());
startDate.setDate(1);
Expand Down
117 changes: 67 additions & 50 deletions api/services/trip/src/actions/BuildExportAction.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,40 @@
import { TerritoryTripsInterface } from './../interfaces/StartTerritoryCountInterface';
import path from 'path';
import os from 'os';
import fs from 'fs';
import { v4 } from 'uuid';
import AdmZip from 'adm-zip';
import { get } from 'lodash';
import csvStringify, { Stringifier } from 'csv-stringify';

import { internalOnlyMiddlewares } from '@pdc/provider-middleware';
import { Action } from '@ilos/core';
import {
handler,
ConfigInterfaceResolver,
ContextType,
handler,
InitHookInterface,
ConfigInterfaceResolver,
KernelInterfaceResolver,
} from '@ilos/common';
import { Action } from '@ilos/core';
import { BucketName, S3StorageProvider } from '@pdc/provider-file';

import { internalOnlyMiddlewares } from '@pdc/provider-middleware';
import AdmZip from 'adm-zip';
import csvStringify, { Stringifier } from 'csv-stringify';
import fs from 'fs';
import { get } from 'lodash';
import os from 'os';
import path from 'path';
import { v4 } from 'uuid';
import { getDefaultEndDate } from '../helpers/getDefaultDates';
import { getOpenDataExportName } from '../helpers/getOpenDataExportName';
import { normalize } from '../helpers/normalizeExportDataHelper';
import { ExportTripInterface } from '../interfaces';
import { TripRepositoryProvider } from '../providers/TripRepositoryProvider';
import {
signature,
FormatInterface,
handlerConfig,
ParamsInterface,
ResultInterface,
QueryInterface,
FormatInterface,
ResultInterface,
signature,
} from '../shared/trip/buildExport.contract';

import {
ParamsInterface as PublishOpenDataParamsInterface,
signature as publishOpenDataSignature,
} from '../shared/trip/publishOpenData.contract';
import { alias } from '../shared/trip/buildExport.schema';
import { TripRepositoryProvider } from '../providers/TripRepositoryProvider';
import { ExportTripInterface } from '../interfaces';
import { normalize } from '../helpers/normalizeExportDataHelper';
import { getOpenDataExportName } from '../helpers/getOpenDataExportName';
import { TerritoryTripsInterface } from '../interfaces/TerritoryTripsInterface';

export interface FlattenTripInterface extends ExportTripInterface<string> {
journey_start_date: string;
Expand Down Expand Up @@ -203,42 +207,36 @@ export class BuildExportAction extends Action implements InitHookInterface {
};

async init(): Promise<void> {
/**
* Activate open data export in production only
*/
if (this.config.get('app.environment') === 'production') {
await this.kernel.notify<ParamsInterface>(
signature,
{
type: 'opendata',
await this.kernel.notify<ParamsInterface>(
signature,
{
type: 'opendata',
},
{
call: {
user: {},
},
{
call: {
user: {},
},
channel: {
service: handlerConfig.service,
metadata: {
repeat: {
cron: '0 5 6 * *',
},
jobId: 'trip.open_data_export',
channel: {
service: handlerConfig.service,
metadata: {
repeat: {
cron: '0 5 6 * *',
},
jobId: 'trip.open_data_export',
},
},
);
}
},
);
}

public async handle(params: ParamsInterface, context: ContextType): Promise<ResultInterface> {
const type = get(params, 'type', 'opendata');
const type = get(params, 'type', 'export');

const queryParam = this.castQueryParams(type, params);
let excluded_territories: TerritoryTripsInterface[];
if (type === 'opendata') {
// eslint-disable-next-line max-len
const excluded_territories: TerritoryTripsInterface[] = await this.tripRepository.getOpenDataExcludedTerritories(
queryParam,
);
excluded_territories = await this.tripRepository.getOpenDataExcludedTerritories(queryParam);
if (excluded_territories.length !== 0) {
queryParam.excluded_start_territory_id = excluded_territories
.filter((t) => t.start_territory_id)
Expand Down Expand Up @@ -278,6 +276,25 @@ export class BuildExportAction extends Action implements InitHookInterface {

const fileKey = await this.fileProvider.upload(BucketName.Export, zippath, zipname);

if (type == 'opendata') {
await this.kernel.notify<PublishOpenDataParamsInterface>(
publishOpenDataSignature,
{
publish: true,
date: queryParam.date.end,
},
{
call: {
user: {},
metadata: { queryParam: queryParam, excludedTerritories: excluded_territories },
},
channel: {
service: handlerConfig.service,
},
},
);
}

return fileKey;
}

Expand All @@ -293,9 +310,7 @@ export class BuildExportAction extends Action implements InitHookInterface {
return params.query;
}

const endDate = new Date();
endDate.setDate(1);
endDate.setHours(0, 0, 0, -1);
const endDate = getDefaultEndDate();
const startDate = new Date(endDate.valueOf());
startDate.setDate(1);
startDate.setHours(0, 0, 0, 0);
Expand All @@ -315,7 +330,9 @@ export class BuildExportAction extends Action implements InitHookInterface {
return {
tz: params.format?.tz ?? 'Europe/Paris',
filename:
params.format?.filename ?? type === 'opendata' ? getOpenDataExportName('csv') : `covoiturage-${v4()}.csv`,
params.format?.filename ?? type === 'opendata'
? getOpenDataExportName('csv', params.query.date.end)
: `covoiturage-${v4()}.csv`,
};
}

Expand Down
25 changes: 15 additions & 10 deletions api/services/trip/src/actions/PublishOpenDataAction.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { internalOnlyMiddlewares } from '@pdc/provider-middleware';
import { ConfigInterfaceResolver, ContextType, handler, NotFoundException } from '@ilos/common';
import { Action } from '@ilos/core';
import { handler, ContextType, ConfigInterfaceResolver, NotFoundException } from '@ilos/common';
import { BucketName, S3StorageProvider } from '@pdc/provider-file';

import { handlerConfig, ParamsInterface, ResultInterface } from '../shared/trip/publishOpenData.contract';
import { alias } from '../shared/trip/publishOpenData.schema';
import { internalOnlyMiddlewares } from '@pdc/provider-middleware';
import { getOpenDataExportName } from '../helpers/getOpenDataExportName';
import { DataGouvProvider } from '../providers/DataGouvProvider';
import { Dataset, Resource } from '../interfaces';
import { DataGouvProvider } from '../providers/DataGouvProvider';
import { handlerConfig, ParamsInterface, ResultInterface } from '../shared/trip/publishOpenData.contract';
import { alias } from '../shared/trip/publishOpenData.schema';
import { AppendMarkdownDescription } from './opendata/AppendMarkdownDescription';

@handler({
...handlerConfig,
Expand All @@ -18,6 +18,7 @@ export class PublishOpenDataAction extends Action {
private file: S3StorageProvider,
private config: ConfigInterfaceResolver,
private datagouv: DataGouvProvider,
private appendMarkdownDescription: AppendMarkdownDescription,
) {
super();
}
Expand All @@ -31,12 +32,16 @@ export class PublishOpenDataAction extends Action {
const resource = await this.createResource(filename);
if (publish) {
await this.ensureExportIsReachable(filename);
const rid = await this.datagouv.publishResource(datasetSlug, resource);
await this.datagouv.checkResource(datasetSlug, rid);
// await this.datagouv.updateDataset(dataset);
await this.datagouv.publishResource(datasetSlug, resource);
const description: string = await this.appendMarkdownDescription.call(
context.call.metadata,
dataset.description,
);
dataset.description = description;
await this.datagouv.updateDataset(dataset);
} else {
await this.datagouv.unpublishResource(datasetSlug, this.findRidFromTitle(dataset, resource.title));
// await this.datagouv.updateDataset(dataset);
await this.datagouv.updateDataset(dataset);
}
} catch (e) {
throw e;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import { OpenDataContextMetadata } from './AppendMarkdownDescription';
/* eslint-disable max-len */
import anyTest, { TestInterface } from 'ava';
import { TripRepositoryProvider } from '../../providers/TripRepositoryProvider';
import { OpenDataTripSearchInterface } from '../../shared/trip/common/interfaces/TripSearchInterface';
import { AppendMarkdownDescription } from './AppendMarkdownDescription';
import sinon, { SinonStub } from 'sinon';

interface Context {
// Injected tokens
tripRepositoryProvider: TripRepositoryProvider;
// Injected tokens method's stubs
tripRepositoryProviderStub: SinonStub;
// Tested token
happenMarkdownDescription: AppendMarkdownDescription;
// Constants
}

const test = anyTest as TestInterface<Partial<Context>>;

test.beforeEach((t) => {
t.context.tripRepositoryProvider = new TripRepositoryProvider(null);
t.context.happenMarkdownDescription = new AppendMarkdownDescription(t.context.tripRepositoryProvider);
t.context.tripRepositoryProviderStub = sinon.stub(t.context.tripRepositoryProvider, 'searchCount');
});

test.afterEach((t) => {});

test('AppendMarkdownDescription: should happen description to existing one', async (t) => {
// Arrange
const existingDescription =
"# Spécificités jeu de données janvier 2021\nLes données concernent également les trajets dont le point de départ OU d'arrivée est situé en dehors du territoire français.\n\n* Nombre trajets collectés et validés par le registre de preuve de covoiturage **96 012**\n* Nombre de trajets exposés dans le jeu de données : **90 443**\n* Nombre de trajets supprimés du jeu de données : **5 569 = 3 103 + 2 998 - 532**\n * Nombre d’occurrences du code INSEE de départ est < 6 : **3 103**\n * Nombre d’occurrences du code INSEE d'arrivée est < 6 : **2 998**\n * Nombre d’occurrences du code INSEE de départ ET d'arrivée est < 6 : **532**";
t.context.tripRepositoryProviderStub.onCall(0).resolves({ count: '21' });
t.context.tripRepositoryProviderStub.onCall(1).resolves({ count: '30' });

const openDataQueryParam: OpenDataTripSearchInterface = {
date: {
start: new Date('2021-09-01'),
end: new Date('2021-09-30'),
},
excluded_start_territory_id: [589, 785, 5, 8],
excluded_end_territory_id: [8888, 77, 5, 8],
};

const openDataContext: OpenDataContextMetadata = {
queryParam: openDataQueryParam,
excludedTerritories: [
{
end_territory_id: 589,
aggregated_trips_journeys: ['trip1', 'trip2'],
},
{
end_territory_id: 785,
aggregated_trips_journeys: ['trip3', 'trip4'],
},
{
end_territory_id: 5,
aggregated_trips_journeys: ['trip5'],
},
{
end_territory_id: 8,
aggregated_trips_journeys: ['trip6'],
},
{
start_territory_id: 8888,
aggregated_trips_journeys: ['trip6'],
},
{
start_territory_id: 77,
aggregated_trips_journeys: ['trip5'],
},
{
start_territory_id: 5,
aggregated_trips_journeys: ['trip7'],
},
{
start_territory_id: 8,
aggregated_trips_journeys: ['trip8', 'trip9'],
},
],
};

// Act
const description: string = await t.context.happenMarkdownDescription.call(openDataContext, existingDescription);

// Assert
const expected_happened_description =
"\n\n# Spécificités jeu de données septembre 2021\nLes données concernent également les trajets dont le point de départ OU d'arrivée est situé en dehors du territoire français.\n\n* Nombre trajets collectés et validés par le registre de preuve de covoiturage **30**\n* Nombre de trajets exposés dans le jeu de données : **21**\n* Nombre de trajets supprimés du jeu de données : **9 = 5 + 6 - 2**\n * Nombre de trajets dont l’occurrence du code INSEE de départ est < 6 : **5**\n * Nombre de trajets dont l’occurrence du code INSEE d'arrivée est < 6 : **6**\n * Nombre de trajets dont l’occurrence du code INSEE de départ ET d'arrivée est < 6 : **2**";
t.deepEqual(description, `${existingDescription}${expected_happened_description}`);
sinon.assert.calledWithMatch(t.context.tripRepositoryProviderStub.firstCall, openDataQueryParam);
const openDataQueryParamCopy: OpenDataTripSearchInterface = {
...openDataQueryParam,
};
delete openDataQueryParamCopy.excluded_end_territory_id;
delete openDataQueryParamCopy.excluded_start_territory_id;
sinon.assert.calledWithMatch(t.context.tripRepositoryProviderStub.secondCall, openDataQueryParamCopy);
});
Loading

0 comments on commit 147f774

Please sign in to comment.