Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RER-24 Graph - "middleware" solution #33

Draft
wants to merge 3 commits into
base: dev
Choose a base branch
from
Draft

RER-24 Graph - "middleware" solution #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
43d4751
WIP add env param and add tests for four endpoints
KalinichenkoP Aug 23, 2021
e720c02
Merge with dev branch
KalinichenkoP Aug 23, 2021
2de9c94
Fix tests
KalinichenkoP Aug 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions app/src/routes/api/v1/graph.router.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,25 @@ class GraphRouter {
const application = ctx.query.application || ctx.query.app || 'rw';
const response = await QueryService.mostLikedDatasets(application);
let data = [];
const datasetIds = [];
if (response) {
data = response.records.map((c) => ({
id: c._fields[0],
count: c._fields[1]
}));
data = response.records.map((c) => {
datasetIds.push(c._fields[0]);
return {
id: c._fields[0],
count: c._fields[1]
};
});
}

let result = [];
if (datasetIds.length > 0) {
result = await datasetService.checkDatasets(datasetIds, ctx.query);
}

ctx.set('cache', 'graph-dataset');
ctx.body = {
data
data: data.filter((el) => result.indexOf(el.dataset) >= 0)
};
}

Expand Down Expand Up @@ -310,9 +320,15 @@ class GraphRouter {
} else if (!datasetIds) {
datasetIds = await QueryService.sortDatasets(sort, datasetIds);
}

let result = [];
if (datasetIds.length > 0) {
result = await datasetService.checkDatasets(datasetIds, ctx.query);
}

ctx.set('cache', 'graph-dataset');
ctx.body = {
data: datasetIds
data: datasetIds.filter((datasetId) => result.indexOf(datasetId) >= 0)
};
}

Expand Down Expand Up @@ -431,7 +447,7 @@ class GraphRouter {

ctx.assert(ctx.query.search, 400, 'search query param required');
const results = await QueryService.querySearchByLabelSynonymons(ctx.query.search ? ctx.query.search.split(' ') : '', application);
logger.debug('AAA', results);

const datasetIds = [];
if (results && results.records) {
results.records.map((el) => {
Expand All @@ -443,9 +459,14 @@ class GraphRouter {
};
});
}

let result = [];
if (datasetIds.length > 0) {
result = await datasetService.checkDatasets(datasetIds, ctx.query);
}
ctx.set('cache', 'graph-default');
ctx.body = {
data: datasetIds
data: datasetIds.filter((datasetId) => result.indexOf(datasetId) >= 0)
};
}

Expand Down
7 changes: 5 additions & 2 deletions app/src/services/dataset.service.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,20 @@ const { RWAPIMicroservice } = require('rw-api-microservice-node');

class DatasetService {

static async checkDatasets(datasets, query) {
static async checkDatasets(datasets, query = {}) {
logger.info('Checking published and other fields of dataset', datasets);
if (query) {
delete query.loggedUser;
}
const env = query.env ? query.env : 'production';

const result = await RWAPIMicroservice.requestToMicroservice({
uri: '/v1/dataset/find-by-ids',
method: 'POST',
json: true,
body: {
ids: datasets
ids: datasets,
env: env.split(',').map((elem) => elem.trim())
},
qs: query
});
Expand Down
256 changes: 256 additions & 0 deletions app/test/e2e/similar-dataset-by-id-including-descendent.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,262 @@ describe('GET similar dataset by id including descendent', () => {
});
});


it('Getting similar datasets by id including descendent with dataset ids should return a 200 (happy case) - filtered by env', async () => {
const records = [
{
keys: [
'dataset',
'dataset_tags',
'number_of_ocurrences'
],
length: 3,
_fields: [
'444138cd-8ef4-48b3-b197-73e324175ad0',
[
'precipitation',
'soil'
],
{
low: 2,
high: 0
}
],
_fieldLookup: {
dataset: 0,
dataset_tags: 1,
number_of_ocurrences: 2
}
},
{
keys: [
'dataset',
'dataset_tags',
'number_of_ocurrences'
],
length: 3,
_fields: [
'4828c405-06a2-4460-a78c-90969bce582b',
[
'water',
'drought'
],
{
low: 2,
high: 0
}
],
_fieldLookup: {
dataset: 0,
dataset_tags: 1,
number_of_ocurrences: 2
}
},
{
keys: [
'dataset',
'dataset_tags',
'number_of_ocurrences'
],
length: 3,
_fields: [
'c0c71e67-0088-4d69-b375-85297f79ee75',
[
'drought',
'precipitation'
],
{
low: 2,
high: 0
}
],
_fieldLookup: {
dataset: 0,
dataset_tags: 1,
number_of_ocurrences: 2
}
}
];
const query = '\nMATCH (d:DATASET)-[:TAGGED_WITH {application: {application}}]->(c:TOPIC)\nWHERE d.id IN {datasets}\nWITH COLLECT(c.id) AS main_tags, d\nMATCH (d2:DATASET)-[:TAGGED_WITH {application: {application}}]->(c1:TOPIC)-[:PART_OF|:IS_A|:QUALITY_OF*1..15]->(c2:TOPIC)\nWHERE (c1.id IN main_tags OR c2.id IN main_tags) AND d2.id <> d.id\nWITH COLLECT(DISTINCT c1.id) AS dataset_tags, d2.id AS dataset\nWITH size(dataset_tags) AS number_of_ocurrences, dataset_tags, dataset\nRETURN dataset, dataset_tags, number_of_ocurrences\nORDER BY number_of_ocurrences DESC\n';
const parameters = {
datasets: [
'e7b9efb2-3836-45ae-8b6a-f8391c7bcd2f'
],
application: 'rw'
};

stubQuery(sandbox, query, parameters, records);
nock(process.env.GATEWAY_URL)
.post('/v1/dataset/find-by-ids', {
// eslint-disable-next-line no-underscore-dangle
ids: records.map((elem) => elem._fields[0]),
env: 'production'
})
.reply(200, {
data: [
{
id: '444138cd-8ef4-48b3-b197-73e324175ad0',
type: 'dataset',
attributes: {
name: 'dis.012.nrt Landslide Hazard Alerts',
slug: 'dis012nrt-Landslide-Hazard-Alerts',
type: 'tabular',
subtitle: '',
application: [
'rw'
],
dataPath: '',
attributesPath: null,
connectorType: 'rest',
provider: 'cartodb',
userId: '58fa22c54eecd907310778cd',
connectorUrl: 'https://rw-nrt.carto.com/tables/dis_012_landslide_hazard_alerts_explore/public',
sources: [],
tableName: 'dis_012_landslide_hazard_alerts_explore',
status: 'saved',
published: true,
overwrite: false,
subscribable: {},
mainDateField: 'datetime',
env: 'production',
geoInfo: true,
protected: false,
legend: {
date: [],
region: [],
country: [],
nested: [],
integer: [],
short: [],
byte: [],
double: [],
float: [],
half_float: [],
scaled_float: [],
boolean: [],
binary: [],
text: [],
keyword: []
},
clonedHost: {},
errorMessage: '',
taskId: null,
createdAt: '2019-06-28T06:59:42.797Z',
updatedAt: '2021-08-13T10:27:01.559Z',
dataLastUpdated: '2021-08-13T02:30:00.000Z',
widgetRelevantProps: [
'datetime',
'nowcast'
],
layerRelevantProps: [
'datetime',
'nowcast'
]
}
},
{
id: '4828c405-06a2-4460-a78c-90969bce582b',
type: 'dataset',
attributes: {
name: 'foo.024.nrt Vegetation Health Index',
slug: 'foo024nrt-Vegetation-Health-Index',
type: 'raster',
subtitle: 'NOAA',
application: [
'prep'
],
dataPath: '',
attributesPath: null,
connectorType: 'rest',
provider: 'gee',
userId: '5899bfbcde3d6e4317ee4ef0',
connectorUrl: '',
sources: [],
tableName: 'users/resourcewatch_wri/foo_024_vegetation_health_index',
status: 'saved',
published: true,
overwrite: false,
subscribable: {},
mainDateField: '',
env: 'production',
applicationConfig: {
rw: {
layerOrder: [
'e9f9d20c-1924-48b2-97ed-6936e233adb2'
]
}
},
geoInfo: true,
protected: false,
legend: {
date: [],
region: [],
country: [],
nested: [],
integer: [],
short: [],
byte: [],
double: [],
float: [],
half_float: [],
scaled_float: [],
boolean: [],
binary: [],
text: [],
keyword: []
},
clonedHost: {},
errorMessage: '',
taskId: null,
createdAt: '2019-07-03T12:34:20.439Z',
updatedAt: '2019-10-29T12:52:47.662Z',
dataLastUpdated: '2019-10-20T00:00:00.000Z',
widgetRelevantProps: [],
layerRelevantProps: []
}
},
],
links: {
self: 'http://api.resourcewatch.org/v1/dataset/find-by-ids?page[number]=1&page[size]=10',
first: 'http://api.resourcewatch.org/v1/dataset/find-by-ids?page[number]=1&page[size]=10',
last: 'http://api.resourcewatch.org/v1/dataset/find-by-ids?page[number]=2&page[size]=10',
prev: 'http://api.resourcewatch.org/v1/dataset/find-by-ids?page[number]=1&page[size]=10',
next: 'http://api.resourcewatch.org/v1/dataset/find-by-ids?page[number]=2&page[size]=10'
},
meta: {
'total-pages': 2,
'total-items': 14,
size: 10
}
});

const response = await requester
.get('/api/v1/graph/query/similar-dataset-including-descendent/e7b9efb2-3836-45ae-8b6a-f8391c7bcd2f');

response.status.should.equal(200);
response.body.should.deep.equal({
data: [
{
concepts: [
'precipitation',
'soil'
],
dataset: '444138cd-8ef4-48b3-b197-73e324175ad0',
numberOfOcurrences: 1,
},
{
concepts: [
'water',
'drought',
],
dataset: '4828c405-06a2-4460-a78c-90969bce582b',
numberOfOcurrences: 1
}
]
});
});


afterEach(() => {
sandbox.restore();

Expand Down
Loading