Skip to content

Commit

Permalink
Merge pull request #4707 from wri/hotfix/ttc-analysis-FLAG-943
Browse files Browse the repository at this point in the history
[FLAG-943] Tropical Tree Cover analysis malfunctioning
  • Loading branch information
willian-viana authored Oct 20, 2023
2 parents 817977f + 84b776d commit 1e19354
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 141 deletions.
1 change: 1 addition & 0 deletions jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ module.exports = {
'!<rootDir>/coverage/**',
'!<rootDir>/cypress/**',
],
moduleDirectories: ['node_modules', '<rootDir>'],
moduleNameMapper: {
// Handle CSS imports (with CSS modules)
// https://jestjs.io/docs/webpack#mocking-css-modules
Expand Down
41 changes: 41 additions & 0 deletions services/__tests__/get-where-query.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { getWHEREQuery } from '../get-where-query';

describe('getWHEREQuery', () => {
it('should return an string with each parameter separated by AND', () => {
const params = {
type: 'country',
adm0: 'BRA',
locationType: 'country',
extentYear: 2000,
thresh: 30,
threshold: 30,
forestType: 'plantations',
dataset: 'annual',
};
const query = getWHEREQuery(params);
const expected =
"WHERE iso = 'BRA' AND umd_tree_cover_density_2000__threshold = 30 AND gfw_planted_forests__type IS NOT NULL ";

expect(query).toEqual(expected);
});

// Tree Cover Density has a default threshold
it('should not return threshold for Tree Cover Density', () => {
const params = {
type: 'country',
adm0: 'PER',
locationType: 'country',
extentYear: 2020,
thresh: '40',
threshold: 40, // passing threshold as parameter from tropical tree cover layer
dataset: 'treeCoverDensity',
};

const query = getWHEREQuery(params);
const expected = "WHERE iso = 'PER' ";
const notExpected = 'AND umd_tree_cover_density_2000__threshold = 40 ';

expect(query).toEqual(expected);
expect(query).not.toEqual(notExpected);
});
});
149 changes: 8 additions & 141 deletions services/analysis-cached.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
import { cartoRequest, dataRequest } from 'utils/request';
import { PROXIES } from 'utils/proxies';

import forestTypes from 'data/forest-types';
import landCategories from 'data/land-categories';
import DATASETS from 'data/analysis-datasets.json';
import DATASETS_VERSIONS from 'data/analysis-datasets-versions.json';

import snakeCase from 'lodash/snakeCase';
import moment from 'moment';

import { getWHEREQuery } from './get-where-query';

const VIIRS_START_YEAR = 2012;

const SQL_QUERIES = {
Expand Down Expand Up @@ -83,51 +87,6 @@ const SQL_QUERIES = {
'SELECT {select_location}, wri_tropical_tree_cover__decile, SUM(wri_tropical_tree_cover_extent__ha) AS wri_tropical_tree_cover_extent__ha FROM data {WHERE} AND wri_tropical_tree_cover__decile >= 0 GROUP BY {location}, wri_tropical_tree_cover__decile ORDER BY {location}, wri_tropical_tree_cover__decile',
};

const ALLOWED_PARAMS = {
annual: ['adm0', 'adm1', 'adm2', 'threshold', 'forestType', 'landCategory'],
integrated_alerts: [
'adm0',
'adm1',
'adm2',
'forestType',
'landCategory',
'is__confirmed_alert',
],
glad: [
'adm0',
'adm1',
'adm2',
'forestType',
'landCategory',
'is__confirmed_alert',
],
viirs: ['adm0', 'adm1', 'adm2', 'forestType', 'landCategory', 'confidence'],
modis: ['adm0', 'adm1', 'adm2', 'forestType', 'landCategory', 'confidence'],
modis_burned_area: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
'confidence',
],
net_change: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
'confidence',
],
tropicalTreeCover: ['adm0', 'adm1', 'adm2', 'threshold', 'forestType'],
};

//
// function for building analysis table queries from params
//

const typeByGrouped = {
global: {
default: 'adm0',
Expand Down Expand Up @@ -305,101 +264,6 @@ const getLocationSelect = ({
);
};

// build {where} statement for query
export const getWHEREQuery = (params) => {
const allPolynames = forestTypes.concat(landCategories);
const paramKeys = params && Object.keys(params);
const allowedParams = ALLOWED_PARAMS[params.dataset || 'annual'];
const paramKeysFiltered = paramKeys.filter(
(p) => (params[p] || p === 'threshold') && allowedParams.includes(p)
);
const { type, dataset } = params || {};
let comparisonString = ' = ';
if (paramKeysFiltered && paramKeysFiltered.length) {
let paramString = 'WHERE ';
paramKeysFiltered.forEach((p, i) => {
const isLast = paramKeysFiltered.length - 1 === i;
const isPolyname = ['forestType', 'landCategory'].includes(p);
const value = isPolyname ? 1 : params[p];
const polynameMeta = allPolynames.find(
(pname) => pname.value === params[p]
);
const tableKey =
polynameMeta &&
(polynameMeta.tableKey || polynameMeta.tableKeys[dataset || 'annual']);

/* TODO
perform better casting / allow to configure types:
AS for example wdpa_protected_area__id needs to be a string,
even that it evaluates AS a number.
Note that the postgres tables will allow us to cast at the query level.
*/
// const zeroString = polynameMeta?.dataType === 'keyword' ? "'0'" : '0';
let isNumericValue = !!(
typeof value === 'number' ||
(!isNaN(value) && !['adm0', 'confidence'].includes(p))
);

let paramKey = p;
if (p === 'confidence') paramKey = 'confidence__cat';
if (p === 'threshold') {
// paramKey = 'umd_tree_cover_density__threshold';
comparisonString = ' = ';

if (dataset === 'tropicalTreeCover') {
paramKey = 'wri_tropical_tree_cover__decile';
} else {
paramKey = 'umd_tree_cover_density_2000__threshold';
}

// }
}
if (p === 'adm0' && type === 'country') paramKey = 'iso';
if (p === 'adm1' && type === 'country') paramKey = 'adm1';
if (p === 'adm2' && type === 'country') paramKey = 'adm2';
if (p === 'adm0' && type === 'geostore') paramKey = 'geostore__id';
if (p === 'adm0' && type === 'wdpa') {
paramKey = 'wdpa_protected_area__id';
isNumericValue = false;
}
if (dataset === 'net_change') {
isNumericValue = false;
}

const polynameString = `
${
isPolyname && tableKey.includes('is__') ? `${tableKey} = 'true'` : ''
}${
isPolyname && !tableKey.includes('is__')
? `${tableKey} IS NOT NULL`
: ''
}${
isPolyname &&
polynameMeta &&
!tableKey.includes('is__') &&
polynameMeta.default &&
polynameMeta.categories
? ` AND ${tableKey} ${polynameMeta.comparison || '='} ${
polynameMeta?.dataType === 'keyword'
? `'${polynameMeta?.default}'`
: `${polynameMeta?.default}`
}`
: ''
}${
!isPolyname
? `${paramKey}${comparisonString}${
isNumericValue ? value : `'${value}'`
}`
: ''
}${isLast ? '' : ' AND '}`;

paramString = paramString.concat(polynameString);
});
return paramString;
}
return '';
};

export const getDatesFilter = ({ startDate }) => {
const startYear = startDate
? moment(startDate).year()
Expand Down Expand Up @@ -2065,7 +1929,10 @@ export const getTreeCoverDensity = (params) => {
getLocationSelect({ ...params, cast: false })
)
.replace(/{location}/g, getLocationSelect({ ...params }))
.replace('{WHERE}', getWHEREQuery({ ...params, dataset: 'annual' }))
.replace(
'{WHERE}',
getWHEREQuery({ ...params, dataset: 'treeCoverDensity' })
)
);

return dataRequest.get(url).then((response) => response.data);
Expand Down
103 changes: 103 additions & 0 deletions services/get-where-query.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import ALLOWED_PARAMS from 'utils/get-where-query-allowed-params';
import { translateParameterKey } from 'utils/get-where-query-translation';

import forestTypes from 'data/forest-types';
import landCategories from 'data/land-categories';

const isNumber = (value) => !!(typeof value === 'number' || !isNaN(value));

// build {where} statement for query
export const getWHEREQuery = (params = {}) => {
const { type, dataset } = params || {};

const allFilterOptions = forestTypes.concat(landCategories);
const allowedParams = ALLOWED_PARAMS[params.dataset || 'annual'];
const isTreeCoverDensity = dataset === 'treeCoverDensity';
const comparisonString = ' = ';

let paramString = 'WHERE ';
let paramKeys = Object.keys(params).filter((parameterName) => {
return (
(params[parameterName] || parameterName === 'threshold') &&
allowedParams.includes(parameterName)
);
});

if (!paramKeys?.length) {
return '';
}

/*
* Removing threshold from Tree Cover Density request
* Tree Cover Density has a default threshold >=10
*/
if (isTreeCoverDensity && paramKeys.includes('threshold')) {
paramKeys = paramKeys.filter((item) => item !== 'threshold');
}

paramKeys.forEach((parameter, index) => {
const isLastParameter = paramKeys.length - 1 === index;
const hasFilterOption = ['forestType', 'landCategory'].includes(parameter);
const value = hasFilterOption ? 1 : params[parameter];
const filterOption = allFilterOptions.find(
(pname) => pname.value === params[parameter]
);

const tableKey =
filterOption &&
(filterOption.tableKey || filterOption.tableKeys[dataset || 'annual']);
let isNumericValue = isNumber(value);

const paramKey = translateParameterKey(parameter, params);

if (parameter === 'adm0' && type === 'wdpa') {
isNumericValue = false;
}

if (dataset === 'net_change') {
isNumericValue = false;
}

const hasPrefixIs__ = hasFilterOption && tableKey.includes('is__');
let WHERE = '';

if (hasFilterOption) {
if (hasPrefixIs__) {
WHERE = `${WHERE}${tableKey} = 'true'`;
}

if (!hasPrefixIs__) {
WHERE = `${WHERE}${tableKey} IS NOT NULL`;
}

if (
filterOption &&
!hasPrefixIs__ &&
filterOption.default &&
filterOption.categories
) {
WHERE = `${WHERE} AND ${tableKey} ${filterOption.comparison || '='} ${
filterOption?.dataType === 'keyword'
? `'${filterOption?.default}'`
: `${filterOption?.default}`
}`;
}
}

if (!hasFilterOption) {
WHERE = `${WHERE}${paramKey}${comparisonString}${
isNumericValue ? value : `'${value}'`
}`;
}

if (isLastParameter) {
WHERE = `${WHERE} `;
} else {
WHERE = `${WHERE} AND `;
}

paramString = paramString.concat(WHERE);
});

return paramString;
};
50 changes: 50 additions & 0 deletions utils/get-where-query-allowed-params.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
const ALLOWED_PARAMS = {
annual: ['adm0', 'adm1', 'adm2', 'threshold', 'forestType', 'landCategory'],
treeCoverDensity: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
],
integrated_alerts: [
'adm0',
'adm1',
'adm2',
'forestType',
'landCategory',
'is__confirmed_alert',
],
glad: [
'adm0',
'adm1',
'adm2',
'forestType',
'landCategory',
'is__confirmed_alert',
],
viirs: ['adm0', 'adm1', 'adm2', 'forestType', 'landCategory', 'confidence'],
modis: ['adm0', 'adm1', 'adm2', 'forestType', 'landCategory', 'confidence'],
modis_burned_area: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
'confidence',
],
net_change: [
'adm0',
'adm1',
'adm2',
'threshold',
'forestType',
'landCategory',
'confidence',
],
tropicalTreeCover: ['adm0', 'adm1', 'adm2', 'threshold', 'forestType'],
};

export default ALLOWED_PARAMS;
Loading

0 comments on commit 1e19354

Please sign in to comment.