Skip to content

Commit

Permalink
[maps] fix no geometry column found when editing ES|QL layers (elasti…
Browse files Browse the repository at this point in the history
…c#176231)

Resolves elastic#176227

PR updates maps ES|QL code to handle response shape when
`drop_null_values` is used. PR adds an integration test that will fail
if the ES|QL response shape changes.

#### Test steps
1. install sample web logs
2. create new map, add `ES|QL` layer
3. Edit query, changing limit from default of "10,000" to "1,000". Click
"run". Verify results are displayed
4. Set query that returns no results like, "bytes > 100000000". Verify
layer displays no results.

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
nreese and kibanamachine authored Feb 7, 2024
1 parent 79dbc47 commit 3e55ab5
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,19 @@
import { parse } from 'wellknown';
import { Feature, FeatureCollection, GeoJsonProperties } from 'geojson';
import type { ESQLSearchReponse } from '@kbn/es-types';
import { getGeometryColumnIndex } from './esql_utils';
import { EMPTY_FEATURE_COLLECTION } from '../../../../common/constants';
import { isGeometryColumn } from './esql_utils';

export function convertToGeoJson(resp: ESQLSearchReponse): FeatureCollection {
const geometryIndex = getGeometryColumnIndex(resp.columns);
const geometryColumnIndex = resp.columns.findIndex(isGeometryColumn);
if (geometryColumnIndex === -1) {
return EMPTY_FEATURE_COLLECTION;
}

const features: Feature[] = [];
for (let i = 0; i < resp.values.length; i++) {
const hit = resp.values[i];
const wkt = hit[geometryIndex];
const wkt = hit[geometryColumnIndex];
if (!wkt) {
continue;
}
Expand All @@ -25,7 +30,7 @@ export function convertToGeoJson(resp: ESQLSearchReponse): FeatureCollection {
const properties: GeoJsonProperties = {};
for (let j = 0; j < hit.length; j++) {
// do not store geometry in properties
if (j === geometryIndex) {
if (j === geometryColumnIndex) {
continue;
}
properties[resp.columns[j].name] = hit[j] as unknown;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import { v4 as uuidv4 } from 'uuid';
import { Adapters } from '@kbn/inspector-plugin/common/adapters';
import { getIndexPatternFromESQLQuery, getLimitFromESQLQuery } from '@kbn/esql-utils';
import { buildEsQuery } from '@kbn/es-query';
import type { BoolQuery, Filter, Query } from '@kbn/es-query';
import type { ESQLSearchReponse } from '@kbn/es-types';
import type { Filter, Query } from '@kbn/es-query';
import type { ESQLSearchParams, ESQLSearchReponse } from '@kbn/es-types';
import { getEsQueryConfig } from '@kbn/data-service/src/es_query';
import { getTime } from '@kbn/data-plugin/public';
import { FIELD_ORIGIN, SOURCE_TYPES, VECTOR_SHAPE_TYPE } from '../../../../common/constants';
Expand All @@ -32,12 +32,7 @@ import type { IField } from '../../fields/field';
import { InlineField } from '../../fields/inline_field';
import { getData, getUiSettings } from '../../../kibana_services';
import { convertToGeoJson } from './convert_to_geojson';
import {
getFieldType,
getGeometryColumnIndex,
ESQL_GEO_POINT_TYPE,
ESQL_GEO_SHAPE_TYPE,
} from './esql_utils';
import { getFieldType, isGeometryColumn, ESQL_GEO_SHAPE_TYPE } from './esql_utils';
import { UpdateSourceEditor } from './update_source_editor';

type ESQLSourceSyncMeta = Pick<
Expand Down Expand Up @@ -128,16 +123,8 @@ export class ESQLSource extends AbstractVectorSource implements IVectorSource {
}

async getSupportedShapeTypes() {
let geomtryColumnType = ESQL_GEO_POINT_TYPE;
try {
const index = getGeometryColumnIndex(this._descriptor.columns);
if (index > -1) {
geomtryColumnType = this._descriptor.columns[index].type;
}
} catch (error) {
// errors for missing geometry columns surfaced in UI by data loading
}
return geomtryColumnType === ESQL_GEO_SHAPE_TYPE
const index = this._descriptor.columns.findIndex(isGeometryColumn);
return index !== -1 && this._descriptor.columns[index].type === ESQL_GEO_SHAPE_TYPE
? [VECTOR_SHAPE_TYPE.POINT, VECTOR_SHAPE_TYPE.LINE, VECTOR_SHAPE_TYPE.POLYGON]
: [VECTOR_SHAPE_TYPE.POINT];
}
Expand All @@ -154,8 +141,9 @@ export class ESQLSource extends AbstractVectorSource implements IVectorSource {
inspectorAdapters: Adapters
): Promise<GeoJsonWithMeta> {
const limit = getLimitFromESQLQuery(this._descriptor.esql);
const params: { query: string; filter?: { bool: BoolQuery } } = {
const params: ESQLSearchParams = {
query: this._descriptor.esql,
dropNullColumns: true,
};

const query: Query[] = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { i18n } from '@kbn/i18n';
import { lastValueFrom } from 'rxjs';
import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import type { ESQLColumn } from '@kbn/es-types';
import type { ESQLColumn, ESQLSearchReponse } from '@kbn/es-types';
import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
import { getData, getIndexPatternService } from '../../../kibana_services';

Expand All @@ -22,21 +22,18 @@ export const ESQL_GEO_POINT_TYPE = 'geo_point';
// ESQL_GEO_SHAPE_TYPE is a column type from an ESQL response
export const ESQL_GEO_SHAPE_TYPE = 'geo_shape';

const NO_GEOMETRY_COLUMN_ERROR_MSG = i18n.translate(
'xpack.maps.source.esql.noGeometryColumnErrorMsg',
{
defaultMessage: 'Elasticsearch ES|QL query does not have a geometry column.',
}
);

export function isGeometryColumn(column: ESQLColumn) {
return [ESQL_GEO_POINT_TYPE, ESQL_GEO_SHAPE_TYPE].includes(column.type);
}

export function verifyGeometryColumn(columns: ESQLColumn[]) {
const geometryColumns = columns.filter(isGeometryColumn);
if (geometryColumns.length === 0) {
throw new Error(NO_GEOMETRY_COLUMN_ERROR_MSG);
throw new Error(
i18n.translate('xpack.maps.source.esql.noGeometryColumnErrorMsg', {
defaultMessage: 'Elasticsearch ES|QL query does not have a geometry column.',
})
);
}

if (geometryColumns.length > 1) {
Expand All @@ -51,14 +48,6 @@ export function verifyGeometryColumn(columns: ESQLColumn[]) {
}
}

export function getGeometryColumnIndex(columns: ESQLColumn[]) {
const index = columns.findIndex(isGeometryColumn);
if (index === -1) {
throw new Error(NO_GEOMETRY_COLUMN_ERROR_MSG);
}
return index;
}

export async function getESQLMeta(esql: string) {
const fields = await getFields(esql);
return {
Expand Down Expand Up @@ -104,7 +93,8 @@ async function getColumns(esql: string) {
)
);

return (resp.rawResponse as unknown as { columns: ESQLColumn[] }).columns;
const searchResponse = resp.rawResponse as unknown as ESQLSearchReponse;
return searchResponse.all_columns ? searchResponse.all_columns : searchResponse.columns;
} catch (error) {
throw new Error(
i18n.translate('xpack.maps.source.esql.getColumnsErrorMsg', {
Expand Down
112 changes: 112 additions & 0 deletions x-pack/test/api_integration/apis/maps/bsearch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import request from 'superagent';
import { inflateResponse } from '@kbn/bfetch-plugin/public/streaming';
import expect from '@kbn/expect';
import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
import { BFETCH_ROUTE_VERSION_LATEST } from '@kbn/bfetch-plugin/common';
import type { FtrProviderContext } from '../../ftr_provider_context';

function parseBfetchResponse(resp: request.Response, compressed: boolean = false) {
return resp.text
.trim()
.split('\n')
.map((item) => {
return JSON.parse(compressed ? inflateResponse<any>(item) : item);
});
}

export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');

describe('bsearch', () => {
describe('ES|QL', () => {
it(`should return getColumns response in expected shape`, async () => {
const resp = await supertest
.post(`/internal/bsearch`)
.set('kbn-xsrf', 'kibana')
.set(ELASTIC_HTTP_VERSION_HEADER, BFETCH_ROUTE_VERSION_LATEST)
.send({
batch: [
{
request: {
params: {
query: 'from logstash-* | keep geo.coordinates | limit 0',
},
},
options: {
strategy: 'esql',
},
},
],
});

const jsonBody = parseBfetchResponse(resp);
expect(resp.status).to.be(200);
expect(jsonBody[0].result.rawResponse).to.eql({
columns: [
{
name: 'geo.coordinates',
type: 'geo_point',
},
],
values: [],
});
});

it(`should return getValues response in expected shape`, async () => {
const resp = await supertest
.post(`/internal/bsearch`)
.set('kbn-xsrf', 'kibana')
.set(ELASTIC_HTTP_VERSION_HEADER, BFETCH_ROUTE_VERSION_LATEST)
.send({
batch: [
{
request: {
params: {
dropNullColumns: true,
query:
'from logstash-* | keep geo.coordinates, @timestamp | sort @timestamp | limit 1',
},
},
options: {
strategy: 'esql',
},
},
],
});

const jsonBody = parseBfetchResponse(resp);
expect(resp.status).to.be(200);
expect(jsonBody[0].result.rawResponse).to.eql({
all_columns: [
{
name: 'geo.coordinates',
type: 'geo_point',
},
{
name: '@timestamp',
type: 'date',
},
],
columns: [
{
name: 'geo.coordinates',
type: 'geo_point',
},
{
name: '@timestamp',
type: 'date',
},
],
values: [['POINT (-120.9871642 38.68407028)', '2015-09-20T00:00:00.000Z']],
});
});
});
});
}
1 change: 1 addition & 0 deletions x-pack/test/api_integration/apis/maps/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export default function ({ loadTestFile, getService }) {
loadTestFile(require.resolve('./migrations'));
loadTestFile(require.resolve('./get_tile'));
loadTestFile(require.resolve('./get_grid_tile'));
loadTestFile(require.resolve('./bsearch'));
});
});
}

0 comments on commit 3e55ab5

Please sign in to comment.