[Security Solution] add initial workflow insights service (elastic#19…

…9606)

## Summary

Adds an SecurityWorkflowInsightsService that is setup during security
solution plugin initialization. The service setup installs the component
templates, index template, and datastream used by the service.

Depends on:
- elastic/elasticsearch#116485


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios


### For maintainers

- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)

Co-authored-by: Konrad Szwarc <[email protected]>

x-pack/plugins/security_solution/server/endpoint/services/index.ts

@@ -9,4 +9,5 @@ export * from './artifacts';
 export * from './actions';
 export * from './agent';
 export * from './artifacts_exception_list';
+export * from './workflow_insights';
 export type { FeatureKeys } from './feature_usage';

x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/constants.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DATA_STREAM_PREFIX = '.security-workflow-insights';
+export const COMPONENT_TEMPLATE_NAME = `${DATA_STREAM_PREFIX}-component-template`;
+export const INDEX_TEMPLATE_NAME = `${DATA_STREAM_PREFIX}-index-template`;
+export const INGEST_PIPELINE_NAME = `${DATA_STREAM_PREFIX}-ingest-pipeline`;
+export const DATA_STREAM_NAME = `${DATA_STREAM_PREFIX}-default`;
+
+export const TOTAL_FIELDS_LIMIT = 2000;

x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/errors.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EndpointError } from '../../../../common/endpoint/errors';
+
+export class SecurityWorkflowInsightsFailedInitialized extends EndpointError {
+  constructor(msg: string) {
+    super(`security workflow insights service failed to initialize with error: ${msg}`);
+  }
+}

x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/field_map_configurations.ts

@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { FieldMap } from '@kbn/data-stream-adapter';
+
+export const securityWorkflowInsightsFieldMap: FieldMap = {
+  '@timestamp': {
+    type: 'date',
+    array: false,
+    required: true,
+  },
+  message: {
+    type: 'text',
+    array: false,
+    required: true,
+  },
+  // endpoint or other part of security
+  category: {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // incompatible_virus, noisy_process_tree, etc
+  type: {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // the creator of the insight
+  source: {
+    type: 'nested',
+    array: false,
+    required: true,
+  },
+  // kibana-insight-task, llm-request-id, etc
+  'source.id': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // endpoint, kibana, llm, etc
+  'source.type': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // starting timestamp of the source data used to generate the insight
+  'source.data_range_start': {
+    type: 'date',
+    array: false,
+    required: true,
+  },
+  // ending timestamp of the source data used to generate the insight
+  'source.data_range_end': {
+    type: 'date',
+    array: false,
+    required: true,
+  },
+  // the target that this insight is created for
+  target: {
+    type: 'nested',
+    array: false,
+    required: true,
+  },
+  // endpoint, policy, etc
+  'target.id': {
+    type: 'keyword',
+    array: true,
+    required: true,
+  },
+  // endpoint ids, policy ids, etc
+  'target.type': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // latest action taken on the insight
+  action: {
+    type: 'nested',
+    array: false,
+    required: true,
+  },
+  // refreshed, remediated, suppressed, dismissed
+  'action.type': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  'action.timestamp': {
+    type: 'date',
+    array: false,
+    required: true,
+  },
+  // unique key for this insight, used for deduplicating insights.
+  // ie. crowdstrike or windows_defender
+  value: {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  // suggested remediation for insight
+  remediation: {
+    type: 'object',
+    array: false,
+    required: true,
+  },
+  // if remediation includes exception list items
+  'remediation.exception_list_items': {
+    type: 'object',
+    array: true,
+    required: false,
+  },
+  'remediation.exception_list_items.list_id': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.name': {
+    type: 'text',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.description': {
+    type: 'text',
+    array: false,
+    required: false,
+  },
+  'remediation.exception_list_items.entries': {
+    type: 'object',
+    array: true,
+    required: true,
+  },
+  'remediation.exception_list_items.entries.field': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.entries.operator': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.entries.type': {
+    type: 'keyword',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.entries.value': {
+    type: 'text',
+    array: false,
+    required: true,
+  },
+  'remediation.exception_list_items.tags': {
+    type: 'keyword',
+    array: true,
+    required: true,
+  },
+  'remediation.exception_list_items.os_types': {
+    type: 'keyword',
+    array: true,
+    required: true,
+  },
+  metadata: {
+    type: 'object',
+    array: false,
+    required: true,
+  },
+  // optional KV for notes
+  'metadata.notes': {
+    type: 'object',
+    array: false,
+    required: false,
+  },
+  // optional i8n variables
+  'metadata.message_variables': {
+    type: 'text',
+    array: true,
+    required: false,
+  },
+} as const;

x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
+import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
+import { kibanaPackageJson } from '@kbn/repo-info';
+
+import { createDatastream, createPipeline } from './helpers';
+import {
+  DATA_STREAM_PREFIX,
+  COMPONENT_TEMPLATE_NAME,
+  INDEX_TEMPLATE_NAME,
+  INGEST_PIPELINE_NAME,
+  TOTAL_FIELDS_LIMIT,
+} from './constants';
+import { securityWorkflowInsightsFieldMap } from './field_map_configurations';
+
+jest.mock('@kbn/data-stream-adapter', () => ({
+  DataStreamSpacesAdapter: jest.fn().mockImplementation(() => ({
+    setComponentTemplate: jest.fn(),
+    setIndexTemplate: jest.fn(),
+  })),
+}));
+
+describe('helpers', () => {
+  describe('createDatastream', () => {
+    it('should create a DataStreamSpacesAdapter with the correct configuration', () => {
+      const kibanaVersion = kibanaPackageJson.version;
+      const ds = createDatastream(kibanaVersion);
+
+      expect(DataStreamSpacesAdapter).toHaveBeenCalledTimes(1);
+      expect(DataStreamSpacesAdapter).toHaveBeenCalledWith(DATA_STREAM_PREFIX, {
+        kibanaVersion,
+        totalFieldsLimit: TOTAL_FIELDS_LIMIT,
+      });
+      expect(ds.setComponentTemplate).toHaveBeenCalledTimes(1);
+      expect(ds.setComponentTemplate).toHaveBeenCalledWith({
+        name: COMPONENT_TEMPLATE_NAME,
+        fieldMap: securityWorkflowInsightsFieldMap,
+      });
+      expect(ds.setIndexTemplate).toHaveBeenCalledTimes(1);
+      expect(ds.setIndexTemplate).toHaveBeenCalledWith({
+        name: INDEX_TEMPLATE_NAME,
+        componentTemplateRefs: [COMPONENT_TEMPLATE_NAME],
+        template: {
+          settings: {
+            default_pipeline: INGEST_PIPELINE_NAME,
+          },
+        },
+        hidden: true,
+      });
+    });
+  });
+
+  describe('createPipeline', () => {
+    let esClient: ElasticsearchClient;
+
+    beforeEach(() => {
+      esClient = elasticsearchServiceMock.createElasticsearchClient();
+    });
+
+    it('should create an ingest pipeline with the correct configuration', async () => {
+      await createPipeline(esClient);
+
+      expect(esClient.ingest.putPipeline).toHaveBeenCalledTimes(1);
+      expect(esClient.ingest.putPipeline).toHaveBeenCalledWith({
+        id: INGEST_PIPELINE_NAME,
+        processors: [],
+        _meta: {
+          managed: true,
+        },
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+
+import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
+
+import {
+  COMPONENT_TEMPLATE_NAME,
+  DATA_STREAM_PREFIX,
+  INDEX_TEMPLATE_NAME,
+  INGEST_PIPELINE_NAME,
+  TOTAL_FIELDS_LIMIT,
+} from './constants';
+import { securityWorkflowInsightsFieldMap } from './field_map_configurations';
+
+export function createDatastream(kibanaVersion: string): DataStreamSpacesAdapter {
+  const ds = new DataStreamSpacesAdapter(DATA_STREAM_PREFIX, {
+    kibanaVersion,
+    totalFieldsLimit: TOTAL_FIELDS_LIMIT,
+  });
+  ds.setComponentTemplate({
+    name: COMPONENT_TEMPLATE_NAME,
+    fieldMap: securityWorkflowInsightsFieldMap,
+  });
+  ds.setIndexTemplate({
+    name: INDEX_TEMPLATE_NAME,
+    componentTemplateRefs: [COMPONENT_TEMPLATE_NAME],
+    template: {
+      settings: {
+        default_pipeline: INGEST_PIPELINE_NAME,
+      },
+    },
+    hidden: true,
+  });
+  return ds;
+}
+
+export async function createPipeline(esClient: ElasticsearchClient): Promise<boolean> {
+  const response = await esClient.ingest.putPipeline({
+    id: INGEST_PIPELINE_NAME,
+    processors: [
+      // requires @elastic/elasticsearch 8.16.0
+      // {
+      //   fingerprint: {
+      //     fields: ['type', 'category', 'value', 'target.type', 'target.id'],
+      //     target_field: '_id',
+      //     method: 'SHA-256',
+      //     if: 'ctx._id == null',
+      //   },
+      // },
+    ],
+    _meta: {
+      managed: true,
+    },
+  });
+  return response.acknowledged;
+}