diff --git a/build.gradle b/build.gradle index f71190ee653..a4410866ca7 100644 --- a/build.gradle +++ b/build.gradle @@ -333,7 +333,6 @@ tasks.register("assembleTarDistribution") { inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") - inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") @@ -350,7 +349,6 @@ tasks.register("assembleOssTarDistribution") { inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") - inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") @@ -365,7 +363,6 @@ tasks.register("assembleZipDistribution") { inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") - inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") @@ -382,7 +379,6 @@ tasks.register("assembleOssZipDistribution") { inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") - inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") diff --git a/config/logstash.yml b/config/logstash.yml index c80f38efade..aeeabaf8743 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -181,38 +181,6 @@ # # api.auth.basic.password_policy.mode: WARN # -# ------------ Module Settings --------------- -# Define modules here. Modules definitions must be defined as an array. -# The simple way to see this is to prepend each `name` with a `-`, and keep -# all associated variables under the `name` they are associated with, and -# above the next, like this: -# -# modules: -# - name: MODULE_NAME -# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE -# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE -# -# Module variable names must be in the format of -# -# var.PLUGIN_TYPE.PLUGIN_NAME.KEY -# -# modules: -# -# ------------ Cloud Settings --------------- -# Define Elastic Cloud settings here. -# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy -# and it may have an label prefix e.g. staging:dXMtZ... -# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' -# cloud.id: -# -# Format of cloud.auth is: : -# This is optional -# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' -# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' -# cloud.auth: elastic: -# # ------------ Queuing Settings -------------- # # Internal queuing model, "memory" for legacy in-memory based queuing and diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index b2e93ad16f5..dc2641edf0b 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -17,11 +17,12 @@ package main import ( "errors" "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "log" "os" "strings" + + "gopkg.in/yaml.v2" ) var validSettings = []string{ @@ -67,7 +68,6 @@ var validSettings = []string{ "log.level", "log.format", "log.format.json.fix_duplicate_message_fields", - "modules", "metric.collect", "path.logs", "path.plugins", @@ -126,8 +126,6 @@ var validSettings = []string{ "xpack.management.elasticsearch.ssl.cipher_suites", "xpack.geoip.download.endpoint", "xpack.geoip.downloader.enabled", - "cloud.id", - "cloud.auth", } // Given a setting name, return a downcased version with delimiters removed. diff --git a/logstash-core/lib/logstash/config/modules_common.rb b/logstash-core/lib/logstash/config/modules_common.rb deleted file mode 100644 index 157ead21b04..00000000000 --- a/logstash-core/lib/logstash/config/modules_common.rb +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/elasticsearch_client" -require "logstash/modules/kibana_client" -require "logstash/modules/elasticsearch_importer" -require "logstash/modules/kibana_importer" -require "logstash/modules/settings_merger" - -module LogStash module Config - class ModulesCommon # extracted here for bwc with 5.x - include LogStash::Util::Loggable - - MODULES_MAX_PIPELINES = 1 - - def self.pipeline_configs(settings) - pipelines = [] - plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules) - - cli_settings = settings.get("modules.cli") - yml_settings = settings.get("modules") - - modules_array = if !(cli_settings.empty? && yml_settings.empty?) - LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings) - elsif cli_settings.empty? - yml_settings - else - cli_settings - end - - if modules_array.empty? - # no specified modules - return pipelines - end - logger.debug("Specified modules", :modules_array => modules_array.to_s) - - module_names = modules_array.collect {|module_hash| module_hash["name"]} - if module_names.size > MODULES_MAX_PIPELINES - error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', ')) - raise LogStash::ConfigLoadingError, error_message - end - - if module_names.length > module_names.uniq.length - duplicate_modules = module_names.group_by(&:to_s).select { |_, v| v.size > 1 }.keys - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules) - end - - available_module_names = plugin_modules.map(&:module_name) - specified_and_available_names = module_names & available_module_names - - if (specified_and_available_names).empty? - i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names} - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", **i18n_opts) - end - - specified_and_available_names.each do |module_name| - connect_fail_args = {} - begin - module_settings = settings.clone - - module_hash = modules_array.find {|m| m["name"] == module_name} - current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name } - - enabled = current_module.is_enabled?(module_settings) - unless enabled - logger.warn("The #{module_name} module is not enabled. Please check the logs for additional information.") - next - end - - alt_name = "module-#{module_name}" - pipeline_id = alt_name - module_settings.set("pipeline.id", pipeline_id) - LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings) - LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash) - current_module.with_settings(module_hash) - config_test = settings.get("config.test_and_exit") - module_setup = settings.get("modules_setup") - # Only import data if it's not a config test and --setup is true - if !config_test && module_setup - logger.info("Setting up the #{module_name} module") - esclient = LogStash::ElasticsearchClient.build(module_hash) - kbnclient = LogStash::Modules::KibanaClient.new(module_hash) - esconnected = esclient.can_connect? - kbnconnected = kbnclient.can_connect? - if esconnected && kbnconnected - current_module.add_kibana_version(kbnclient.version_parts) - current_module.import( - LogStash::Modules::ElasticsearchImporter.new(esclient), - LogStash::Modules::KibanaImporter.new(kbnclient) - ) - else - connect_fail_args[:module_name] = module_name - connect_fail_args[:elasticsearch_hosts] = esclient.host_settings - connect_fail_args[:kibana_hosts] = kbnclient.host_settings - end - else - logger.info("Starting the #{module_name} module") - end - - config_string = current_module.config_string - pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => module_settings} - rescue => e - new_error = LogStash::ConfigLoadingError.new(I18n.t("logstash.modules.configuration.parse-failed", :error => e.message)) - new_error.set_backtrace(e.backtrace) - raise new_error - end - - if !connect_fail_args.empty? - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", **connect_fail_args) - end - end - pipelines - end - - end -end end diff --git a/logstash-core/lib/logstash/config/source/base.rb b/logstash-core/lib/logstash/config/source/base.rb index 0db79bc474e..b891551a534 100644 --- a/logstash-core/lib/logstash/config/source/base.rb +++ b/logstash-core/lib/logstash/config/source/base.rb @@ -71,37 +71,5 @@ def config_path def config_path? !(config_path.nil? || config_path.empty?) end - - def modules_cli_setting - @settings.get_setting("modules.cli") - end - - def modules_cli - modules_cli_setting.value - end - - def modules_cli? - !(modules_cli.nil? || modules_cli.empty?) - end - - def modules_setting - @settings.get_setting("modules") - end - - def modules - modules_setting.value - end - - def modules? - !(modules.nil? || modules.empty?) - end - - def both_module_configs? - modules_cli? && modules? - end - - def modules_defined? - modules_cli? || modules? - end end end end end diff --git a/logstash-core/lib/logstash/config/source/local.rb b/logstash-core/lib/logstash/config/source/local.rb index e9601c2b5f5..f994130d509 100644 --- a/logstash-core/lib/logstash/config/source/local.rb +++ b/logstash-core/lib/logstash/config/source/local.rb @@ -179,7 +179,7 @@ def pipeline_configs def match? # see basic settings predicates and getters defined in the base class - (config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string? + (config_string? || config_path?) && !automatic_reload_with_config_string? end def config_conflict? diff --git a/logstash-core/lib/logstash/config/source/modules.rb b/logstash-core/lib/logstash/config/source/modules.rb deleted file mode 100644 index 47794465c54..00000000000 --- a/logstash-core/lib/logstash/config/source/modules.rb +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/config/source/base" -require "logstash/config/modules_common" - -module LogStash module Config module Source - class Modules < Base - include LogStash::Util::Loggable - def pipeline_configs - if config_conflict? # double check - raise ConfigurationError, @conflict_messages.join(", ") - end - - pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings) - pipelines.map do |hash| - org.logstash.config.ir.PipelineConfig.new(self.class, hash["pipeline_id"].to_sym, - org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]), - hash["settings"]) - end - end - - def match? - # see basic settings predicates and getters defined in the base class - (modules_cli? || modules?) && !(config_string? || config_path?) && !automatic_reload_with_modules? - end - - def config_conflict? - @conflict_messages.clear - # Make note that if modules are configured in both cli and logstash.yml that cli module - # settings will overwrite the logstash.yml modules settings - if modules_cli? && modules? - logger.info(I18n.t("logstash.runner.cli-module-override")) - end - - if automatic_reload_with_modules? - @conflict_messages << I18n.t("logstash.runner.reload-with-modules") - end - - # Check if config (-f or -e) and modules are configured - if (modules_cli? || modules?) && (config_string? || config_path?) - @conflict_messages << I18n.t("logstash.runner.config-module-exclusive") - end - - @conflict_messages.any? - end - - private - - def automatic_reload_with_modules? - (modules_cli? || modules?) && config_reload_automatic? - end - end -end end end diff --git a/logstash-core/lib/logstash/config/source/multi_local.rb b/logstash-core/lib/logstash/config/source/multi_local.rb index 14697835cba..1a7904e296d 100644 --- a/logstash-core/lib/logstash/config/source/multi_local.rb +++ b/logstash-core/lib/logstash/config/source/multi_local.rb @@ -48,7 +48,7 @@ def pipeline_configs end def match? - if modules_cli? || modules? || config_string? || config_path? + if config_string? || config_path? return false end detect_pipelines if !@detect_pipelines_called @@ -62,7 +62,7 @@ def invalid_pipelines_detected? def config_conflict? @conflict_messages.clear # are there any auto-reload conflicts? - if !(modules_cli? || modules? || config_string? || config_path?) + if !(config_string? || config_path?) detect_pipelines if !@detect_pipelines_called if @detected_marker.nil? @conflict_messages << I18n.t("logstash.runner.config-pipelines-failed-read", :path => pipelines_yaml_location) @@ -74,7 +74,7 @@ def config_conflict? @conflict_messages << @detected_marker.message end else - do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because modules or command line options are specified") + do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because command line options are specified") end @conflict_messages.any? end diff --git a/logstash-core/lib/logstash/config/source_loader.rb b/logstash-core/lib/logstash/config/source_loader.rb index 7de6685a53a..1774f2fd78f 100644 --- a/logstash-core/lib/logstash/config/source_loader.rb +++ b/logstash-core/lib/logstash/config/source_loader.rb @@ -16,7 +16,6 @@ # under the License. require "logstash/config/source/local" -require "logstash/config/source/modules" require "logstash/config/source/multi_local" require "thread" require "set" diff --git a/logstash-core/lib/logstash/elasticsearch_client.rb b/logstash-core/lib/logstash/elasticsearch_client.rb deleted file mode 100644 index 7a79efa1666..00000000000 --- a/logstash-core/lib/logstash/elasticsearch_client.rb +++ /dev/null @@ -1,157 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "elasticsearch" -require "elasticsearch/transport/transport/http/manticore" -require 'logstash/util/manticore_ssl_config_helper' -require 'logstash/util/password' - -module LogStash class ElasticsearchClient - include LogStash::Util::Loggable - - class Response - # duplicated here from Elasticsearch::Transport::Transport::Response - # to create a normalised response across different client IMPL - attr_reader :status, :body, :headers - - def initialize(status, body, headers = {}) - @status, @body, @headers = status, body, headers - @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding) - end - end - - def self.build(settings) - new(RubyClient.new(settings, logger)) - end - - class RubyClient - include LogStash::Util::ManticoreSSLConfigHelper - - def initialize(settings, logger) - @settings = settings - @logger = logger - @client_args = client_args - - ssl_options = manticore_ssl_options_from_config('elasticsearch', settings) - - @client_args[:ssl] = ssl_options - - username = @settings["var.elasticsearch.username"] - if username - password = @settings["var.elasticsearch.password"] - if password.is_a?(LogStash::Util::Password) - password = password.value - end - @client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64("#{username}:#{password}").chomp } } - end - - @client = Elasticsearch::Client.new(@client_args) - end - - def can_connect? - begin - head(SecureRandom.hex(32).prepend('_')) - rescue Elasticsearch::Transport::Transport::Errors::BadRequest - true - rescue Manticore::SocketException - false - end - end - - def host_settings - @client_args[:hosts] - end - - def delete(path) - begin - normalize_response(@client.perform_request('DELETE', path, {}, nil)) - rescue Exception => e - if e.class.to_s =~ /NotFound/ || e.message =~ /Not\s*Found|404/i - Response.new(404, "", {}) - else - raise e - end - end - end - - def put(path, content) - normalize_response(@client.perform_request('PUT', path, {}, content)) - end - - def head(path) - begin - normalize_response(@client.perform_request('HEAD', path, {}, nil)) - rescue Exception => e - if is_404_error?(e) - Response.new(404, "", {}) - else - raise e - end - end - end - - private - - def is_404_error?(error) - error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i - end - - def normalize_response(response) - Response.new(response.status, response.body, response.headers) - end - - def client_args - { - :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore, - :hosts => [*unpack_hosts], - # :logger => @logger, # silence the client logging - } - end - - def unpack_hosts - setting = @settings.fetch("var.elasticsearch.hosts", "localhost:9200") - if setting.is_a?(String) - return setting.split(',').map(&:strip) - end - setting - end - end - - def initialize(client) - @client = client - end - - def delete(path) - @client.delete(path) - end - - def put(path, content) - @client.put(path, content) - end - - def head(path) - @client.head(path) - end - - def can_connect? - @client.can_connect? - end - - def host_settings - @client.host_settings - end -end end diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 025d5d6a58a..c125da1ec8f 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -20,7 +20,6 @@ require "logstash/settings" require "logstash/util/cloud_setting_id" require "logstash/util/cloud_setting_auth" -require "logstash/util/modules_setting_array" require "socket" require "stud/temporary" @@ -39,13 +38,6 @@ module Environment Setting::NullableString.new("path.config", nil, false), Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")), Setting::NullableString.new("config.string", nil, false), - Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []), - Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []), - Setting.new("modules_list", Array, []), - Setting.new("modules_variable_list", Array, []), - Setting::Modules.new("cloud.id", LogStash::Util::CloudSettingId), - Setting::Modules.new("cloud.auth", LogStash::Util::CloudSettingAuth), - Setting::Boolean.new("modules_setup", false), Setting::Boolean.new("config.test_and_exit", false), Setting::Boolean.new("config.reload.automatic", false), Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds diff --git a/logstash-core/lib/logstash/modules/cli_parser.rb b/logstash-core/lib/logstash/modules/cli_parser.rb deleted file mode 100644 index f69d0884fd3..00000000000 --- a/logstash-core/lib/logstash/modules/cli_parser.rb +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module LogStash module Modules class CLIParser - include LogStash::Util::Loggable - - attr_reader :output - - def initialize(module_names, module_variables) - @output = [] - # The #compact here catches instances when module_variables may be nil or - # [nil] and sets it to [] - parse_it(module_names, Array(module_variables).compact) - end - - def parse_modules(module_list) - parsed_modules = [] - module_list.each do |module_value| - # Calling --modules but not filling it results in [nil], so skip that. - next if module_value.nil? - # Catch if --modules was launched empty but an option/flag (-something) - # follows immediately after - if module_value.start_with?('-') - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-invalid-name", :module_name => module_value) - end - parsed_modules.concat module_value.split(',') - end - parsed_modules - end - - def get_kv(module_name, unparsed) - # Ensure that there is at least 1 equals sign in our variable string - # Using String#partition to split on the first '=' - # This hackery is to catch the possibility of an equals (`=`) sign - # in a passphrase, which might result in an incomplete key. The - # portion before the first `=` should always be the key, leaving - # the rest to be the value - k, op, rest = unparsed.partition('=') - if rest.size.zero? - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => (module_name + '.' + unparsed)) - end - return k.strip, rest.strip - end - - def name_splitter(unparsed) - # It must have at least `modulename.something` - module_name, dot, rest = unparsed.partition('.') - if rest.count('.') >= 1 - return module_name, rest - else - raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => unparsed) - end - end - - def parse_vars(module_name, vars_list) - module_hash = {"name" => module_name} - vars_list.each do |unparsed| - extracted_name, modvar = name_splitter(unparsed) - next if extracted_name != module_name - k, v = get_kv(extracted_name, modvar) - module_hash[k] = v - end - module_hash - end - - def parse_it(module_list, module_variable_list) - if module_list.is_a?(Array) - parse_modules(module_list).each do |module_name| - @output << parse_vars(module_name, module_variable_list) - end - end - end -end end end diff --git a/logstash-core/lib/logstash/modules/elasticsearch_config.rb b/logstash-core/lib/logstash/modules/elasticsearch_config.rb deleted file mode 100644 index 683442122b9..00000000000 --- a/logstash-core/lib/logstash/modules/elasticsearch_config.rb +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "elasticsearch_resource" - -module LogStash module Modules class ElasticsearchConfig - attr_reader :index_name - - # We name it `modul` here because `module` has meaning in Ruby. - def initialize(modul, settings) - @directory = ::File.join(modul.directory, "elasticsearch") - @name = modul.module_name - @settings = settings - @full_path = ::File.join(@directory, "#{@name}.json") - @index_name = @settings.fetch("elasticsearch.template_path", "_template") - end - - def resources - [ElasticsearchResource.new(@index_name, "not-used", @full_path)] - end -end end end diff --git a/logstash-core/lib/logstash/modules/elasticsearch_importer.rb b/logstash-core/lib/logstash/modules/elasticsearch_importer.rb deleted file mode 100644 index 07c68cea796..00000000000 --- a/logstash-core/lib/logstash/modules/elasticsearch_importer.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module LogStash module Modules class ElasticsearchImporter - include LogStash::Util::Loggable - - def initialize(client) - @client = client - end - - def put(resource, overwrite = true) - path = resource.import_path - logger.debug("Attempting PUT", :url_path => path, :file_path => resource.content_path) - if !overwrite && content_exists?(path) - logger.debug("Found existing Elasticsearch resource.", :resource => path) - return - end - put_overwrite(path, resource.content) - end - - private - - def put_overwrite(path, content) - if content_exists?(path) - response = @client.delete(path) - end - # hmmm, versioning? - @client.put(path, content).status == 201 - end - - def content_exists?(path) - response = @client.head(path) - response.status >= 200 && response.status < 300 - end -end end end # class LogStash::Modules::Importer diff --git a/logstash-core/lib/logstash/modules/elasticsearch_resource.rb b/logstash-core/lib/logstash/modules/elasticsearch_resource.rb deleted file mode 100644 index 21ccbc3697c..00000000000 --- a/logstash-core/lib/logstash/modules/elasticsearch_resource.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "resource_base" - -module LogStash module Modules class ElasticsearchResource - include ResourceBase - def import_path - base + "/" + content_id - end -end end end diff --git a/logstash-core/lib/logstash/modules/file_reader.rb b/logstash-core/lib/logstash/modules/file_reader.rb deleted file mode 100644 index e43a9133e81..00000000000 --- a/logstash-core/lib/logstash/modules/file_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/json" - -module LogStash module Modules class FileReader - # stub these methods for testing - include LogStash::Util::Loggable - - def self.read(path) - begin - ::File.read(path) - rescue => e - logger.error("Error when reading file from path", :path => path) - "" - end - end - - def self.read_json(path) - json = read(path) - begin - LogStash::Json.load(json) - rescue => e - logger.error("Error when parsing json from path", :path => path) - return {} - end - end - - def self.glob(path) - files = Dir.glob(path) - if files.empty? - logger.warn("No files found for glob", :pattern => path) - end - files - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_base.rb b/logstash-core/lib/logstash/modules/kibana_base.rb deleted file mode 100644 index 54e23d98af0..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_base.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/json" - -module LogStash module Modules class KibanaBase - attr_reader :import_path, :content - - def initialize(import_path, content) - @import_path, @content = import_path, content - end - - def import(client) - raise NotImplementedError, "#{self.class.name} needs to implement `#import`" - end - - def to_s - import_path - end - - def content_as_object - return content unless content.is_a?(String) - LogStash::Json.load(content) rescue nil - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_client.rb b/logstash-core/lib/logstash/modules/kibana_client.rb deleted file mode 100644 index 1dce000c69c..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_client.rb +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/json" -require "manticore" -require 'logstash/util/manticore_ssl_config_helper' -require 'logstash/util/password' - -module LogStash module Modules class KibanaClient - include LogStash::Util::Loggable - - include LogStash::Util::ManticoreSSLConfigHelper - - class Response - # to create a custom response with body as an Object (Hash or Array) - attr_reader :status, :body, :headers - - def initialize(status, body, headers = {}) - @status, @body, @headers = status, body, headers - @body = body.is_a?(String) ? LogStash::Json.load(body) : body - end - - def succeeded? - @status >= 200 && @status < 300 - end - - def failed? - !succeeded? - end - end - - SCHEME_REGEX = /^https?$/ - - attr_reader :version, :endpoint - - def initialize(settings, client = nil) # allow for test mock injection - @settings = settings - - client_options = { - request_timeout: 5, - connect_timeout: 5, - socket_timeout: 5, - pool_max: 10, - pool_max_per_route: 2 - } - - ssl_options = manticore_ssl_options_from_config('kibana', settings) - ssl_enabled = ssl_options.any? - - client_options[:ssl] = ssl_options - - @host = @settings.fetch("var.kibana.host", "localhost:5601") - implicit_scheme, colon_slash_slash, host = @host.partition("://") - explicit_scheme = @settings["var.kibana.scheme"] - @scheme = "http" - if !colon_slash_slash.empty? - if !explicit_scheme.nil? && implicit_scheme != explicit_scheme - # both are set and not the same - error - msg = sprintf("Detected differing Kibana host schemes as sourced from var.kibana.host: '%s' and var.kibana.scheme: '%s'", implicit_scheme, explicit_scheme) - raise ArgumentError.new(msg) - end - @scheme = implicit_scheme - @host = host - elsif !explicit_scheme.nil? - @scheme = explicit_scheme - end - - if SCHEME_REGEX.match(@scheme).nil? - msg = sprintf("Kibana host scheme given is invalid, given value: '%s' - acceptable values: 'http', 'https'", @scheme) - raise ArgumentError.new(msg) - end - - if ssl_enabled && @scheme != "https" - @scheme = "https" - end - - @endpoint = "#{@scheme}://#{@host}" - - @client = client || Manticore::Client.new(client_options) - @http_options = {:headers => {'Content-Type' => 'application/json'}} - username = @settings["var.kibana.username"] - if username - password = @settings["var.kibana.password"] - if password.is_a?(LogStash::Util::Password) - password = password.value - end - @http_options[:headers]['Authorization'] = 'Basic ' + Base64.encode64("#{username}:#{password}").chomp - end - - # e.g. {"name":"Elastics-MacBook-Pro.local","version":{"number":"6.0.0-beta1","build_hash":"41e69","build_number":15613,"build_snapshot":true}..} - @version = "0.0.0" - response = get("api/status") - if response.succeeded? - status = response.body - if status["version"].is_a?(Hash) - @version = status["version"]["number"] - if status["version"]["build_snapshot"] - @version.concat("-SNAPSHOT") - end - end - end - @http_options[:headers]['kbn-version'] = @version - end - - def version_parts - @version.split(/[.-]/) - end - - def host_settings - "[\"#{@host}\"]" - end - - def get(relative_path) - # e.g. api/kibana/settings - safely(:get, relative_path, @http_options) - end - - # content will be converted to a json string - def post(relative_path, content, headers = nil) - body = content.is_a?(String) ? content : LogStash::Json.dump(content) - options = {:body => body}.merge(headers || @http_options) - safely(:post, relative_path, options) - end - - def head(relative_path) - safely(:head, relative_path, @http_options) - end - - def can_connect? - head("api/status").succeeded? - end - - private - - def safely(method_sym, relative_path, options = {}) - begin - resp = @client.http(method_sym, full_url(relative_path), options).call - Response.new(resp.code, resp.body, resp.headers) - rescue Manticore::ManticoreException => e - logger.error("Error when executing Kibana client request", :error => e) - body = {"statusCode" => 0, "error" => e.message} - Response.new(0, body, {}) - end - end - - def full_url(relative) - "#{@endpoint}/#{relative}" - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_config.rb b/logstash-core/lib/logstash/modules/kibana_config.rb deleted file mode 100644 index bcfa4297fa0..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_config.rb +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "file_reader" -require_relative "kibana_settings" -require_relative "kibana_dashboards" -require_relative "kibana_resource" - -module LogStash module Modules class KibanaConfig - include LogStash::Util::Loggable - - ALLOWED_DIRECTORIES = ["search", "visualization"] - attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig - - # We name it `modul` here because `module` has meaning in Ruby. - def initialize(modul, settings) - build_versioned_directory(modul) - @name = modul.module_name - @settings = settings - @index_name = "kibana" - @pattern_name = "#{@name}-*" - @kibana_settings = [ - KibanaSettings::Setting.new("defaultIndex", @pattern_name) - ] - end - - def dashboards - # there can be more than one dashboard to load - filenames = FileReader.read_json(dynamic("dashboard")) - filenames.map do |filename| - KibanaResource.new(@index_name, "dashboard", dynamic("dashboard", filename)) - end - end - - def index_pattern - [KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"), nil, @pattern_name)] - end - - def resources - list = index_pattern - dashboards.each do |board| - list << board - extract_panels_into(board, list) - end - list.concat(extract_saved_searches_into(list)) - [ - KibanaSettings.new("api/kibana/settings", @kibana_settings), - KibanaDashboards.new("api/kibana/dashboards/import", list) - ] - end - - private - - def build_versioned_directory(modul) - # try to detect which directory holds the config for the kibana version - base_dir = ::File.join(modul.directory, "kibana") - maj, min, patch = modul.kibana_version_parts - version_dir = "#{maj}.#{min}.#{patch}" - @directory = ::File.join(base_dir, version_dir) - return if ::File.directory?(@directory) - version_dir = "#{maj}.#{min}.x" - @directory = ::File.join(base_dir, version_dir) - return if ::File.directory?(@directory) - version_dir = "#{maj}.x" - @directory = ::File.join(base_dir, version_dir) - unless ::File.directory?(@directory) - logger.error("Cannot find kibana version sub-directory", :module => @name, :base_directory => base_dir) - end - end - - def dynamic(dynamic_folder, filename = @name) - ::File.join(@directory, dynamic_folder, "#{filename}.json") - end - - def extract_panels_into(dashboard, list) - dash = dashboard.content_as_object - - if !dash.is_a?(Hash) - logger.warn("Kibana dashboard JSON is not an Object", :module => @name) - return - end - - panelsjson = dash["panelsJSON"] - - if panelsjson.nil? - logger.info("No panelJSON key found in kibana dashboard", :module => @name) - return - end - - begin - panels = LogStash::Json.load(panelsjson) - rescue => e - logger.error("JSON parse error when reading kibana panelsJSON", :module => @name) - return - end - - panels.each do |panel| - panel_type = panel["type"] - if ALLOWED_DIRECTORIES.member?(panel_type) - list << KibanaResource.new(@index_name, panel_type, dynamic(panel_type, panel["id"])) - else - logger.warn("panelJSON contained unknown type", :type => panel_type) - end - end - end - - def extract_saved_searches_into(list) - result = [] # must not add to list while iterating - list.each do |resource| - content = resource.content_as_object - next if content.nil? - next unless content.keys.include?("savedSearchId") - saved_search = content["savedSearchId"] - next if saved_search.nil? - ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search)) - next if list.member?(ss_resource) || result.member?(ss_resource) - result << ss_resource - end - result - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_dashboards.rb b/logstash-core/lib/logstash/modules/kibana_dashboards.rb deleted file mode 100644 index c34fceb4efd..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_dashboards.rb +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "kibana_base" - -module LogStash module Modules class KibanaDashboards < KibanaBase - include LogStash::Util::Loggable - - attr_reader :import_path, :content - - # content is a list of kibana file resources - def initialize(import_path, content) - @import_path, @content = import_path, content - end - - def import(client) - # e.g. curl "http://localhost:5601/api/kibana/dashboards/import" - # extract and prepare all objects - objects = [] - content.each do |resource| - hash = { - "id" => resource.content_id, - "type" => resource.content_type, - "version" => 1, - "attributes" => resource.content_as_object - } - objects << hash - end - body = {"version": client.version, "objects": objects} - response = client.post(import_path, body) - if response.failed? - logger.error("Attempted POST failed", :url_path => import_path, :response => response.body) - end - response - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_importer.rb b/logstash-core/lib/logstash/modules/kibana_importer.rb deleted file mode 100644 index 403217f5384..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_importer.rb +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module LogStash module Modules class KibanaImporter - include LogStash::Util::Loggable - - def initialize(client) - @client = client - end - - def put(via_kibana) - path = via_kibana.import_path - logger.debug("Attempting POST", :url_path => path, :content => via_kibana.content) - via_kibana.import(@client) - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_resource.rb b/logstash-core/lib/logstash/modules/kibana_resource.rb deleted file mode 100644 index ef98c3d36a2..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_resource.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "resource_base" - -module LogStash module Modules class KibanaResource - include ResourceBase - def import_path - base + "/" + content_type + "/" + content_id - end -end end end diff --git a/logstash-core/lib/logstash/modules/kibana_settings.rb b/logstash-core/lib/logstash/modules/kibana_settings.rb deleted file mode 100644 index d2ed4f24be1..00000000000 --- a/logstash-core/lib/logstash/modules/kibana_settings.rb +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "kibana_base" - -module LogStash module Modules class KibanaSettings < KibanaBase - include LogStash::Util::Loggable - - class Setting - attr_reader :name, :value - - def initialize(name, value) - @name, @value = name, value - end - end - - attr_reader :import_path, :content - - # content is an array of Setting required for this module - def initialize(import_path, content) - @import_path, @content = import_path, content - end - - def import(client) - # e.g. curl "http://localhost:5601/api/kibana/settings" - # 6.0.0-beta1 -> {"settings":{"buildNum":{"userValue":15613},"defaultIndex":{"userValue":"arcsight-*"}}} - # 5.4 -> {"settings":{"defaultIndex":{"userValue":"cef-*"},"metrics:max_buckets":{"userValue":"600000"}}} - # array of Setting objects - # The POST api body { "changes": { "defaultIndex": "arcsight-*", "metrics:max_buckets": "400" } } - settings = {} - content.each do |setting| - settings[setting.name] = "#{setting.value}" - end - body = {"changes" => settings} - response = client.post(import_path, body) - if response.failed? - logger.error("Attempted POST failed", :url_path => import_path, :response => response.body) - end - response - end -end end end diff --git a/logstash-core/lib/logstash/modules/logstash_config.rb b/logstash-core/lib/logstash/modules/logstash_config.rb deleted file mode 100644 index a9848b9f128..00000000000 --- a/logstash-core/lib/logstash/modules/logstash_config.rb +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "file_reader" -require "logstash/settings" - -module LogStash module Modules class LogStashConfig - include LogStash::Util::Loggable - # We name it `modul` here because `module` has meaning in Ruby. - def initialize(modul, settings) - @directory = ::File.join(modul.directory, "logstash") - @name = modul.module_name - @settings = settings - end - - def template - ::File.join(@directory, "#{@name}.conf.erb") - end - - def configured_inputs(default = [], aliases = {}) - name = "var.inputs" - values = get_setting(LogStash::Setting::SplittableStringArray.new(name, String, default)) - aliases.each { |k, v| values << v if values.include?(k) } - aliases.invert.each { |k, v| values << v if values.include?(k) } - values.flatten.uniq - end - - def alias_settings_keys!(aliases) - aliased_settings = alias_matching_keys(aliases, @settings) - @settings = alias_matching_keys(aliases.invert, aliased_settings) - end - - def array_to_string(array) - "[#{array.collect { |i| "'#{i}'" }.join(", ")}]" - end - - def csv_string(array) - "'#{array.join(',')}'" - end - - def get_setting(setting_class) - raw_value = @settings[setting_class.name] - # If we dont check for NIL, the Settings class will try to coerce the value - # and most of the it will fails when a NIL value is explicitly set. - # This will be fixed once we wrap the plugins settings into a Settings class - setting_class.set(raw_value) unless raw_value.nil? - setting_class.value - end - - def setting(name, default) - # by default we use the more permissive setting which is a `NullableString` - # This is fine because the end format of the logstash configuration is a string representation - # of the pipeline. There is a good reason why I think we should use the settings classes, we - # can `preprocess` a template and generate a configuration from the defined settings - # validate the values and replace them in the template. - case default - when String - get_setting(LogStash::Setting::NullableString.new(name, default.to_s)) - when Numeric - get_setting(LogStash::Setting::Numeric.new(name, default)) - when true, false - get_setting(LogStash::Setting::Boolean.new(name, default)) - else - get_setting(LogStash::Setting::NullableString.new(name, default.to_s)) - end - end - - def has_setting?(name) - @settings.key?(name) - end - - def raw_setting(name) - @settings[name] - end - - def fetch_raw_setting(name, default) - @settings.fetch(name, default) - end - - def elasticsearch_output_config(type_string = nil, index_suffix = "-%{+YYYY.MM.dd}") - hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"]))) - index = "#{@name}#{index_suffix}" - user = @settings["var.elasticsearch.username"] - password = @settings["var.elasticsearch.password"] - lines = ["hosts => #{hosts}", "index => \"#{index}\""] - lines.push(user ? "user => \"#{user}\"" : nil) - lines.push(password ? "password => \"#{password.value}\"" : nil) - lines.push(type_string ? "document_type => #{type_string}" : nil) - lines.push("ssl_enabled => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}") - if cacert = @settings["var.elasticsearch.ssl.certificate_authority"] - lines.push("ssl_certificate_authorities => \"#{cacert}\"") if cacert - end - # NOTE: the first line should be indented in the conf.erb - <<-CONF -elasticsearch { - #{lines.compact.join("\n ")} - manage_template => false - } -CONF - end - - def config_string - # process the template and settings - # send back as a string - renderer = ERB.new(FileReader.read(template)) - renderer.result(binding) - end - - private - # For a first version we are copying the values of the original hash, - # this might become problematic if we users changes the values of the - # settings in the template, which could result in an inconsistent view of the original data - # - # For v1 of the feature I think its an OK compromise, v2 we have a more advanced hash that - # support alias. - def alias_matching_keys(aliases, target) - aliased_target = target.dup - - aliases.each do |matching_key_prefix, new_key_prefix| - target.each do |k, v| - re = /^#{matching_key_prefix}\./ - - if k =~ re - alias_key = k.gsub(re, "#{new_key_prefix}.") - - # If the user setup the same values twices with different values lets just halt. - raise "Cannot create an alias, the destination key has already a value set: original key: #{k}, alias key: #{alias_key}" if (!aliased_target[alias_key].nil? && aliased_target[alias_key] != v) - aliased_target[alias_key] = v unless v.nil? - end - end - end - - aliased_target - end -end end end diff --git a/logstash-core/lib/logstash/modules/resource_base.rb b/logstash-core/lib/logstash/modules/resource_base.rb deleted file mode 100644 index 5e351b06409..00000000000 --- a/logstash-core/lib/logstash/modules/resource_base.rb +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/json" -require_relative "file_reader" - -module LogStash module Modules module ResourceBase - attr_reader :base, :content_type, :content_path, :content_id - - def initialize(base, content_type, content_path, content = nil, content_id = nil) - @base, @content_type, @content_path = base, content_type, content_path - @content_id = content_id || ::File.basename(@content_path, ".*") - # content at this time will be a JSON string - @content = content - if !@content.nil? - @content_as_object = LogStash::Json.load(@content) rescue {} - end - end - - def content - @content ||= FileReader.read(@content_path) - end - - def to_s - "#{base}, #{content_type}, #{content_path}, #{content_id}" - end - - def content_as_object - @content_as_object ||= FileReader.read_json(@content_path) rescue nil - end - - def <=>(other) - to_s <=> other.to_s - end - - def ==(other) - to_s == other.to_s - end -end end end diff --git a/logstash-core/lib/logstash/modules/scaffold.rb b/logstash-core/lib/logstash/modules/scaffold.rb deleted file mode 100644 index 548825be651..00000000000 --- a/logstash-core/lib/logstash/modules/scaffold.rb +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "erb" - -require_relative "elasticsearch_config" -require_relative "kibana_config" -require_relative "logstash_config" - -module LogStash module Modules class Scaffold - include LogStash::Util::Loggable - - attr_reader :directory, :module_name, :kibana_version_parts - attr_reader :kibana_configuration, :logstash_configuration, :elasticsearch_configuration - - def initialize(name, directory) - @module_name = name - @directory = directory # this is the 'configuration folder in the GEM root.' - @kibana_version_parts = "6.0.0".split('.') # this is backup in case kibana client fails to connect - logger.debug("Found module", :module_name => name, :directory => directory) - end - - def add_kibana_version(version_parts) - @kibana_version_parts = version_parts - end - - def import(import_engine, kibana_import_engine) - @elasticsearch_configuration.resources.each do |resource| - import_engine.put(resource) - end - @kibana_configuration.resources.each do |resource| - kibana_import_engine.put(resource) - end - end - - def with_settings(module_settings) - @logstash_configuration = LogStashConfig.new(self, module_settings) - @kibana_configuration = KibanaConfig.new(self, module_settings) - @elasticsearch_configuration = ElasticsearchConfig.new(self, module_settings) - self - end - - def config_string() - # settings should be set earlier by the caller. - # settings should be the subset from the YAML file with a structure like - # {"name" => "plugin name", "k1" => "v1", "k2" => "v2"}, etc. - return nil if @logstash_configuration.nil? - @logstash_configuration.config_string - end - - # subclass may override - def is_enabled?(settings) - true - end -end end end # class LogStash::Modules::Scaffold diff --git a/logstash-core/lib/logstash/modules/settings_merger.rb b/logstash-core/lib/logstash/modules/settings_merger.rb deleted file mode 100644 index 05a5151aab6..00000000000 --- a/logstash-core/lib/logstash/modules/settings_merger.rb +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "logstash/util" - -module LogStash module Modules module SettingsMerger - include LogStash::Util::Loggable - extend self - - # cli_settings Array or LogStash::Util::ModulesSettingArray - # yml_settings Array or LogStash::Util::ModulesSettingArray - def merge(cli_settings, yml_settings) - # both args are arrays of hashes, e.g. - # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}] - # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}] - merged = [] - # union and group_by preserves order - # union will also coalesce identical hashes - # this "|" operator is provided to Java List by RubyJavaIntegration - union_of_settings = (cli_settings | yml_settings) - grouped_by_name = union_of_settings.group_by {|e| e["name"]} - grouped_by_name.each do |_, array| - if array.size == 2 - merged << array.last.merge(array.first) - else - merged.concat(array) - end - end - merged - end - - def merge_cloud_settings(module_settings, logstash_settings) - cloud_id = logstash_settings.get("cloud.id") - cloud_auth = logstash_settings.get("cloud.auth") - if cloud_id.nil? - if cloud_auth.nil? - return # user did not specify cloud settings - else - raise ArgumentError.new("Cloud Auth without Cloud Id") - end - end - if logger.debug? - settings_copy = LogStash::Util.deep_clone(module_settings) - end - - module_settings["var.kibana.scheme"] = cloud_id.kibana_scheme - module_settings["var.kibana.host"] = cloud_id.kibana_host - # elasticsearch client does not use scheme, it URI parses the host setting - module_settings["var.elasticsearch.hosts"] = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}" - unless cloud_auth.nil? - module_settings["var.elasticsearch.username"] = cloud_auth.username - module_settings["var.elasticsearch.password"] = cloud_auth.password - module_settings["var.kibana.username"] = cloud_auth.username - module_settings["var.kibana.password"] = cloud_auth.password - end - if logger.debug? - format_module_settings(settings_copy, module_settings).each {|line| logger.debug(line)} - end - end - - def merge_kibana_auth!(module_settings) - module_settings["var.kibana.username"] = module_settings["var.elasticsearch.username"] if module_settings["var.kibana.username"].nil? - module_settings["var.kibana.password"] = module_settings["var.elasticsearch.password"] if module_settings["var.kibana.password"].nil? - end - - def format_module_settings(settings_before, settings_after) - output = [] - output << "-------- Module Settings ---------" - settings_after.each do |setting_name, setting| - setting_before = settings_before.fetch(setting_name, "") - line = "#{setting_name}: '#{setting}'" - if setting_before != setting - line.concat(", was: '#{setting_before}'") - end - output << line - end - output << "-------- Module Settings ---------" - output - end -end end end diff --git a/logstash-core/lib/logstash/modules/util.rb b/logstash-core/lib/logstash/modules/util.rb deleted file mode 100644 index 64dea4996d4..00000000000 --- a/logstash-core/lib/logstash/modules/util.rb +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "scaffold" - -# This module function should be used when gems or -# x-pack defines modules in their folder structures. -module LogStash module Modules module Util - def self.register_local_modules(path) - modules_path = ::File.join(path, "modules") - ::Dir.foreach(modules_path) do |item| - # Ignore unix relative path ids - next if item == '.' or item == '..' - # Ignore non-directories - next if !::File.directory?(::File.join(modules_path, ::File::Separator, item)) - LogStash::PLUGIN_REGISTRY.add(:modules, item, Scaffold.new(item, ::File.join(modules_path, item, "configuration"))) - end - end -end end end diff --git a/logstash-core/lib/logstash/plugins/registry.rb b/logstash-core/lib/logstash/plugins/registry.rb index 849e5011919..680b9c39640 100644 --- a/logstash-core/lib/logstash/plugins/registry.rb +++ b/logstash-core/lib/logstash/plugins/registry.rb @@ -17,7 +17,6 @@ require "rubygems/package" require "logstash/plugin" -require "logstash/modules/scaffold" require "logstash/codecs/base" require "logstash/filters/base" require "logstash/outputs/base" diff --git a/logstash-core/lib/logstash/runner.rb b/logstash-core/lib/logstash/runner.rb index 68409ba92e2..1df665712ac 100644 --- a/logstash-core/lib/logstash/runner.rb +++ b/logstash-core/lib/logstash/runner.rb @@ -34,7 +34,6 @@ require "clamp" require "logstash-core/logstash-core" require "logstash/environment" -require "logstash/modules/cli_parser" require "logstash/util/settings_helper" require "logstash/util/jackson" @@ -46,7 +45,6 @@ require "logstash/settings" require "logstash/version" require 'logstash/plugins' -require "logstash/modules/util" require "logstash/bootstrap_check/default_config" require 'logstash/deprecation_message' @@ -92,31 +90,6 @@ class LogStash::Runner < Clamp::StrictCommand :default => LogStash::SETTINGS.get_default("config.field_reference.escape_style"), :attribute_name => "config.field_reference.escape_style" - - # Module settings - option ["--modules"], "MODULES", - I18n.t("logstash.runner.flag.modules"), - :multivalued => true, - :attribute_name => "modules_list" - - option ["-M", "--modules.variable"], "MODULES_VARIABLE", - I18n.t("logstash.runner.flag.modules_variable"), - :multivalued => true, - :attribute_name => "modules_variable_list" - - option ["--setup"], :flag, - I18n.t("logstash.runner.flag.modules_setup"), - :default => LogStash::SETTINGS.get_default("modules_setup"), - :attribute_name => "modules_setup" - - option ["--cloud.id"], "CLOUD_ID", - I18n.t("logstash.runner.flag.cloud_id"), - :attribute_name => "cloud.id" - - option ["--cloud.auth"], "CLOUD_AUTH", - I18n.t("logstash.runner.flag.cloud_auth"), - :attribute_name => "cloud.auth" - # Pipeline settings option ["--pipeline.id"], "ID", I18n.t("logstash.runner.flag.pipeline-id"), @@ -266,7 +239,6 @@ def initialize(*args) # Default we check local sources: `-e`, `-f` and the logstash.yml options. @source_loader = LogStash::Config::SourceLoader.new(@settings) @source_loader.add_source(LogStash::Config::Source::Local.new(@settings)) - @source_loader.add_source(LogStash::Config::Source::Modules.new(@settings)) @source_loader.add_source(LogStash::Config::Source::MultiLocal.new(@settings)) super(*args) @@ -318,8 +290,6 @@ def execute jvmArgs = ManagementFactory.getRuntimeMXBean().getInputArguments() logger.info "JVM bootstrap flags: #{jvmArgs}" - # Add local modules to the registry before everything else - LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME) # Verify the Jackson defaults LogStash::Util::Jackson.verify_jackson_overrides @@ -340,10 +310,6 @@ def execute return start_shell(setting("interactive"), binding) if setting("interactive") - module_parser = LogStash::Modules::CLIParser.new(setting("modules_list"), setting("modules_variable_list")) - # Now populate Setting for modules.list with our parsed array. - @settings.set("modules.cli", module_parser.output) - begin @bootstrap_checks.each { |bootstrap| bootstrap.check(@settings) } rescue LogStash::BootstrapCheckError => e diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index d340390646b..0447a2f0331 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -800,30 +800,6 @@ def validate(value) end end - class Modules < Coercible - def initialize(name, klass, default = nil) - super(name, klass, default, false) - end - - def set(value) - coerced_value = coerce(value) - @wrapped_setting.set(coerced_value) - coerced_value - end - - def coerce(value) - if value.is_a?(@klass) - return value - end - @klass.new(value) - end - - protected - def validate(value) - coerce(value) - end - end - # @see Setting#nullable # @api internal class Nullable < SimpleDelegator diff --git a/logstash-core/lib/logstash/util/manticore_ssl_config_helper.rb b/logstash-core/lib/logstash/util/manticore_ssl_config_helper.rb deleted file mode 100644 index 2654c05d43b..00000000000 --- a/logstash-core/lib/logstash/util/manticore_ssl_config_helper.rb +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module LogStash; module Util; module ManticoreSSLConfigHelper - extend self - - ## - # Extract Manticore-style SSL directives from the given configuration. - # - # @param namespace [String] a string namespace (e.g., `kibana` in `var.kibana.ssl.*`) - # @param settings [Hash] a collection of Manticore-friendly SSL directives. - # if SSL explicitly disabled, an _empty_ hash will be returned. - # - # @return [Hash] - def manticore_ssl_options_from_config(namespace, settings) - ssl_settings = strip_prefix(settings, "var.#{namespace}.ssl.") - - # boolean settings may be strings if set through the cli - # or booleans if set through the yaml file, so we use .to_s - if ssl_settings.include?('enabled') && !coerce_boolean(ssl_settings['enabled']) - logger.warn('SSL explicitly disabled; other SSL settings will be ignored') if logger && ssl_settings.size > 1 - return {} - end - - { - :verify => ssl_settings.fetch('verification_mode', :strict).to_sym, - :ca_file => ssl_settings.fetch('certificate_authority', nil), - :client_cert => ssl_settings.fetch('certificate', nil), - :client_key => ssl_settings.fetch('key', nil), - } - end - - private - - ## - # Returns the subset of the hash whose keys match the given prefix, with the prefix removed - # - # @param hash [Hash] - # @param prefix [String] - # @return [Hash] - def strip_prefix(hash, prefix) - hash.each_with_object({}) do |(key, value), memo| - next unless key.start_with?(prefix) - unprefixed_key = key[prefix.length..-1] - memo[unprefixed_key] = value - end - end - - ## - # Coerces the non-nil input to boolean - # - # @param value [Boolean,String,Integer] - # @return [Boolean] - def coerce_boolean(value) - case value - when true, "true", "T", 1 then true - when false, "false", "F", 0 then false - else - fail("Boolean value required, received `#{value}`") - end - end - - ## - # Adapter to enable logging via the including class' `#logger` method or `@logger` instance variable - # - # @return [Logger,nil] - def logger - return super if defined?(super) - - @logger - end -end end end diff --git a/logstash-core/lib/logstash/util/modules_setting_array.rb b/logstash-core/lib/logstash/util/modules_setting_array.rb deleted file mode 100644 index d092f7e3633..00000000000 --- a/logstash-core/lib/logstash/util/modules_setting_array.rb +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module LogStash; module Util - java_import "org.logstash.util.ModulesSettingArray" -end; end diff --git a/logstash-core/locales/en.yml b/logstash-core/locales/en.yml index eddc2b2210e..4985edbd2f0 100644 --- a/logstash-core/locales/en.yml +++ b/logstash-core/locales/en.yml @@ -67,33 +67,11 @@ en: logging: unrecognized_option: |- unrecognized option [%{option}] - modules: - configuration: - parse-failed: |- - Failed to parse the module configuration: [%{error}] - modules-must-be-unique: >- - Only a single instance of any module can be run at a time. Duplicate - modules: %{duplicate_modules} - modules-invalid-name: >- - Invalid module name: %{module_name} - modules-variables-malformed: >- - Failed to parse module variable %{rawvar}. Must be in -M - "MODULE_NAME.KEY.SUBKEY=VALUE" format - modules-unavailable: >- - The modules specified are not available yet. - Specified modules: %{specified_modules} - Available modules: %{available_modules} - elasticsearch_connection_failed: >- - Failed to import module configurations to Elasticsearch and/or Kibana. - Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts} - modules-too-many-specified: >- - Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules} runner: short-help: |- usage: bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG] - bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG] bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG] bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace] bin/logstash -V [--log.level fatal|error|warn|info|debug|trace] @@ -108,14 +86,6 @@ en: the '-f yourlogstash.conf' flag? config-string-path-exclusive: Settings 'path.config' (-f) and 'config.string' (-e) can't be used simultaneously. - config-module-exclusive: >- - Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with - (--modules) or the "modules:" block in the logstash.yml file. - reload-with-modules: >- - Configuration reloading can't be used with command-line or logstash.yml specified modules. - cli-module-override: >- - Both command-line and logstash.yml modules configurations detected. - Using command-line module configuration to override logstash.yml module configuration. config-pipelines-failed-read: >- Failed to read pipelines yaml file. Location: %{path} config-pipelines-failed-read-with-exception: >- @@ -244,40 +214,6 @@ en: HTML-style ampersand-hash encoding notation representing decimal unicode codepoints (`[` is `[`; `]` is `]`). - modules: |+ - Load Logstash modules. - Modules can be defined using multiple instances - '--modules module1 --modules module2', - or comma-separated syntax - '--modules=module1,module2' - Cannot be used in conjunction with '-e' or '-f' - Use of '--modules' will override modules declared - in the 'logstash.yml' file. - modules_variable: |+ - Load variables for module template. - Multiple instances of '-M' or - '--modules.variable' are supported. - Ignored if '--modules' flag is not used. - Should be in the format of - '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"' - as in - '-M "example.var.filter.mutate.fieldname=fieldvalue"' - modules_setup: |+ - Load index template into Elasticsearch, and saved searches, - index-pattern, visualizations, and dashboards into Kibana when - running modules. - cloud_id: |+ - Sets the elasticsearch and kibana host settings for - module connections in Elastic Cloud. - Your Elastic Cloud User interface or the Cloud support - team should provide this. - Add an optional label prefix '