From 47d44ccd145f9da2373cf2fdf43fea7815b26ffd Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 20 Oct 2022 17:56:59 +0000 Subject: [PATCH 1/7] test setup: ensure presence of /etc/protocols --- .ci/setup.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 .ci/setup.sh diff --git a/.ci/setup.sh b/.ci/setup.sh new file mode 100755 index 0000000..54a42db --- /dev/null +++ b/.ci/setup.sh @@ -0,0 +1,11 @@ +# user_agent requires /etc/protocols, which is provided by netbase. +# https://github.com/jruby/jruby/issues/3955 +if [ ! -f "/etc/protocols" ]; then + if [ $(command -v apt-get) ]; then + echo "installing netbase with apt-get" + sudo apt-get install -y netbase + else + echo "installing netbase with yum" + sudo yum install -y netbase + fi +fi \ No newline at end of file From 1782d09418109f4c130412f3c519f4336198c7ac Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 26 Dec 2024 22:44:56 +0000 Subject: [PATCH 2/7] test setup: actually run secure_integration tests When SECURE_INTEGRATION is speicified, the (non-secure) `:integration` specs are excluded, so we cannot have the `:secure_integration` specs wrapped in a context flagged as `:integration`. --- spec/inputs/integration/elasticsearch_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index 6330190..2e15349 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -4,7 +4,7 @@ require "logstash/inputs/elasticsearch" require_relative "../../../spec/es_helper" -describe LogStash::Inputs::Elasticsearch, :integration => true do +describe LogStash::Inputs::Elasticsearch do SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true' From 8caeb9f22fb77eadd9af7f7d1f177aed4bc6b7e4 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 26 Dec 2024 22:29:28 +0000 Subject: [PATCH 3/7] test setup: regnerate test certs (and add regen script) --- spec/fixtures/test_certs/GENERATED_AT | 1 + spec/fixtures/test_certs/ca.crt | 35 +++++++++++++------------- spec/fixtures/test_certs/ca.der.sha256 | 2 +- spec/fixtures/test_certs/es.crt | 35 +++++++++++++------------- spec/fixtures/test_certs/renew.sh | 14 +++++++++++ 5 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 spec/fixtures/test_certs/GENERATED_AT create mode 100755 spec/fixtures/test_certs/renew.sh diff --git a/spec/fixtures/test_certs/GENERATED_AT b/spec/fixtures/test_certs/GENERATED_AT new file mode 100644 index 0000000..8bfc704 --- /dev/null +++ b/spec/fixtures/test_certs/GENERATED_AT @@ -0,0 +1 @@ +2024-12-26T22:27:15+00:00 diff --git a/spec/fixtures/test_certs/ca.crt b/spec/fixtures/test_certs/ca.crt index e9312d7..87e96b7 100644 --- a/spec/fixtures/test_certs/ca.crt +++ b/spec/fixtures/test_certs/ca.crt @@ -1,20 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL -BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l -cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD -VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr -8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr -PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd -OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe -oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz -YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC -AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY -MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI -hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB -G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL -DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL -sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36 -NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz -emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw= +MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz +dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy +MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm +aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/ +QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH +90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv +99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h +i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98 +m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B +AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv +lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz +p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp +E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0 +IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1 +T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q== -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/ca.der.sha256 b/spec/fixtures/test_certs/ca.der.sha256 index 2f04c23..a2d2cd5 100644 --- a/spec/fixtures/test_certs/ca.der.sha256 +++ b/spec/fixtures/test_certs/ca.der.sha256 @@ -1 +1 @@ -195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283 +b1e955819b0d14f64f863adb103c248ddacf2e17bea48d04ee4b57c64814ccc4 diff --git a/spec/fixtures/test_certs/es.crt b/spec/fixtures/test_certs/es.crt index 228dccd..7676eac 100644 --- a/spec/fixtures/test_certs/es.crt +++ b/spec/fixtures/test_certs/es.crt @@ -1,20 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL -BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l -cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD -VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu -1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T -Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt -CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ -2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX -LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ -tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX -MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs -YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc -7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW -f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE -Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+ -LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA -qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX -qi02i4q6meHGcw== +MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz +dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy +MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V +V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz +36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c +MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH +/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR +hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY +BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU +K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB +wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp +CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU +T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b +byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P +UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J +avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3 -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/renew.sh b/spec/fixtures/test_certs/renew.sh new file mode 100755 index 0000000..11d0147 --- /dev/null +++ b/spec/fixtures/test_certs/renew.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e +cd "$(dirname "$0")" + +openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr +openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr +openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256 + +openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr +openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr + +# output ISO8601 timestamp to file +date -Iseconds > GENERATED_AT \ No newline at end of file From 79cfe8596c41820005df5f3fc30874580883e7e4 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 26 Dec 2024 22:33:17 +0000 Subject: [PATCH 4/7] test setup: give ES the full cert chain In order for the `ca_trusted_fingerprint` specs to work with the CA's fingerprint, ES needs to be configured to present a cert chain that includes the CA. --- .ci/Dockerfile.elasticsearch | 2 +- spec/fixtures/test_certs/es.chain.crt | 38 +++++++++++++++++++++++++++ spec/fixtures/test_certs/renew.sh | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/test_certs/es.chain.crt diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch index 8ff2b75..fb9f58c 100644 --- a/.ci/Dockerfile.elasticsearch +++ b/.ci/Dockerfile.elasticsearch @@ -13,7 +13,7 @@ COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/con RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.enabled: true" >> $es_yml; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/es.key" >> $es_yml; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/es.crt" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/es.chain.crt" >> $es_yml; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.verification_mode: certificate" >> $es_yml; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ -n "$ES_SSL_SUPPORTED_PROTOCOLS" ] ; then echo "xpack.security.http.ssl.supported_protocols: ${ES_SSL_SUPPORTED_PROTOCOLS}" >> $es_yml; fi diff --git a/spec/fixtures/test_certs/es.chain.crt b/spec/fixtures/test_certs/es.chain.crt new file mode 100644 index 0000000..334de63 --- /dev/null +++ b/spec/fixtures/test_certs/es.chain.crt @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz +dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy +MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V +V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz +36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c +MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH +/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR +hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY +BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU +K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB +wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp +CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU +T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b +byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P +UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J +avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz +dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy +MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm +aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/ +QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH +90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv +99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h +i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98 +m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B +AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv +lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz +p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp +E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0 +IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1 +T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q== +-----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/renew.sh b/spec/fixtures/test_certs/renew.sh index 11d0147..8ef56cc 100755 --- a/spec/fixtures/test_certs/renew.sh +++ b/spec/fixtures/test_certs/renew.sh @@ -9,6 +9,7 @@ openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr +cat es.crt ca.crt > es.chain.crt # output ISO8601 timestamp to file date -Iseconds > GENERATED_AT \ No newline at end of file From 8912943f79402e71cdddbe3352ae0abb7cee7146 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 20 Oct 2022 18:03:50 +0000 Subject: [PATCH 5/7] resilience: prevent failures from crashing plugin When an Event cannot be created directly from the hit, or when the docinfo cannot be merged into a non-hash field in the hit, emit an Event tagged with `_elasticsearch_input_failure` that contains the JSON-encoded hit in `[event][original]` instead of crashing. --- CHANGELOG.md | 4 + docs/index.asciidoc | 10 +++ lib/logstash/inputs/elasticsearch.rb | 30 ++++++-- logstash-input-elasticsearch.gemspec | 2 +- spec/inputs/elasticsearch_spec.rb | 105 ++++++++++++++++++++++++++- 5 files changed, 141 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5acdb..f4afe71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.21.1 + - Fix: prevent plugin crash when hits contain illegal structure + - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit. + ## 4.21.0 - Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 597e08b..fb516c8 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -93,6 +93,16 @@ The plugin logs a warning when ECS is enabled and `target` isn't set. TIP: Set the `target` option to avoid potential schema conflicts. +[id="plugins-{type}s-{plugin}-failure-handling"] +==== Failure handling + +When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit. + +Common causes are: + + - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <> directive to avoid conflicts with the top-level namespace. + - When <> is enabled and the docinfo fields cannot be merged into the hit result. Combine <> and <> to avoid conflict. + [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Input configuration options diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index e624e84..34c0c92 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -350,24 +350,42 @@ def run(output_queue) end ## + # @param output_queue [#<<] + # @param scroll_id [String]: a scroll id to resume + # @return [Array(Boolean,String)]: a tuple representing whether the response + # + def process_next_scroll(output_queue, scroll_id) + r = scroll_request(scroll_id) + r['hits']['hits'].each { |hit| push_hit(hit, output_queue) } + [r['hits']['hits'].any?, r['_scroll_id']] + end + # This can be called externally from the query_executor public def push_hit(hit, output_queue, root_field = '_source') - event = targeted_event_factory.new_event hit[root_field] - set_docinfo_fields(hit, event) if @docinfo + event = event_from_hit(hit, root_field) decorate(event) output_queue << event end + def event_from_hit(hit, root_field) + event = targeted_event_factory.new_event hit[root_field] + set_docinfo_fields(hit, event) if @docinfo + + event + rescue => e + serialized_hit = hit.to_json + logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit) + return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure']) + end + def set_docinfo_fields(hit, event) # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event. docinfo_target = event.get(@docinfo_target) || {} unless docinfo_target.is_a?(Hash) - @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata) - - # TODO: (colin) I am not sure raising is a good strategy here? - raise Exception.new("Elasticsearch input: incompatible event") + # expect error to be handled by `#event_from_hit` + fail RuntimeError, "Incompatible event; unable to merge docinfo fields into docinfo_target=`#{@docinfo_target}`" end @docinfo_fields.each do |field| diff --git a/logstash-input-elasticsearch.gemspec b/logstash-input-elasticsearch.gemspec index 8818301..9dc4ebc 100644 --- a/logstash-input-elasticsearch.gemspec +++ b/logstash-input-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-elasticsearch' - s.version = '4.21.0' + s.version = '4.21.1' s.licenses = ['Apache License (2.0)'] s.summary = "Reads query results from an Elasticsearch cluster" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/elasticsearch_spec.rb b/spec/inputs/elasticsearch_spec.rb index f8a9a31..31477a2 100644 --- a/spec/inputs/elasticsearch_spec.rb +++ b/spec/inputs/elasticsearch_spec.rb @@ -653,11 +653,28 @@ def synchronize_method!(object, method_name) context 'if the `docinfo_target` exist but is not of type hash' do let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' } let(:do_register) { false } + let(:mock_queue) { double('Queue', :<< => nil) } + let(:hit) { response.dig('hits', 'hits').first } + + it 'emits a tagged event with JSON-serialized event in [event][original]' do + allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object) - it 'raises an exception if the `docinfo_target` exist but is not of type hash' do - expect(client).not_to receive(:clear_scroll) plugin.register - expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/) + plugin.run(mock_queue) + + expect(mock_queue).to have_received(:<<) do |event| + expect(event).to be_a_kind_of LogStash::Event + + expect(event.get('tags')).to include("_elasticsearch_input_failure") + expect(event.get('[event][original]')).to be_a_kind_of String + expect(JSON.load(event.get('[event][original]'))).to eq hit + end + + expect(plugin.logger) + .to have_received(:warn).with( + a_string_including("Event creation error, original data now in [event][original] field"), + a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'), + :data => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"'))) end end @@ -1235,6 +1252,88 @@ def wait_receive_request end end + context '#push_hit' do + let(:config) do + { + 'docinfo' => true, # include ids + 'docinfo_target' => '[@metadata][docinfo]' + } + end + + let(:hit) do + JSON.load(<<~EOJSON) + { + "_index" : "test_bulk_index_2", + "_type" : "_doc", + "_id" : "sHe6A3wBesqF7ydicQvG", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2021-09-20T15:02:02.557Z", + "message" : "ping", + "@version" : "17", + "sequence" : 7, + "host" : { + "name" : "maybe.local", + "ip" : "127.0.0.1" + } + } + } + EOJSON + end + + let(:mock_queue) { double('queue', :<< => nil) } + + it 'pushes a generated event to the queue' do + plugin.send(:push_hit, hit, mock_queue) + expect(mock_queue).to have_received(:<<) do |event| + expect(event).to be_a_kind_of LogStash::Event + + # fields overriding defaults + expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z") + expect(event.get('@version')).to eq("17") + + # structure from hit's _source + expect(event.get('message')).to eq("ping") + expect(event.get('sequence')).to eq(7) + expect(event.get('[host][name]')).to eq("maybe.local") + expect(event.get('[host][ip]')).to eq("127.0.0.1") + + # docinfo fields + expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2") + expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc") + expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG") + end + end + + context 'when event creation fails' do + before(:each) do + allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object) + + allow(plugin.event_factory).to receive(:new_event).and_call_original + allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional') + end + + it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do + plugin.send(:push_hit, hit, mock_queue) + + expect(mock_queue).to have_received(:<<) do |event| + expect(event).to be_a_kind_of LogStash::Event + + expect(event.get('tags')).to include("_elasticsearch_input_failure") + expect(event.get('[event][original]')).to be_a_kind_of String + expect(JSON.load(event.get('[event][original]'))).to eq hit + end + + expect(plugin.logger) + .to have_received(:warn).with( + a_string_including("Event creation error, original data now in [event][original] field"), + a_hash_including(:message => a_string_including('intentional'), + :data => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"'))) + + end + end + end + # @note can be removed once we depends on elasticsearch gem >= 6.x def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client client.transport.respond_to?(:transport) ? client.transport.transport : client.transport From 5529740adc631a6c4e7ed963ece7d8991ed61378 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Fri, 27 Dec 2024 00:13:48 +0000 Subject: [PATCH 6/7] add link to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4afe71..f3273f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 4.21.1 - - Fix: prevent plugin crash when hits contain illegal structure + - Fix: prevent plugin crash when hits contain illegal structure [#183](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183) - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit. ## 4.21.0 From 4172d62b6c5b486a1bfa2726f4df27e84f162e39 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Mon, 30 Dec 2024 16:45:21 -0800 Subject: [PATCH 7/7] remove orphan method from refactor --- lib/logstash/inputs/elasticsearch.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index 34c0c92..32948d2 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -350,16 +350,6 @@ def run(output_queue) end ## - # @param output_queue [#<<] - # @param scroll_id [String]: a scroll id to resume - # @return [Array(Boolean,String)]: a tuple representing whether the response - # - def process_next_scroll(output_queue, scroll_id) - r = scroll_request(scroll_id) - r['hits']['hits'].each { |hit| push_hit(hit, output_queue) } - [r['hits']['hits'].any?, r['_scroll_id']] - end - # This can be called externally from the query_executor public def push_hit(hit, output_queue, root_field = '_source')