diff --git a/Rakefile b/Rakefile index f61fb20..f2a00ff 100644 --- a/Rakefile +++ b/Rakefile @@ -1,23 +1,18 @@ -require "bundler/gem_tasks" -require 'jars/version' -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new(:spec) -rescue LoadError -end - -task default: "spec" +# encoding: utf-8 +require "jars/installer" +require "fileutils" -require 'jars/installer' -desc 'Install the JAR dependencies to vendor/' -task :install_jars do - # We actually want jar-dependencies will download the jars and place it in - # vendor/jar-dependencies/runtime-jars - Jars::Installer.new.vendor_jars!(false, 'vendor/jar-dependencies/runtime-jars') +task :default do + system('rake -vT') end -task build: :install_jars -require "logstash/devutils/rake" -task vendor: :install_jars +task :vendor do + exit(1) unless system './gradlew vendor' +end +task :clean do + ["vendor/jar-dependencies", "Gemfile.lock"].each do |p| + FileUtils.rm_rf(p) + end +end diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..0899109 --- /dev/null +++ b/build.gradle @@ -0,0 +1,97 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +apply plugin: "java" +apply plugin: 'maven' +apply plugin: "distribution" +apply plugin: "idea" + +group "org.logstash.inputs" + +sourceCompatibility = JavaVersion.VERSION_1_8 + +buildscript { + repositories { + mavenCentral() + jcenter() + } + +} + +repositories { + mavenCentral() +} + +task wrapper(type: Wrapper) { + gradleVersion = '4.0' +} + +dependencies { + compile 'com.amazonaws:amazon-kinesis-client:1.7.0' + compile 'com.amazonaws:aws-java-sdk-core:1.11.16' + compile 'com.amazonaws:aws-java-sdk-dynamodb:1.11.14' + compile 'com.amazonaws:aws-java-sdk-s3:1.11.14' + compile 'com.amazonaws:aws-java-sdk-kms:1.11.14' + compile 'com.amazonaws:aws-java-sdk-core:1.11.16' + compile 'commons-logging:commons-logging:1.2' + compile 'org.apache.httpcomponents:httpclient:4.5.2' + compile 'org.apache.httpcomponents:httpcore:4.4.4' + compile 'commons-logging:commons-logging:1.2' + compile 'commons-codec:commons-codec:1.9' + compile 'com.fasterxml.jackson.core:jackson-databind:2.6.6' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0' + compile 'com.fasterxml.jackson.core:jackson-core:2.6.6' + compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.6' + compile 'com.fasterxml.jackson.core:jackson-core:2.6.6' + compile 'joda-time:joda-time:2.8.1' + compile 'com.amazonaws:aws-java-sdk-kinesis:1.11.14' + compile 'com.amazonaws:aws-java-sdk-cloudwatch:1.11.14' + compile 'com.google.guava:guava:18.0' + compile 'com.google.protobuf:protobuf-java:2.6.1' + compile 'commons-lang:commons-lang:2.6' +} + +task generateGemJarRequiresFile { + doLast { + File jars_file = file('lib/logstash-input-kinesis_jars.rb') + jars_file.newWriter().withWriter { w -> + w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n" + w << "require \'jar_dependencies\'\n" + configurations.runtime.allDependencies.each { + w << "require_jar(\'${it.group}\', \'${it.name}\', \'${it.version}\')\n" + } + } + } +} + +task vendor { + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + configurations.runtime.allDependencies.each { dep -> + File f = configurations.runtime.filter { it.absolutePath.contains("${dep.group}/${dep.name}/${dep.version}") }.singleFile + String groupPath = dep.group.replaceAll('\\.', '/') + File newJarFile = file("${vendorPathPrefix}/${groupPath}/${dep.name}/${dep.version}/${dep.name}-${dep.version}.jar") + newJarFile.mkdirs() + Files.copy(f.toPath(), newJarFile.toPath(), REPLACE_EXISTING) + } + } +} + +vendor.dependsOn(generateGemJarRequiresFile) diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..6b1823d --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.daemon=false diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..d0bd875 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..8de2fda --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jun 21 11:39:16 CEST 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.0-all.zip diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..cccdd3d --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..f955316 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lib/logstash-input-kinesis_jars.rb b/lib/logstash-input-kinesis_jars.rb new file mode 100644 index 0000000..e93d925 --- /dev/null +++ b/lib/logstash-input-kinesis_jars.rb @@ -0,0 +1,22 @@ +# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT. + +require 'jar_dependencies' +require_jar('com.amazonaws', 'amazon-kinesis-client', '1.7.0') +require_jar('com.amazonaws', 'aws-java-sdk-core', '1.11.16') +require_jar('com.amazonaws', 'aws-java-sdk-dynamodb', '1.11.14') +require_jar('com.amazonaws', 'aws-java-sdk-s3', '1.11.14') +require_jar('com.amazonaws', 'aws-java-sdk-kms', '1.11.14') +require_jar('commons-logging', 'commons-logging', '1.2') +require_jar('org.apache.httpcomponents', 'httpclient', '4.5.2') +require_jar('org.apache.httpcomponents', 'httpcore', '4.4.4') +require_jar('commons-codec', 'commons-codec', '1.9') +require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.6.6') +require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.6.0') +require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.6.6') +require_jar('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.6.6') +require_jar('joda-time', 'joda-time', '2.8.1') +require_jar('com.amazonaws', 'aws-java-sdk-kinesis', '1.11.14') +require_jar('com.amazonaws', 'aws-java-sdk-cloudwatch', '1.11.14') +require_jar('com.google.guava', 'guava', '18.0') +require_jar('com.google.protobuf', 'protobuf-java', '2.6.1') +require_jar('commons-lang', 'commons-lang', '2.6') diff --git a/logstash-input-kinesis.gemspec b/logstash-input-kinesis.gemspec index 0d56e60..3d4c660 100644 --- a/logstash-input-kinesis.gemspec +++ b/logstash-input-kinesis.gemspec @@ -15,16 +15,13 @@ Gem::Specification.new do |spec| spec.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) - spec.require_paths = ['lib', 'vendor/jar-dependencies/runtime-jars'] + spec.require_paths = ['lib', 'vendor/jar-dependencies/'] # Special flag to let us know this is actually a logstash plugin spec.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } spec.platform = 'java' - spec.requirements << "jar 'com.amazonaws:amazon-kinesis-client', '1.7.0'" - spec.requirements << "jar 'com.amazonaws:aws-java-sdk-core', '1.11.16'" - spec.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" spec.add_development_dependency 'logstash-devutils'