Merge pull request #63 from unifio/yl-wip

v0.8.0

.circleci/config.yml

@@ -5,11 +5,12 @@ jobs:
     working_directory: ~/repo
 
     docker:
-      - image: circleci/ruby:2.3.6
+      - image: circleci/ruby:2.5.1
 
     environment:
       COVALENCE_VERSION: 0.7.8
       TERRAFORM_VERSION: 0.11.7
+      SOPS_VERSION: 3.0.5
 
     steps:
       - checkout
@@ -21,9 +22,33 @@ jobs:
             wget -q "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
             unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip
 
+            # Install Sops
+            sudo wget -q "https://github.com/mozilla/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux" -O /usr/local/bin/sops
+            sudo chmod +x /usr/local/bin/sops
+
             # Install gem bundle
             bundle install
 
       - run:
           name: Run tests
           command: bundle exec rake ci:covalence
+
+      - run:
+          name: Setup gem credentials
+          command: |
+            mkdir -p ~/.gem/
+            cat <<EOF >~/.gem/credentials
+            ---
+            :rubygems_api_key: ${RUBYGEMS_API_KEY}
+            EOF
+            chmod 600 ~/.gem/credentials
+
+      - deploy:
+          name: Publish gem to rubygems
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              gem signin
+              gem build covalence.gemspec
+              COVALENCE_VERSION=$(ruby -e "require \"#{Dir.pwd}/lib/covalence/version\"; puts Covalence::VERSION")
+              gem push covalence-${COVALENCE_VERSION}.gem
+            fi

.env.sample

@@ -1,3 +1,6 @@
 COVALENCE_WORKSPACE = $PWD
+AWS_PROFILE = "<likely, if you're going to use sops>"
 AWS_REGION = "us-west-2"
 ATLAS_TOKEN = "<insert_token_here>"
+SOPS_KMS_ARN = "https://github.com/mozilla/sops#2usage"
+CONSUL_KV_FILE = "<$WORKSPACE_DIR/consul-kv.yml>"

.env.test

@@ -3,5 +3,8 @@ COVALENCE_CONFIG = "spec/fixtures/covalence_spec.yaml"
 COVALENCE_TERRAFORM_DIR = "spec/fixtures/terraform"
 COVALENCE_PACKER_DIR = "spec/fixtures"
 AWS_REGION = "us-west-2"
+AWS_PROFILE = "unifio"
 ATLAS_TOKEN = "<insert_token_here>"
 RAKE_ENV = "test"
+SOPS_KMS_ARN = "<some fun arn in our aws acct>"
+CONSUL_KV_FILE = "spec/fixtures/data/consul-kv.yml"

.ruby-version

@@ -1 +1 @@
-2.3.6
+2.5.1

Gemfile.lock

@@ -1,48 +1,55 @@
 PATH
   remote: .
   specs:
-    covalence (0.7.8)
-      activemodel (~> 4.2.6)
-      activesupport (~> 4.2.6)
-      aws-sdk (~> 2.9.5)
-      deep_merge (~> 1.0.1)
-      hiera (~> 3.3.0)
-      highline (~> 1.6.0)
-      json (~> 1.8.3)
+    covalence (0.8.0)
+      activemodel (~> 5.2.0)
+      activesupport (~> 5.2.0)
+      aws-sdk-s3 (~> 1)
+      consul_loader (~> 1.0.0)
+      deep_merge (~> 1.2.1)
+      hiera (~> 3.4.3)
+      highline (~> 1.7.10)
+      json (~> 2.1.0)
       rake (>= 11.1.2)
       rest-client (~> 2.0.0.rc3)
-      semantic (~> 1.4.1)
-      slop (~> 4.4.1)
+      semantic (~> 1.6.1)
+      slop (~> 4.6.2)
       virtus (~> 1.0.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (4.2.10)
-      activesupport (= 4.2.10)
-      builder (~> 3.1)
-    activesupport (4.2.10)
-      i18n (~> 0.7)
+    activemodel (5.2.0)
+      activesupport (= 5.2.0)
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
-    awesome_print (1.7.0)
-    aws-sdk (2.9.44)
-      aws-sdk-resources (= 2.9.44)
-    aws-sdk-core (2.9.44)
+    awesome_print (1.8.0)
+    aws-eventstream (1.0.0)
+    aws-partitions (1.87.0)
+    aws-sdk-core (3.21.2)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.9.44)
-      aws-sdk-core (= 2.9.44)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.13.0)
+      aws-sdk-core (~> 3, >= 3.21.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
     builder (3.2.3)
-    byebug (9.0.6)
+    byebug (10.0.2)
     ci_reporter (2.0.0)
       builder (>= 2.1.2)
     ci_reporter_rspec (1.0.0)
@@ -51,47 +58,35 @@ GEM
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     concurrent-ruby (1.0.5)
+    consul_loader (1.0.0)
+      rest-client
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    deep_merge (1.0.1)
+    deep_merge (1.2.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
-    docile (1.1.5)
+    docile (1.3.0)
     domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
-    dotenv (2.1.2)
+    dotenv (2.4.0)
     equalizer (0.0.11)
-    fabrication (2.15.2)
-    faraday (0.9.2)
-      multipart-post (>= 1.2, < 3)
-    gemfury (0.6.0)
-      faraday (>= 0.9.0, < 0.10.0.pre)
-      highline (~> 1.6.0)
-      multi_json (~> 1.10)
-      netrc (~> 0.10.0)
-      thor (>= 0.14.0, < 1.0.0.pre)
+    fabrication (2.20.1)
     hashdiff (0.3.7)
-    hiera (3.3.3)
-    highline (1.6.21)
+    hiera (3.4.3)
+    highline (1.7.10)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (0.9.5)
+    i18n (1.0.1)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     jmespath (1.4.0)
-    json (1.8.6)
+    json (2.1.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
     minitest (5.11.3)
-    multi_json (1.13.1)
-    multipart-post (2.0.0)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
-    net-ssh (4.2.0)
-    net-telnet (0.1.1)
-    netrc (0.10.3)
+    netrc (0.11.0)
     public_suffix (3.0.2)
     rake (12.3.1)
     rest-client (2.0.2)
@@ -107,33 +102,18 @@ GEM
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
-    rspec-its (1.2.0)
-      rspec-core (>= 3.0.0)
-      rspec-expectations (>= 3.0.0)
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
     safe_yaml (1.0.4)
-    semantic (1.4.1)
-    serverspec (2.36.1)
-      multi_json
-      rspec (~> 3.0)
-      rspec-its
-      specinfra (~> 2.53)
-    sfl (2.3)
-    simplecov (0.12.0)
-      docile (~> 1.1.0)
+    semantic (1.6.1)
+    simplecov (0.16.1)
+      docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    slop (4.4.3)
-    specinfra (2.73.3)
-      net-scp
-      net-ssh (>= 2.7, < 5.0)
-      net-telnet
-      sfl
-    thor (0.20.0)
+    slop (4.6.2)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
@@ -145,7 +125,7 @@ GEM
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
-    webmock (2.0.3)
+    webmock (3.4.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -154,18 +134,16 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  awesome_print (~> 1.7.0)
+  awesome_print (~> 1.8.0)
   bundler (>= 1.9.0)
-  byebug (~> 9.0.5)
+  byebug (~> 10.0.2)
   ci_reporter_rspec (~> 1.0.0)
   covalence!
-  dotenv (~> 2.1.0)
-  fabrication (~> 2.15.2)
-  gemfury (~> 0.6.0)
+  dotenv (~> 2.4.0)
+  fabrication (~> 2.20.1)
   rspec (~> 3.5)
-  serverspec (~> 2.36.0)
-  simplecov (~> 0.12.0)
-  webmock (~> 2.0.3)
+  simplecov (~> 0.16.1)
+  webmock (~> 3.4.1)
 
 BUNDLED WITH
    1.16.1

README.md

@@ -1,5 +1,6 @@
 # Covalence
 [![CircleCI](https://circleci.com/gh/unifio/covalence.svg?style=svg)](https://circleci.com/gh/unifio/covalence)
+[![Dependency Status](https://gemnasium.com/badges/github.com/unifio/covalence.svg)](https://gemnasium.com/github.com/unifio/covalence)
 
 A tool for the management and orchestration of data used by HashiCorp infrastructure tooling.
 
@@ -503,8 +504,11 @@ vpc_id:
 
 Covalence is packaged as a Ruby Gem.
 
+You will probably need the following packages installed locally
+- Terraform
+- Packer
+- Sops
+
 Execute the following to build the gem:
 
 `$ gem build covalence.gemspec`
-
-Gem artifacts are hosted at https://repo.fury.io/unifio/.

covalence.gemspec

@@ -16,40 +16,33 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://unif.io"
   spec.license       = "MPL-2.0"
 
-  if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://repo.fury.io/unifio/"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
-  end
-
   spec.files         = Dir["*.md", "#{gem_root}/**/*"]
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = [ gem_root ]
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_dependency "deep_merge", "~> 1.0.1"
-  spec.add_dependency "hiera", "~> 3.3.0"
-  spec.add_dependency "json", "~> 1.8.3"
+  spec.add_dependency "deep_merge", "~> 1.2.1"
+  spec.add_dependency "hiera", "~> 3.4.3"
+  spec.add_dependency "json", "~> 2.1.0"
   spec.add_dependency "rest-client", "~> 2.0.0.rc3"
   spec.add_dependency "rake", ">= 11.1.2"
-  spec.add_dependency "aws-sdk", "~> 2.9.5"
+  spec.add_dependency "aws-sdk-s3", "~> 1"
   spec.add_dependency "virtus", "~> 1.0.5"
-  spec.add_dependency "activesupport", "~> 4.2.6"
-  spec.add_dependency "activemodel", "~> 4.2.6"
-  spec.add_dependency "semantic", "~> 1.4.1"
-  spec.add_dependency "slop", "~> 4.4.1"
-  spec.add_dependency "highline", "~> 1.6.0"
+  spec.add_dependency "activesupport", "~> 5.2.0"
+  spec.add_dependency "activemodel", "~> 5.2.0"
+  spec.add_dependency "semantic", "~> 1.6.1"
+  spec.add_dependency "slop", "~> 4.6.2"
+  spec.add_dependency "highline", "~> 1.7.10"
+  spec.add_dependency "consul_loader", "~> 1.0.0"
 
   Covalence::Helpers::SpecDependencies.dependencies.each do |name, requirement|
     spec.add_development_dependency name, requirement
   end
-  spec.add_development_dependency "awesome_print", "~> 1.7.0"
+  spec.add_development_dependency "awesome_print", "~> 1.8.0"
   spec.add_development_dependency "bundler", ">= 1.9.0"
-  spec.add_development_dependency "dotenv", "~> 2.1.0"
-  spec.add_development_dependency "byebug", "~> 9.0.5"
-  spec.add_development_dependency "serverspec", "~> 2.36.0"
-  spec.add_development_dependency "webmock", "~> 2.0.3"
-  spec.add_development_dependency "gemfury", "~> 0.6.0"
-  spec.add_development_dependency "fabrication", "~> 2.15.2"
-  spec.add_development_dependency "simplecov", "~> 0.12.0"
+  spec.add_development_dependency "dotenv", "~> 2.4.0"
+  spec.add_development_dependency "byebug", "~> 10.0.2"
+  spec.add_development_dependency "webmock", "~> 3.4.1"
+  spec.add_development_dependency "fabrication", "~> 2.20.1"
+  spec.add_development_dependency "simplecov", "~> 0.16.1"
 end

example/Rakefile

@@ -2,15 +2,5 @@ require 'rake'
 require 'rspec/core/rake_task'
 require 'covalence/environment_tasks'
 require 'covalence/spec_tasks'
-require 'consul_loader'
-
-def load_yaml(filename)
-  consul_loader = ConsulLoader::Loader.new(ConsulLoader::ConfigParser.new)
-  consul_server = "http://#{ENV['CONSUL_HTTP_ADDR']}"
-  consul_loader.load_config(filename, consul_server)
-end
-
-desc 'Load K/V data into Consul service'
-task 'consul_load' do
-  load_yaml("#{ENV['CONSUL_KV_FILE']}")
-end
+require 'covalence/consul_tasks'
+require 'covalence/sops_tasks'

lib/covalence.rb

@@ -26,6 +26,11 @@ module Covalence
 
   PACKER_CMD = ENV['PACKER_CMD'] || "packer"
 
+  SOPS_CMD = ENV['SOPS_CMD'] || "sops"
+  SOPS_VERSION = ENV['SOPS_VERSION'] || (`#{SOPS_CMD} --version`.gsub(/[^\d\.]/, '') rescue "0.0.0")
+  SOPS_ENCRYPTED_SUFFIX = ENV['SOPS_ENCRYPTED_SUFFIX'] || "-encrypted"
+  SOPS_DECRYPTED_SUFFIX = ENV['SOPS_DECRYPTED_SUFFIX'] || "-decrypted"
+
   # No-op shell command. Should not need to modify for most unix shells.
   DRY_RUN_CMD = (ENV['COVALENCE_DRY_RUN_CMD'] || ":")
   DEBUG_CLI = (ENV['COVALENCE_DEBUG'] || 'false') =~ (/(true|t|yes|y|1)$/i)