From 6a9b4819bacf6404369e7d581b6a74d6321e7458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pacana?= Date: Fri, 14 Sep 2018 01:15:25 +0200 Subject: [PATCH 1/2] Initial pass of EncryptionMapper. - relies on repository of external encryption keys (in-memory for testing and reference is provided) - keys are matched with attributes to encrypt via encryption schema definition - each attribute is encrypted with different iv, event when the same key used https://security.stackexchange.com/questions/6058/is-real-salt-the-same-as-initialization-vectors/6059#6059 - cipher being used is persisted per attribute to allow decrypting when new encryption cipher is introduced, that helps changing ciphers and gradual cryptogram updates after particular cipher becomes to weak (cryptoperiod) https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-57p1r3.pdf - unreadable data is a null that prints as a string - before calling Cipher#random_iv or Cipher#random_key, first Cipher#encrypt must be called, these are ignored in context of mutant http://ruby-doc.org/stdlib-2.5.0/libdoc/openssl/rdoc/OpenSSL/Cipher.html#method-i-random_iv http://ruby-doc.org/stdlib-2.5.0/libdoc/openssl/rdoc/OpenSSL/Cipher.html#method-i-random_key --- ruby_event_store/Makefile | 6 +- ruby_event_store/lib/ruby_event_store.rb | 1 + .../mappers/encryption_mapper.rb | 218 +++++++++++ .../ruby_event_store/mappers/null_mapper.rb | 1 - .../spec/mappers/encryption_mapper_spec.rb | 349 ++++++++++++++++++ 5 files changed, 573 insertions(+), 2 deletions(-) create mode 100644 ruby_event_store/lib/ruby_event_store/mappers/encryption_mapper.rb create mode 100644 ruby_event_store/spec/mappers/encryption_mapper_spec.rb diff --git a/ruby_event_store/Makefile b/ruby_event_store/Makefile index e26486a070..a26f2dbdf0 100644 --- a/ruby_event_store/Makefile +++ b/ruby_event_store/Makefile @@ -4,7 +4,11 @@ REQUIRE = $(GEM_NAME) IGNORE = RubyEventStore::InMemoryRepository\#append_with_synchronize \ RubyEventStore::Client::Within\#add_thread_subscribers \ RubyEventStore::Client::Within\#add_thread_global_subscribers \ - RubyEventStore::Client::Within\#call + RubyEventStore::Client::Within\#call \ + RubyEventStore::Mappers::InMemoryEncryptionKeyRepository\#prepare_encrypt \ + RubyEventStore::Mappers::EncryptionKey\#prepare_encrypt \ + RubyEventStore::Mappers::EncryptionKey\#prepare_decrypt + SUBJECT ?= RubyEventStore* include ../lib/install.mk diff --git a/ruby_event_store/lib/ruby_event_store.rb b/ruby_event_store/lib/ruby_event_store.rb index fd19d882e3..d09cf3c23b 100644 --- a/ruby_event_store/lib/ruby_event_store.rb +++ b/ruby_event_store/lib/ruby_event_store.rb @@ -19,6 +19,7 @@ require 'ruby_event_store/mappers/protobuf' require 'ruby_event_store/mappers/null_mapper' require 'ruby_event_store/mappers/instrumented_mapper' +require 'ruby_event_store/mappers/encryption_mapper' require 'ruby_event_store/batch_enumerator' require 'ruby_event_store/correlated_commands' require 'ruby_event_store/link_by_metadata' diff --git a/ruby_event_store/lib/ruby_event_store/mappers/encryption_mapper.rb b/ruby_event_store/lib/ruby_event_store/mappers/encryption_mapper.rb new file mode 100644 index 0000000000..65ae0cbf22 --- /dev/null +++ b/ruby_event_store/lib/ruby_event_store/mappers/encryption_mapper.rb @@ -0,0 +1,218 @@ +module RubyEventStore + module Mappers + class ForgottenData + FORGOTTEN_DATA = 'FORGOTTEN_DATA'.freeze + + def initialize(string = FORGOTTEN_DATA) + @string = string + end + + def to_s + @string + end + + def ==(other) + @string == other + end + + def method_missing(*) + self + end + + def respond_to_missing?(*) + true + end + end + + class InMemoryEncryptionKeyRepository + DEFAULT_CIPHER = 'aes-256-cbc'.freeze + + def initialize + @keys = {} + end + + def key_of(identifier, cipher: DEFAULT_CIPHER) + @keys[[identifier, cipher]] + end + + def create(identifier, cipher: DEFAULT_CIPHER) + crypto = prepare_encrypt(cipher) + @keys[[identifier, cipher]] = EncryptionKey.new(cipher: cipher, key: crypto.random_key) + end + + def forget(identifier) + @keys = @keys.reject { |(id, _)| id.eql?(identifier) } + end + + private + + def prepare_encrypt(cipher) + crypto = OpenSSL::Cipher.new(cipher) + crypto.encrypt + crypto + end + end + + class EncryptionKey + def initialize(cipher:, key:) + @cipher = cipher + @key = key + end + + def encrypt(message, iv) + crypto = prepare_encrypt(cipher) + crypto.iv = iv + crypto.key = key + crypto.update(message) + crypto.final + end + + def decrypt(message, iv) + crypto = prepare_decrypt(cipher) + crypto.iv = iv + crypto.key = key + crypto.update(message) + crypto.final + end + + def random_iv + crypto = prepare_encrypt(cipher) + crypto.random_iv + end + + attr_reader :cipher, :key + + private + + def prepare_encrypt(cipher) + crypto = OpenSSL::Cipher.new(cipher) + crypto.encrypt + crypto + end + + def prepare_decrypt(cipher) + crypto = OpenSSL::Cipher.new(cipher) + crypto.decrypt + crypto + end + end + + class MissingEncryptionKey < StandardError + def initialize(key_identifier) + super %Q|Could not find encryption key for '#{key_identifier}'| + end + end + + class EncryptionMapper + class Leaf + def self.===(hash) + hash.keys.sort.eql? %i(cipher identifier iv) + end + end + private_constant :Leaf + + def initialize(key_repository, serializer: YAML, forgotten_data: ForgottenData.new) + @key_repository = key_repository + @serializer = serializer + @forgotten_data = forgotten_data + end + + def event_to_serialized_record(domain_event) + metadata = domain_event.metadata.to_h + crypto_description = encryption_metadata(domain_event.data, encryption_schema(domain_event)) + metadata[:encryption] = crypto_description unless crypto_description.empty? + + SerializedRecord.new( + event_id: domain_event.event_id, + metadata: serializer.dump(metadata), + data: serializer.dump(encrypt_data(deep_dup(domain_event.data), crypto_description)), + event_type: domain_event.class.to_s + ) + end + + def serialized_record_to_event(record) + metadata = serializer.load(record.metadata) + crypto_description = Hash(metadata.delete(:encryption)) + + Object.const_get(record.event_type).new( + event_id: record.event_id, + data: decrypt_data(serializer.load(record.data), crypto_description), + metadata: metadata, + ) + end + + private + attr_reader :key_repository, :serializer, :forgotten_data + + def encryption_schema(event) + event.class.respond_to?(:encryption_schema) ? event.class.encryption_schema : {} + end + + def deep_dup(hash) + duplicate = hash.dup + duplicate.each do |k, v| + duplicate[k] = v.instance_of?(Hash) ? deep_dup(v) : v + end + duplicate + end + + def encryption_metadata(data, schema) + schema.inject({}) do |acc, (key, value)| + case value + when Hash + acc[key] = encryption_metadata(data, value) + when Proc + key_identifier = value.call(data) + encryption_key = key_repository.key_of(key_identifier) or raise MissingEncryptionKey.new(key_identifier) + acc[key] = { + cipher: encryption_key.cipher, + iv: encryption_key.random_iv, + identifier: key_identifier, + } + end + acc + end + end + + def encrypt_data(data, meta) + meta.reduce(data) do |acc, (key, value)| + acc[key] = encrypt_attribute(acc, key, value) + acc + end + end + + def decrypt_data(data, meta) + meta.reduce(data) do |acc, (key, value)| + acc[key] = decrypt_attribute(data, key, value) + acc + end + end + + def encrypt_attribute(data, attribute, meta) + case meta + when Leaf + value = data.fetch(attribute) + return unless value + + encryption_key = key_repository.key_of(meta.fetch(:identifier)) + encryption_key.encrypt(serializer.dump(value), meta.fetch(:iv)) + when Hash + encrypt_data(data.fetch(attribute), meta) + end + end + + def decrypt_attribute(data, attribute, meta) + case meta + when Leaf + cryptogram = data.fetch(attribute) + return unless cryptogram + + encryption_key = key_repository.key_of(meta.fetch(:identifier), cipher: meta.fetch(:cipher)) or return forgotten_data + serializer.load(encryption_key.decrypt(cryptogram, meta.fetch(:iv))) + when Hash + decrypt_data(data.fetch(attribute), meta) + end + rescue OpenSSL::Cipher::CipherError + forgotten_data + end + end + end +end \ No newline at end of file diff --git a/ruby_event_store/lib/ruby_event_store/mappers/null_mapper.rb b/ruby_event_store/lib/ruby_event_store/mappers/null_mapper.rb index b4e1fa3fb2..be705dc0bf 100644 --- a/ruby_event_store/lib/ruby_event_store/mappers/null_mapper.rb +++ b/ruby_event_store/lib/ruby_event_store/mappers/null_mapper.rb @@ -1,7 +1,6 @@ module RubyEventStore module Mappers class NullMapper - def event_to_serialized_record(domain_event) domain_event end diff --git a/ruby_event_store/spec/mappers/encryption_mapper_spec.rb b/ruby_event_store/spec/mappers/encryption_mapper_spec.rb new file mode 100644 index 0000000000..d975345fe6 --- /dev/null +++ b/ruby_event_store/spec/mappers/encryption_mapper_spec.rb @@ -0,0 +1,349 @@ +require 'spec_helper' +require 'openssl' + +module RubyEventStore + module Mappers + TicketTransferred = Class.new(RubyEventStore::Event) do + def self.encryption_schema + { + sender: { + name: ->(data) { data.dig(:sender, :user_id) }, + email: ->(data) { data.dig(:sender, :user_id) }, + }, + recipient: { + name: ->(data) { data.dig(:recipient, :user_id) }, + email: ->(data) { data.dig(:recipient, :user_id) }, + } + } + end + end + + TicketCancelled = Class.new(RubyEventStore::Event) + + TicketHolderEmailProvided = Class.new(RubyEventStore::Event) do + def self.encryption_schema + { + email: ->(data) do + data.fetch(:user_id) + end + } + end + end + + RSpec.describe EncryptionMapper do + let(:key_repository) { InMemoryEncryptionKeyRepository.new } + let(:serializer) { YAML } + let(:mapper) { EncryptionMapper.new(key_repository, serializer: serializer) } + let(:sender_id) { SecureRandom.uuid } + let(:recipient_id) { SecureRandom.uuid } + let(:event_id) { SecureRandom.uuid } + let(:ticket_id) { SecureRandom.uuid } + let(:correlation_id) { SecureRandom.uuid } + let(:sender_email) { 'alice@universe' } + let(:recipient) do + { + user_id: recipient_id, + name: 'Bob', + email: 'bob@universe', + } + end + let(:sender) do + { + user_id: sender_id, + name: 'Alice', + email: sender_email + } + end + let(:data) do + { + ticket_id: ticket_id, + sender: sender, + recipient: recipient + } + end + let(:metadata) do + { + correlation_id: correlation_id + } + end + + let(:ticket_transferred) do + TicketTransferred.new( + event_id: event_id, + data: data, + metadata: metadata + ) + end + + let(:ticket_cancelled) do + TicketCancelled.new( + event_id: event_id, + data: { + ticket_id: ticket_id, + }, + metadata: metadata + ) + end + + def encrypt(e) + mapper.event_to_serialized_record(e) + end + + def decrypt(r) + mapper.serialized_record_to_event(r) + end + + specify 'decrypts encrypted fields in presence of encryption keys' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + event = decrypt(encrypt(ticket_transferred)) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + sender: sender, + recipient: recipient + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'obfuscates data for missing keys on decryption' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + record = encrypt(ticket_transferred) + key_repository.forget(sender_id.dup) + event = decrypt(record) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + sender: { + user_id: sender_id, + name: ForgottenData::FORGOTTEN_DATA, + email: ForgottenData::FORGOTTEN_DATA, + }, + recipient: recipient + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'obfuscates data for incorrect keys on decryption' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + record = encrypt(ticket_transferred) + key_repository.forget(sender_id) + key_repository.create(sender_id) + event = decrypt(record) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + sender: { + user_id: sender_id, + name: ForgottenData::FORGOTTEN_DATA, + email: ForgottenData::FORGOTTEN_DATA, + }, + recipient: recipient + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'no-op for events without encryption schema' do + event = decrypt(encrypt(ticket_cancelled)) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'raises error on encryption with missing encryption key' do + expect do + encrypt(ticket_transferred) + end.to raise_error(MissingEncryptionKey, "Could not find encryption key for '#{sender_id}'") + end + + specify 'does not modify original event' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + event = ticket_transferred + encrypt(event) + + expect(event.data.dig(:sender, :name)).to eq('Alice') + expect(event.data.dig(:sender, :email)).to eq(sender_email) + expect(event.data.dig(:recipient, :name)).to eq('Bob') + expect(event.data.dig(:recipient, :email)).to eq('bob@universe') + expect(event.metadata).not_to have_key(:encryption) + end + + specify 'does not modify original record' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + record = encrypt(ticket_transferred) + data = serializer.load(record.data) + metadata = serializer.load(record.metadata) + decrypt(record) + + expect(serializer.load(record.data)).to eq(data) + expect(serializer.load(record.metadata)).to eq(metadata) + end + + specify 'two cryptograms of the same input and key are not alike' do + key_repository.create(sender_id) + + record = encrypt( + TicketTransferred.new( + event_id: event_id, + data: { + ticket_id: ticket_id, + sender: sender, + recipient: sender + }, + metadata: metadata + ) + ) + data = serializer.load(record.data) + + expect(data.dig(:sender, :name)).not_to eq(data.dig(:recipient, :name)) + expect(data.dig(:sender, :email)).not_to eq(data.dig(:recipient, :email)) + end + + specify 'handles non-nested encryption schema' do + key_repository.create(sender_id) + + event = + decrypt( + encrypt( + TicketHolderEmailProvided.new( + event_id: event_id, + data: { + ticket_id: ticket_id, + user_id: sender_id, + email: sender_email + }, + metadata: metadata + ) + ) + ) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + user_id: sender_id, + email: sender_email + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'handles non-string values' do + key_repository.create(sender_id) + + event = + decrypt( + encrypt( + TicketHolderEmailProvided.new( + event_id: event_id, + data: { + ticket_id: ticket_id, + user_id: sender_id, + email: [sender_email] + }, + metadata: metadata + ) + ) + ) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + user_id: sender_id, + email: [sender_email] + }) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'no-op for nil value' do + key_repository.create(sender_id) + + record = + encrypt( + TicketHolderEmailProvided.new( + event_id: event_id, + data: { + ticket_id: ticket_id, + user_id: sender_id, + email: nil + }, + metadata: metadata + ) + ) + event = decrypt(record) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + user_id: sender_id, + email: nil + }) + expect(serializer.load(record.data)).to eq(event.data) + expect(event.metadata.to_h).to eq(metadata) + end + + specify 'defaults' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + record = + EncryptionMapper + .new(key_repository) + .event_to_serialized_record(ticket_transferred) + event = + EncryptionMapper + .new(key_repository) + .serialized_record_to_event(record) + + expect(event).to eq(ticket_transferred) + end + + specify 'handles decryption after changing cipher' do + key_repository.create(sender_id) + key_repository.create(recipient_id) + + record = encrypt(ticket_transferred) + silence_warnings { InMemoryEncryptionKeyRepository::DEFAULT_CIPHER = 'aes-128-cbc' } + event = decrypt(record) + + expect(event.event_id).to eq(event_id) + expect(event.data).to eq({ + ticket_id: ticket_id, + sender: sender, + recipient: recipient + }) + expect(event.metadata.to_h).to eq(metadata) + end + end + + RSpec.describe ForgottenData do + specify 'compares with string' do + expect(ForgottenData.new).to eq(ForgottenData::FORGOTTEN_DATA) + expect(ForgottenData.new('bazinga')).to eq('bazinga') + expect(ForgottenData.new('bazinga')).not_to eq(ForgottenData::FORGOTTEN_DATA) + end + + specify "prints as string" do + expect{ print(ForgottenData.new) }.to output(ForgottenData::FORGOTTEN_DATA).to_stdout + expect{ print(ForgottenData.new('bazinga')) }.to output('bazinga').to_stdout + end + + specify 'behaves like null object' do + data = ForgottenData.new + expect(data.foo.bar[:baz]).to eq(data) + end + end + end +end From 47281e469c5786de6816ff139136fabb4999f0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pacana?= Date: Fri, 8 Mar 2019 15:23:58 +0100 Subject: [PATCH 2/2] Test to verify no unnecessary encryption metadata. --- ruby_event_store/spec/mappers/encryption_mapper_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ruby_event_store/spec/mappers/encryption_mapper_spec.rb b/ruby_event_store/spec/mappers/encryption_mapper_spec.rb index d975345fe6..9358ddc86b 100644 --- a/ruby_event_store/spec/mappers/encryption_mapper_spec.rb +++ b/ruby_event_store/spec/mappers/encryption_mapper_spec.rb @@ -161,6 +161,11 @@ def decrypt(r) expect(event.metadata.to_h).to eq(metadata) end + specify 'no encryption metadata without encryption schema' do + record = encrypt(ticket_cancelled) + expect(serializer.load(record.metadata)).to eq(metadata) + end + specify 'raises error on encryption with missing encryption key' do expect do encrypt(ticket_transferred)