diff --git a/CMakeLists.txt b/CMakeLists.txt index e697be3..82b0387 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ set(EXTENSION_SOURCES src/core/functions/function_data/encrypt_function_data.cpp src/core/functions/cast/varchar_cast.cpp src/core/functions/table/encrypt_table.cpp + src/core/functions/secrets/authentication.cpp src/core/utils/simple_encryption_utils.cpp src/core/crypto/crypto_primitives.cpp) diff --git a/src/core/crypto/crypto_primitives.cpp b/src/core/crypto/crypto_primitives.cpp index 50c1f0a..63a1e6f 100644 --- a/src/core/crypto/crypto_primitives.cpp +++ b/src/core/crypto/crypto_primitives.cpp @@ -11,30 +11,6 @@ namespace duckdb { -void sha256(const char *in, size_t in_len, hash_bytes &out) { - duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out); -} - -void hmac256(const std::string &message, const char *secret, size_t secret_len, - hash_bytes &out) { - duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), - message.size(), (char *)out); -} - -void hmac256(std::string message, hash_bytes secret, hash_bytes &out) { - hmac256(message, (char *)secret, sizeof(hash_bytes), out); -} - -void hex256(hash_bytes &in, hash_str &out) { - const char *hex = "0123456789abcdef"; - unsigned char *pin = in; - unsigned char *pout = out; - for (; pin < in + sizeof(in); pout += 2, pin++) { - pout[0] = hex[(*pin >> 4) & 0xF]; - pout[1] = hex[*pin & 0xF]; - } -} - const EVP_CIPHER *GetCipher(const string &key, AESStateSSL::Algorithm algorithm) { diff --git a/src/core/functions/scalar/encrypt_to_etype.cpp b/src/core/functions/scalar/encrypt_to_etype.cpp index ded8766..356e58d 100644 --- a/src/core/functions/scalar/encrypt_to_etype.cpp +++ b/src/core/functions/scalar/encrypt_to_etype.cpp @@ -1,8 +1,5 @@ #define DUCKDB_EXTENSION_MAIN -// what is the maximum size of biggest type in duckdb -#define MAX_BUFFER_SIZE 1024 - #include "duckdb.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/types.hpp" @@ -22,8 +19,6 @@ #include "simple_encryption/core/types.hpp" #include "duckdb/common/vector_operations/generic_executor.hpp" -// temporary - namespace simple_encryption { namespace core { @@ -79,6 +74,7 @@ ProcessAndCastDecrypt(shared_ptr encryption_state, size_t decrypted_size = encrypted_size; Blob::FromBase64(base64_data, reinterpret_cast(buffer_p), encrypted_size); + D_ASSERT(encrypted_size <= base64_data.GetSize()); string_t decrypted_data = diff --git a/src/core/functions/secrets/authentication.cpp b/src/core/functions/secrets/authentication.cpp new file mode 100644 index 0000000..e8bd07a --- /dev/null +++ b/src/core/functions/secrets/authentication.cpp @@ -0,0 +1,109 @@ +#include "simple_encryption/core/functions/secrets.hpp" +#include "simple_encryption/common.hpp" +#include "simple_encryption/core/utils/simple_encryption_utils.hpp" +#include "simple_encryption/core/crypto/crypto_primitives.hpp" +#include "simple_encryption/core/functions/scalar.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/main/secret/secret.hpp" +#include "duckdb/main/extension_util.hpp" +#include "openssl/rand.h" +#include +#include + +namespace simple_encryption { + +namespace core { + + string_t GenerateDataEncryptionKey(const uint32_t size){ + + unsigned char* key = new unsigned char[size]; + + // generate random bytes with OpenSSL function + RAND_bytes(key, size); + + // cast back to string ('normal' string, not duckdb string) + std::string key_string(reinterpret_cast(key), size); + + return key_string; +} + +string_t GetDataEncryptionKey(const uint32_t size){ + + switch(size){ + case 16: + return GenerateDataEncryptionKey(16); + case 24: + return GenerateDataEncryptionKey(24); + case 32: + return GenerateDataEncryptionKey(32); + default: + throw InvalidInputException("Invalid size for data encryption key: '%d', expected: 16, 24, or 32", size); + } +} + +// This code partly copied / inspired by the gsheets extension for duckdb +static void CopySecret(const std::string &key, const CreateSecretInput &input, + KeyValueSecret &result) { + + // this method checks whether a secret_param is present in the secret_map + auto val = input.options.find(key); + // does this also take a key, value or list struct? + if (val != input.options.end()) { + result.secret_map[key] = val->second; + } +} + +static void RegisterCommonSecretParameters(CreateSecretFunction &function) { + // Register named parameters for encryption of columns + // key encryption key (kek) (keks encrypt/decrypt data encryption keys (deks)) + function.named_parameters["kek"] = LogicalType::VARCHAR; + function.named_parameters["column"] = LogicalType::VARCHAR; + function.named_parameters["key_id"] = LogicalType::VARCHAR; +} + +static void InsertColumnKeys(KeyValueSecret &result, string column_key_name) { + result.redact_keys.insert(column_key_name); +} + +static unique_ptr +CreateKeyEncryptionKey(ClientContext &context, CreateSecretInput &input) { + + auto scope = input.scope; + + auto result = + make_uniq(scope, input.type, input.provider, input.name); + + // Manage specific secret option + CopySecret("column", input, *result); + CopySecret("key_id", input, *result); + + // Redact sensible keys +// RedactCommonKeys(*result); + + result->redact_keys.insert("column"); + + return std::move(result); +} + +void CoreSecretFunctions::RegisterStoreEncryptSecretFunction(DatabaseInstance &db) { + + string type = "internal"; + + // Register the new secret type + SecretType secret_type; + secret_type.name = type; + secret_type.deserializer = KeyValueSecret::Deserialize; + secret_type.default_provider = "client"; + ExtensionUtil::RegisterSecretType(db, secret_type); + + // Register the key_encryption_key secret provider + CreateSecretFunction key_encryption_key = {type, "client", CreateKeyEncryptionKey}; + + key_encryption_key.named_parameters["client"] = LogicalType::VARCHAR; + RegisterCommonSecretParameters(key_encryption_key); + ExtensionUtil::RegisterFunction(db, key_encryption_key); + +} + +} +} \ No newline at end of file diff --git a/src/core/module.cpp b/src/core/module.cpp index c715339..9596bae 100644 --- a/src/core/module.cpp +++ b/src/core/module.cpp @@ -1,12 +1,14 @@ #include "simple_encryption/core/module.hpp" #include "simple_encryption/common.hpp" #include "simple_encryption/core/functions/scalar.hpp" +#include "simple_encryption/core/functions/secrets.hpp" namespace simple_encryption { namespace core { void CoreModule::Register(DatabaseInstance &db) { CoreScalarFunctions::Register(db); + CoreSecretFunctions::Register(db); } } // namespace core } // namespace simple_encryption \ No newline at end of file diff --git a/src/include/simple_encryption/core/crypto/crypto_primitives.hpp b/src/include/simple_encryption/core/crypto/crypto_primitives.hpp index b9c2cdb..ff6a3ca 100644 --- a/src/include/simple_encryption/core/crypto/crypto_primitives.hpp +++ b/src/include/simple_encryption/core/crypto/crypto_primitives.hpp @@ -10,18 +10,6 @@ typedef struct evp_cipher_ctx_st EVP_CIPHER_CTX; namespace duckdb { -typedef unsigned char hash_bytes[32]; -typedef unsigned char hash_str[64]; - -void sha256(const char *in, size_t in_len, hash_bytes &out); - -void hmac256(const std::string &message, const char *secret, size_t secret_len, - hash_bytes &out); - -void hmac256(std::string message, hash_bytes secret, hash_bytes &out); - -void hex256(hash_bytes &in, hash_str &out); - class DUCKDB_EXTENSION_API AESStateSSL : public duckdb::EncryptionState { public: diff --git a/src/include/simple_encryption/core/functions/scalar.hpp b/src/include/simple_encryption/core/functions/scalar.hpp index c15efe3..4c12e97 100644 --- a/src/include/simple_encryption/core/functions/scalar.hpp +++ b/src/include/simple_encryption/core/functions/scalar.hpp @@ -14,8 +14,7 @@ struct CoreScalarFunctions { private: static void RegisterEncryptDataScalarFunction(duckdb::DatabaseInstance &db); - static void - RegisterEncryptDataStructScalarFunction(duckdb::DatabaseInstance &db); + static void RegisterEncryptDataStructScalarFunction(duckdb::DatabaseInstance &db); }; } // namespace core diff --git a/src/include/simple_encryption/core/functions/secrets.hpp b/src/include/simple_encryption/core/functions/secrets.hpp new file mode 100644 index 0000000..742c536 --- /dev/null +++ b/src/include/simple_encryption/core/functions/secrets.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include "duckdb/main/database.hpp" + +namespace simple_encryption { + +namespace core { + +std::string read_token_from_file(const std::string &file_path); +duckdb::string_t GenerateDataEncryptionKey(); + +struct CoreSecretFunctions { +public: + //! Register all CreateSecretFunctions + static void Register(duckdb::DatabaseInstance &db){ + RegisterStoreEncryptSecretFunction(db); +// RegisterGetEncryptSecretStructFunction(db); +// RegisterDeleteEncryptSecretStructFunction(db); + } + +private: + static void RegisterStoreEncryptSecretFunction(duckdb::DatabaseInstance &db); +// static void RegisterGetEncryptSecretStructFunction(duckdb::DatabaseInstance &db); +// static void RegisterDeleteEncryptSecretStructFunction(duckdb::DatabaseInstance &db); +}; + +} +} diff --git a/test/sql/bugfix_varchar_struct.test b/test/sql/encryption/bugfix_varchar_struct.test similarity index 100% rename from test/sql/bugfix_varchar_struct.test rename to test/sql/encryption/bugfix_varchar_struct.test diff --git a/test/sql/bulk_encryption.test b/test/sql/encryption/bulk_encryption.test similarity index 100% rename from test/sql/bulk_encryption.test rename to test/sql/encryption/bulk_encryption.test diff --git a/test/sql/encrypt_column.test b/test/sql/encryption/encrypt_column.test similarity index 100% rename from test/sql/encrypt_column.test rename to test/sql/encryption/encrypt_column.test diff --git a/test/sql/encrypt_decrypt.test b/test/sql/encryption/encrypt_decrypt.test similarity index 100% rename from test/sql/encrypt_decrypt.test rename to test/sql/encryption/encrypt_decrypt.test diff --git a/test/sql/simple_encryption.test b/test/sql/encryption/simple_encryption.test similarity index 100% rename from test/sql/simple_encryption.test rename to test/sql/encryption/simple_encryption.test diff --git a/test/sql/simple_struct_encryption.test b/test/sql/encryption/simple_struct_encryption.test similarity index 100% rename from test/sql/simple_struct_encryption.test rename to test/sql/encryption/simple_struct_encryption.test diff --git a/test/sql/secrets/simple_secrets_test.test b/test/sql/secrets/simple_secrets_test.test new file mode 100644 index 0000000..71fce72 --- /dev/null +++ b/test/sql/secrets/simple_secrets_test.test @@ -0,0 +1,20 @@ +# name: test/sql/secrets/simple_secrets_test.test +# description: Test secret creation for internal encryption +# group: [simple-encryption/secrets] + +statement ok +PRAGMA enable_verification; + +require simple_encryption + +# Ensure any currently stored secrets don't interfere with the test +statement ok +set allow_persistent_secrets=false; + +# Create an internal secret (for internal encryption of columns) +statement ok +CREATE SECRET ( + TYPE INTERNAL, + KEY_ID 'key_id', + COLUMN 'column' +); \ No newline at end of file