Adding SECRET syntax

CMakeLists.txt

@@ -30,6 +30,7 @@ set(EXTENSION_SOURCES
     src/core/functions/function_data/encrypt_function_data.cpp
     src/core/functions/cast/varchar_cast.cpp
     src/core/functions/table/encrypt_table.cpp
+    src/core/functions/secrets/authentication.cpp
     src/core/utils/simple_encryption_utils.cpp
     src/core/crypto/crypto_primitives.cpp)
 

src/core/crypto/crypto_primitives.cpp

@@ -11,30 +11,6 @@
 
 namespace duckdb {
 
-void sha256(const char *in, size_t in_len, hash_bytes &out) {
-  duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
-}
-
-void hmac256(const std::string &message, const char *secret, size_t secret_len,
-             hash_bytes &out) {
-  duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(),
-                                          message.size(), (char *)out);
-}
-
-void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
-  hmac256(message, (char *)secret, sizeof(hash_bytes), out);
-}
-
-void hex256(hash_bytes &in, hash_str &out) {
-  const char *hex = "0123456789abcdef";
-  unsigned char *pin = in;
-  unsigned char *pout = out;
-  for (; pin < in + sizeof(in); pout += 2, pin++) {
-    pout[0] = hex[(*pin >> 4) & 0xF];
-    pout[1] = hex[*pin & 0xF];
-  }
-}
-
 const EVP_CIPHER *GetCipher(const string &key,
                             AESStateSSL::Algorithm algorithm) {
 

src/core/functions/scalar/encrypt_to_etype.cpp

@@ -1,8 +1,5 @@
 #define DUCKDB_EXTENSION_MAIN
 
-// what is the maximum size of biggest type in duckdb
-#define MAX_BUFFER_SIZE 1024
-
 #include "duckdb.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types.hpp"
@@ -22,8 +19,6 @@
 #include "simple_encryption/core/types.hpp"
 #include "duckdb/common/vector_operations/generic_executor.hpp"
 
-// temporary
-
 namespace simple_encryption {
 
 namespace core {
@@ -79,6 +74,7 @@ ProcessAndCastDecrypt(shared_ptr<EncryptionState> encryption_state,
   size_t decrypted_size = encrypted_size;
   Blob::FromBase64(base64_data, reinterpret_cast<data_ptr_t>(buffer_p),
                    encrypted_size);
+
   D_ASSERT(encrypted_size <= base64_data.GetSize());
 
   string_t decrypted_data =

src/core/functions/secrets/authentication.cpp

@@ -0,0 +1,109 @@
+#include "simple_encryption/core/functions/secrets.hpp"
+#include "simple_encryption/common.hpp"
+#include "simple_encryption/core/utils/simple_encryption_utils.hpp"
+#include "simple_encryption/core/crypto/crypto_primitives.hpp"
+#include "simple_encryption/core/functions/scalar.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/main/secret/secret.hpp"
+#include "duckdb/main/extension_util.hpp"
+#include "openssl/rand.h"
+#include <fstream>
+#include <cstdlib>
+
+namespace simple_encryption {
+
+namespace core {
+
+ string_t GenerateDataEncryptionKey(const uint32_t size){
+
+   unsigned char* key = new unsigned char[size];
+
+   // generate random bytes with OpenSSL function
+   RAND_bytes(key, size);
+
+   // cast back to string ('normal' string, not duckdb string)
+   std::string key_string(reinterpret_cast<char*>(key), size);
+
+  return key_string;
+}
+
+string_t GetDataEncryptionKey(const uint32_t size){
+
+  switch(size){
+  case 16:
+    return GenerateDataEncryptionKey(16);
+  case 24:
+    return GenerateDataEncryptionKey(24);
+  case 32:
+    return GenerateDataEncryptionKey(32);
+  default:
+    throw InvalidInputException("Invalid size for data encryption key: '%d', expected: 16, 24, or 32", size);
+  }
+}
+
+// This code partly copied / inspired by the gsheets extension for duckdb
+static void CopySecret(const std::string &key, const CreateSecretInput &input,
+                       KeyValueSecret &result) {
+
+  // this method checks whether a secret_param is present in the secret_map
+  auto val = input.options.find(key);
+  // does this also take a key, value or list struct?
+  if (val != input.options.end()) {
+    result.secret_map[key] = val->second;
+  }
+}
+
+static void RegisterCommonSecretParameters(CreateSecretFunction &function) {
+  // Register named parameters for encryption of columns
+  // key encryption key (kek) (keks encrypt/decrypt data encryption keys (deks))
+  function.named_parameters["kek"] = LogicalType::VARCHAR;
+  function.named_parameters["column"] = LogicalType::VARCHAR;
+  function.named_parameters["key_id"] = LogicalType::VARCHAR;
+}
+
+static void InsertColumnKeys(KeyValueSecret &result, string column_key_name) {
+  result.redact_keys.insert(column_key_name);
+}
+
+static unique_ptr<BaseSecret>
+CreateKeyEncryptionKey(ClientContext &context, CreateSecretInput &input) {
+
+  auto scope = input.scope;
+
+  auto result =
+      make_uniq<KeyValueSecret>(scope, input.type, input.provider, input.name);
+
+  // Manage specific secret option
+  CopySecret("column", input, *result);
+  CopySecret("key_id", input, *result);
+
+  // Redact sensible keys
+//  RedactCommonKeys(*result);
+
+  result->redact_keys.insert("column");
+
+  return std::move(result);
+}
+
+void CoreSecretFunctions::RegisterStoreEncryptSecretFunction(DatabaseInstance &db) {
+
+  string type = "internal";
+
+  // Register the new secret type
+  SecretType secret_type;
+  secret_type.name = type;
+  secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
+  secret_type.default_provider = "client";
+  ExtensionUtil::RegisterSecretType(db, secret_type);
+
+  // Register the key_encryption_key secret provider
+  CreateSecretFunction key_encryption_key = {type, "client", CreateKeyEncryptionKey};
+
+  key_encryption_key.named_parameters["client"] = LogicalType::VARCHAR;
+  RegisterCommonSecretParameters(key_encryption_key);
+  ExtensionUtil::RegisterFunction(db, key_encryption_key);
+
+}
+
+}
+}

src/core/module.cpp

@@ -1,12 +1,14 @@
 #include "simple_encryption/core/module.hpp"
 #include "simple_encryption/common.hpp"
 #include "simple_encryption/core/functions/scalar.hpp"
+#include "simple_encryption/core/functions/secrets.hpp"
 
 namespace simple_encryption {
 namespace core {
 
 void CoreModule::Register(DatabaseInstance &db) {
   CoreScalarFunctions::Register(db);
+  CoreSecretFunctions::Register(db);
 }
 } // namespace core
 } // namespace simple_encryption

src/include/simple_encryption/core/crypto/crypto_primitives.hpp

@@ -10,18 +10,6 @@ typedef struct evp_cipher_ctx_st EVP_CIPHER_CTX;
 
 namespace duckdb {
 
-typedef unsigned char hash_bytes[32];
-typedef unsigned char hash_str[64];
-
-void sha256(const char *in, size_t in_len, hash_bytes &out);
-
-void hmac256(const std::string &message, const char *secret, size_t secret_len,
-             hash_bytes &out);
-
-void hmac256(std::string message, hash_bytes secret, hash_bytes &out);
-
-void hex256(hash_bytes &in, hash_str &out);
-
 class DUCKDB_EXTENSION_API AESStateSSL : public duckdb::EncryptionState {
 
 public:

src/include/simple_encryption/core/functions/scalar.hpp

@@ -14,8 +14,7 @@ struct CoreScalarFunctions {
 
 private:
   static void RegisterEncryptDataScalarFunction(duckdb::DatabaseInstance &db);
-  static void
-  RegisterEncryptDataStructScalarFunction(duckdb::DatabaseInstance &db);
+  static void RegisterEncryptDataStructScalarFunction(duckdb::DatabaseInstance &db);
 };
 
 } // namespace core

src/include/simple_encryption/core/functions/secrets.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+#include "duckdb/main/database.hpp"
+
+namespace simple_encryption {
+
+namespace core {
+
+std::string read_token_from_file(const std::string &file_path);
+duckdb::string_t GenerateDataEncryptionKey();
+
+struct CoreSecretFunctions {
+public:
+  //! Register all CreateSecretFunctions
+  static void Register(duckdb::DatabaseInstance &db){
+    RegisterStoreEncryptSecretFunction(db);
+//    RegisterGetEncryptSecretStructFunction(db);
+//    RegisterDeleteEncryptSecretStructFunction(db);
+  }
+
+private:
+  static void RegisterStoreEncryptSecretFunction(duckdb::DatabaseInstance &db);
+//  static void RegisterGetEncryptSecretStructFunction(duckdb::DatabaseInstance &db);
+//  static void RegisterDeleteEncryptSecretStructFunction(duckdb::DatabaseInstance &db);
+};
+
+}
+}

test/sql/secrets/simple_secrets_test.test

@@ -0,0 +1,20 @@
+# name: test/sql/secrets/simple_secrets_test.test
+# description: Test secret creation for internal encryption
+# group: [simple-encryption/secrets]
+
+statement ok
+PRAGMA enable_verification;
+
+require simple_encryption
+
+# Ensure any currently stored secrets don't interfere with the test
+statement ok
+set allow_persistent_secrets=false;
+
+# Create an internal secret (for internal encryption of columns)
+statement ok
+CREATE SECRET (
+	TYPE INTERNAL,
+    KEY_ID 'key_id',
+    COLUMN 'column'
+);