Skip to content

Commit

Permalink
Merge pull request #7218 from jackyalbo/jacky-root-key-rotate
Browse files Browse the repository at this point in the history
Root key rotate - core side
  • Loading branch information
jackyalbo authored Apr 19, 2023
2 parents f4c47f1 + 65d0f56 commit 57b0dde
Show file tree
Hide file tree
Showing 14 changed files with 356 additions and 133 deletions.
37 changes: 33 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ NAME_POSTFIX?="$(shell ${CONTAINER_ENGINE} ps -a | wc -l | xargs)"
BUILDER_TAG?="noobaa-builder"
TESTER_TAG?="noobaa-tester"
POSTGRES_IMAGE?="centos/postgresql-12-centos7"
MONGO_IMAGE?="centos/mongodb-36-centos7"
NOOBAA_TAG?="noobaa"
NOOBAA_BASE_TAG?="noobaa-base"
SUPPRESS_LOGS?=""
Expand Down Expand Up @@ -79,13 +80,39 @@ tester: noobaa
.PHONY: tester

test: tester
@echo "\033[1;34mRunning tests.\033[0m"
$(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" $(TESTER_TAG)
@echo "\033[1;34mRunning tests with Mongo.\033[0m"
@echo "\033[1;34mCreating docker network\033[0m"
$(CONTAINER_ENGINE) network create noobaa-net || true
@echo "\033[1;34mRunning Mongo container\033[0m"
$(CONTAINER_ENGINE) run -d $(CPUSET) --network noobaa-net --name coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX) --env "MONGODB_ADMIN_PASSWORD=noobaa" --env "MONGODB_DATABASE=coretest" --env "MONGODB_USER=noobaa" --env "MONGODB_PASSWORD=noobaa" $(MONGO_IMAGE)
@echo "\033[1;34mRunning tests\033[0m"
$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG)
@echo "\033[1;34mStopping/removing test container\033[0m"
$(CONTAINER_ENGINE) stop noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
$(CONTAINER_ENGINE) rm noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
@echo "\033[1;34mStopping/removing Mongo container\033[0m"
$(CONTAINER_ENGINE) stop coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
$(CONTAINER_ENGINE) rm coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
@echo "\033[1;34mRemove docker network\033[0m"
$(CONTAINER_ENGINE) network rm noobaa-net
.PHONY: test

run-single-test: tester
@echo "\033[1;34mRunning single test.\033[0m"
$(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname)
@echo "\033[1;34mRunning tests with Mongo.\033[0m"
@echo "\033[1;34mCreating docker network\033[0m"
$(CONTAINER_ENGINE) network create noobaa-net || true
@echo "\033[1;34mRunning Mongo container\033[0m"
$(CONTAINER_ENGINE) run -d $(CPUSET) --network noobaa-net --name coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX) --env "MONGODB_ADMIN_PASSWORD=noobaa" --env "MONGODB_DATABASE=coretest" --env "MONGODB_USER=noobaa" --env "MONGODB_PASSWORD=noobaa" $(MONGO_IMAGE)
@echo "\033[1;34mRunning tests\033[0m"
$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname)
@echo "\033[1;34mStopping/removing test container\033[0m"
$(CONTAINER_ENGINE) stop noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
$(CONTAINER_ENGINE) rm noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
@echo "\033[1;34mStopping/removing Mongo container\033[0m"
$(CONTAINER_ENGINE) stop coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
$(CONTAINER_ENGINE) rm coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
@echo "\033[1;34mRemove docker network\033[0m"
$(CONTAINER_ENGINE) network rm noobaa-net
.PHONY: run-single-test


Expand All @@ -95,6 +122,8 @@ run-single-test-postgres: tester
$(CONTAINER_ENGINE) network create noobaa-net || true
@echo "\033[1;34mRunning Postgres container\033[0m"
$(CONTAINER_ENGINE) run -d $(CPUSET) --network noobaa-net --name coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX) --env "POSTGRESQL_DATABASE=coretest" --env "POSTGRESQL_USER=noobaa" --env "POSTGRESQL_PASSWORD=noobaa" --env "LC_COLLATE=C" $(POSTGRES_IMAGE)
@echo "\033[1;34mWaiting for postgres to start..\033[0m"
sleep 20
@echo "\033[1;34mRunning tests\033[0m"
$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "PG_ENABLE_QUERY_LOG=true" --env "PG_EXPLAIN_QUERIES=true" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname)
@echo "\033[1;34mStopping/removing test container\033[0m"
Expand Down
11 changes: 11 additions & 0 deletions config.js
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ config.JWT_SECRET = process.env.JWT_SECRET || _get_data_from_file(`/etc/noobaa-s
config.SERVER_SECRET = process.env.SERVER_SECRET || _get_data_from_file(`/etc/noobaa-server/server_secret`);
config.NOOBAA_AUTH_TOKEN = process.env.NOOBAA_AUTH_TOKEN || _get_data_from_file(`/etc/noobaa-auth-token/auth_token`);

config.ROOT_KEY_MOUNT = '/etc/noobaa-server/root_keys';

///////////////
// MD CONFIG //
///////////////
Expand Down Expand Up @@ -348,6 +350,7 @@ config.DEFAULT_S3_AUTH_METHOD = {
//////////////////////

config.LIFECYCLE_INTERVAL = 8 * 60 * 60 * 1000; // 8h
config.LIFECYCLE_ENABLED = true;

//////////////////////////
// STATISTICS_COLLECTOR //
Expand Down Expand Up @@ -554,6 +557,14 @@ config.LOG_REPLICATION_ENABLED = true;
config.AWS_LOG_CANDIDATES_LIMIT = 10;
config.BUCKET_LOG_REPLICATOR_DELAY = 5 * 60 * 1000;

///////////////////////////
// KEY ROTATOR //
///////////////////////////

config.KEY_ROTATOR_ENABLED = true;
config.KEY_ROTATOR_RUN_INTERVAL = 24 * 60 * 60 * 1000; // Once a day,
config.KEY_ROTATOR_ERROR_DELAY = 10 * 60 * 1000; // Run again in 10 minutes

///////////////////////
// NAMESPACE CACHING //
///////////////////////
Expand Down
14 changes: 0 additions & 14 deletions src/api/system_api.js
Original file line number Diff line number Diff line change
Expand Up @@ -115,20 +115,6 @@ module.exports = {
}
},

rotate_root_key: {
method: 'PUT',
params: {
type: 'object',
required: ['new_root_key'],
properties: {
new_root_key: { $ref: 'common_api#/definitions/secret_enc_key' },
}
},
auth: {
system: 'admin'
}
},

rotate_master_key: {
method: 'PUT',
params: {
Expand Down
10 changes: 0 additions & 10 deletions src/deploy/NVA_build/Tests.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,8 @@ ENV TEST_CONTAINER true
# Cache: rebuild when we adding/removing requirments
##############################################################

# RUN dnf install -y ntpdate vim && \
COPY ./src/deploy/NVA_build/set_mongo_repo.sh /tmp/
RUN chmod +x /tmp/set_mongo_repo.sh && \
/bin/bash -xc "/tmp/set_mongo_repo.sh"

RUN dnf group install -y -q "Development Tools" && \
dnf install -y -q --nogpgcheck vim \
mongodb-org-3.6.3 \
mongodb-org-server-3.6.3 \
mongodb-org-shell-3.6.3 \
mongodb-org-mongos-3.6.3 \
mongodb-org-tools-3.6.3 \
which python3-virtualenv python36-devel libevent-devel libffi-devel libxml2-devel libxslt-devel zlib-devel \
git && \
dnf clean all
Expand Down
68 changes: 68 additions & 0 deletions src/server/bg_services/key_rotator.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/* Copyright (C) 2016 NooBaa */
'use strict';

const _ = require('lodash');

const dbg = require('../../util/debug_module')(__filename);
const system_store = require('../system_services/system_store').get_instance();
const system_utils = require('../utils/system_utils');
const config = require('../../../config');

class KeyRotator {

constructor({ name }) {
this.name = name;
}

async run_batch() {
if (!this._can_run()) return config.KEY_ROTATOR_ERROR_DELAY;
const mkm = system_store.master_key_manager;
await mkm.load_root_keys_from_mount();

dbg.log0('KeyRotator: new rotate cycle has started');
const system = system_store.data.systems[0];
// if the system root_key_id exist and enabled - nothing to do
if (system.master_key_id.root_key_id && !mkm.is_m_key_disabled(system.master_key_id.root_key_id)) {
return config.KEY_ROTATOR_RUN_INTERVAL;
}
try {
const active_root_key_id = mkm.get_root_key_id();
const reencrypted = mkm._reencrypt_master_key_by_current_root(
system.master_key_id._id,
active_root_key_id
);
const unset = system.master_key_id.master_key_id && { master_key_id: 1 };
await system_store.make_changes({
update: {
master_keys: [_.omitBy({
_id: system.master_key_id._id,
$set: {
root_key_id: active_root_key_id,
cipher_key: reencrypted
},
$unset: unset,
}, _.isUndefined)]
}
});
} catch (err) {
dbg.error(`KeyRotator: got error when trying to rotate system ${system._id} root key:`, err);
return config.KEY_ROTATOR_ERROR_DELAY;
}
dbg.log0(`KeyRotator: root key for system ${system._id} was rotated succesfully`);
return config.KEY_ROTATOR_RUN_INTERVAL;
}

_can_run() {
if (!system_store.is_finished_initial_load) {
dbg.log0('KeyRotator: system_store did not finish initial load');
return false;
}
const system = system_store.data.systems[0];
if (!system || system_utils.system_in_maintenance(system._id)) return false;

return true;
}
}

// EXPORTS
exports.KeyRotator = KeyRotator;
12 changes: 10 additions & 2 deletions src/server/bg_workers.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ const background_scheduler = require('../util/background_scheduler').get_instanc
const stats_collector = require('./bg_services/stats_collector');
const dedup_indexer = require('./bg_services/dedup_indexer');
const db_cleaner = require('./bg_services/db_cleaner');
const { KeyRotator } = require('./bg_services/key_rotator');
const prom_reporting = require('./analytic_services/prometheus_reporting');

const MASTER_BG_WORKERS = [
Expand All @@ -53,7 +54,8 @@ const MASTER_BG_WORKERS = [
'db_cleaner',
'aws_usage_metering',
'agent_blocks_verifier',
'agent_blocks_reclaimer'
'agent_blocks_reclaimer',
'key_rotator'
];

if (process.env.NOOBAA_LOG_LEVEL) {
Expand Down Expand Up @@ -211,7 +213,7 @@ function run_master_workers() {
dbg.warn('AGENT BLOCKS RECLAIMER NOT ENABLED');
}

if (config.LIFECYCLE_DISABLED !== 'true') {
if (config.LIFECYCLE_ENABLED) {
register_bg_worker({
name: 'lifecycle',
delay: config.LIFECYCLE_INTERVAL,
Expand All @@ -231,6 +233,12 @@ function run_master_workers() {
delay: config.AWS_METERING_INTERVAL
}, aws_usage_metering.background_worker);
}

if (config.KEY_ROTATOR_ENABLED) {
register_bg_worker(new KeyRotator({ name: 'key rotator' }));
} else {
dbg.warn('KEY ROATATION NOT ENABLED');
}
}

async function start_bg_workers_server() {
Expand Down
Loading

0 comments on commit 57b0dde

Please sign in to comment.