Skip to content

Commit

Permalink
gNOI Warm Reboot - Added tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@rkavitha-hcl
rkavitha-hcl authored and KAVITHA RAMALINGAM committed Dec 5, 2024
1 parent ec885af commit ad96646
Show file tree
Hide file tree
Showing 3 changed files with 246 additions and 7 deletions.
9 changes: 5 additions & 4 deletions src/sonic-framework/rebootbackend/rebootbe.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ RebootBE::RebootBE(DbusInterface &dbus_interface)
}

RebootBE::RebManagerStatus RebootBE::GetCurrentStatus() {
const std::lock_guard<std::mutex> lock(m_StatusMutex);
return m_CurrentStatus;
}

Expand All @@ -43,6 +42,9 @@ void RebootBE::SetCurrentStatus(RebManagerStatus newStatus) {
void RebootBE::Start() {
SWSS_LOG_ENTER();
SWSS_LOG_NOTICE("--- Starting rebootbackend ---");
swss::WarmStart::initialize("rebootbackend", "sonic-framework");
swss::WarmStart::checkWarmStart("rebootbackend", "sonic-framework",
/*incr_restore_cnt=*/false);

swss::Select s;
s.addSelectable(&m_NotificationConsumer);
Expand All @@ -51,7 +53,6 @@ void RebootBE::Start() {


if (swss::WarmStart::isWarmStart()) {
SWSS_LOG_NOTICE("Launching init thread for warm start");
SetCurrentStatus(RebManagerStatus::WARM_INIT_WAIT);
} else {
SWSS_LOG_NOTICE("Warm restart not enabled");
Expand Down Expand Up @@ -176,9 +177,9 @@ bool RebootBE::RebootAllowed(const gnoi::system::RebootMethod rebMethod) {
case RebManagerStatus::WARM_REBOOT_IN_PROGRESS: {
return false;
}
case RebManagerStatus::WARM_INIT_WAIT: {
case RebManagerStatus::WARM_INIT_WAIT: {
return rebMethod == gnoi::system::RebootMethod::COLD;
}
}
case RebManagerStatus::IDLE: {
return true;
}
Expand Down
65 changes: 65 additions & 0 deletions src/sonic-framework/tests/reboot_thread_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,34 @@ TEST_F(RebootStatusTest, TestGetStatus) {
EXPECT_EQ(0, response.when());
}

TEST_F(RebootStatusTest, TestGetWarmStatus) {
std::chrono::nanoseconds curr_ns = std::chrono::high_resolution_clock::now().time_since_epoch();

m_status.set_start_status(gnoi::system::RebootMethod::WARM, "reboot because");

gnoi::system::RebootStatusResponse response = m_status.get_response();
EXPECT_EQ(response.status().status(),
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN);

m_status.set_completed_status(
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS, "anything");

response = m_status.get_response();

// message should be empty while reboot is active
EXPECT_THAT(response.status().message(), StrEq(""));

uint64_t reboot_ns = response.when();
EXPECT_TRUE(reboot_ns > (uint64_t)curr_ns.count());

m_status.set_inactive();
response = m_status.get_response();
EXPECT_THAT(response.status().message(), StrEq("anything"));
EXPECT_EQ(response.status().status(),
gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS);
EXPECT_EQ(0, response.when());
}

class RebootThreadTest : public ::testing::Test {
protected:
RebootThreadTest()
Expand Down Expand Up @@ -278,4 +306,41 @@ TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) {
IsStatus(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_F(RebootThreadTest, TestNoWarmIfNonRetriableFailure) {
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
set_completed_status(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"failed to warm reboot");
force_inactive();

gnoi::system::RebootRequest request;
request.set_method(gnoi::system::RebootMethod::WARM);

NotificationResponse response = m_reboot_thread.Start(request);
EXPECT_EQ(response.status, swss::StatusCode::SWSS_RC_FAILED_PRECONDITION);
EXPECT_EQ(response.json_string,
"RebootThread: last WARM reboot failed with non-retriable failure");
}

TEST_F(RebootThreadTest, TestSigTermStartofDoReboot) {
sigterm_requested = true;
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
do_reboot();
force_inactive();
gnoi::system::RebootStatusResponse response = m_reboot_thread.GetResponse();
EXPECT_THAT(
response,
IsStatus(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_F(RebootThreadTest, TestWaitForRebootPositive) {
overwrite_reboot_timeout(1);
set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot");
swss::Select s;
swss::SelectableEvent m_stop;
s.addSelectable(&m_stop);
RebootThread::Progress progress = wait_for_platform_reboot(s);
EXPECT_EQ(progress, RebootThread::Progress::PROCEED);
}


} // namespace rebootbackend
179 changes: 176 additions & 3 deletions src/sonic-framework/tests/rebootbe_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#include <string>
#include <thread>
#include <vector>
#include<iostream>

#include "mock_reboot_interfaces.h"
#include "reboot_common.h"
Expand Down Expand Up @@ -63,6 +62,7 @@ class RebootBETestWithoutStop : public ::testing::Test {
m_rebootbeReponseChannel(&m_db, REBOOT_RESPONSE_NOTIFICATION_CHANNEL),
m_rebootbe(m_dbus_interface) {
sigterm_requested = false;
// TestUtils::clear_tables(m_db);


m_s.addSelectable(&m_rebootbeReponseChannel);
Expand All @@ -74,6 +74,15 @@ class RebootBETestWithoutStop : public ::testing::Test {
}
virtual ~RebootBETestWithoutStop() = default;

void force_warm_start_state(bool enabled) {
swss::Table enable_table(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME);
enable_table.hset("system", "enable", enabled ? "true" : "false");
enable_table.hset("sonic-framework", "enable", enabled ? "true" : "false");

swss::Table restart_table(&m_db, STATE_WARM_RESTART_TABLE_NAME);
restart_table.hset("rebootbackend", "restore_count", enabled ? "0" : "");
}

void start_rebootbe() {
m_rebootbe_thread =
std::make_unique<std::thread>(&RebootBE::Start, &m_rebootbe);
Expand Down Expand Up @@ -202,18 +211,62 @@ class RebootBETest : public RebootBETestWithoutStop {
}
};

TEST_F(RebootBETest, WarmbootInProgressBlocksNewWarmboot) {
force_warm_start_state(true);

start_rebootbe();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);

// Send a warmboot request, confirm it fails.
RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request, swss::StatusCode::SWSS_RC_IN_USE);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);
force_warm_start_state(false);
}


TEST_F(RebootBETest, ColdbootWhileWarmbootInProgress) {
force_warm_start_state(true);
set_mock_defaults();

start_rebootbe();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);

// Send a coldboot request, confirm it starts.
RebootRequest request;
request.set_method(RebootMethod::COLD);
start_reboot_via_rpc(request);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::COLD_REBOOT_IN_PROGRESS);

// Cleanup without going through the whole reboot.
send_stop_reboot_thread();
force_warm_start_state(false);
}


// Test fixture to skip through the startup sequence into the main loop.
// Param indicates if RebootBE should be initialized into a state where the
// system came up in warmboot.
class RebootBEAutoStartTest : public RebootBETest,
public ::testing::WithParamInterface<bool> {
protected:
RebootBEAutoStartTest() {
//force_warm_start_state(GetParam());

start_rebootbe();

std::this_thread::sleep_for(std::chrono::milliseconds(50));
//std::this_thread::sleep_for(std::chrono::milliseconds(50));
std::this_thread::sleep_for(std::chrono::milliseconds(ONE_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
}
};
Expand Down Expand Up @@ -312,6 +365,31 @@ TEST_P(RebootBEAutoStartTest, TestColdRebootDbusToCompletion) {
"platform failed to reboot"));
}


TEST_P(RebootBEAutoStartTest, TestWarmRebootDbusToCompletion) {
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
EXPECT_CALL(m_dbus_interface, Reboot(_))
.Times(1)
.WillRepeatedly(Return(dbus_response));

overwrite_reboot_timeout(1);
RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

sleep(TWO_SECONDS);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"failed to warm reboot"));

}

TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
sigterm_requested = true;
set_mock_defaults();
Expand All @@ -331,6 +409,25 @@ TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestWarmBootSigterm) {
sigterm_requested = true;
set_mock_defaults();
overwrite_reboot_timeout(1);

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

sleep(ONE_SECOND);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(
second_resp,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
// Return FAIL from dbus reboot call.
DbusInterface::DbusResponse dbus_response{
Expand All @@ -353,6 +450,28 @@ TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
"dbus reboot failed"));
}

TEST_P(RebootBEAutoStartTest, TestWarmBootDbusError) {
// Return FAIL from dbus reboot call.
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_FAIL, "dbus reboot failed"};
EXPECT_CALL(m_dbus_interface, Reboot(_))
.Times(1)
.WillOnce(Return(dbus_response));

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

sleep(TWO_SECONDS);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(second_resp,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"dbus reboot failed"));
}

TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
set_mock_defaults();

Expand All @@ -374,12 +493,66 @@ TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestStopDuringWarmBoot) {
set_mock_defaults();

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);
// std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

send_stop_reboot_thread();
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);

gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(
response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestInvalidJsonRebootRequest) {
std::string json_request = "abcd";
NotificationResponse response = handle_reboot_request(json_request);
EXPECT_EQ(swss::StatusCode::SWSS_RC_INTERNAL, response.status);
}

TEST_P(RebootBEAutoStartTest, TestWarmFailureFollowedByColdBoot) {
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
overwrite_reboot_timeout(1);

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));


request.set_method(RebootMethod::COLD);
start_reboot_via_rpc(request);

// We have to wait for the 1 second reboot Timeout
std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 2, RebootMethod::COLD));
EXPECT_THAT(response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"platform failed to reboot"));
}


INSTANTIATE_TEST_SUITE_P(TestWithStartupWarmbootEnabledState,
RebootBEAutoStartTest, testing::Values(true, false));

Expand Down

0 comments on commit ad96646

Please sign in to comment.