Skip to content

Commit

Permalink
gNOI Warm Reboot - Added tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@rkavitha-hcl
rkavitha-hcl committed Nov 14, 2024
1 parent 8c2b361 commit 6fa9455
Show file tree
Hide file tree
Showing 4 changed files with 302 additions and 4 deletions.
67 changes: 66 additions & 1 deletion src/sonic-framework/tests/reboot_thread_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,34 @@ TEST_F(RebootStatusTest, TestGetStatus) {
EXPECT_EQ(0, response.when());
}

TEST_F(RebootStatusTest, TestGetWarmStatus) {
std::chrono::nanoseconds curr_ns = std::chrono::high_resolution_clock::now().time_since_epoch();

m_status.set_start_status(RebootMethod::WARM, "reboot because");

RebootStatusResponse response = m_status.get_response();
EXPECT_EQ(response.status().status(),
RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN);

m_status.set_completed_status(
RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS, "anything");

response = m_status.get_response();

// message should be empty while reboot is active
EXPECT_THAT(response.status().message(), StrEq(""));

uint64_t reboot_ns = response.when();
EXPECT_TRUE(reboot_ns > (uint64_t)curr_ns.count());

m_status.set_inactive();
response = m_status.get_response();
EXPECT_THAT(response.status().message(), StrEq("anything"));
EXPECT_EQ(response.status().status(),
RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS);
EXPECT_EQ(0, response.when());
}

class RebootThreadTest : public ::testing::Test {
protected:
RebootThreadTest()
Expand Down Expand Up @@ -263,7 +291,7 @@ TEST_F(RebootThreadTest, TestUnsupportedRebootType) {
"RebootThread: Start rx'd unsupported method");
}

TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) {
TEST_F(RebootThreadTest, TestInvalidMethodDoReboot) {
set_start_status(RebootMethod::POWERUP, "time to reboot");
do_reboot();
force_inactive();
Expand All @@ -273,4 +301,41 @@ TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) {
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_F(RebootThreadTest, TestNoWarmIfNonRetriableFailure) {
set_start_status(RebootMethod::WARM, "time to reboot");
set_completed_status(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"failed to warm reboot");
force_inactive();

RebootRequest request;
request.set_method(RebootMethod::WARM);

NotificationResponse response = m_reboot_thread.Start(request);
EXPECT_EQ(response.status, swss::StatusCode::SWSS_RC_FAILED_PRECONDITION);
EXPECT_EQ(response.json_string,
"RebootThread: last WARM reboot failed with non-retriable failure");
}

TEST_F(RebootThreadTest, TestSigTermStartofDoReboot) {
sigterm_requested = true;
set_start_status(RebootMethod::WARM, "time to reboot");
do_reboot();
force_inactive();
RebootStatusResponse response = m_reboot_thread.GetResponse();
EXPECT_THAT(
response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_F(RebootThreadTest, TestWaitForRebootPositive) {
overwrite_reboot_timeout(1);
set_start_status(RebootMethod::WARM, "time to reboot");
swss::Select s;
swss::SelectableEvent m_stop;
s.addSelectable(&m_stop);
RebootThread::Progress progress = wait_for_platform_reboot(s);
EXPECT_EQ(progress, RebootThread::Progress::PROCEED);
}


} // namespace rebootbackend
179 changes: 176 additions & 3 deletions src/sonic-framework/tests/rebootbe_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#include <string>
#include <thread>
#include <vector>
#include<iostream>

#include "mock_reboot_interfaces.h"
#include "reboot_common.h"
Expand Down Expand Up @@ -63,6 +62,7 @@ class RebootBETestWithoutStop : public ::testing::Test {
m_rebootbeReponseChannel(&m_db, REBOOT_RESPONSE_NOTIFICATION_CHANNEL),
m_rebootbe(m_dbus_interface) {
sigterm_requested = false;
// TestUtils::clear_tables(m_db);


m_s.addSelectable(&m_rebootbeReponseChannel);
Expand All @@ -74,6 +74,15 @@ class RebootBETestWithoutStop : public ::testing::Test {
}
virtual ~RebootBETestWithoutStop() = default;

void force_warm_start_state(bool enabled) {
swss::Table enable_table(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME);
enable_table.hset("system", "enable", enabled ? "true" : "false");
enable_table.hset("sonic-framework", "enable", enabled ? "true" : "false");

swss::Table restart_table(&m_db, STATE_WARM_RESTART_TABLE_NAME);
restart_table.hset("rebootbackend", "restore_count", enabled ? "0" : "");
}

void start_rebootbe() {
m_rebootbe_thread =
std::make_unique<std::thread>(&RebootBE::Start, &m_rebootbe);
Expand Down Expand Up @@ -202,18 +211,62 @@ class RebootBETest : public RebootBETestWithoutStop {
}
};

TEST_F(RebootBETest, WarmbootInProgressBlocksNewWarmboot) {
force_warm_start_state(true);

start_rebootbe();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);

// Send a warmboot request, confirm it fails.
RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request, swss::StatusCode::SWSS_RC_IN_USE);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);
force_warm_start_state(false);
}


TEST_F(RebootBETest, ColdbootWhileWarmbootInProgress) {
force_warm_start_state(true);
set_mock_defaults();

start_rebootbe();
std::this_thread::sleep_for(std::chrono::milliseconds(100));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_INIT_WAIT);

// Send a coldboot request, confirm it starts.
RebootRequest request;
request.set_method(RebootMethod::COLD);
start_reboot_via_rpc(request);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::COLD_REBOOT_IN_PROGRESS);

// Cleanup without going through the whole reboot.
send_stop_reboot_thread();
force_warm_start_state(false);
}


// Test fixture to skip through the startup sequence into the main loop.
// Param indicates if RebootBE should be initialized into a state where the
// system came up in warmboot.
class RebootBEAutoStartTest : public RebootBETest,
public ::testing::WithParamInterface<bool> {
protected:
RebootBEAutoStartTest() {
//force_warm_start_state(GetParam());

start_rebootbe();

std::this_thread::sleep_for(std::chrono::milliseconds(50));
//std::this_thread::sleep_for(std::chrono::milliseconds(50));
std::this_thread::sleep_for(std::chrono::milliseconds(ONE_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
}
};
Expand Down Expand Up @@ -312,6 +365,31 @@ TEST_P(RebootBEAutoStartTest, TestColdRebootDbusToCompletion) {
"platform failed to reboot"));
}


TEST_P(RebootBEAutoStartTest, TestWarmRebootDbusToCompletion) {
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
EXPECT_CALL(m_dbus_interface, Reboot(_))
.Times(1)
.WillRepeatedly(Return(dbus_response));

overwrite_reboot_timeout(1);
RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

sleep(TWO_SECONDS);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"failed to warm reboot"));

}

TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
sigterm_requested = true;
set_mock_defaults();
Expand All @@ -331,6 +409,25 @@ TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) {
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestWarmBootSigterm) {
sigterm_requested = true;
set_mock_defaults();
overwrite_reboot_timeout(1);

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

sleep(ONE_SECOND);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(
second_resp,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
// Return FAIL from dbus reboot call.
DbusInterface::DbusResponse dbus_response{
Expand All @@ -353,6 +450,28 @@ TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) {
"dbus reboot failed"));
}

TEST_P(RebootBEAutoStartTest, TestWarmBootDbusError) {
// Return FAIL from dbus reboot call.
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_FAIL, "dbus reboot failed"};
EXPECT_CALL(m_dbus_interface, Reboot(_))
.Times(1)
.WillOnce(Return(dbus_response));

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

sleep(TWO_SECONDS);

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc();
EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(second_resp,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"dbus reboot failed"));
}

TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
set_mock_defaults();

Expand All @@ -374,12 +493,66 @@ TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) {
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestStopDuringWarmBoot) {
set_mock_defaults();

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);
// std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

send_stop_reboot_thread();
std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);

gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));
EXPECT_THAT(
response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, ""));
}

TEST_P(RebootBEAutoStartTest, TestInvalidJsonRebootRequest) {
std::string json_request = "abcd";
NotificationResponse response = handle_reboot_request(json_request);
EXPECT_EQ(swss::StatusCode::SWSS_RC_INTERNAL, response.status);
}

TEST_P(RebootBEAutoStartTest, TestWarmFailureFollowedByColdBoot) {
DbusInterface::DbusResponse dbus_response{
DbusInterface::DbusStatus::DBUS_SUCCESS, ""};
overwrite_reboot_timeout(1);

RebootRequest request;
request.set_method(RebootMethod::WARM);
start_reboot_via_rpc(request);

std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS));
EXPECT_EQ(m_rebootbe.GetCurrentStatus(),
RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS);

std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));
gnoi::system::RebootStatusResponse response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM));


request.set_method(RebootMethod::COLD);
start_reboot_via_rpc(request);

// We have to wait for the 1 second reboot Timeout
std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS));

EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE);
response = do_reboot_status_rpc();
EXPECT_THAT(response, ActiveCountMethod(false, 2, RebootMethod::COLD));
EXPECT_THAT(response,
IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE,
"platform failed to reboot"));
}


INSTANTIATE_TEST_SUITE_P(TestWithStartupWarmbootEnabledState,
RebootBEAutoStartTest, testing::Values(true, false));

Expand Down
43 changes: 43 additions & 0 deletions src/sonic-framework/tests/redis_utils_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,52 @@

namespace rebootbackend {

using WarmStartState = ::swss::WarmStart::WarmStartState;

using ::testing::AllOf;
using ::testing::HasSubstr;
using ::testing::StrEq;

class RedisTest : public ::testing::Test {
protected:
RedisTest() : m_db("STATE_DB", 0) {
TestUtils::clear_tables(m_db);
}

swss::DBConnector m_db;
};

TEST_F(RedisTest, testSetWarmRestartEnable) {
swss::Table warmRestartTable(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME);

for (const auto &enabled : {true, false}) {
warmRestartTable.del("system");

set_warm_restart_enable(m_db, enabled);

std::string value;
bool ret = warmRestartTable.hget("system", "enable", value);
EXPECT_TRUE(ret);
EXPECT_EQ(value, enabled ? "true" : "false");
}
}

TEST_F(RedisTest, GetWarmRestartCounter) {
EXPECT_THAT(get_warm_restart_counter(m_db), StrEq(""));
for (int i = 0; i < 5; i++) {
set_warm_restart_counter(m_db, i);
EXPECT_THAT(get_warm_restart_counter(m_db), StrEq(std::to_string(i)));
}
}

class RedisTestWithWarmStartState
: public RedisTest,
public ::testing::WithParamInterface<WarmStartState> {};

GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(RedisTestWithWarmStartState);

INSTANTIATE_TEST_SUITE_P(TestOverWarmStartStates,
RedisTestWithWarmStartState, testing::Values(true, false));


} // namespace rebootbackend
Loading

0 comments on commit 6fa9455

Please sign in to comment.