From 6fa945564cec98b961b47d652f92cfa974d77db7 Mon Sep 17 00:00:00 2001 From: Kavitha Ramalingam Date: Thu, 14 Nov 2024 16:21:30 +0530 Subject: [PATCH] gNOI Warm Reboot - Added tests --- .../tests/reboot_thread_test.cpp | 67 ++++++- src/sonic-framework/tests/rebootbe_test.cpp | 179 +++++++++++++++++- .../tests/redis_utils_test.cpp | 43 +++++ .../tests/test_utils_common.cpp | 17 ++ 4 files changed, 302 insertions(+), 4 deletions(-) diff --git a/src/sonic-framework/tests/reboot_thread_test.cpp b/src/sonic-framework/tests/reboot_thread_test.cpp index 35778ff864a1..5b82e50aab8f 100644 --- a/src/sonic-framework/tests/reboot_thread_test.cpp +++ b/src/sonic-framework/tests/reboot_thread_test.cpp @@ -89,6 +89,34 @@ TEST_F(RebootStatusTest, TestGetStatus) { EXPECT_EQ(0, response.when()); } +TEST_F(RebootStatusTest, TestGetWarmStatus) { + std::chrono::nanoseconds curr_ns = std::chrono::high_resolution_clock::now().time_since_epoch(); + + m_status.set_start_status(RebootMethod::WARM, "reboot because"); + + RebootStatusResponse response = m_status.get_response(); + EXPECT_EQ(response.status().status(), + RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN); + + m_status.set_completed_status( + RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS, "anything"); + + response = m_status.get_response(); + + // message should be empty while reboot is active + EXPECT_THAT(response.status().message(), StrEq("")); + + uint64_t reboot_ns = response.when(); + EXPECT_TRUE(reboot_ns > (uint64_t)curr_ns.count()); + + m_status.set_inactive(); + response = m_status.get_response(); + EXPECT_THAT(response.status().message(), StrEq("anything")); + EXPECT_EQ(response.status().status(), + RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS); + EXPECT_EQ(0, response.when()); +} + class RebootThreadTest : public ::testing::Test { protected: RebootThreadTest() @@ -263,7 +291,7 @@ TEST_F(RebootThreadTest, TestUnsupportedRebootType) { "RebootThread: Start rx'd unsupported method"); } -TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) { +TEST_F(RebootThreadTest, TestInvalidMethodDoReboot) { set_start_status(RebootMethod::POWERUP, "time to reboot"); do_reboot(); force_inactive(); @@ -273,4 +301,41 @@ TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) { IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_F(RebootThreadTest, TestNoWarmIfNonRetriableFailure) { + set_start_status(RebootMethod::WARM, "time to reboot"); + set_completed_status(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "failed to warm reboot"); + force_inactive(); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + + NotificationResponse response = m_reboot_thread.Start(request); + EXPECT_EQ(response.status, swss::StatusCode::SWSS_RC_FAILED_PRECONDITION); + EXPECT_EQ(response.json_string, + "RebootThread: last WARM reboot failed with non-retriable failure"); +} + +TEST_F(RebootThreadTest, TestSigTermStartofDoReboot) { + sigterm_requested = true; + set_start_status(RebootMethod::WARM, "time to reboot"); + do_reboot(); + force_inactive(); + RebootStatusResponse response = m_reboot_thread.GetResponse(); + EXPECT_THAT( + response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + +TEST_F(RebootThreadTest, TestWaitForRebootPositive) { + overwrite_reboot_timeout(1); + set_start_status(RebootMethod::WARM, "time to reboot"); + swss::Select s; + swss::SelectableEvent m_stop; + s.addSelectable(&m_stop); + RebootThread::Progress progress = wait_for_platform_reboot(s); + EXPECT_EQ(progress, RebootThread::Progress::PROCEED); +} + + } // namespace rebootbackend diff --git a/src/sonic-framework/tests/rebootbe_test.cpp b/src/sonic-framework/tests/rebootbe_test.cpp index a4eb79b642c1..ecacf386235d 100644 --- a/src/sonic-framework/tests/rebootbe_test.cpp +++ b/src/sonic-framework/tests/rebootbe_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include "mock_reboot_interfaces.h" #include "reboot_common.h" @@ -63,6 +62,7 @@ class RebootBETestWithoutStop : public ::testing::Test { m_rebootbeReponseChannel(&m_db, REBOOT_RESPONSE_NOTIFICATION_CHANNEL), m_rebootbe(m_dbus_interface) { sigterm_requested = false; +// TestUtils::clear_tables(m_db); m_s.addSelectable(&m_rebootbeReponseChannel); @@ -74,6 +74,15 @@ class RebootBETestWithoutStop : public ::testing::Test { } virtual ~RebootBETestWithoutStop() = default; + void force_warm_start_state(bool enabled) { + swss::Table enable_table(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME); + enable_table.hset("system", "enable", enabled ? "true" : "false"); + enable_table.hset("sonic-framework", "enable", enabled ? "true" : "false"); + + swss::Table restart_table(&m_db, STATE_WARM_RESTART_TABLE_NAME); + restart_table.hset("rebootbackend", "restore_count", enabled ? "0" : ""); + } + void start_rebootbe() { m_rebootbe_thread = std::make_unique(&RebootBE::Start, &m_rebootbe); @@ -202,6 +211,50 @@ class RebootBETest : public RebootBETestWithoutStop { } }; +TEST_F(RebootBETest, WarmbootInProgressBlocksNewWarmboot) { + force_warm_start_state(true); + + start_rebootbe(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + + // Send a warmboot request, confirm it fails. + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request, swss::StatusCode::SWSS_RC_IN_USE); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + force_warm_start_state(false); +} + + +TEST_F(RebootBETest, ColdbootWhileWarmbootInProgress) { + force_warm_start_state(true); + set_mock_defaults(); + + start_rebootbe(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + + // Send a coldboot request, confirm it starts. + RebootRequest request; + request.set_method(RebootMethod::COLD); + start_reboot_via_rpc(request); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::COLD_REBOOT_IN_PROGRESS); + + // Cleanup without going through the whole reboot. + send_stop_reboot_thread(); + force_warm_start_state(false); +} + + // Test fixture to skip through the startup sequence into the main loop. // Param indicates if RebootBE should be initialized into a state where the // system came up in warmboot. @@ -209,11 +262,11 @@ class RebootBEAutoStartTest : public RebootBETest, public ::testing::WithParamInterface { protected: RebootBEAutoStartTest() { - //force_warm_start_state(GetParam()); start_rebootbe(); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + //std::this_thread::sleep_for(std::chrono::milliseconds(50)); + std::this_thread::sleep_for(std::chrono::milliseconds(ONE_SECOND_MS)); EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); } }; @@ -312,6 +365,31 @@ TEST_P(RebootBEAutoStartTest, TestColdRebootDbusToCompletion) { "platform failed to reboot")); } + +TEST_P(RebootBEAutoStartTest, TestWarmRebootDbusToCompletion) { + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_SUCCESS, ""}; + EXPECT_CALL(m_dbus_interface, Reboot(_)) + .Times(1) + .WillRepeatedly(Return(dbus_response)); + + overwrite_reboot_timeout(1); + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + sleep(TWO_SECONDS); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT(response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "failed to warm reboot")); + +} + TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) { sigterm_requested = true; set_mock_defaults(); @@ -331,6 +409,25 @@ TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) { IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_P(RebootBEAutoStartTest, TestWarmBootSigterm) { + sigterm_requested = true; + set_mock_defaults(); + overwrite_reboot_timeout(1); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + sleep(ONE_SECOND); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc(); + EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT( + second_resp, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) { // Return FAIL from dbus reboot call. DbusInterface::DbusResponse dbus_response{ @@ -353,6 +450,28 @@ TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) { "dbus reboot failed")); } +TEST_P(RebootBEAutoStartTest, TestWarmBootDbusError) { + // Return FAIL from dbus reboot call. + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_FAIL, "dbus reboot failed"}; + EXPECT_CALL(m_dbus_interface, Reboot(_)) + .Times(1) + .WillOnce(Return(dbus_response)); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + sleep(TWO_SECONDS); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc(); + EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT(second_resp, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "dbus reboot failed")); +} + TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) { set_mock_defaults(); @@ -374,12 +493,66 @@ TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) { IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_P(RebootBEAutoStartTest, TestStopDuringWarmBoot) { + set_mock_defaults(); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); +// std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + send_stop_reboot_thread(); + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT( + response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + TEST_P(RebootBEAutoStartTest, TestInvalidJsonRebootRequest) { std::string json_request = "abcd"; NotificationResponse response = handle_reboot_request(json_request); EXPECT_EQ(swss::StatusCode::SWSS_RC_INTERNAL, response.status); } +TEST_P(RebootBEAutoStartTest, TestWarmFailureFollowedByColdBoot) { + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_SUCCESS, ""}; + overwrite_reboot_timeout(1); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS)); + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + + + request.set_method(RebootMethod::COLD); + start_reboot_via_rpc(request); + + // We have to wait for the 1 second reboot Timeout + std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS)); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 2, RebootMethod::COLD)); + EXPECT_THAT(response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "platform failed to reboot")); +} + + INSTANTIATE_TEST_SUITE_P(TestWithStartupWarmbootEnabledState, RebootBEAutoStartTest, testing::Values(true, false)); diff --git a/src/sonic-framework/tests/redis_utils_test.cpp b/src/sonic-framework/tests/redis_utils_test.cpp index 00864059a21e..40397476beba 100644 --- a/src/sonic-framework/tests/redis_utils_test.cpp +++ b/src/sonic-framework/tests/redis_utils_test.cpp @@ -15,9 +15,52 @@ namespace rebootbackend { +using WarmStartState = ::swss::WarmStart::WarmStartState; + using ::testing::AllOf; using ::testing::HasSubstr; using ::testing::StrEq; +class RedisTest : public ::testing::Test { + protected: + RedisTest() : m_db("STATE_DB", 0) { + TestUtils::clear_tables(m_db); + } + + swss::DBConnector m_db; +}; + +TEST_F(RedisTest, testSetWarmRestartEnable) { + swss::Table warmRestartTable(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME); + + for (const auto &enabled : {true, false}) { + warmRestartTable.del("system"); + + set_warm_restart_enable(m_db, enabled); + + std::string value; + bool ret = warmRestartTable.hget("system", "enable", value); + EXPECT_TRUE(ret); + EXPECT_EQ(value, enabled ? "true" : "false"); + } +} + +TEST_F(RedisTest, GetWarmRestartCounter) { + EXPECT_THAT(get_warm_restart_counter(m_db), StrEq("")); + for (int i = 0; i < 5; i++) { + set_warm_restart_counter(m_db, i); + EXPECT_THAT(get_warm_restart_counter(m_db), StrEq(std::to_string(i))); + } +} + +class RedisTestWithWarmStartState + : public RedisTest, + public ::testing::WithParamInterface {}; + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(RedisTestWithWarmStartState); + +INSTANTIATE_TEST_SUITE_P(TestOverWarmStartStates, + RedisTestWithWarmStartState, testing::Values(true, false)); + } // namespace rebootbackend diff --git a/src/sonic-framework/tests/test_utils_common.cpp b/src/sonic-framework/tests/test_utils_common.cpp index ef0088b7bf05..1b3754454890 100644 --- a/src/sonic-framework/tests/test_utils_common.cpp +++ b/src/sonic-framework/tests/test_utils_common.cpp @@ -27,4 +27,21 @@ void TestUtils::wait_for_finish(swss::Select &s, EXPECT_EQ(sel, &finished); } +void TestUtils::clear_tables(swss::DBConnector &db) { + const std::vector kTablesToClear = { + "BOOT_INFO", + STATE_WARM_RESTART_TABLE_NAME, + STATE_WARM_RESTART_ENABLE_TABLE_NAME }; + + for (const auto &table_name : kTablesToClear) { + swss::Table table(&db, table_name); + std::vector keys; + table.getKeys(keys); + for (const auto &key : keys) { + table.del(key); + } + } +} + + } // namespace rebootbackend