Skip to content

Commit

Permalink
tmp: fix and align app update size check
Browse files Browse the repository at this point in the history
Signed-off-by: Mike Sul <[email protected]>
  • Loading branch information
mike-sul committed Aug 31, 2023
1 parent 4f1b06a commit 0c200a7
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 83 deletions.
29 changes: 9 additions & 20 deletions src/docker/restorableappengine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,8 @@ RestorableAppEngine::StorageSpaceFunc RestorableAppEngine::GetDefStorageSpaceFun
std::to_string(watermark));
}

return [watermark](const boost::filesystem::path& path) {
boost::system::error_code ec;
const boost::filesystem::space_info store_info{boost::filesystem::space(path, ec)};
if (ec.failed()) {
throw std::runtime_error("Failed to get an available storage size: " + ec.message());
}

boost::uintmax_t allowed_for_usage{static_cast<boost::uintmax_t>((float(watermark) / 100) * store_info.available)};
return std::tuple<boost::uintmax_t, boost::uintmax_t>(
store_info.available /* overall available */,
allowed_for_usage /* available for usage taking into account a highe level watermark*/);
return [watermark](const boost::filesystem::path& path, storage::Volume::UsageInfo& usage_info) {
storage::Volume::getUsageInfo(path.string(), (100 - watermark), usage_info);
};
}

Expand Down Expand Up @@ -963,15 +954,13 @@ void RestorableAppEngine::checkAvailableStorageInStores(const std::string& app_n
const uint64_t& docker_required_storage) const {
auto checkRoomInStore = [&](const std::string& store_name, const uint64_t& required_storage,
const boost::filesystem::path& store_path) {
boost::uintmax_t capacity;
boost::uintmax_t available;

std::tie(capacity, available) = storage_space_func_(store_path);
LOG_INFO << app_name << " -> " << store_name << " store total update size: " << required_storage
<< " bytes; available: " << available << " (out of " << capacity << ") "
<< ", path: " << store_path.string();
if (required_storage > available) {
throw InsufficientSpaceError(store_name, store_path.string(), required_storage, available);
storage::Volume::UsageInfo usage_info;
storage_space_func_(store_path, usage_info);
LOG_INFO << app_name << " -> " << store_name
<< " store total update size: " << usage_info.relativeStrRepr(required_storage) << "; "
<< usage_info.toStr();
if (required_storage > usage_info.available.first) {
throw InsufficientSpaceError(store_name, store_path.string(), required_storage, usage_info.available.first);
}
};

Expand Down
4 changes: 2 additions & 2 deletions src/docker/restorableappengine.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include "docker/docker.h"
#include "docker/dockerclient.h"
#include "storage/stat.h"

namespace Docker {

Expand Down Expand Up @@ -74,8 +75,7 @@ namespace Docker {
class RestorableAppEngine : public AppEngine {
public:
static const std::string ComposeFile;
using StorageSpaceFunc =
std::function<std::tuple<boost::uintmax_t, boost::uintmax_t>(const boost::filesystem::path&)>;
using StorageSpaceFunc = std::function<void(const boost::filesystem::path&, storage::Volume::UsageInfo&)>;
using ClientImageSrcFunc = std::function<std::string(const Docker::Uri&, const std::string&)>;

static const int LowWatermarkLimit{20};
Expand Down
7 changes: 7 additions & 0 deletions src/storage/stat.cc
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ Volume::UsageInfo::Type Volume::UsageInfo::toFullType(const uint64_t& val) const
return {val, std::ceil(((double)val / size.first) * 100)};
}

std::string Volume::UsageInfo::relativeStrRepr(const uint64_t& val) const {
Type rel_val{val, std::ceil(((double)val / size.first) * 100)};
std::stringstream ss;
ss << rel_val;
return ss.str();
}

} // namespace storage

std::ostream& operator<<(std::ostream& os, const storage::Volume::UsageInfo::Type& t) {
Expand Down
1 change: 1 addition & 0 deletions src/storage/stat.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ struct Volume {

std::string toStr() const;
Type toFullType(const uint64_t& val) const;
std::string relativeStrRepr(const uint64_t& val) const;
};

static void getUsageInfo(const std::string& path, unsigned int reserved_percentage, UsageInfo& usage_info);
Expand Down
115 changes: 61 additions & 54 deletions tests/fixtures/aklitetest.cc
Original file line number Diff line number Diff line change
@@ -1,54 +1,61 @@
#include "fixtures/composeappenginetest.cc"
#include "fixtures/liteclienttest.cc"

class AkliteTest : public fixtures::ClientTest,
public fixtures::AppEngineTest,
public ::testing::WithParamInterface<std::string> {
protected:
void SetUp() override {
fixtures::AppEngineTest::SetUp();

const auto app_engine_type{GetParam()};

if (app_engine_type == "ComposeAppEngine") {
app_engine =
std::make_shared<Docker::ComposeAppEngine>(apps_root_dir, compose_cmd, docker_client_, registry_client_);
} else if (app_engine_type == "RestorableAppEngine") {
setAvailableStorageSpace(std::numeric_limits<boost::uintmax_t>::max());
app_engine = std::make_shared<Docker::RestorableAppEngine>(
fixtures::ClientTest::test_dir_.Path() / "apps-store", apps_root_dir, daemon_.dataRoot(), registry_client_,
docker_client_, registry.getSkopeoClient(), daemon_.getUrl(), compose_cmd,
[this](const boost::filesystem::path& path) {
return std::tuple<boost::uintmax_t, boost::uintmax_t>{this->available_storage_space_,
(this->watermark_ * this->available_storage_space_)};
});
} else {
throw std::invalid_argument("Unsupported AppEngine type: " + app_engine_type);
}
}

std::shared_ptr<LiteClient> createLiteClient(InitialVersion initial_version = InitialVersion::kOn,
boost::optional<std::vector<std::string>> apps = boost::none,
bool finalize = true) override {
const auto app_engine_type{GetParam()};

if (app_engine_type == "ComposeAppEngine") {
return ClientTest::createLiteClient(app_engine, initial_version, apps, apps_root_dir.string(), boost::none,
create_containers_before_reboot_, finalize);
} else if (app_engine_type == "RestorableAppEngine") {
return ClientTest::createLiteClient(app_engine, initial_version, apps, apps_root_dir.string(),
!!apps ? apps : std::vector<std::string>{""},
create_containers_before_reboot_, finalize);
} else {
throw std::invalid_argument("Unsupported AppEngine type: " + app_engine_type);
}
}

void setAvailableStorageSpace(const boost::uintmax_t& space_size) { available_storage_space_ = space_size; }
void setCreateContainersBeforeReboot(bool value) { create_containers_before_reboot_ = value; }

private:
boost::uintmax_t available_storage_space_;
float watermark_{0.8};
bool create_containers_before_reboot_{true};
};
#include "fixtures/composeappenginetest.cc"
#include "fixtures/liteclienttest.cc"

class AkliteTest : public fixtures::ClientTest,
public fixtures::AppEngineTest,
public ::testing::WithParamInterface<std::string> {
protected:
void SetUp() override {
fixtures::AppEngineTest::SetUp();

const auto app_engine_type{GetParam()};

if (app_engine_type == "ComposeAppEngine") {
app_engine =
std::make_shared<Docker::ComposeAppEngine>(apps_root_dir, compose_cmd, docker_client_, registry_client_);
} else if (app_engine_type == "RestorableAppEngine") {
setAvailableStorageSpace(std::numeric_limits<boost::uintmax_t>::max());
app_engine = std::make_shared<Docker::RestorableAppEngine>(
fixtures::ClientTest::test_dir_.Path() / "apps-store", apps_root_dir, daemon_.dataRoot(), registry_client_,
docker_client_, registry.getSkopeoClient(), daemon_.getUrl(), compose_cmd,
[this](const boost::filesystem::path& path, storage::Volume::UsageInfo& usage_info) {

usage_info.path = path.string();
usage_info.size = {this->available_storage_space_, 100};
usage_info.reserved = {(1 - this->watermark_) * this->available_storage_space_, (1 - this->watermark_) * 100};
usage_info.free = {this->available_storage_space_ - usage_info.reserved.first, ((double)(this->available_storage_space_ - usage_info.reserved.first)/this->available_storage_space_)*100};
usage_info.available = {
(usage_info.free.first > usage_info.reserved.first) ? (usage_info.free.first - usage_info.reserved.first) : 0,
(usage_info.free.second > usage_info.reserved.second) ? (usage_info.free.second - usage_info.reserved.second)
: 0};
});
} else {
throw std::invalid_argument("Unsupported AppEngine type: " + app_engine_type);
}
}

std::shared_ptr<LiteClient> createLiteClient(InitialVersion initial_version = InitialVersion::kOn,
boost::optional<std::vector<std::string>> apps = boost::none,
bool finalize = true) override {
const auto app_engine_type{GetParam()};

if (app_engine_type == "ComposeAppEngine") {
return ClientTest::createLiteClient(app_engine, initial_version, apps, apps_root_dir.string(), boost::none,
create_containers_before_reboot_, finalize);
} else if (app_engine_type == "RestorableAppEngine") {
return ClientTest::createLiteClient(app_engine, initial_version, apps, apps_root_dir.string(),
!!apps ? apps : std::vector<std::string>{""},
create_containers_before_reboot_, finalize);
} else {
throw std::invalid_argument("Unsupported AppEngine type: " + app_engine_type);
}
}

void setAvailableStorageSpace(const boost::uintmax_t& space_size) { available_storage_space_ = space_size; }
void setCreateContainersBeforeReboot(bool value) { create_containers_before_reboot_ = value; }

private:
boost::uintmax_t available_storage_space_;
float watermark_{0.8};
bool create_containers_before_reboot_{true};
};
37 changes: 30 additions & 7 deletions tests/restorableappengine_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,21 @@ class RestorableAppEngineTest : public fixtures::AppEngineTest {

app_engine = std::make_shared<Docker::RestorableAppEngine>(
skopeo_store_root_, apps_root_dir, daemon_.dataRoot(), registry_client_, docker_client_,
registry.getSkopeoClient(), daemon_.getUrl(), compose_cmd, [this](const boost::filesystem::path& path) {
return std::tuple<boost::uintmax_t, boost::uintmax_t>{this->available_storage_space_,
(this->watermark_ * this->available_storage_space_)};
registry.getSkopeoClient(), daemon_.getUrl(), compose_cmd,
[this](const boost::filesystem::path& path, storage::Volume::UsageInfo& usage_info) {
usage_info.path = path.string();
usage_info.size = {this->available_storage_space_, 100};
usage_info.reserved = {(1 - this->watermark_) * this->available_storage_space_, (1 - this->watermark_) * 100};
usage_info.free = {
this->available_storage_space_ - usage_info.reserved.first,
((double)(this->available_storage_space_ - usage_info.reserved.first) / this->available_storage_space_) *
100};
usage_info.available = {(usage_info.free.first > usage_info.reserved.first)
? (usage_info.free.first - usage_info.reserved.first)
: 0,
(usage_info.free.second > usage_info.reserved.second)
? (usage_info.free.second - usage_info.reserved.second)
: 0};
});
}

Expand Down Expand Up @@ -80,9 +92,20 @@ class RestorableAppEngineTestParameterized : public RestorableAppEngineTest,
app_engine = std::make_shared<Docker::RestorableAppEngine>(
skopeo_store_root_, apps_root_dir, docker_data_root_path.empty() ? daemon_.dataRoot() : docker_data_root_path,
registry_client_, docker_client_, registry.getSkopeoClient(), daemon_.getUrl(), compose_cmd,
[this](const boost::filesystem::path& path) {
return std::tuple<boost::uintmax_t, boost::uintmax_t>{this->available_storage_space_,
(this->watermark_ * this->available_storage_space_)};
[this](const boost::filesystem::path& path, storage::Volume::UsageInfo& usage_info) {
usage_info.path = path.string();
usage_info.size = {this->available_storage_space_, 100};
usage_info.reserved = {(1 - this->watermark_) * this->available_storage_space_, (1 - this->watermark_) * 100};
usage_info.free = {
this->available_storage_space_ - usage_info.reserved.first,
((double)(this->available_storage_space_ - usage_info.reserved.first) / this->available_storage_space_) *
100};
usage_info.available = {(usage_info.free.first > usage_info.reserved.first)
? (usage_info.free.first - usage_info.reserved.first)
: 0,
(usage_info.free.second > usage_info.reserved.second)
? (usage_info.free.second - usage_info.reserved.second)
: 0};
});
}
};
Expand Down Expand Up @@ -340,7 +363,7 @@ TEST_F(RestorableAppEngineTest, FetchAndCheckSizeInsufficientSpaceIfWatermark) {

{
const auto compose_app{fixtures::ComposeApp::createAppWithCustomeLayers("app-01", layers)};
setAvailableStorageSpace(6144);
setAvailableStorageSpace(7100);
auto app = registry.addApp(compose_app);
ASSERT_TRUE(app_engine->fetch(app));
ASSERT_TRUE(app_engine->isFetched(app));
Expand Down

0 comments on commit 0c200a7

Please sign in to comment.