Skip to content

Commit

Permalink
sysroot: Align storage space related params
Browse files Browse the repository at this point in the history 
- Change the meaning of the storage space "watermark" for delta to the
  same as the meaning of the storage space "watermark" for ostree
  objects. Now, both of the parameters indicate amount of reserved
  storage in percentage of the overall storage capacity.

- Rename these two params according to their meaning, what they actually
  set/define. Use the term "reserved" instead of "min" or "watermark".

- Adjust existing code and tests to the changes.

Signed-off-by: Mike Sul <[email protected]>
  • Loading branch information
@mike-sul
mike-sul committed Aug 31, 2023
1 parent 275cbe2 commit f02e7d2
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 89 deletions.
97 changes: 52 additions & 45 deletions src/ostree/sysroot.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,120 +7,127 @@

namespace OSTree {

const unsigned int Sysroot::Config::DefaultStorageWatermark{90};
const unsigned int Sysroot::Config::MinStorageWatermark{50};
const unsigned int Sysroot::Config::MaxStorageWatermark{95};
const unsigned int Sysroot::Config::DefaultReservedStorageSpacePercentageDelta{5};
const unsigned int Sysroot::Config::MinReservedStorageSpacePercentageDelta{3};
const unsigned int Sysroot::Config::MaxReservedStorageSpacePercentageDelta{50};

const unsigned int Sysroot::Config::MinFreeSpacePercent{3};
const unsigned int Sysroot::Config::MaxFreeSpacePercent{50};
const unsigned int Sysroot::Config::MinReservedStorageSpacePercentageOstree{3};
const unsigned int Sysroot::Config::MaxReservedStorageSpacePercentageOstree{50};

Sysroot::Config::Config(const PackageConfig& pconfig) {
Path = pconfig.sysroot.string();
Type = pconfig.booted;
OsName = pconfig.os.empty() ? "lmp" : pconfig.os;
path = pconfig.sysroot.string();
type = pconfig.booted;
osname = pconfig.os.empty() ? "lmp" : pconfig.os;

if (pconfig.extra.count(StorageWatermarkParamName) == 1) {
const std::string val_str{pconfig.extra.at(StorageWatermarkParamName)};
if (pconfig.extra.count(ReservedStorageSpacePercentageDeltaParamName) == 1) {
const std::string val_str{pconfig.extra.at(ReservedStorageSpacePercentageDeltaParamName)};
try {
const auto val{boost::lexical_cast<unsigned int>(val_str)};
if (val < MinStorageWatermark) {
LOG_ERROR << "Value of `" << StorageWatermarkParamName << "` parameter is too low: " << val_str
<< "; setting it the minimum allowed: " << MinStorageWatermark;
StorageWatermark = MinStorageWatermark;
} else if (val > MaxStorageWatermark) {
LOG_ERROR << "Value of `" << StorageWatermarkParamName << "` parameter is too high: " << val_str
<< "; setting it the maximum allowed: " << MaxStorageWatermark;
StorageWatermark = MaxStorageWatermark;
if (val < MinReservedStorageSpacePercentageDelta) {
LOG_ERROR << "Value of `" << ReservedStorageSpacePercentageDeltaParamName
<< "` parameter is too low: " << val_str
<< "; setting it the minimum allowed: " << MinReservedStorageSpacePercentageDelta;
ReservedStorageSpacePercentageDelta = MinReservedStorageSpacePercentageDelta;
} else if (val > MaxReservedStorageSpacePercentageDelta) {
LOG_ERROR << "Value of `" << ReservedStorageSpacePercentageDeltaParamName
<< "` parameter is too high: " << val_str
<< "; setting it the maximum allowed: " << MaxReservedStorageSpacePercentageDelta;
ReservedStorageSpacePercentageDelta = MaxReservedStorageSpacePercentageDelta;
} else {
StorageWatermark = val;
ReservedStorageSpacePercentageDelta = val;
}
} catch (const std::exception& exc) {
LOG_ERROR << "Invalid value of `" << StorageWatermarkParamName << "` parameter: " << val_str
<< "; setting it the default value: " << DefaultStorageWatermark;
LOG_ERROR << "Invalid value of `" << ReservedStorageSpacePercentageDeltaParamName << "` parameter: " << val_str
<< "; setting it the default value: " << DefaultReservedStorageSpacePercentageDelta;
}
}

if (pconfig.extra.count(StorageFreeSpacePercentParamName) == 1) {
const std::string val_str{pconfig.extra.at(StorageFreeSpacePercentParamName)};
if (pconfig.extra.count(ReservedStorageSpacePercentageOstreeParamName) == 1) {
const std::string val_str{pconfig.extra.at(ReservedStorageSpacePercentageOstreeParamName)};
try {
const auto val{boost::lexical_cast<unsigned int>(val_str)};
if (val < MinFreeSpacePercent) {
LOG_ERROR << "Value of `" << StorageFreeSpacePercentParamName << "` parameter is too low: " << val_str
<< "; won't override the value set in the ostree config";
} else if (val > MaxFreeSpacePercent) {
LOG_ERROR << "Value of `" << StorageFreeSpacePercentParamName << "` parameter is too high: " << val_str
<< "; won't override the value set in the ostree config";
if (val < MinReservedStorageSpacePercentageOstree) {
LOG_ERROR << "Value of `" << ReservedStorageSpacePercentageOstreeParamName
<< "` parameter is too low: " << val_str << "; won't override the value set in the ostree config";
} else if (val > MaxReservedStorageSpacePercentageOstree) {
LOG_ERROR << "Value of `" << ReservedStorageSpacePercentageOstreeParamName
<< "` parameter is too high: " << val_str << "; won't override the value set in the ostree config";
} else {
StorageFreeSpacePercent = val;
ReservedStorageSpacePercentageOstree = val;
}
} catch (const std::exception& exc) {
LOG_ERROR << "Invalid value of `" << StorageFreeSpacePercentParamName << "` parameter: " << val_str
LOG_ERROR << "Invalid value of `" << ReservedStorageSpacePercentageOstreeParamName << "` parameter: " << val_str
<< "; won't override the value set in the ostree config";
}
}
}

Sysroot::Sysroot(const PackageConfig& pconfig)
: cfg_{pconfig},
repo_path_{cfg_.Path + "/ostree/repo"},
deployment_path_{cfg_.Path + "/ostree/deploy/" + cfg_.OsName + "/deploy"} {
repo_path_{cfg_.path + "/ostree/repo"},
deployment_path_{cfg_.path + "/ostree/deploy/" + cfg_.osname + "/deploy"} {
Repo repo{repo_path_};
const auto ostree_min_free_space{repo.getFreeSpacePercent()};
if (-1 == cfg_.StorageFreeSpacePercent) {
if (-1 == cfg_.ReservedStorageSpacePercentageOstree) {
LOG_DEBUG
<< "Minimum free space percentage is not overridden, applying the one that is configured in the ostree config: "
<< ostree_min_free_space;
} else {
try {
repo.setFreeSpacePercent(cfg_.StorageFreeSpacePercent);
repo.setFreeSpacePercent(cfg_.ReservedStorageSpacePercentageOstree);
LOG_DEBUG << "Minimum free space percentage is overridden; "
<< "from " << ostree_min_free_space << " to " << cfg_.StorageFreeSpacePercent;
<< "from " << ostree_min_free_space << " to " << cfg_.ReservedStorageSpacePercentageOstree;
} catch (const std::exception& exc) {
LOG_ERROR << "Failed to override `min-free-space-percent` value in the ostree config, applying the one that is "
"configured in the ostree config: "
<< ostree_min_free_space << "; err: " << exc.what();
}
}
sysroot_ = OstreeManager::LoadSysroot(cfg_.Path);
sysroot_ = OstreeManager::LoadSysroot(cfg_.path);
}

bool Sysroot::reload() {
// Just reload for the booted env. In non-booted env "pending" deployment becomes "current" just after installation
// without a need to reboot. It in turns invalidates "getCurrent" value for tests at the stage after installation and
// before reboot.
if (cfg_.Type == BootedType::kBooted) {
if (cfg_.type == BootedType::kBooted) {
return static_cast<bool>(ostree_sysroot_load_if_changed(sysroot_.get(), nullptr, nullptr, nullptr));
}
return true;
}

unsigned int Sysroot::reservedStorageSpacePercentageOstree() const {
OSTree::Repo repo{repoPath()};
return repo.getFreeSpacePercent();
}

std::string Sysroot::getDeploymentHash(Deployment deployment_type) const {
std::string deployment_hash;
g_autoptr(GPtrArray) deployments = nullptr;
OstreeDeployment* deployment = nullptr;

switch (cfg_.Type) {
switch (cfg_.type) {
case BootedType::kBooted:
deployment = getDeploymentIfBooted(sysroot_.get(), cfg_.OsName.c_str(), deployment_type);
deployment = getDeploymentIfBooted(sysroot_.get(), cfg_.osname.c_str(), deployment_type);
break;
case BootedType::kStaged:
if (deployment_type == Deployment::kPending) {
OstreeDeployment* cur_deployment = getDeploymentIfStaged(sysroot_.get(), cfg_.OsName.c_str(), deployment_type);
OstreeDeployment* cur_deployment = getDeploymentIfStaged(sysroot_.get(), cfg_.osname.c_str(), deployment_type);
// Load the sysroot to make sure we get its latest state, so we can get real "pending" deployment caused by
// successful installation
GObjectUniquePtr<OstreeSysroot> changed_sysroot = OstreeManager::LoadSysroot(cfg_.Path);
GObjectUniquePtr<OstreeSysroot> changed_sysroot = OstreeManager::LoadSysroot(cfg_.path);
OstreeDeployment* pend_deployment =
getDeploymentIfStaged(changed_sysroot.get(), cfg_.OsName.c_str(), deployment_type);
getDeploymentIfStaged(changed_sysroot.get(), cfg_.osname.c_str(), deployment_type);
deployment =
(strcmp(ostree_deployment_get_csum(pend_deployment), ostree_deployment_get_csum(cur_deployment)) == 0)
? nullptr
: pend_deployment;
} else {
deployment = getDeploymentIfStaged(sysroot_.get(), cfg_.OsName.c_str(), deployment_type);
deployment = getDeploymentIfStaged(sysroot_.get(), cfg_.osname.c_str(), deployment_type);
}
break;
default:
throw std::runtime_error("Invalid boot type: " + std::to_string(static_cast<int>(cfg_.Type)));
throw std::runtime_error("Invalid boot type: " + std::to_string(static_cast<int>(cfg_.type)));
}

if (deployment != nullptr) {
Expand Down
57 changes: 33 additions & 24 deletions src/ostree/sysroot.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,44 +14,53 @@ class Sysroot {
public:
explicit Config(const PackageConfig& pconfig);

static constexpr const char* const StorageWatermarkParamName{"sysroot_storage_watermark"};
static const unsigned int DefaultStorageWatermark;
static const unsigned int MinStorageWatermark;
static const unsigned int MaxStorageWatermark;

static constexpr const char* const StorageFreeSpacePercentParamName{"min_free_space_percent"};
static const unsigned int MinFreeSpacePercent;
static const unsigned int MaxFreeSpacePercent;

// A high watermark for storage usage, expressed as a percentage,
// in other words, up to X% of the overall volume capacity can be used.
// The volume on which the sysroot is persisted is what is meant in this context.
unsigned int StorageWatermark{DefaultStorageWatermark};

// A low watermark for storage free space expressed as a percentage.
// In other words, at least X% of the overall volume capacity must be free, or
// up to (100 - X%) of the overall volume capacity can be used.
// This parameter is intended solely for the non-delta case.
static constexpr const char* const ReservedStorageSpacePercentageDeltaParamName{
"sysroot_reserved_space_percentage"};
static const unsigned int DefaultReservedStorageSpacePercentageDelta;
static const unsigned int MinReservedStorageSpacePercentageDelta;
static const unsigned int MaxReservedStorageSpacePercentageDelta;

static constexpr const char* const ReservedStorageSpacePercentageOstreeParamName{
"sysroot_ostree_reserved_space_percentage"};
static const unsigned int MinReservedStorageSpacePercentageOstree;
static const unsigned int MaxReservedStorageSpacePercentageOstree;

// This variable represents the reserved amount of storage, expressed as a percentage
// of the overall capacity of the volume where the sysroot/ostree repo is located.
// The reserved percentage is only considered when performing a delta-based ostree pull.
// The downloader verifies that the reserved storage will remain untouched prior to initiating a delta-based ostree
// pull. If the available free space, in addition to the reserved space, is insufficient to fit delta files, then
// the downloader will reject the download and exit with an error.
unsigned int ReservedStorageSpacePercentageDelta{DefaultReservedStorageSpacePercentageDelta};

// This variable represents the reserved amount of storage, expressed as a percentage
// of the overall capacity of the volume where the sysroot/ostree repo is located.
// The reserved percentage is considered in the both cases, during performing
// an object-based ostree pull and delta-based ostree pull.
// The downloader guarantees that the reserved storage is untouched when ostree objects are being committed to an
// ostree repo. If the available free space, in addition to the reserved space, is insufficient to fit object files,
// then the downloader will reject the download and exit with an error.
// Effectively, it enforces setting of the ostree repo config param `core.min-free-space-percent`.
int StorageFreeSpacePercent{-1};
int ReservedStorageSpacePercentageOstree{-1};

std::string Path;
BootedType Type;
std::string OsName;
std::string path;
BootedType type;
std::string osname;
};

enum class Deployment { kCurrent = 0, kPending, kRollback };
using Ptr = std::shared_ptr<Sysroot>;

explicit Sysroot(const PackageConfig& pconfig);

const std::string& path() const { return cfg_.Path; }
const std::string& path() const { return cfg_.path; }
const std::string& repoPath() const { return repo_path_; }
const std::string& deployment_path() const { return deployment_path_; }

virtual std::string getDeploymentHash(Deployment deployment_type) const;
bool reload();
unsigned int storageWatermark() const { return cfg_.StorageWatermark; }
unsigned int reservedStorageSpacePercentageDelta() const { return cfg_.ReservedStorageSpacePercentageDelta; }
unsigned int reservedStorageSpacePercentageOstree() const;

private:
static OstreeDeployment* getDeploymentIfBooted(OstreeSysroot* sysroot, const char* os_name,
Expand Down
2 changes: 1 addition & 1 deletion src/rootfstreemanager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ bool RootfsTreeManager::getDeltaStatIfAvailable(const TufTarget& target, const R
bool RootfsTreeManager::canDeltaFitOnDisk(const DeltaStat& delta_stat, UpdateStat& update_stat) const {
StorageStat storage{};
getStorageStat(sysroot_->repoPath(), storage);
const auto highWatermark{getStorageHighWatermark()};
const auto highWatermark{100 - sysroot_->reservedStorageSpacePercentageDelta()};

const uint64_t max_blocks_available = std::floor(storage.blockNumb * (static_cast<double>(highWatermark) / 100));
const uint64_t blocks_in_use = storage.blockNumb - storage.freeBlockNumb;
Expand Down
1 change: 0 additions & 1 deletion src/rootfstreemanager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ class RootfsTreeManager : public OstreeManager, public Downloader {
data::InstallationResult verifyBootloaderUpdate(const Uptane::Target& target) const;
bool getDeltaStatIfAvailable(const TufTarget& target, const Remote& remote, DeltaStat& delta_stat) const;
bool canDeltaFitOnDisk(const DeltaStat& delta_stat, UpdateStat& update_stat) const;
unsigned int getStorageHighWatermark() const { return sysroot_->storageWatermark(); };

static bool getDeltaStatsRef(const Json::Value& json, DeltaStatsRef& ref);
static Json::Value downloadDeltaStats(const DeltaStatsRef& ref, const Remote& remote);
Expand Down
39 changes: 21 additions & 18 deletions tests/nospace_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class NoSpaceTest : public fixtures::ClientTest {

void tweakConf(Config& cfg) override {
if (!min_free_space_.empty()) {
cfg.pacman.extra[OSTree::Sysroot::Config::StorageFreeSpacePercentParamName] = min_free_space_;
cfg.pacman.extra[OSTree::Sysroot::Config::ReservedStorageSpacePercentageOstreeParamName] = min_free_space_;
}
}
void setMinFreeSpace(const std::string& min_free_space) { min_free_space_ = min_free_space; }
Expand All @@ -75,42 +75,45 @@ class NoSpaceTest : public fixtures::ClientTest {
std::string min_free_space_;
};

TEST_F(NoSpaceTest, SysrootStorageWatermarkParam) {
TEST_F(NoSpaceTest, ReservedStorageSpacePercentageDeltaParam) {
{
// check default value
const auto cfg{OSTree::Sysroot::Config(PackageConfig{})};
ASSERT_EQ(OSTree::Sysroot::Config::DefaultStorageWatermark, cfg.StorageWatermark);
ASSERT_EQ(OSTree::Sysroot::Config::DefaultReservedStorageSpacePercentageDelta,
cfg.ReservedStorageSpacePercentageDelta);
}
{
// check if set to the default value if the specified param value is ivalid
PackageConfig pacmancfg;
pacmancfg.extra[OSTree::Sysroot::Config::StorageWatermarkParamName] = "10foo";
pacmancfg.extra[OSTree::Sysroot::Config::ReservedStorageSpacePercentageDeltaParamName] = "10foo";
const auto cfg{OSTree::Sysroot::Config(pacmancfg)};
ASSERT_EQ(OSTree::Sysroot::Config::DefaultStorageWatermark, cfg.StorageWatermark);
ASSERT_EQ(OSTree::Sysroot::Config::DefaultReservedStorageSpacePercentageDelta,
cfg.ReservedStorageSpacePercentageDelta);
}
{
// check if set to the min allowed value if the specified param value is lower than the one
PackageConfig pacmancfg;
pacmancfg.extra[OSTree::Sysroot::Config::StorageWatermarkParamName] =
std::to_string(OSTree::Sysroot::Config::MinStorageWatermark - 1);
pacmancfg.extra[OSTree::Sysroot::Config::ReservedStorageSpacePercentageDeltaParamName] =
std::to_string(OSTree::Sysroot::Config::MinReservedStorageSpacePercentageDelta - 1);
const auto cfg{OSTree::Sysroot::Config(pacmancfg)};
ASSERT_EQ(OSTree::Sysroot::Config::MinStorageWatermark, cfg.StorageWatermark);
ASSERT_EQ(OSTree::Sysroot::Config::MinReservedStorageSpacePercentageDelta, cfg.ReservedStorageSpacePercentageDelta);
}
{
// check if set to the max allowed value if the specified param value is higher than the one
PackageConfig pacmancfg;
pacmancfg.extra[OSTree::Sysroot::Config::StorageWatermarkParamName] =
std::to_string(OSTree::Sysroot::Config::MaxStorageWatermark + 1);
pacmancfg.extra[OSTree::Sysroot::Config::ReservedStorageSpacePercentageDeltaParamName] =
std::to_string(OSTree::Sysroot::Config::MaxReservedStorageSpacePercentageDelta + 1);
const auto cfg{OSTree::Sysroot::Config(pacmancfg)};
ASSERT_EQ(OSTree::Sysroot::Config::MaxStorageWatermark, cfg.StorageWatermark);
ASSERT_EQ(OSTree::Sysroot::Config::MaxReservedStorageSpacePercentageDelta, cfg.ReservedStorageSpacePercentageDelta);
}
{
// check if a custom valid value can be set
PackageConfig pacmancfg;
const unsigned int my_watermark{93};
pacmancfg.extra[OSTree::Sysroot::Config::StorageWatermarkParamName] = std::to_string(my_watermark);
const unsigned int my_watermark{43};
pacmancfg.extra[OSTree::Sysroot::Config::ReservedStorageSpacePercentageDeltaParamName] =
std::to_string(my_watermark);
const auto cfg{OSTree::Sysroot::Config(pacmancfg)};
ASSERT_EQ(my_watermark, cfg.StorageWatermark);
ASSERT_EQ(my_watermark, cfg.ReservedStorageSpacePercentageDelta);
}
}

Expand Down Expand Up @@ -219,15 +222,15 @@ TEST_F(NoSpaceTest, OstreeUpdateNoSpaceIfStaticDeltaStats) {
ASSERT_TRUE(targetsMatch(client->getCurrent(), getInitialTarget()));
}
{
// not enough free blocks taking into account the default watermark 90%,
// (20 - 11 = 9) - 9% of blocks will be free after the update, we need 10%
SetFreeBlockNumb(20, 100);
// not enough free blocks taking into account the default reserved 5%,
// (15 - 11 = 4) - 4% of blocks will be free after the update, we need 5%
SetFreeBlockNumb(15, 100);
update(*client, getInitialTarget(), new_target, data::ResultCode::Numeric::kDownloadFailed,
{DownloadResult::Status::DownloadFailed_NoSpace, "Insufficient storage available"});
const auto events{device_gateway_.getEvents()};
const std::string event_err_msg{events[events.size() - 1]["event"]["details"].asString()};
ASSERT_TRUE(std::string::npos !=
event_err_msg.find("available 40960 out of 368640(90% of the volume capacity 409600)"))
event_err_msg.find("available 40960 out of 389120(95% of the volume capacity 409600)"))
<< event_err_msg;
ASSERT_TRUE(targetsMatch(client->getCurrent(), getInitialTarget()));
}
Expand Down

0 comments on commit f02e7d2

Please sign in to comment.