Skip to content

Commit

Permalink
feat: Linux Control Group version 2 API support (cgroup v2) (aws-gree…
Browse files Browse the repository at this point in the history 
…ngrass#1262)
  • Loading branch information
@ChangxinDong
ChangxinDong committed Oct 9, 2022
1 parent 5013bf9 commit cfeef06
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 102 deletions.
34 changes: 13 additions & 21 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,27 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.util.Utils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

/**
* Represents Linux cgroup v2 subsystems.
* Represents Linux cgroup v2.
*/
@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path "
+ "cannot be relative")
public enum CgroupV2 {
Memory("memory"), CPU("cpu"), Freezer("freezer"), Any("any");
public final class CgroupV2 {

private static final String CGROUP_ROOT = "/sys/fs/cgroup";
private static final String GG_NAMESPACE = "greengrass";
private static final String CPU_MAX = "cpu.max";
private static final String MEMORY_MAX = "memory.max";
private static final String CGROUP_PROCS = "cgroup.procs";
private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control";
private static final String FREEZER_STATE_FILE = "freezer.state";
private static final String CGROUP_FREEZE = "cgroup.freeze";
private final String osString;

CgroupV2(String str) {
this.osString = str;
private CgroupV2() {
}

public static Path getRootPath() {
Expand All @@ -44,39 +36,39 @@ public static String rootMountCmd() {
return String.format("mount -t cgroup2 none %s", CGROUP_ROOT);
}

public Path getSubsystemRootPath() {
public static Path getSubsystemRootPath() {
return Paths.get(CGROUP_ROOT);
}

public Path getRootSubTreeControlPath() {
public static Path getRootSubTreeControlPath() {
return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL);
}

public Path getSubsystemGGPath() {
public static Path getSubsystemGGPath() {
return getSubsystemRootPath().resolve(GG_NAMESPACE);
}

public Path getGGSubTreeControlPath() {
public static Path getGGSubTreeControlPath() {
return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL);
}

public Path getSubsystemComponentPath(String componentName) {
public static Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
}

public Path getComponentCpuMaxPath(String componentName) {
public static Path getComponentCpuMaxPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CPU_MAX);
}

public Path getComponentMemoryMaxPath(String componentName) {
public static Path getComponentMemoryMaxPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX);
}

public Path getCgroupProcsPath(String componentName) {
public static Path getCgroupProcsPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS);
}
public Path getCgroupFreezePath(String componentName) {

public static Path getCgroupFreezePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE);
}
}
135 changes: 54 additions & 81 deletions ...in/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,11 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
Expand All @@ -42,9 +40,6 @@ public class LinuxSystemResourceControllerV2 implements SystemResourceController

private static final String UNICODE_SPACE = "\\040";
private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids";
private static final List<CgroupV2> RESOURCE_LIMIT_CGROUPS = Arrays.asList(CgroupV2.Memory, CgroupV2.CPU);

private final CopyOnWriteArrayList<CgroupV2> usedCgroups = new CopyOnWriteArrayList<>();

protected final LinuxPlatform platform;

Expand All @@ -54,30 +49,28 @@ public LinuxSystemResourceControllerV2(LinuxPlatform platform) {

@Override
public void removeResourceController(GreengrassService component) {
usedCgroups.forEach(cg -> {
try {
// Assumes processes belonging to cgroups would already be terminated/killed.
Files.deleteIfExists(cg.getSubsystemComponentPath(component.getServiceName()));
} catch (IOException e) {
logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName())
.log("Failed to remove the resource controller");
}
});
try {
// Assumes processes belonging to cgroups would already be terminated/killed.
Files.deleteIfExists(CgroupV2.getSubsystemComponentPath(component.getServiceName()));
} catch (IOException e) {
logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName())
.log("Failed to remove the resource controller");
}
}

@Override
public void updateResourceLimits(GreengrassService component, Map<String, Object> resourceLimit) {
try {
if (!Files.exists(CgroupV2.Any.getSubsystemComponentPath(component.getServiceName()))) {
initializeCgroup(component, CgroupV2.Any);
if (!Files.exists(CgroupV2.getSubsystemComponentPath(component.getServiceName()))) {
initializeCgroup(component);
}

if (resourceLimit.containsKey(MEMORY_KEY)) {
long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY));

if (memoryLimitInKB > 0) {
String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB);
Files.write(CgroupV2.Memory.getComponentMemoryMaxPath(component.getServiceName()),
Files.write(CgroupV2.getComponentMemoryMaxPath(component.getServiceName()),
memoryLimit.getBytes(StandardCharsets.UTF_8));
} else {
logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB)
Expand All @@ -89,7 +82,7 @@ public void updateResourceLimits(GreengrassService component, Map<String, Object
double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY));
if (cpu > 0) {
byte[] content = Files.readAllBytes(
CgroupV2.CPU.getComponentCpuMaxPath(component.getServiceName()));
CgroupV2.getComponentCpuMaxPath(component.getServiceName()));
String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim();
String[] cpuMaxContentArr = cpuMaxContent.split(" ");
String cpuMaxStr = "max";
Expand All @@ -107,7 +100,7 @@ public void updateResourceLimits(GreengrassService component, Map<String, Object
}

String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr);
Files.write(CgroupV2.CPU.getComponentCpuMaxPath(component.getServiceName()),
Files.write(CgroupV2.getComponentCpuMaxPath(component.getServiceName()),
latestCpuMaxContent.getBytes(StandardCharsets.UTF_8));
} else {
logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu)
Expand All @@ -122,68 +115,64 @@ public void updateResourceLimits(GreengrassService component, Map<String, Object

@Override
public void resetResourceLimits(GreengrassService component) {
for (CgroupV2 cg : RESOURCE_LIMIT_CGROUPS) {
try {
if (Files.exists(cg.getSubsystemComponentPath(component.getServiceName()))) {
Files.delete(cg.getSubsystemComponentPath(component.getServiceName()));
Files.createDirectory(cg.getSubsystemComponentPath(component.getServiceName()));
}
} catch (IOException e) {
logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName())
.log("Failed to remove the resource controller");
try {
if (Files.exists(CgroupV2.getSubsystemComponentPath(component.getServiceName()))) {
Files.delete(CgroupV2.getSubsystemComponentPath(component.getServiceName()));
Files.createDirectory(CgroupV2.getSubsystemComponentPath(component.getServiceName()));
}
} catch (IOException e) {
logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName())
.log("Failed to remove the resource controller");
}
}

@Override
public void addComponentProcess(GreengrassService component, Process process) {
RESOURCE_LIMIT_CGROUPS.forEach(cg -> {
try {
addComponentProcessToCgroup(component.getServiceName(), process, cg);

// Child processes of a process in a cgroup are auto-added to the same cgroup by linux kernel. But in
// case of a race condition in starting a child process and us adding pids to cgroup, neither us nor
// the linux kernel will add it to the cgroup. To account for this, re-list all pids for the component
// after 1 second and add to cgroup again so that all component processes are resource controlled.
component.getContext().get(ScheduledExecutorService.class).schedule(() -> {
try {
addComponentProcessToCgroup(component.getServiceName(), process, cg);
} catch (IOException e) {
handleErrorAddingPidToCgroup(e, component.getServiceName());
}
}, 1, TimeUnit.SECONDS);
try {
addComponentProcessToCgroup(component.getServiceName(), process);

// Child processes of a process in a cgroup are auto-added to the same cgroup by linux kernel. But in
// case of a race condition in starting a child process and us adding pids to cgroup, neither us nor
// the linux kernel will add it to the cgroup. To account for this, re-list all pids for the component
// after 1 second and add to cgroup again so that all component processes are resource controlled.
component.getContext().get(ScheduledExecutorService.class).schedule(() -> {
try {
addComponentProcessToCgroup(component.getServiceName(), process);
} catch (IOException e) {
handleErrorAddingPidToCgroup(e, component.getServiceName());
}
}, 1, TimeUnit.SECONDS);

} catch (IOException e) {
handleErrorAddingPidToCgroup(e, component.getServiceName());
}
});
} catch (IOException e) {
handleErrorAddingPidToCgroup(e, component.getServiceName());
}
}

@Override
public void pauseComponentProcesses(GreengrassService component, List<Process> processes) throws IOException {
initializeCgroup(component, CgroupV2.Freezer);
initializeCgroup(component);

for (Process process : processes) {
addComponentProcessToCgroup(component.getServiceName(), process, CgroupV2.Freezer);
addComponentProcessToCgroup(component.getServiceName(), process);
}

Files.write(freezerCgroupStateFile(component.getServiceName()),
LinuxSystemResourceControllerV2.CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8),
String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

@Override
public void resumeComponentProcesses(GreengrassService component) throws IOException {
Files.write(freezerCgroupStateFile(component.getServiceName()),
LinuxSystemResourceControllerV2.CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8),
String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

private void addComponentProcessToCgroup(String component, Process process, CgroupV2 cg)
private void addComponentProcessToCgroup(String component, Process process)
throws IOException {

if (!Files.exists(cg.getSubsystemComponentPath(component))) {
logger.atDebug().kv(COMPONENT_NAME, component).kv("resource-controller", cg.toString())
if (!Files.exists(CgroupV2.getSubsystemComponentPath(component))) {
logger.atDebug().kv(COMPONENT_NAME, component)
.log("Resource controller is not enabled");
return;
}
Expand All @@ -192,7 +181,7 @@ private void addComponentProcessToCgroup(String component, Process process, Cgro
try {
Set<Integer> childProcesses = platform.getChildPids(process);
childProcesses.add(PidUtil.getPid(process));
Set<Integer> pidsInCgroup = pidsInComponentCgroup(cg, component);
Set<Integer> pidsInCgroup = pidsInComponentCgroup(component);
if (!Utils.isEmpty(childProcesses) && Objects.nonNull(pidsInCgroup)
&& !childProcesses.equals(pidsInCgroup)) {

Expand All @@ -204,7 +193,7 @@ private void addComponentProcessToCgroup(String component, Process process, Cgro
continue;
}

Files.write(cg.getCgroupProcsPath(component),
Files.write(CgroupV2.getCgroupProcsPath(component),
Integer.toString(pid).getBytes(StandardCharsets.UTF_8));
}
}
Expand Down Expand Up @@ -253,48 +242,32 @@ private Set<String> getMountedPaths() throws IOException {
return mountedPaths;
}

private void initializeCgroup(GreengrassService component, CgroupV2 cgroup) throws IOException {
private void initializeCgroup(GreengrassService component) throws IOException {
Set<String> mounts = getMountedPaths();

if (!mounts.contains(CgroupV2.getRootPath().toString())) {
platform.runCmd(CgroupV2.rootMountCmd(), o -> {
}, "Failed to mount cgroup2 root");
Utils.createPaths(cgroup.getSubsystemRootPath());
Utils.createPaths(CgroupV2.getSubsystemRootPath());
}

//Enable controllers for root group
Files.write(cgroup.getRootSubTreeControlPath(),
Files.write(CgroupV2.getRootSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));

Utils.createPaths(cgroup.getSubsystemGGPath());
Utils.createPaths(CgroupV2.getSubsystemGGPath());
//Enable controllers for gg group
Files.write(cgroup.getGGSubTreeControlPath(),
Files.write(CgroupV2.getGGSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));
Utils.createPaths(cgroup.getSubsystemComponentPath(component.getServiceName()));

usedCgroups.add(cgroup);
Utils.createPaths(CgroupV2.getSubsystemComponentPath(component.getServiceName()));
}

private Set<Integer> pidsInComponentCgroup(CgroupV2 cgroup, String component) throws IOException {
return Files.readAllLines(cgroup.getCgroupProcsPath(component))
private Set<Integer> pidsInComponentCgroup(String component) throws IOException {
return Files.readAllLines(CgroupV2.getCgroupProcsPath(component))
.stream().map(Integer::parseInt).collect(Collectors.toSet());
}

private Path freezerCgroupStateFile(String component) {
return CgroupV2.Freezer.getCgroupFreezePath(component);
}

private LinuxSystemResourceControllerV2.CgroupFreezerState currentFreezerCgroupState(String component)
throws IOException {
List<String> stateFileContent =
Files.readAllLines(freezerCgroupStateFile(component));
if (Utils.isEmpty(stateFileContent) || stateFileContent.size() != 1) {
throw new IOException("Unexpected error reading freezer cgroup state");
}
return LinuxSystemResourceControllerV2.CgroupFreezerState.valueOf(stateFileContent.get(0).trim());
}

public enum CgroupFreezerState {
THAWED, FREEZING, FROZEN
return CgroupV2.getCgroupFreezePath(component);
}
}

0 comments on commit cfeef06

Please sign in to comment.