diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java index 27f2c3ec3a..ff98909911 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java @@ -5,22 +5,17 @@ package com.aws.greengrass.util.platforms.unix.linux; -import com.aws.greengrass.util.Utils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; /** - * Represents Linux cgroup v2 subsystems. + * Represents Linux cgroup v2. */ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path " + "cannot be relative") -public enum CgroupV2 { - Memory("memory"), CPU("cpu"), Freezer("freezer"), Any("any"); +public final class CgroupV2 { private static final String CGROUP_ROOT = "/sys/fs/cgroup"; private static final String GG_NAMESPACE = "greengrass"; @@ -28,12 +23,9 @@ public enum CgroupV2 { private static final String MEMORY_MAX = "memory.max"; private static final String CGROUP_PROCS = "cgroup.procs"; private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; - private static final String FREEZER_STATE_FILE = "freezer.state"; private static final String CGROUP_FREEZE = "cgroup.freeze"; - private final String osString; - CgroupV2(String str) { - this.osString = str; + private CgroupV2() { } public static Path getRootPath() { @@ -44,39 +36,39 @@ public static String rootMountCmd() { return String.format("mount -t cgroup2 none %s", CGROUP_ROOT); } - public Path getSubsystemRootPath() { + public static Path getSubsystemRootPath() { return Paths.get(CGROUP_ROOT); } - public Path getRootSubTreeControlPath() { + public static Path getRootSubTreeControlPath() { return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); } - public Path getSubsystemGGPath() { + public static Path getSubsystemGGPath() { return getSubsystemRootPath().resolve(GG_NAMESPACE); } - public Path getGGSubTreeControlPath() { + public static Path getGGSubTreeControlPath() { return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); } - public Path getSubsystemComponentPath(String componentName) { + public static Path getSubsystemComponentPath(String componentName) { return getSubsystemGGPath().resolve(componentName); } - public Path getComponentCpuMaxPath(String componentName) { + public static Path getComponentCpuMaxPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CPU_MAX); } - public Path getComponentMemoryMaxPath(String componentName) { + public static Path getComponentMemoryMaxPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); } - public Path getCgroupProcsPath(String componentName) { + public static Path getCgroupProcsPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); } - - public Path getCgroupFreezePath(String componentName) { + + public static Path getCgroupFreezePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java index f3231f61cf..74bdce25fd 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java @@ -20,13 +20,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -42,9 +40,6 @@ public class LinuxSystemResourceControllerV2 implements SystemResourceController private static final String UNICODE_SPACE = "\\040"; private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; - private static final List RESOURCE_LIMIT_CGROUPS = Arrays.asList(CgroupV2.Memory, CgroupV2.CPU); - - private final CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); protected final LinuxPlatform platform; @@ -54,22 +49,20 @@ public LinuxSystemResourceControllerV2(LinuxPlatform platform) { @Override public void removeResourceController(GreengrassService component) { - usedCgroups.forEach(cg -> { - try { - // Assumes processes belonging to cgroups would already be terminated/killed. - Files.deleteIfExists(cg.getSubsystemComponentPath(component.getServiceName())); - } catch (IOException e) { - logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) - .log("Failed to remove the resource controller"); - } - }); + try { + // Assumes processes belonging to cgroups would already be terminated/killed. + Files.deleteIfExists(CgroupV2.getSubsystemComponentPath(component.getServiceName())); + } catch (IOException e) { + logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) + .log("Failed to remove the resource controller"); + } } @Override public void updateResourceLimits(GreengrassService component, Map resourceLimit) { try { - if (!Files.exists(CgroupV2.Any.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, CgroupV2.Any); + if (!Files.exists(CgroupV2.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component); } if (resourceLimit.containsKey(MEMORY_KEY)) { @@ -77,7 +70,7 @@ public void updateResourceLimits(GreengrassService component, Map 0) { String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); - Files.write(CgroupV2.Memory.getComponentMemoryMaxPath(component.getServiceName()), + Files.write(CgroupV2.getComponentMemoryMaxPath(component.getServiceName()), memoryLimit.getBytes(StandardCharsets.UTF_8)); } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) @@ -89,7 +82,7 @@ public void updateResourceLimits(GreengrassService component, Map 0) { byte[] content = Files.readAllBytes( - CgroupV2.CPU.getComponentCpuMaxPath(component.getServiceName())); + CgroupV2.getComponentCpuMaxPath(component.getServiceName())); String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); String[] cpuMaxContentArr = cpuMaxContent.split(" "); String cpuMaxStr = "max"; @@ -107,7 +100,7 @@ public void updateResourceLimits(GreengrassService component, Map { - try { - addComponentProcessToCgroup(component.getServiceName(), process, cg); - - // Child processes of a process in a cgroup are auto-added to the same cgroup by linux kernel. But in - // case of a race condition in starting a child process and us adding pids to cgroup, neither us nor - // the linux kernel will add it to the cgroup. To account for this, re-list all pids for the component - // after 1 second and add to cgroup again so that all component processes are resource controlled. - component.getContext().get(ScheduledExecutorService.class).schedule(() -> { - try { - addComponentProcessToCgroup(component.getServiceName(), process, cg); - } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); - } - }, 1, TimeUnit.SECONDS); + try { + addComponentProcessToCgroup(component.getServiceName(), process); + + // Child processes of a process in a cgroup are auto-added to the same cgroup by linux kernel. But in + // case of a race condition in starting a child process and us adding pids to cgroup, neither us nor + // the linux kernel will add it to the cgroup. To account for this, re-list all pids for the component + // after 1 second and add to cgroup again so that all component processes are resource controlled. + component.getContext().get(ScheduledExecutorService.class).schedule(() -> { + try { + addComponentProcessToCgroup(component.getServiceName(), process); + } catch (IOException e) { + handleErrorAddingPidToCgroup(e, component.getServiceName()); + } + }, 1, TimeUnit.SECONDS); - } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); - } - }); + } catch (IOException e) { + handleErrorAddingPidToCgroup(e, component.getServiceName()); + } } @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - initializeCgroup(component, CgroupV2.Freezer); + initializeCgroup(component); for (Process process : processes) { - addComponentProcessToCgroup(component.getServiceName(), process, CgroupV2.Freezer); + addComponentProcessToCgroup(component.getServiceName(), process); } Files.write(freezerCgroupStateFile(component.getServiceName()), - LinuxSystemResourceControllerV2.CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8), + String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING); } @Override public void resumeComponentProcesses(GreengrassService component) throws IOException { Files.write(freezerCgroupStateFile(component.getServiceName()), - LinuxSystemResourceControllerV2.CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8), + String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING); } - private void addComponentProcessToCgroup(String component, Process process, CgroupV2 cg) + private void addComponentProcessToCgroup(String component, Process process) throws IOException { - if (!Files.exists(cg.getSubsystemComponentPath(component))) { - logger.atDebug().kv(COMPONENT_NAME, component).kv("resource-controller", cg.toString()) + if (!Files.exists(CgroupV2.getSubsystemComponentPath(component))) { + logger.atDebug().kv(COMPONENT_NAME, component) .log("Resource controller is not enabled"); return; } @@ -192,7 +181,7 @@ private void addComponentProcessToCgroup(String component, Process process, Cgro try { Set childProcesses = platform.getChildPids(process); childProcesses.add(PidUtil.getPid(process)); - Set pidsInCgroup = pidsInComponentCgroup(cg, component); + Set pidsInCgroup = pidsInComponentCgroup(component); if (!Utils.isEmpty(childProcesses) && Objects.nonNull(pidsInCgroup) && !childProcesses.equals(pidsInCgroup)) { @@ -204,7 +193,7 @@ private void addComponentProcessToCgroup(String component, Process process, Cgro continue; } - Files.write(cg.getCgroupProcsPath(component), + Files.write(CgroupV2.getCgroupProcsPath(component), Integer.toString(pid).getBytes(StandardCharsets.UTF_8)); } } @@ -253,48 +242,32 @@ private Set getMountedPaths() throws IOException { return mountedPaths; } - private void initializeCgroup(GreengrassService component, CgroupV2 cgroup) throws IOException { + private void initializeCgroup(GreengrassService component) throws IOException { Set mounts = getMountedPaths(); if (!mounts.contains(CgroupV2.getRootPath().toString())) { platform.runCmd(CgroupV2.rootMountCmd(), o -> { }, "Failed to mount cgroup2 root"); - Utils.createPaths(cgroup.getSubsystemRootPath()); + Utils.createPaths(CgroupV2.getSubsystemRootPath()); } //Enable controllers for root group - Files.write(cgroup.getRootSubTreeControlPath(), + Files.write(CgroupV2.getRootSubTreeControlPath(), CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - Utils.createPaths(cgroup.getSubsystemGGPath()); + Utils.createPaths(CgroupV2.getSubsystemGGPath()); //Enable controllers for gg group - Files.write(cgroup.getGGSubTreeControlPath(), + Files.write(CgroupV2.getGGSubTreeControlPath(), CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - Utils.createPaths(cgroup.getSubsystemComponentPath(component.getServiceName())); - - usedCgroups.add(cgroup); + Utils.createPaths(CgroupV2.getSubsystemComponentPath(component.getServiceName())); } - private Set pidsInComponentCgroup(CgroupV2 cgroup, String component) throws IOException { - return Files.readAllLines(cgroup.getCgroupProcsPath(component)) + private Set pidsInComponentCgroup(String component) throws IOException { + return Files.readAllLines(CgroupV2.getCgroupProcsPath(component)) .stream().map(Integer::parseInt).collect(Collectors.toSet()); } private Path freezerCgroupStateFile(String component) { - return CgroupV2.Freezer.getCgroupFreezePath(component); - } - - private LinuxSystemResourceControllerV2.CgroupFreezerState currentFreezerCgroupState(String component) - throws IOException { - List stateFileContent = - Files.readAllLines(freezerCgroupStateFile(component)); - if (Utils.isEmpty(stateFileContent) || stateFileContent.size() != 1) { - throw new IOException("Unexpected error reading freezer cgroup state"); - } - return LinuxSystemResourceControllerV2.CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); - } - - public enum CgroupFreezerState { - THAWED, FREEZING, FROZEN + return CgroupV2.getCgroupFreezePath(component); } }