diff --git a/Makefile b/Makefile index e17bd7769a..4fe8f8a353 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,7 @@ binaries: podman podman-remote ## Build podman and podman-remote binaries else ifneq (, $(findstring $(GOOS),darwin windows)) binaries: podman-remote ## Build podman-remote (client) only binaries else -binaries: podman podman-remote podmansh rootlessport quadlet ## Build podman, podman-remote and rootlessport binaries quadlet +binaries: podman podman-remote podman-testing podmansh rootlessport quadlet ## Build podman, podman-remote and rootlessport binaries quadlet endif # Extract text following double-# for targets, as their description for @@ -457,6 +457,16 @@ rootlessport: bin/rootlessport podmansh: bin/podman if [ ! -f bin/podmansh ]; then ln -s podman bin/podmansh; fi +$(SRCBINDIR)/podman-testing: $(SOURCES) go.mod go.sum + $(GOCMD) build \ + $(BUILDFLAGS) \ + $(GO_LDFLAGS) '$(LDFLAGS_PODMAN)' \ + -tags "${BUILDTAGS}" \ + -o $@ ./cmd/podman-testing + +.PHONY: podman-testing +podman-testing: bin/podman-testing + ### ### Secondary binary-build targets ### @@ -877,6 +887,11 @@ ifneq ($(shell uname -s),FreeBSD) install ${SELINUXOPT} -m 644 contrib/tmpfile/podman.conf $(DESTDIR)${TMPFILESDIR}/podman.conf endif +.PHONY: install.testing +install.testing: + install ${SELINUXOPT} -d -m 755 $(DESTDIR)$(BINDIR) + install ${SELINUXOPT} -m 755 bin/podman-testing $(DESTDIR)$(BINDIR)/podman-testing + .PHONY: install.modules-load install.modules-load: # This should only be used by distros which might use iptables-legacy, this is not needed on RHEL install ${SELINUXOPT} -m 755 -d $(DESTDIR)${MODULESLOADDIR} diff --git a/cmd/podman-testing/create.go b/cmd/podman-testing/create.go new file mode 100644 index 0000000000..1bdd14c5f9 --- /dev/null +++ b/cmd/podman-testing/create.go @@ -0,0 +1,127 @@ +package main + +import ( + "fmt" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/validate" + "github.com/containers/podman/v5/internal/domain/entities" + "github.com/spf13/cobra" +) + +var ( + createStorageLayerDescription = `Create an unmanaged layer in local storage.` + createStorageLayerCmd = &cobra.Command{ + Use: "create-storage-layer [options]", + Args: validate.NoArgs, + Short: "Create an unmanaged layer", + Long: createStorageLayerDescription, + RunE: createStorageLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-storage-layer`, + } + + createStorageLayerOpts entities.CreateStorageLayerOptions + + createLayerDescription = `Create an unused layer in local storage.` + createLayerCmd = &cobra.Command{ + Use: "create-layer [options]", + Args: validate.NoArgs, + Short: "Create an unused layer", + Long: createLayerDescription, + RunE: createLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-layer`, + } + + createLayerOpts entities.CreateLayerOptions + + createImageDescription = `Create an image in local storage.` + createImageCmd = &cobra.Command{ + Use: "create-image [options]", + Args: validate.NoArgs, + Short: "Create an image", + Long: createImageDescription, + RunE: createImage, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-image`, + } + + createImageOpts entities.CreateImageOptions + + createContainerDescription = `Create a container in local storage.` + createContainerCmd = &cobra.Command{ + Use: "create-container [options]", + Args: validate.NoArgs, + Short: "Create a container", + Long: createContainerDescription, + RunE: createContainer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-container`, + } + + createContainerOpts entities.CreateContainerOptions +) + +func init() { + mainCmd.AddCommand(createStorageLayerCmd) + flags := createStorageLayerCmd.Flags() + flags.StringVarP(&createStorageLayerOpts.ID, "id", "i", "", "ID to assign the new layer (default random)") + flags.StringVarP(&createStorageLayerOpts.Parent, "parent", "p", "", "ID of parent of new layer (default none)") + + mainCmd.AddCommand(createLayerCmd) + flags = createLayerCmd.Flags() + flags.StringVarP(&createLayerOpts.ID, "id", "i", "", "ID to assign the new layer (default random)") + flags.StringVarP(&createLayerOpts.Parent, "parent", "p", "", "ID of parent of new layer (default none)") + + mainCmd.AddCommand(createImageCmd) + flags = createImageCmd.Flags() + flags.StringVarP(&createImageOpts.ID, "id", "i", "", "ID to assign the new image (default random)") + flags.StringVarP(&createImageOpts.Layer, "layer", "l", "", "ID of image's main layer (default none)") + + mainCmd.AddCommand(createContainerCmd) + flags = createContainerCmd.Flags() + flags.StringVarP(&createContainerOpts.ID, "id", "i", "", "ID to assign the new container (default random)") + flags.StringVarP(&createContainerOpts.Image, "image", "b", "", "ID of containers's base image (default none)") + flags.StringVarP(&createContainerOpts.Layer, "layer", "l", "", "ID of containers's read-write layer (default none)") +} + +func createStorageLayer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.CreateStorageLayer(mainContext, createStorageLayerOpts) + if err != nil { + return err + } + + fmt.Println(results.ID) + return nil +} + +func createLayer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.CreateLayer(mainContext, createLayerOpts) + if err != nil { + return err + } + + fmt.Println(results.ID) + return nil +} + +func createImage(cmd *cobra.Command, args []string) error { + results, err := testingEngine.CreateImage(mainContext, createImageOpts) + if err != nil { + return err + } + + fmt.Println(results.ID) + return nil +} + +func createContainer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.CreateContainer(mainContext, createContainerOpts) + if err != nil { + return err + } + + fmt.Println(results.ID) + return nil +} diff --git a/cmd/podman-testing/data.go b/cmd/podman-testing/data.go new file mode 100644 index 0000000000..6fe2099c00 --- /dev/null +++ b/cmd/podman-testing/data.go @@ -0,0 +1,405 @@ +package main + +import ( + "errors" + "os" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/validate" + "github.com/containers/podman/v5/internal/domain/entities" + "github.com/spf13/cobra" +) + +var ( + createLayerDataDescription = `Create data for a layer in local storage.` + createLayerDataCmd = &cobra.Command{ + Use: "create-layer-data [options]", + Args: validate.NoArgs, + Short: "Create data for a layer", + Long: createLayerDataDescription, + RunE: createLayerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-layer-data`, + } + + createLayerDataOpts entities.CreateLayerDataOptions + createLayerDataKey string + createLayerDataValue string + createLayerDataFile string + + createImageDataDescription = `Create data for an image in local storage.` + createImageDataCmd = &cobra.Command{ + Use: "create-image-data [options]", + Args: validate.NoArgs, + Short: "Create data for an image", + Long: createImageDataDescription, + RunE: createImageData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-image-data`, + } + + createImageDataOpts entities.CreateImageDataOptions + createImageDataKey string + createImageDataValue string + createImageDataFile string + + createContainerDataDescription = `Create data for a container in local storage.` + createContainerDataCmd = &cobra.Command{ + Use: "create-container-data [options]", + Args: validate.NoArgs, + Short: "Create data for a container", + Long: createContainerDataDescription, + RunE: createContainerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing create-container-data`, + } + + createContainerDataOpts entities.CreateContainerDataOptions + createContainerDataKey string + createContainerDataValue string + createContainerDataFile string + + modifyLayerDataDescription = `Modify data for a layer in local storage, corrupting it.` + modifyLayerDataCmd = &cobra.Command{ + Use: "modify-layer-data [options]", + Args: validate.NoArgs, + Short: "Modify data for a layer", + Long: modifyLayerDataDescription, + RunE: modifyLayerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing modify-layer-data`, + } + + modifyLayerDataOpts entities.ModifyLayerDataOptions + modifyLayerDataValue string + modifyLayerDataFile string + + modifyImageDataDescription = `Modify data for an image in local storage, corrupting it.` + modifyImageDataCmd = &cobra.Command{ + Use: "modify-image-data [options]", + Args: validate.NoArgs, + Short: "Modify data for an image", + Long: modifyImageDataDescription, + RunE: modifyImageData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing modify-image-data`, + } + + modifyImageDataOpts entities.ModifyImageDataOptions + modifyImageDataValue string + modifyImageDataFile string + + modifyContainerDataDescription = `Modify data for a container in local storage, corrupting it.` + modifyContainerDataCmd = &cobra.Command{ + Use: "modify-container-data [options]", + Args: validate.NoArgs, + Short: "Modify data for a container", + Long: modifyContainerDataDescription, + RunE: modifyContainerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing modify-container-data`, + } + + modifyContainerDataOpts entities.ModifyContainerDataOptions + modifyContainerDataValue string + modifyContainerDataFile string + + removeLayerDataDescription = `Remove data from a layer in local storage, corrupting it.` + removeLayerDataCmd = &cobra.Command{ + Use: "remove-layer-data [options]", + Args: validate.NoArgs, + Short: "Remove data for a layer", + Long: removeLayerDataDescription, + RunE: removeLayerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-layer-data`, + } + + removeLayerDataOpts entities.RemoveLayerDataOptions + + removeImageDataDescription = `Remove data from an image in local storage, corrupting it.` + removeImageDataCmd = &cobra.Command{ + Use: "remove-image-data [options]", + Args: validate.NoArgs, + Short: "Remove data from an image", + Long: removeImageDataDescription, + RunE: removeImageData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-image-data`, + } + + removeImageDataOpts entities.RemoveImageDataOptions + + removeContainerDataDescription = `Remove data from a container in local storage, corrupting it.` + removeContainerDataCmd = &cobra.Command{ + Use: "remove-container-data [options]", + Args: validate.NoArgs, + Short: "Remove data from a container", + Long: removeContainerDataDescription, + RunE: removeContainerData, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-container-data`, + } + + removeContainerDataOpts entities.RemoveContainerDataOptions +) + +func init() { + mainCmd.AddCommand(createLayerDataCmd) + flags := createLayerDataCmd.Flags() + flags.StringVarP(&createLayerDataOpts.ID, "layer", "i", "", "ID of the layer") + flags.StringVarP(&createLayerDataKey, "key", "k", "", "Name of the data item") + flags.StringVarP(&createLayerDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&createLayerDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(createImageDataCmd) + flags = createImageDataCmd.Flags() + flags.StringVarP(&createImageDataOpts.ID, "image", "i", "", "ID of the image") + flags.StringVarP(&createImageDataKey, "key", "k", "", "Name of the data item") + flags.StringVarP(&createImageDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&createImageDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(createContainerDataCmd) + flags = createContainerDataCmd.Flags() + flags.StringVarP(&createContainerDataOpts.ID, "container", "i", "", "ID of the container") + flags.StringVarP(&createContainerDataKey, "key", "k", "", "Name of the data item") + flags.StringVarP(&createContainerDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&createContainerDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(modifyLayerDataCmd) + flags = modifyLayerDataCmd.Flags() + flags.StringVarP(&modifyLayerDataOpts.ID, "layer", "i", "", "ID of the layer") + flags.StringVarP(&modifyLayerDataOpts.Key, "key", "k", "", "Name of the data item") + flags.StringVarP(&modifyLayerDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&modifyLayerDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(modifyImageDataCmd) + flags = modifyImageDataCmd.Flags() + flags.StringVarP(&modifyImageDataOpts.ID, "image", "i", "", "ID of the image") + flags.StringVarP(&modifyImageDataOpts.Key, "key", "k", "", "Name of the data item") + flags.StringVarP(&modifyImageDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&modifyImageDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(modifyContainerDataCmd) + flags = modifyContainerDataCmd.Flags() + flags.StringVarP(&modifyContainerDataOpts.ID, "container", "i", "", "ID of the container") + flags.StringVarP(&modifyContainerDataOpts.Key, "key", "k", "", "Name of the data item") + flags.StringVarP(&modifyContainerDataValue, "value", "v", "", "Value of the data item") + flags.StringVarP(&modifyContainerDataFile, "file", "f", "", "File containing the data item") + + mainCmd.AddCommand(removeLayerDataCmd) + flags = removeLayerDataCmd.Flags() + flags.StringVarP(&removeLayerDataOpts.ID, "layer", "i", "", "ID of the layer") + flags.StringVarP(&removeLayerDataOpts.Key, "key", "k", "", "Name of the data item") + + mainCmd.AddCommand(removeImageDataCmd) + flags = removeImageDataCmd.Flags() + flags.StringVarP(&removeImageDataOpts.ID, "image", "i", "", "ID of the image") + flags.StringVarP(&removeImageDataOpts.Key, "key", "k", "", "Name of the data item") + + mainCmd.AddCommand(removeContainerDataCmd) + flags = removeContainerDataCmd.Flags() + flags.StringVarP(&removeContainerDataOpts.ID, "container", "i", "", "ID of the container") + flags.StringVarP(&removeContainerDataOpts.Key, "key", "k", "", "Name of the data item") +} + +func createLayerData(cmd *cobra.Command, args []string) error { + if createLayerDataOpts.ID == "" { + return errors.New("layer ID not specified") + } + if createLayerDataKey == "" { + return errors.New("layer data name not specified") + } + if createLayerDataValue == "" && createLayerDataFile == "" { + return errors.New("neither layer data value nor file specified") + } + createLayerDataOpts.Data = make(map[string][]byte) + if createLayerDataValue != "" { + createLayerDataOpts.Data[createLayerDataKey] = []byte(createLayerDataValue) + } + if createLayerDataFile != "" { + buf, err := os.ReadFile(createLayerDataFile) + if err != nil { + return err + } + createLayerDataOpts.Data[createLayerDataKey] = buf + } + _, err := testingEngine.CreateLayerData(mainContext, createLayerDataOpts) + if err != nil { + return err + } + return nil +} + +func createImageData(cmd *cobra.Command, args []string) error { + if createImageDataOpts.ID == "" { + return errors.New("image ID not specified") + } + if createImageDataKey == "" { + return errors.New("image data name not specified") + } + if createImageDataValue == "" && createImageDataFile == "" { + return errors.New("neither image data value nor file specified") + } + createImageDataOpts.Data = make(map[string][]byte) + if createImageDataValue != "" { + createImageDataOpts.Data[createImageDataKey] = []byte(createImageDataValue) + } + if createImageDataFile != "" { + d, err := os.ReadFile(createImageDataFile) + if err != nil { + return err + } + createImageDataOpts.Data[createImageDataKey] = d + } + _, err := testingEngine.CreateImageData(mainContext, createImageDataOpts) + if err != nil { + return err + } + return nil +} + +func createContainerData(cmd *cobra.Command, args []string) error { + if createContainerDataOpts.ID == "" { + return errors.New("container ID not specified") + } + if createContainerDataKey == "" { + return errors.New("container data name not specified") + } + if createContainerDataValue == "" && createContainerDataFile == "" { + return errors.New("neither container data value nor file specified") + } + createContainerDataOpts.Data = make(map[string][]byte) + if createContainerDataValue != "" { + createContainerDataOpts.Data[createContainerDataKey] = []byte(createContainerDataValue) + } + if createContainerDataFile != "" { + d, err := os.ReadFile(createContainerDataFile) + if err != nil { + return err + } + createContainerDataOpts.Data[createContainerDataKey] = d + } + _, err := testingEngine.CreateContainerData(mainContext, createContainerDataOpts) + if err != nil { + return err + } + return nil +} + +func modifyLayerData(cmd *cobra.Command, args []string) error { + if modifyLayerDataOpts.ID == "" { + return errors.New("layer ID not specified") + } + if modifyLayerDataOpts.Key == "" { + return errors.New("layer data name not specified") + } + if modifyLayerDataValue == "" && modifyLayerDataFile == "" { + return errors.New("neither layer data value nor file specified") + } + modifyLayerDataOpts.Data = []byte(modifyLayerDataValue) + if modifyLayerDataFile != "" { + d, err := os.ReadFile(modifyLayerDataFile) + if err != nil { + return err + } + modifyLayerDataOpts.Data = d + } + _, err := testingEngine.ModifyLayerData(mainContext, modifyLayerDataOpts) + if err != nil { + return err + } + return nil +} + +func modifyImageData(cmd *cobra.Command, args []string) error { + if modifyImageDataOpts.ID == "" { + return errors.New("image ID not specified") + } + if modifyImageDataOpts.Key == "" { + return errors.New("image data name not specified") + } + if modifyImageDataValue == "" && modifyImageDataFile == "" { + return errors.New("neither image data value nor file specified") + } + modifyImageDataOpts.Data = []byte(modifyImageDataValue) + if modifyImageDataFile != "" { + d, err := os.ReadFile(modifyImageDataFile) + if err != nil { + return err + } + modifyImageDataOpts.Data = d + } + _, err := testingEngine.ModifyImageData(mainContext, modifyImageDataOpts) + if err != nil { + return err + } + return nil +} + +func modifyContainerData(cmd *cobra.Command, args []string) error { + if modifyContainerDataOpts.ID == "" { + return errors.New("container ID not specified") + } + if modifyContainerDataOpts.Key == "" { + return errors.New("container data name not specified") + } + if modifyContainerDataValue == "" && modifyContainerDataFile == "" { + return errors.New("neither container data value nor file specified") + } + modifyContainerDataOpts.Data = []byte(modifyContainerDataValue) + if modifyContainerDataFile != "" { + d, err := os.ReadFile(modifyContainerDataFile) + if err != nil { + return err + } + modifyContainerDataOpts.Data = d + } + _, err := testingEngine.ModifyContainerData(mainContext, modifyContainerDataOpts) + if err != nil { + return err + } + return nil +} + +func removeLayerData(cmd *cobra.Command, args []string) error { + if removeLayerDataOpts.ID == "" { + return errors.New("layer ID not specified") + } + if removeLayerDataOpts.Key == "" { + return errors.New("layer data name not specified") + } + _, err := testingEngine.RemoveLayerData(mainContext, removeLayerDataOpts) + if err != nil { + return err + } + return nil +} + +func removeImageData(cmd *cobra.Command, args []string) error { + if removeImageDataOpts.ID == "" { + return errors.New("image ID not specified") + } + if removeImageDataOpts.Key == "" { + return errors.New("image data name not specified") + } + _, err := testingEngine.RemoveImageData(mainContext, removeImageDataOpts) + if err != nil { + return err + } + return nil +} + +func removeContainerData(cmd *cobra.Command, args []string) error { + if removeContainerDataOpts.ID == "" { + return errors.New("container ID not specified") + } + if removeContainerDataOpts.Key == "" { + return errors.New("container data name not specified") + } + _, err := testingEngine.RemoveContainerData(mainContext, removeContainerDataOpts) + if err != nil { + return err + } + return nil +} diff --git a/cmd/podman-testing/layer.go b/cmd/podman-testing/layer.go new file mode 100644 index 0000000000..ae4de28ca1 --- /dev/null +++ b/cmd/podman-testing/layer.go @@ -0,0 +1,91 @@ +package main + +import ( + "errors" + "os" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/validate" + "github.com/containers/podman/v5/internal/domain/entities" + "github.com/spf13/cobra" +) + +var ( + populateLayerDescription = `Populate a layer in local storage.` + populateLayerCmd = &cobra.Command{ + Use: "populate-layer [options]", + Args: validate.NoArgs, + Short: "Populate a layer", + Long: populateLayerDescription, + RunE: populateLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing populate-layer`, + } + + populateLayerOpts entities.PopulateLayerOptions + populateLayerFile string + + modifyLayerDescription = `Modify a layer in local storage, corrupting it.` + modifyLayerCmd = &cobra.Command{ + Use: "modify-layer [options]", + Args: validate.NoArgs, + Short: "Modify the contents of a layer", + Long: modifyLayerDescription, + RunE: modifyLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing modify-layer`, + } + + modifyLayerOpts entities.ModifyLayerOptions + modifyLayerFile string +) + +func init() { + mainCmd.AddCommand(populateLayerCmd) + flags := populateLayerCmd.Flags() + flags.StringVarP(&populateLayerOpts.ID, "layer", "l", "", "ID of layer to be populated") + flags.StringVarP(&populateLayerFile, "file", "f", "", "archive of contents to extract in layer") + + mainCmd.AddCommand(modifyLayerCmd) + flags = modifyLayerCmd.Flags() + flags.StringVarP(&modifyLayerOpts.ID, "layer", "l", "", "ID of layer to be modified") + flags.StringVarP(&modifyLayerFile, "file", "f", "", "archive of contents to extract over layer") +} + +func populateLayer(cmd *cobra.Command, args []string) error { + if populateLayerOpts.ID == "" { + return errors.New("layer ID not specified") + } + if populateLayerFile == "" { + return errors.New("layer contents file not specified") + } + buf, err := os.ReadFile(populateLayerFile) + if err != nil { + return err + } + populateLayerOpts.ContentsArchive = buf + _, err = testingEngine.PopulateLayer(mainContext, populateLayerOpts) + if err != nil { + return err + } + return nil +} + +func modifyLayer(cmd *cobra.Command, args []string) error { + if modifyLayerOpts.ID == "" { + return errors.New("layer ID not specified") + } + if modifyLayerFile == "" { + return errors.New("layer contents file not specified") + } + buf, err := os.ReadFile(modifyLayerFile) + if err != nil { + return err + } + modifyLayerOpts.ContentsArchive = buf + _, err = testingEngine.ModifyLayer(mainContext, modifyLayerOpts) + if err != nil { + return err + } + return nil +} diff --git a/cmd/podman-testing/main.go b/cmd/podman-testing/main.go new file mode 100644 index 0000000000..ccc1ff7454 --- /dev/null +++ b/cmd/podman-testing/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "syscall" + + "github.com/containers/common/pkg/config" + _ "github.com/containers/podman/v5/cmd/podman/completion" + ientities "github.com/containers/podman/v5/internal/domain/entities" + "github.com/containers/podman/v5/internal/domain/infra" + "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/storage" + "github.com/containers/storage/pkg/reexec" + "github.com/containers/storage/pkg/unshare" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + mainCmd = &cobra.Command{ + Use: "podman-testing", + Long: "Assorted tools for use in testing podman", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return before() + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + return after() + }, + SilenceUsage: true, + SilenceErrors: true, + } + mainContext = context.Background() + podmanConfig entities.PodmanConfig + globalStorageOptions storage.StoreOptions + globalLogLevel string + testingEngine ientities.TestingEngine +) + +func init() { + podmanConfig.FlagSet = mainCmd.PersistentFlags() + fl := mainCmd.PersistentFlags() + fl.StringVar(&podmanConfig.DockerConfig, "docker-config", os.Getenv("DOCKER_CONFIG"), "path to .docker/config") + fl.StringVar(&globalLogLevel, "log-level", "warn", "logging level") + fl.StringVar(&podmanConfig.URI, "url", "", "URL to access Podman service") + fl.StringVar(&podmanConfig.RegistriesConf, "registries-conf", os.Getenv("REGISTRIES_CONF"), "path to registries.conf (REGISTRIES_CONF)") +} + +func before() error { + if globalLogLevel != "" { + parsedLogLevel, err := logrus.ParseLevel(globalLogLevel) + if err != nil { + return fmt.Errorf("parsing log level %q: %w", globalLogLevel, err) + } + logrus.SetLevel(parsedLogLevel) + } + if err := storeBefore(); err != nil { + return fmt.Errorf("setting up storage: %w", err) + } + + podmanConfig.EngineMode = engineMode + podmanConfig.Remote = podmanConfig.URI != "" + + containersConf, err := config.Default() + if err != nil { + return fmt.Errorf("loading default configuration (may reference $CONTAINERS_CONF): %w", err) + } + podmanConfig.ContainersConfDefaultsRO = containersConf + containersConf, err = config.New(nil) + if err != nil { + return fmt.Errorf("loading default configuration (may reference $CONTAINERS_CONF): %w", err) + } + podmanConfig.ContainersConf = containersConf + + podmanConfig.StorageDriver = globalStorageOptions.GraphDriverName + podmanConfig.GraphRoot = globalStorageOptions.GraphRoot + podmanConfig.Runroot = globalStorageOptions.RunRoot + podmanConfig.ImageStore = globalStorageOptions.ImageStore + podmanConfig.StorageOpts = globalStorageOptions.GraphDriverOptions + podmanConfig.TransientStore = globalStorageOptions.TransientStore + + te, err := infra.NewTestingEngine(&podmanConfig) + if err != nil { + return fmt.Errorf("initializing libpod: %w", err) + } + testingEngine = te + return nil +} + +func after() error { + if err := storeAfter(); err != nil { + return fmt.Errorf("shutting down storage: %w", err) + } + return nil +} + +func main() { + if reexec.Init() { + // We were invoked with a different argv[0] indicating that we + // had a specific job to do as a subprocess, and it's done. + return + } + unshare.MaybeReexecUsingUserNamespace(false) + + exitCode := 1 + if err := mainCmd.Execute(); err != nil { + if logrus.IsLevelEnabled(logrus.TraceLevel) { + fmt.Fprintf(os.Stderr, "Error: %+v\n", err) + } else { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } + var ee *exec.ExitError + if errors.As(err, &ee) { + if w, ok := ee.Sys().(syscall.WaitStatus); ok { + exitCode = w.ExitStatus() + } + } + } else { + exitCode = 0 + } + os.Exit(exitCode) +} diff --git a/cmd/podman-testing/remove.go b/cmd/podman-testing/remove.go new file mode 100644 index 0000000000..59dadfd2d0 --- /dev/null +++ b/cmd/podman-testing/remove.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/validate" + "github.com/containers/podman/v5/internal/domain/entities" + "github.com/spf13/cobra" +) + +var ( + removeStorageLayerDescription = `Remove an unmanaged layer in local storage, potentially corrupting it.` + removeStorageLayerCmd = &cobra.Command{ + Use: "remove-storage-layer [options]", + Args: validate.NoArgs, + Short: "Remove an unmanaged layer", + Long: removeStorageLayerDescription, + RunE: removeStorageLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-storage-layer`, + } + + removeStorageLayerOpts entities.RemoveStorageLayerOptions + + removeLayerDescription = `Remove a layer in local storage, potentially corrupting it.` + removeLayerCmd = &cobra.Command{ + Use: "remove-layer [options]", + Args: validate.NoArgs, + Short: "Remove a layer", + Long: removeLayerDescription, + RunE: removeLayer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-layer`, + } + + removeLayerOpts entities.RemoveLayerOptions + + removeImageDescription = `Remove an image in local storage, potentially corrupting it.` + removeImageCmd = &cobra.Command{ + Use: "remove-image [options]", + Args: validate.NoArgs, + Short: "Remove an image", + Long: removeImageDescription, + RunE: removeImage, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-image`, + } + + removeImageOpts entities.RemoveImageOptions + + removeContainerDescription = `Remove a container in local storage, potentially corrupting it.` + removeContainerCmd = &cobra.Command{ + Use: "remove-container [options]", + Args: validate.NoArgs, + Short: "Remove an container", + Long: removeContainerDescription, + RunE: removeContainer, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman testing remove-container`, + } + + removeContainerOpts entities.RemoveContainerOptions +) + +func init() { + mainCmd.AddCommand(removeStorageLayerCmd) + flags := removeStorageLayerCmd.Flags() + flags.StringVarP(&removeStorageLayerOpts.ID, "layer", "i", "", "ID of the layer to remove") + + mainCmd.AddCommand(removeLayerCmd) + flags = removeLayerCmd.Flags() + flags.StringVarP(&removeLayerOpts.ID, "layer", "i", "", "ID of the layer to remove") + + mainCmd.AddCommand(removeImageCmd) + flags = removeImageCmd.Flags() + flags.StringVarP(&removeImageOpts.ID, "image", "i", "", "ID of the image to remove") + + mainCmd.AddCommand(removeContainerCmd) + flags = removeContainerCmd.Flags() + flags.StringVarP(&removeContainerOpts.ID, "container", "i", "", "ID of the container to remove") +} + +func removeStorageLayer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.RemoveStorageLayer(mainContext, removeStorageLayerOpts) + if err != nil { + return err + } + fmt.Println(results.ID) + return nil +} + +func removeLayer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.RemoveLayer(mainContext, removeLayerOpts) + if err != nil { + return err + } + fmt.Println(results.ID) + return nil +} + +func removeImage(cmd *cobra.Command, args []string) error { + results, err := testingEngine.RemoveImage(mainContext, removeImageOpts) + if err != nil { + return err + } + fmt.Println(results.ID) + return nil +} + +func removeContainer(cmd *cobra.Command, args []string) error { + results, err := testingEngine.RemoveContainer(mainContext, removeContainerOpts) + if err != nil { + return err + } + fmt.Println(results.ID) + return nil +} diff --git a/cmd/podman-testing/store_supported.go b/cmd/podman-testing/store_supported.go new file mode 100644 index 0000000000..b8e2fac5a7 --- /dev/null +++ b/cmd/podman-testing/store_supported.go @@ -0,0 +1,65 @@ +//go:build linux && !remote +// +build linux,!remote + +package main + +import ( + "fmt" + "os" + + "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/storage" + "github.com/containers/storage/types" +) + +var ( + globalStore storage.Store + engineMode = entities.ABIMode +) + +func init() { + if defaultStoreOptions, err := storage.DefaultStoreOptions(); err == nil { + globalStorageOptions = defaultStoreOptions + } + if storageConf, ok := os.LookupEnv("CONTAINERS_STORAGE_CONF"); ok { + options := globalStorageOptions + if types.ReloadConfigurationFileIfNeeded(storageConf, &options) == nil { + globalStorageOptions = options + } + } + fl := mainCmd.PersistentFlags() + fl.StringVar(&globalStorageOptions.GraphDriverName, "storage-driver", "", "storage driver used to manage images and containers") + fl.StringVar(&globalStorageOptions.GraphRoot, "root", "", "where images and containers will be stored") + fl.StringVar(&globalStorageOptions.RunRoot, "runroot", "", "where volatile state information will be stored") + fl.StringArrayVar(&globalStorageOptions.GraphDriverOptions, "storage-opt", nil, "storage driver options") + fl.StringVar(&globalStorageOptions.ImageStore, "imagestore", "", "where to store just some parts of images") + fl.BoolVar(&globalStorageOptions.TransientStore, "transient-store", false, "enable transient container storage") +} + +func storeBefore() error { + defaultStoreOptions, err := storage.DefaultStoreOptions() + if err != nil { + fmt.Fprintf(os.Stderr, "selecting storage options: %v", err) + return nil + } + globalStorageOptions = defaultStoreOptions + store, err := storage.GetStore(globalStorageOptions) + if err != nil { + return err + } + globalStore = store + if podmanConfig.URI != "" { + engineMode = entities.TunnelMode + } else { + engineMode = entities.ABIMode + } + return nil +} + +func storeAfter() error { + if globalStore != nil { + _, err := globalStore.Shutdown(false) + return err + } + return nil +} diff --git a/cmd/podman-testing/store_unsupported.go b/cmd/podman-testing/store_unsupported.go new file mode 100644 index 0000000000..de79ed88b0 --- /dev/null +++ b/cmd/podman-testing/store_unsupported.go @@ -0,0 +1,16 @@ +//go:build !linux || remote +// +build !linux remote + +package main + +import "github.com/containers/podman/v5/pkg/domain/entities" + +const engineMode = entities.TunnelMode + +func storeBefore() error { + return nil +} + +func storeAfter() error { + return nil +} diff --git a/cmd/podman/system/check.go b/cmd/podman/system/check.go new file mode 100644 index 0000000000..3a3f0348bc --- /dev/null +++ b/cmd/podman/system/check.go @@ -0,0 +1,138 @@ +package system + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/registry" + "github.com/containers/podman/v5/cmd/podman/validate" + "github.com/containers/podman/v5/pkg/domain/entities/types" + multierror "github.com/hashicorp/go-multierror" + "github.com/spf13/cobra" +) + +var ( + checkOptions = types.SystemCheckOptions{} + checkDescription = ` + podman system check + + Check storage for consistency and remove anything that looks damaged +` + + checkCommand = &cobra.Command{ + Use: "check [options]", + Short: "Check storage consistency", + Args: validate.NoArgs, + Long: checkDescription, + RunE: check, + ValidArgsFunction: completion.AutocompleteNone, + Example: `podman system check`, + } +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: checkCommand, + Parent: systemCmd, + }) + flags := checkCommand.Flags() + flags.BoolVarP(&checkOptions.Quick, "quick", "q", false, "Skip time-consuming checks. The default is to include time-consuming checks") + flags.BoolVarP(&checkOptions.Repair, "repair", "r", false, "Remove inconsistent images") + flags.BoolVarP(&checkOptions.RepairLossy, "force", "f", false, "Remove inconsistent images and containers") + flags.DurationP("max", "m", 24*time.Hour, "Maximum allowed age of unreferenced layers") + _ = checkCommand.RegisterFlagCompletionFunc("max", completion.AutocompleteNone) +} + +func check(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + if flags.Changed("max") { + maxAge, err := flags.GetDuration("max") + if err != nil { + return err + } + checkOptions.UnreferencedLayerMaximumAge = &maxAge + } + response, err := registry.ContainerEngine().SystemCheck(context.Background(), checkOptions) + if err != nil { + return err + } + + if err = printSystemCheckResults(response); err != nil { + return err + } + + if !checkOptions.Repair && !checkOptions.RepairLossy && response.Errors { + return errors.New("damage detected in local storage") + } + + recheckOptions := checkOptions + recheckOptions.Repair = false + recheckOptions.RepairLossy = false + if response, err = registry.ContainerEngine().SystemCheck(context.Background(), recheckOptions); err != nil { + return err + } + if response.Errors { + return errors.New("damage in local storage still present after repair attempt") + } + + return nil +} + +func printSystemCheckResults(report *types.SystemCheckReport) error { + if !report.Errors { + return nil + } + errorSlice := func(strs []string) []error { + if strs == nil { + return nil + } + errs := make([]error, len(strs)) + for i, s := range strs { + errs[i] = errors.New(s) + } + return errs + } + for damagedLayer, errorsSlice := range report.Layers { + merr := multierror.Append(nil, errorSlice(errorsSlice)...) + if err := merr.ErrorOrNil(); err != nil { + fmt.Printf("Damaged layer %s:\n%s", damagedLayer, err) + } + } + for _, removedLayer := range report.RemovedLayers { + fmt.Printf("Deleted damaged layer: %s\n", removedLayer) + } + for damagedROLayer, errorsSlice := range report.ROLayers { + merr := multierror.Append(nil, errorSlice(errorsSlice)...) + if err := merr.ErrorOrNil(); err != nil { + fmt.Printf("Damaged read-only layer %s:\n%s", damagedROLayer, err) + } + } + for damagedImage, errorsSlice := range report.Images { + merr := multierror.Append(nil, errorSlice(errorsSlice)...) + if err := merr.ErrorOrNil(); err != nil { + fmt.Printf("Damaged image %s:\n%s", damagedImage, err) + } + } + for removedImage := range report.RemovedImages { + fmt.Printf("Deleted damaged image: %s\n", removedImage) + } + for damagedROImage, errorsSlice := range report.ROImages { + merr := multierror.Append(nil, errorSlice(errorsSlice)...) + if err := merr.ErrorOrNil(); err != nil { + fmt.Printf("Damaged read-only image %s\n%s", damagedROImage, err) + } + } + for damagedContainer, errorsSlice := range report.Containers { + merr := multierror.Append(nil, errorSlice(errorsSlice)...) + if err := merr.ErrorOrNil(); err != nil { + fmt.Printf("Damaged container %s:\n%s", damagedContainer, err) + } + } + for removedContainer := range report.RemovedContainers { + fmt.Printf("Deleted damaged container: %s\n", removedContainer) + } + return nil +} diff --git a/docs/source/markdown/podman-system-check.1.md b/docs/source/markdown/podman-system-check.1.md new file mode 100644 index 0000000000..1abe83ce6c --- /dev/null +++ b/docs/source/markdown/podman-system-check.1.md @@ -0,0 +1,59 @@ +% podman-system-check 1 + +## NAME +podman\-system\-check - Perform consistency checks on image and container storage + +## SYNOPSIS +**podman system check** [*options*] + +## DESCRIPTION +Perform consistency checks on image and container storage, reporting images and +containers which have identified issues. + +## OPTIONS + +#### **--force**, **-f** + +When attempting to remove damaged images, also remove containers which depend +on those images. By default, damaged images which are being used by containers +are left alone. + +Containers which depend on damaged images do so regardless of which engine +created them, but because podman only "knows" how to shut down containers that +it started, the effect on still-running containers which were started by other +engines is difficult to predict. + +#### **--max**, **-m**=*duration* + +When considering layers which are not used by any images or containers, assume +that any layers which are more than *duration* old are the results of canceled +attempts to pull images, and should be treated as though they are damaged. + +#### **--quick**, **-q** + +Skip checks which are known to be time-consuming. This will prevent some types +of errors from being detected. + +#### **--repair**, **-r** + +Remove any images which are determined to have been damaged in some way, unless +they are in use by containers. Use **--force** to remove containers which +depend on damaged images, and those damaged images, as well. + +## EXAMPLE + +A reasonably quick check: +``` +podman system check --quick --repair --force +``` + +A more thorough check: +``` +podman system check --repair --max=1h --force +``` + +## SEE ALSO +**[podman(1)](podman.1.md)**, **[podman-system(1)](podman-system.1.md)** + +## HISTORY +April 2024 diff --git a/docs/source/markdown/podman-system.1.md b/docs/source/markdown/podman-system.1.md index 103b877aa2..0f20ced2c4 100644 --- a/docs/source/markdown/podman-system.1.md +++ b/docs/source/markdown/podman-system.1.md @@ -13,6 +13,7 @@ The system command allows management of the podman systems | Command | Man Page | Description | | ------- | ------------------------------------------------------------ | ------------------------------------------------------------------------ | +| check | [podman-system-check(1)](podman-system-check.1.md) | Perform consistency checks on image and container storage. | connection | [podman-system-connection(1)](podman-system-connection.1.md) | Manage the destination(s) for Podman service(s) | | df | [podman-system-df(1)](podman-system-df.1.md) | Show podman disk usage. | | events | [podman-events(1)](podman-events.1.md) | Monitor Podman events | diff --git a/hack/golangci-lint.sh b/hack/golangci-lint.sh index 35fdcde554..8803b68bd3 100755 --- a/hack/golangci-lint.sh +++ b/hack/golangci-lint.sh @@ -18,8 +18,8 @@ BUILD_TAGS_TUNNEL="$BUILD_TAGS_DEFAULT,remote" BUILD_TAGS_REMOTE="remote,containers_image_openpgp" SKIP_DIRS_ABI="" -SKIP_DIRS_TUNNEL="pkg/api,pkg/domain/infra/abi" -SKIP_DIRS_REMOTE="libpod/events,pkg/api,pkg/domain/infra/abi,pkg/machine/qemu,pkg/trust,test" +SKIP_DIRS_TUNNEL="pkg/api,pkg/domain/infra/abi,internal/domain/infra/abi" +SKIP_DIRS_REMOTE="libpod/events,pkg/api,pkg/domain/infra/abi,internal/domain/infra/abi,pkg/machine/qemu,pkg/trust,test" declare -a to_lint to_lint=(ABI TUNNEL REMOTE) diff --git a/internal/domain/entities/engine_testing.go b/internal/domain/entities/engine_testing.go new file mode 100644 index 0000000000..9ad9ee18bb --- /dev/null +++ b/internal/domain/entities/engine_testing.go @@ -0,0 +1,27 @@ +package entities + +import ( + "context" +) + +type TestingEngine interface { //nolint:interfacebloat + CreateStorageLayer(ctx context.Context, opts CreateStorageLayerOptions) (*CreateStorageLayerReport, error) + CreateLayer(ctx context.Context, opts CreateLayerOptions) (*CreateLayerReport, error) + CreateLayerData(ctx context.Context, opts CreateLayerDataOptions) (*CreateLayerDataReport, error) + CreateImage(ctx context.Context, opts CreateImageOptions) (*CreateImageReport, error) + CreateImageData(ctx context.Context, opts CreateImageDataOptions) (*CreateImageDataReport, error) + CreateContainer(ctx context.Context, opts CreateContainerOptions) (*CreateContainerReport, error) + CreateContainerData(ctx context.Context, opts CreateContainerDataOptions) (*CreateContainerDataReport, error) + ModifyLayer(ctx context.Context, opts ModifyLayerOptions) (*ModifyLayerReport, error) + PopulateLayer(ctx context.Context, opts PopulateLayerOptions) (*PopulateLayerReport, error) + RemoveStorageLayer(ctx context.Context, opts RemoveStorageLayerOptions) (*RemoveStorageLayerReport, error) + RemoveLayer(ctx context.Context, opts RemoveLayerOptions) (*RemoveLayerReport, error) + RemoveImage(ctx context.Context, opts RemoveImageOptions) (*RemoveImageReport, error) + RemoveContainer(ctx context.Context, opts RemoveContainerOptions) (*RemoveContainerReport, error) + RemoveLayerData(ctx context.Context, opts RemoveLayerDataOptions) (*RemoveLayerDataReport, error) + RemoveImageData(ctx context.Context, opts RemoveImageDataOptions) (*RemoveImageDataReport, error) + RemoveContainerData(ctx context.Context, opts RemoveContainerDataOptions) (*RemoveContainerDataReport, error) + ModifyLayerData(ctx context.Context, opts ModifyLayerDataOptions) (*ModifyLayerDataReport, error) + ModifyImageData(ctx context.Context, opts ModifyImageDataOptions) (*ModifyImageDataReport, error) + ModifyContainerData(ctx context.Context, opts ModifyContainerDataOptions) (*ModifyContainerDataReport, error) +} diff --git a/internal/domain/entities/testing.go b/internal/domain/entities/testing.go new file mode 100644 index 0000000000..754c4927e7 --- /dev/null +++ b/internal/domain/entities/testing.go @@ -0,0 +1,153 @@ +package entities + +type CreateStorageLayerOptions struct { + Parent string + ID string + ContentsArchive []byte +} + +type CreateStorageLayerReport struct { + ID string +} + +type CreateLayerOptions struct { + Parent string + ID string +} + +type CreateLayerReport struct { + ID string +} + +type CreateLayerDataOptions struct { + ID string + Data map[string][]byte +} + +type CreateLayerDataReport struct{} + +type CreateImageOptions struct { + Layer string + Names []string + ID string +} + +type CreateImageReport struct { + ID string +} + +type CreateImageDataOptions struct { + ID string + Data map[string][]byte +} + +type CreateImageDataReport struct{} + +type CreateContainerOptions struct { + Layer string + Image string + Names []string + ID string +} + +type CreateContainerReport struct { + ID string +} + +type CreateContainerDataOptions struct { + ID string + Data map[string][]byte +} + +type CreateContainerDataReport struct{} + +type ModifyLayerOptions struct { + ID string + ContentsArchive []byte +} + +type ModifyLayerReport struct{} + +type PopulateLayerOptions struct { + ID string + ContentsArchive []byte +} + +type PopulateLayerReport struct{} + +type RemoveStorageLayerOptions struct { + ID string +} + +type RemoveStorageLayerReport struct { + ID string +} + +type RemoveLayerOptions struct { + ID string +} + +type RemoveLayerReport struct { + ID string +} + +type RemoveImageOptions struct { + ID string +} + +type RemoveImageReport struct { + ID string +} + +type RemoveContainerOptions struct { + ID string +} + +type RemoveContainerReport struct { + ID string +} + +type RemoveLayerDataOptions struct { + ID string + Key string +} + +type RemoveLayerDataReport struct{} + +type RemoveImageDataOptions struct { + ID string + Key string +} + +type RemoveImageDataReport struct{} + +type RemoveContainerDataOptions struct { + ID string + Key string +} + +type RemoveContainerDataReport struct{} + +type ModifyLayerDataOptions struct { + ID string + Key string + Data []byte +} + +type ModifyLayerDataReport struct{} + +type ModifyImageDataOptions struct { + ID string + Key string + Data []byte +} + +type ModifyImageDataReport struct{} + +type ModifyContainerDataOptions struct { + ID string + Key string + Data []byte +} + +type ModifyContainerDataReport struct{} diff --git a/internal/domain/infra/abi/testing.go b/internal/domain/infra/abi/testing.go new file mode 100644 index 0000000000..4cf49d0801 --- /dev/null +++ b/internal/domain/infra/abi/testing.go @@ -0,0 +1,220 @@ +package abi + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + + "github.com/containers/image/v5/manifest" + "github.com/containers/podman/v5/internal/domain/entities" + "github.com/containers/podman/v5/libpod" + "github.com/containers/storage" + graphdriver "github.com/containers/storage/drivers" + "github.com/containers/storage/pkg/chrootarchive" + "github.com/containers/storage/pkg/stringid" +) + +type TestingEngine struct { + Libpod *libpod.Runtime + Store storage.Store +} + +func (te *TestingEngine) CreateStorageLayer(ctx context.Context, opts entities.CreateStorageLayerOptions) (*entities.CreateStorageLayerReport, error) { + driver, err := te.Store.GraphDriver() + if err != nil { + return nil, err + } + id := opts.ID + if id == "" { + id = stringid.GenerateNonCryptoID() + } + if err := driver.CreateReadWrite(id, opts.Parent, &graphdriver.CreateOpts{}); err != nil { + return nil, err + } + return &entities.CreateStorageLayerReport{ID: id}, nil +} + +func (te *TestingEngine) CreateLayer(ctx context.Context, opts entities.CreateLayerOptions) (*entities.CreateLayerReport, error) { + layer, err := te.Store.CreateLayer(opts.ID, opts.Parent, nil, "", true, nil) + if err != nil { + return nil, err + } + return &entities.CreateLayerReport{ID: layer.ID}, nil +} + +func (te *TestingEngine) CreateLayerData(ctx context.Context, opts entities.CreateLayerDataOptions) (*entities.CreateLayerDataReport, error) { + for key, data := range opts.Data { + if err := te.Store.SetLayerBigData(opts.ID, key, bytes.NewReader(data)); err != nil { + return nil, err + } + } + return &entities.CreateLayerDataReport{}, nil +} + +func (te *TestingEngine) ModifyLayer(ctx context.Context, opts entities.ModifyLayerOptions) (*entities.ModifyLayerReport, error) { + mnt, err := te.Store.Mount(opts.ID, "") + if err != nil { + return nil, err + } + modifyError := chrootarchive.UntarWithRoot(bytes.NewReader(opts.ContentsArchive), mnt, nil, mnt) + if _, err := te.Store.Unmount(opts.ID, false); err != nil { + return nil, err + } + if modifyError != nil { + return nil, modifyError + } + return &entities.ModifyLayerReport{}, nil +} + +func (te *TestingEngine) PopulateLayer(ctx context.Context, opts entities.PopulateLayerOptions) (*entities.PopulateLayerReport, error) { + if _, err := te.Store.ApplyDiff(opts.ID, bytes.NewReader(opts.ContentsArchive)); err != nil { + return nil, err + } + return &entities.PopulateLayerReport{}, nil +} + +func (te *TestingEngine) CreateImage(ctx context.Context, opts entities.CreateImageOptions) (*entities.CreateImageReport, error) { + image, err := te.Store.CreateImage(opts.ID, opts.Names, opts.Layer, "", nil) + if err != nil { + return nil, err + } + return &entities.CreateImageReport{ID: image.ID}, nil +} + +func (te *TestingEngine) CreateImageData(ctx context.Context, opts entities.CreateImageDataOptions) (*entities.CreateImageDataReport, error) { + for key, data := range opts.Data { + if err := te.Store.SetImageBigData(opts.ID, key, data, manifest.Digest); err != nil { + return nil, err + } + } + return &entities.CreateImageDataReport{}, nil +} + +func (te *TestingEngine) CreateContainer(ctx context.Context, opts entities.CreateContainerOptions) (*entities.CreateContainerReport, error) { + image, err := te.Store.CreateContainer(opts.ID, opts.Names, opts.Image, opts.Layer, "", nil) + if err != nil { + return nil, err + } + return &entities.CreateContainerReport{ID: image.ID}, nil +} + +func (te *TestingEngine) CreateContainerData(ctx context.Context, opts entities.CreateContainerDataOptions) (*entities.CreateContainerDataReport, error) { + for key, data := range opts.Data { + if err := te.Store.SetContainerBigData(opts.ID, key, data); err != nil { + return nil, err + } + } + return &entities.CreateContainerDataReport{}, nil +} + +func (te *TestingEngine) RemoveStorageLayer(ctx context.Context, opts entities.RemoveStorageLayerOptions) (*entities.RemoveStorageLayerReport, error) { + driver, err := te.Store.GraphDriver() + if err != nil { + return nil, err + } + if err := driver.Remove(opts.ID); err != nil { + return nil, err + } + return &entities.RemoveStorageLayerReport{ID: opts.ID}, nil +} + +func (te *TestingEngine) RemoveLayer(ctx context.Context, opts entities.RemoveLayerOptions) (*entities.RemoveLayerReport, error) { + if err := te.Store.Delete(opts.ID); err != nil { + return nil, err + } + return &entities.RemoveLayerReport{ID: opts.ID}, nil +} + +func (te *TestingEngine) RemoveImage(ctx context.Context, opts entities.RemoveImageOptions) (*entities.RemoveImageReport, error) { + if err := te.Store.Delete(opts.ID); err != nil { + return nil, err + } + return &entities.RemoveImageReport{ID: opts.ID}, nil +} + +func (te *TestingEngine) RemoveContainer(ctx context.Context, opts entities.RemoveContainerOptions) (*entities.RemoveContainerReport, error) { + if err := te.Store.Delete(opts.ID); err != nil { + return nil, err + } + return &entities.RemoveContainerReport{ID: opts.ID}, nil +} + +func (te *TestingEngine) datapath(itemType, id, key string) (string, error) { + switch itemType { + default: + return "", fmt.Errorf("unknown item type %q", itemType) + case "layer", "image", "container": + } + driverName := te.Store.GraphDriverName() + graphRoot := te.Store.GraphRoot() + datapath := filepath.Join(graphRoot, driverName+"-"+itemType+"s", id, key) // more or less accurate for keys whose names are [.a-z0-9]+ + return datapath, nil +} + +func (te *TestingEngine) RemoveLayerData(ctx context.Context, opts entities.RemoveLayerDataOptions) (*entities.RemoveLayerDataReport, error) { + datapath, err := te.datapath("layer", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.Remove(datapath); err != nil { + return nil, err + } + return &entities.RemoveLayerDataReport{}, nil +} + +func (te *TestingEngine) RemoveImageData(ctx context.Context, opts entities.RemoveImageDataOptions) (*entities.RemoveImageDataReport, error) { + datapath, err := te.datapath("image", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.Remove(datapath); err != nil { + return nil, err + } + return &entities.RemoveImageDataReport{}, nil +} + +func (te *TestingEngine) RemoveContainerData(ctx context.Context, opts entities.RemoveContainerDataOptions) (*entities.RemoveContainerDataReport, error) { + datapath, err := te.datapath("container", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.Remove(datapath); err != nil { + return nil, err + } + return &entities.RemoveContainerDataReport{}, nil +} + +func (te *TestingEngine) ModifyLayerData(ctx context.Context, opts entities.ModifyLayerDataOptions) (*entities.ModifyLayerDataReport, error) { + datapath, err := te.datapath("layer", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.WriteFile(datapath, opts.Data, 0o0600); err != nil { + return nil, err + } + return &entities.ModifyLayerDataReport{}, nil +} + +func (te *TestingEngine) ModifyImageData(ctx context.Context, opts entities.ModifyImageDataOptions) (*entities.ModifyImageDataReport, error) { + datapath, err := te.datapath("image", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.WriteFile(datapath, opts.Data, 0o0600); err != nil { + return nil, err + } + return &entities.ModifyImageDataReport{}, nil +} + +func (te *TestingEngine) ModifyContainerData(ctx context.Context, opts entities.ModifyContainerDataOptions) (*entities.ModifyContainerDataReport, error) { + datapath, err := te.datapath("container", opts.ID, opts.Key) + if err != nil { + return nil, err + } + if err = os.WriteFile(datapath, opts.Data, 0o0600); err != nil { + return nil, err + } + return &entities.ModifyContainerDataReport{}, nil +} diff --git a/internal/domain/infra/abi/testing_test.go b/internal/domain/infra/abi/testing_test.go new file mode 100644 index 0000000000..75c6f4f542 --- /dev/null +++ b/internal/domain/infra/abi/testing_test.go @@ -0,0 +1,5 @@ +package abi + +import "github.com/containers/podman/v5/internal/domain/entities" + +var _ entities.TestingEngine = &TestingEngine{} diff --git a/internal/domain/infra/runtime_abi.go b/internal/domain/infra/runtime_abi.go new file mode 100644 index 0000000000..9dab3b190c --- /dev/null +++ b/internal/domain/infra/runtime_abi.go @@ -0,0 +1,26 @@ +//go:build !remote + +package infra + +import ( + "context" + "fmt" + + ientities "github.com/containers/podman/v5/internal/domain/entities" + "github.com/containers/podman/v5/internal/domain/infra/tunnel" + "github.com/containers/podman/v5/pkg/bindings" + "github.com/containers/podman/v5/pkg/domain/entities" +) + +// NewTestingEngine factory provides a libpod runtime for testing-specific operations +func NewTestingEngine(facts *entities.PodmanConfig) (ientities.TestingEngine, error) { + switch facts.EngineMode { + case entities.ABIMode: + r, err := NewLibpodTestingRuntime(facts.FlagSet, facts) + return r, err + case entities.TunnelMode: + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.URI, facts.Identity, facts.MachineMode) + return &tunnel.TestingEngine{ClientCtx: ctx}, err + } + return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) +} diff --git a/internal/domain/infra/runtime_proxy.go b/internal/domain/infra/runtime_proxy.go new file mode 100644 index 0000000000..2e2600cd65 --- /dev/null +++ b/internal/domain/infra/runtime_proxy.go @@ -0,0 +1,26 @@ +//go:build !remote + +package infra + +import ( + "context" + + ientities "github.com/containers/podman/v5/internal/domain/entities" + "github.com/containers/podman/v5/internal/domain/infra/abi" + "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/containers/podman/v5/pkg/domain/infra" + "github.com/containers/storage" + flag "github.com/spf13/pflag" +) + +func NewLibpodTestingRuntime(flags *flag.FlagSet, opts *entities.PodmanConfig) (ientities.TestingEngine, error) { + r, err := infra.GetRuntime(context.Background(), flags, opts) + if err != nil { + return nil, err + } + store, err := storage.GetStore(r.StorageConfig()) + if err != nil { + return nil, err + } + return &abi.TestingEngine{Libpod: r, Store: store}, nil +} diff --git a/internal/domain/infra/runtime_tunnel.go b/internal/domain/infra/runtime_tunnel.go new file mode 100644 index 0000000000..5bade4eddc --- /dev/null +++ b/internal/domain/infra/runtime_tunnel.go @@ -0,0 +1,25 @@ +//go:build remote + +package infra + +import ( + "context" + "fmt" + + ientities "github.com/containers/podman/v5/internal/domain/entities" + "github.com/containers/podman/v5/internal/domain/infra/tunnel" + "github.com/containers/podman/v5/pkg/bindings" + "github.com/containers/podman/v5/pkg/domain/entities" +) + +// NewTestingEngine factory provides a libpod runtime for testing-specific operations +func NewTestingEngine(facts *entities.PodmanConfig) (ientities.TestingEngine, error) { + switch facts.EngineMode { + case entities.ABIMode: + return nil, fmt.Errorf("direct image runtime not supported") + case entities.TunnelMode: + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.URI, facts.Identity, facts.MachineMode) + return &tunnel.TestingEngine{ClientCtx: ctx}, err + } + return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) +} diff --git a/internal/domain/infra/tunnel/testing.go b/internal/domain/infra/tunnel/testing.go new file mode 100644 index 0000000000..8efc6c3727 --- /dev/null +++ b/internal/domain/infra/tunnel/testing.go @@ -0,0 +1,88 @@ +package tunnel + +import ( + "context" + "syscall" + + "github.com/containers/podman/v5/internal/domain/entities" +) + +type TestingEngine struct { + ClientCtx context.Context +} + +func (te *TestingEngine) CreateStorageLayer(ctx context.Context, opts entities.CreateStorageLayerOptions) (*entities.CreateStorageLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateLayer(ctx context.Context, opts entities.CreateLayerOptions) (*entities.CreateLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateLayerData(ctx context.Context, opts entities.CreateLayerDataOptions) (*entities.CreateLayerDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) ModifyLayer(ctx context.Context, opts entities.ModifyLayerOptions) (*entities.ModifyLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) PopulateLayer(ctx context.Context, opts entities.PopulateLayerOptions) (*entities.PopulateLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveStorageLayer(ctx context.Context, opts entities.RemoveStorageLayerOptions) (*entities.RemoveStorageLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateImage(ctx context.Context, opts entities.CreateImageOptions) (*entities.CreateImageReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateImageData(ctx context.Context, opts entities.CreateImageDataOptions) (*entities.CreateImageDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveLayer(ctx context.Context, opts entities.RemoveLayerOptions) (*entities.RemoveLayerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveImage(ctx context.Context, opts entities.RemoveImageOptions) (*entities.RemoveImageReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveContainer(ctx context.Context, opts entities.RemoveContainerOptions) (*entities.RemoveContainerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateContainer(ctx context.Context, opts entities.CreateContainerOptions) (*entities.CreateContainerReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) CreateContainerData(ctx context.Context, opts entities.CreateContainerDataOptions) (*entities.CreateContainerDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveLayerData(ctx context.Context, opts entities.RemoveLayerDataOptions) (*entities.RemoveLayerDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveImageData(ctx context.Context, opts entities.RemoveImageDataOptions) (*entities.RemoveImageDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) RemoveContainerData(ctx context.Context, opts entities.RemoveContainerDataOptions) (*entities.RemoveContainerDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) ModifyLayerData(ctx context.Context, opts entities.ModifyLayerDataOptions) (*entities.ModifyLayerDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) ModifyImageData(ctx context.Context, opts entities.ModifyImageDataOptions) (*entities.ModifyImageDataReport, error) { + return nil, syscall.ENOSYS +} + +func (te *TestingEngine) ModifyContainerData(ctx context.Context, opts entities.ModifyContainerDataOptions) (*entities.ModifyContainerDataReport, error) { + return nil, syscall.ENOSYS +} diff --git a/internal/domain/infra/tunnel/testing_test.go b/internal/domain/infra/tunnel/testing_test.go new file mode 100644 index 0000000000..d8cfa120f4 --- /dev/null +++ b/internal/domain/infra/tunnel/testing_test.go @@ -0,0 +1,5 @@ +package tunnel + +import "github.com/containers/podman/v5/internal/domain/entities" + +var _ entities.TestingEngine = &TestingEngine{} diff --git a/libpod/runtime.go b/libpod/runtime.go index b9a89ae6c2..7ad2c23bbf 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -31,6 +31,7 @@ import ( "github.com/containers/podman/v5/libpod/lock" "github.com/containers/podman/v5/libpod/plugin" "github.com/containers/podman/v5/libpod/shutdown" + "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/pkg/rootless" "github.com/containers/podman/v5/pkg/systemd" "github.com/containers/podman/v5/pkg/util" @@ -39,9 +40,11 @@ import ( "github.com/containers/storage/pkg/lockfile" "github.com/containers/storage/pkg/unshare" "github.com/docker/docker/pkg/namesgenerator" + "github.com/hashicorp/go-multierror" jsoniter "github.com/json-iterator/go" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "golang.org/x/exp/slices" ) // Set up the JSON library for all of Libpod @@ -1249,3 +1252,133 @@ func (r *Runtime) LockConflicts() (map[uint32][]string, []uint32, error) { return toReturn, locksHeld, nil } + +// SystemCheck checks our storage for consistency, and depending on the options +// specified, will attempt to remove anything which fails consistency checks. +func (r *Runtime) SystemCheck(ctx context.Context, options entities.SystemCheckOptions) (entities.SystemCheckReport, error) { + what := storage.CheckEverything() + if options.Quick { + what = storage.CheckMost() + } + if options.UnreferencedLayerMaximumAge != nil { + tmp := *options.UnreferencedLayerMaximumAge + what.LayerUnreferencedMaximumAge = &tmp + } + storageReport, err := r.store.Check(what) + if err != nil { + return entities.SystemCheckReport{}, err + } + if len(storageReport.Containers) == 0 && + len(storageReport.Layers) == 0 && + len(storageReport.ROLayers) == 0 && + len(storageReport.Images) == 0 && + len(storageReport.ROImages) == 0 { + // no errors detected + return entities.SystemCheckReport{}, nil + } + mapErrorSlicesToStringSlices := func(m map[string][]error) map[string][]string { + if len(m) == 0 { + return nil + } + mapped := make(map[string][]string, len(m)) + for k, errs := range m { + strs := make([]string, len(errs)) + for i, e := range errs { + strs[i] = e.Error() + } + mapped[k] = strs + } + return mapped + } + + report := entities.SystemCheckReport{ + Errors: true, + Layers: mapErrorSlicesToStringSlices(storageReport.Layers), + ROLayers: mapErrorSlicesToStringSlices(storageReport.ROLayers), + Images: mapErrorSlicesToStringSlices(storageReport.Images), + ROImages: mapErrorSlicesToStringSlices(storageReport.ROImages), + Containers: mapErrorSlicesToStringSlices(storageReport.Containers), + } + if !options.Repair && report.Errors { + // errors detected, no corrective measures to be taken + return report, err + } + + // get a list of images that we knew of before we tried to clean up any + // that were damaged + imagesBefore, err := r.store.Images() + if err != nil { + return report, fmt.Errorf("getting a list of images before attempting repairs: %w", err) + } + + repairOptions := storage.RepairOptions{ + RemoveContainers: options.RepairLossy, + } + var containers []*Container + if repairOptions.RemoveContainers { + // build a list of the containers that we claim as ours that we + // expect to be removing in a bit + for containerID := range storageReport.Containers { + ctr, lookupErr := r.state.LookupContainer(containerID) + if lookupErr != nil { + // we're about to remove it, so it's okay that + // it isn't even one of ours + continue + } + containers = append(containers, ctr) + } + } + + // run the cleanup + merr := multierror.Append(nil, r.store.Repair(storageReport, &repairOptions)...) + + if repairOptions.RemoveContainers { + // get the list of containers that storage will still admit to knowing about + containersAfter, err := r.store.Containers() + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("getting a list of containers after attempting repairs: %w", err)) + } + for _, ctr := range containers { + // if one of our containers that we tried to remove is + // still on disk, report an error + if slices.IndexFunc(containersAfter, func(containerAfter storage.Container) bool { + return containerAfter.ID == ctr.ID() + }) != -1 { + merr = multierror.Append(merr, fmt.Errorf("clearing storage for container %s: %w", ctr.ID(), err)) + continue + } + // remove the container from our database + if removeErr := r.state.RemoveContainer(ctr); removeErr != nil { + merr = multierror.Append(merr, fmt.Errorf("updating state database to reflect removal of container %s: %w", ctr.ID(), removeErr)) + continue + } + if report.RemovedContainers == nil { + report.RemovedContainers = make(map[string]string) + } + report.RemovedContainers[ctr.ID()] = ctr.config.Name + } + } + + // get a list of images that are still around after we clean up any + // that were damaged + imagesAfter, err := r.store.Images() + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("getting a list of images after attempting repairs: %w", err)) + } + for _, imageBefore := range imagesBefore { + if slices.IndexFunc(imagesAfter, func(imageAfter storage.Image) bool { + return imageAfter.ID == imageBefore.ID + }) == -1 { + if report.RemovedImages == nil { + report.RemovedImages = make(map[string][]string) + } + report.RemovedImages[imageBefore.ID] = slices.Clone(imageBefore.Names) + } + } + + if merr != nil { + err = merr.ErrorOrNil() + } + + return report, err +} diff --git a/pkg/api/handlers/libpod/system.go b/pkg/api/handlers/libpod/system.go index 8e9f939588..c6444fd901 100644 --- a/pkg/api/handlers/libpod/system.go +++ b/pkg/api/handlers/libpod/system.go @@ -3,6 +3,7 @@ package libpod import ( "fmt" "net/http" + "time" "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/pkg/api/handlers/utils" @@ -65,3 +66,46 @@ func DiskUsage(w http.ResponseWriter, r *http.Request) { } utils.WriteResponse(w, http.StatusOK, response) } + +func SystemCheck(w http.ResponseWriter, r *http.Request) { + decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + + query := struct { + Quick bool `schema:"quick"` + Repair bool `schema:"repair"` + RepairLossy bool `schema:"repair_lossy"` + UnreferencedLayerMaximumAge string `schema:"unreferenced_layer_max_age"` + }{} + + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, http.StatusBadRequest, + fmt.Errorf("failed to parse parameters for %s: %w", r.URL.String(), err)) + return + } + + containerEngine := abi.ContainerEngine{Libpod: runtime} + + var unreferencedLayerMaximumAge *time.Duration + if query.UnreferencedLayerMaximumAge != "" { + duration, err := time.ParseDuration(query.UnreferencedLayerMaximumAge) + if err != nil { + utils.Error(w, http.StatusBadRequest, + fmt.Errorf("failed to parse unreferenced_layer_max_age parameter %q for %s: %w", query.UnreferencedLayerMaximumAge, r.URL.String(), err)) + } + unreferencedLayerMaximumAge = &duration + } + checkOptions := entities.SystemCheckOptions{ + Quick: query.Quick, + Repair: query.Repair, + RepairLossy: query.RepairLossy, + UnreferencedLayerMaximumAge: unreferencedLayerMaximumAge, + } + report, err := containerEngine.SystemCheck(r.Context(), checkOptions) + if err != nil { + utils.InternalServerError(w, err) + return + } + + utils.WriteResponse(w, http.StatusOK, report) +} diff --git a/pkg/api/handlers/swagger/responses.go b/pkg/api/handlers/swagger/responses.go index 6c52a00588..c41a83a38e 100644 --- a/pkg/api/handlers/swagger/responses.go +++ b/pkg/api/handlers/swagger/responses.go @@ -188,6 +188,13 @@ type versionResponse struct { Body entities.ComponentVersion } +// Check +// swagger:response +type systemCheckResponse struct { + // in:body + Body entities.SystemCheckReport +} + // Disk usage // swagger:response type systemDiskUsage struct { diff --git a/pkg/api/server/register_system.go b/pkg/api/server/register_system.go index 2034c2a7b6..4010dfb0e0 100644 --- a/pkg/api/server/register_system.go +++ b/pkg/api/server/register_system.go @@ -25,6 +25,39 @@ func (s *APIServer) registerSystemHandlers(r *mux.Router) error { r.Handle(VersionedPath("/system/df"), s.APIHandler(compat.GetDiskUsage)).Methods(http.MethodGet) // Added non version path to URI to support docker non versioned paths r.Handle("/system/df", s.APIHandler(compat.GetDiskUsage)).Methods(http.MethodGet) + // swagger:operation POST /libpod/system/check libpod SystemCheckLibpod + // --- + // tags: + // - system + // summary: Performs consistency checks on storage, optionally removing items which fail checks + // parameters: + // - in: query + // name: quick + // type: boolean + // description: Skip time-consuming checks + // - in: query + // name: repair + // type: boolean + // description: Remove inconsistent images + // - in: query + // name: repair_lossy + // type: boolean + // description: Remove inconsistent containers and images + // - in: query + // name: unreferenced_layer_max_age + // type: string + // description: Maximum allowed age of unreferenced layers + // default: 24h0m0s + // produces: + // - application/json + // responses: + // 200: + // $ref: '#/responses/systemCheckResponse' + // 400: + // $ref: "#/responses/badParamError" + // 500: + // $ref: "#/responses/internalError" + r.Handle(VersionedPath("/libpod/system/check"), s.APIHandler(libpod.SystemCheck)).Methods(http.MethodPost) // swagger:operation POST /libpod/system/prune libpod SystemPruneLibpod // --- // tags: diff --git a/pkg/bindings/system/system.go b/pkg/bindings/system/system.go index dbe5c318ea..a41bfb1f8e 100644 --- a/pkg/bindings/system/system.go +++ b/pkg/bindings/system/system.go @@ -87,6 +87,26 @@ func Prune(ctx context.Context, options *PruneOptions) (*types.SystemPruneReport return &report, response.Process(&report) } +func Check(ctx context.Context, options *CheckOptions) (*types.SystemCheckReport, error) { + var report types.SystemCheckReport + + conn, err := bindings.GetClient(ctx) + if err != nil { + return nil, err + } + params, err := options.ToParams() + if err != nil { + return nil, err + } + response, err := conn.DoRequest(ctx, nil, http.MethodPost, "/system/check", params, nil) + if err != nil { + return nil, err + } + defer response.Body.Close() + + return &report, response.Process(&report) +} + func Version(ctx context.Context, options *VersionOptions) (*types.SystemVersionReport, error) { var ( component types.SystemComponentVersion diff --git a/pkg/bindings/system/types.go b/pkg/bindings/system/types.go index 89e093f688..2342f7c497 100644 --- a/pkg/bindings/system/types.go +++ b/pkg/bindings/system/types.go @@ -38,3 +38,13 @@ type DiskOptions struct { //go:generate go run ../generator/generator.go InfoOptions type InfoOptions struct { } + +// CheckOptions are optional options for storage consistency check/repair +// +//go:generate go run ../generator/generator.go CheckOptions +type CheckOptions struct { + Quick *bool `schema:"quick"` + Repair *bool `schema:"repair"` + RepairLossy *bool `schema:"repair_lossy"` + UnreferencedLayerMaximumAge *string `schema:"unreferenced_layer_max_age"` +} diff --git a/pkg/bindings/system/types_check_options.go b/pkg/bindings/system/types_check_options.go new file mode 100644 index 0000000000..374f142d80 --- /dev/null +++ b/pkg/bindings/system/types_check_options.go @@ -0,0 +1,78 @@ +// Code generated by go generate; DO NOT EDIT. +package system + +import ( + "net/url" + + "github.com/containers/podman/v5/pkg/bindings/internal/util" +) + +// Changed returns true if named field has been set +func (o *CheckOptions) Changed(fieldName string) bool { + return util.Changed(o, fieldName) +} + +// ToParams formats struct fields to be passed to API service +func (o *CheckOptions) ToParams() (url.Values, error) { + return util.ToParams(o) +} + +// WithQuick set field Quick to given value +func (o *CheckOptions) WithQuick(value bool) *CheckOptions { + o.Quick = &value + return o +} + +// GetQuick returns value of field Quick +func (o *CheckOptions) GetQuick() bool { + if o.Quick == nil { + var z bool + return z + } + return *o.Quick +} + +// WithRepair set field Repair to given value +func (o *CheckOptions) WithRepair(value bool) *CheckOptions { + o.Repair = &value + return o +} + +// GetRepair returns value of field Repair +func (o *CheckOptions) GetRepair() bool { + if o.Repair == nil { + var z bool + return z + } + return *o.Repair +} + +// WithRepairLossy set field RepairLossy to given value +func (o *CheckOptions) WithRepairLossy(value bool) *CheckOptions { + o.RepairLossy = &value + return o +} + +// GetRepairLossy returns value of field RepairLossy +func (o *CheckOptions) GetRepairLossy() bool { + if o.RepairLossy == nil { + var z bool + return z + } + return *o.RepairLossy +} + +// WithUnreferencedLayerMaximumAge set field UnreferencedLayerMaximumAge to given value +func (o *CheckOptions) WithUnreferencedLayerMaximumAge(value string) *CheckOptions { + o.UnreferencedLayerMaximumAge = &value + return o +} + +// GetUnreferencedLayerMaximumAge returns value of field UnreferencedLayerMaximumAge +func (o *CheckOptions) GetUnreferencedLayerMaximumAge() string { + if o.UnreferencedLayerMaximumAge == nil { + var z string + return z + } + return *o.UnreferencedLayerMaximumAge +} diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index 712fdd7cdd..0e798be4e8 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -103,6 +103,7 @@ type ContainerEngine interface { //nolint:interfacebloat SecretExists(ctx context.Context, nameOrID string) (*BoolReport, error) Shutdown(ctx context.Context) SystemDf(ctx context.Context, options SystemDfOptions) (*SystemDfReport, error) + SystemCheck(ctx context.Context, options SystemCheckOptions) (*SystemCheckReport, error) Unshare(ctx context.Context, args []string, options SystemUnshareOptions) error Version(ctx context.Context) (*SystemVersionReport, error) VolumeCreate(ctx context.Context, opts VolumeCreateOptions) (*IDOrNameResponse, error) diff --git a/pkg/domain/entities/system.go b/pkg/domain/entities/system.go index e62214e7c4..ab9c861a02 100644 --- a/pkg/domain/entities/system.go +++ b/pkg/domain/entities/system.go @@ -9,6 +9,8 @@ type ServiceOptions = types.ServiceOptions type SystemPruneOptions = types.SystemPruneOptions type SystemPruneReport = types.SystemPruneReport type SystemMigrateOptions = types.SystemMigrateOptions +type SystemCheckOptions = types.SystemCheckOptions +type SystemCheckReport = types.SystemCheckReport type SystemDfOptions = types.SystemDfOptions type SystemDfReport = types.SystemDfReport type SystemDfImageReport = types.SystemDfImageReport diff --git a/pkg/domain/entities/types/system.go b/pkg/domain/entities/types/system.go index 3d1361ef01..6c331cd50e 100644 --- a/pkg/domain/entities/types/system.go +++ b/pkg/domain/entities/types/system.go @@ -15,6 +15,28 @@ type ServiceOptions struct { URI string // Path to unix domain socket service should listen on } +// SystemCheckOptions provides options for checking storage consistency. +type SystemCheckOptions struct { + Quick bool // skip the most time-intensive checks + Repair bool // remove damaged images + RepairLossy bool // remove damaged containers + UnreferencedLayerMaximumAge *time.Duration // maximum allowed age for unreferenced layers +} + +// SystemCheckReport provides a report of what a storage consistency check +// found, and if we removed anything that was damaged, what we removed. +type SystemCheckReport struct { + Errors bool // any errors were detected + Layers map[string][]string // layer ID → what was detected + ROLayers map[string][]string // layer ID → what was detected + RemovedLayers []string // layer ID + Images map[string][]string // image ID → what was detected + ROImages map[string][]string // image ID → what was detected + RemovedImages map[string][]string // image ID → names + Containers map[string][]string // container ID → what was detected + RemovedContainers map[string]string // container ID → name +} + // SystemPruneOptions provides options to prune system. type SystemPruneOptions struct { All bool diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index 8a43c8b8f5..2da7b6e5fc 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -337,3 +337,11 @@ func (ic ContainerEngine) Locks(ctx context.Context) (*entities.LocksReport, err report.LocksHeld = held return &report, nil } + +func (ic ContainerEngine) SystemCheck(ctx context.Context, options entities.SystemCheckOptions) (*entities.SystemCheckReport, error) { + report, err := ic.Libpod.SystemCheck(ctx, options) + if err != nil { + return nil, err + } + return &report, nil +} diff --git a/pkg/domain/infra/tunnel/system.go b/pkg/domain/infra/tunnel/system.go index f091fc79cb..025e902620 100644 --- a/pkg/domain/infra/tunnel/system.go +++ b/pkg/domain/infra/tunnel/system.go @@ -23,6 +23,15 @@ func (ic *ContainerEngine) SystemPrune(ctx context.Context, opts entities.System return system.Prune(ic.ClientCtx, options) } +func (ic *ContainerEngine) SystemCheck(ctx context.Context, opts entities.SystemCheckOptions) (*entities.SystemCheckReport, error) { + options := new(system.CheckOptions).WithQuick(opts.Quick).WithRepair(opts.Repair).WithRepairLossy(opts.RepairLossy) + if opts.UnreferencedLayerMaximumAge != nil { + duration := *opts.UnreferencedLayerMaximumAge + options = options.WithUnreferencedLayerMaximumAge(duration.String()) + } + return system.Check(ic.ClientCtx, options) +} + func (ic *ContainerEngine) Migrate(ctx context.Context, options entities.SystemMigrateOptions) error { return errors.New("runtime migration is not supported on remote clients") } diff --git a/rpm/podman.spec b/rpm/podman.spec index fb4ca390a3..a35690382e 100644 --- a/rpm/podman.spec +++ b/rpm/podman.spec @@ -240,6 +240,10 @@ export BUILDTAGS="$BASEBUILDTAGS exclude_graphdriver_btrfs btrfs_noversion remot export BUILDTAGS="$BASEBUILDTAGS $(hack/btrfs_installed_tag.sh) $(hack/btrfs_tag.sh)" %gobuild -o bin/quadlet ./cmd/quadlet +# build %%{name}-testing +export BUILDTAGS="$BASEBUILDTAGS $(hack/btrfs_installed_tag.sh) $(hack/btrfs_tag.sh)" +%gobuild -o bin/podman-testing ./cmd/podman-testing + # reset LDFLAGS for plugins binaries LDFLAGS='' @@ -255,8 +259,9 @@ PODMAN_VERSION=%{version} %{__make} DESTDIR=%{buildroot} PREFIX=%{_prefix} ETCDI install.docker \ install.docker-docs \ install.remote \ + install.testing \ %if %{defined _modulesloaddir} - install.modules-load + install.modules-load %endif sed -i 's;%{buildroot};;g' %{buildroot}%{_bindir}/docker @@ -314,6 +319,7 @@ cp -pav test/system %{buildroot}/%{_datadir}/%{name}/test/ %{_datadir}/zsh/site-functions/_%{name}-remote %files tests +%{_bindir}/%{name}-testing %{_datadir}/%{name}/test %files -n %{name}sh diff --git a/test/system/331-system-check.bats b/test/system/331-system-check.bats new file mode 100644 index 0000000000..7308fc98a9 --- /dev/null +++ b/test/system/331-system-check.bats @@ -0,0 +1,248 @@ +#!/usr/bin/env bats -*- bats -*- +# +# Creates errors that should be caught by `system check`, and verifies +# that they are caught and remedied, even if it requires discarding some +# data in read-write layers. +# + +load helpers + +@test "podman system check - unmanaged layers" { + run_podman_testing create-storage-layer + layerID="$output" + run_podman_testing create-storage-layer --parent=$layerID + run_podman 125 system check + assert "$output" =~ "layer in lower level storage driver not accounted for" "output from 'podman system check' with unmanaged layers" + run_podman system check -r + run_podman system check +} + +@test "podman system check - unused layers" { + run_podman_testing create-layer + layerID="$output" + run_podman_testing create-layer --parent=$layerID + run_podman system check + run_podman 125 system check -m 0 + assert "$output" =~ "layer not referenced" "output from 'podman system check' with unused layers" + run_podman system check -m 0 -r + run_podman system check -m 0 +} + +@test "podman system check - layer content digest changed" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + make_layer_blob 1 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing modify-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman 125 system check + assert "$output" =~ "checksum failed" "output from 'podman system check' with modified layer contents" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - layer content added" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + make_layer_blob 100 101 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing modify-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman 125 system check + assert "$output" =~ "content modified" "output from 'podman system check' with unexpected content added to layer" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - storage image layer missing" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + run_podman_testing remove-layer --layer=$layerID + run_podman 125 system check + assert "$output" =~ "image layer is missing" "output from 'podman system check' with missing layer" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - storage container image missing" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + run_podman_testing remove-image --image=$imageID + run_podman 125 system check -m 0 + assert "$output" =~ "image missing" "output from 'podman system check' with missing image" + run_podman 125 system check -r -m 0 + run_podman 0+w system check -r -f -m 0 + run_podman system check -m 0 +} + +@test "podman system check - storage layer data missing" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing create-layer-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --layer=$layerID + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + run_podman_testing remove-layer-data --key=foo --layer=$layerID + run_podman 125 system check + assert "$output" =~ "layer data item is missing" "output from 'podman system check' with missing layer data" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - storage image data missing" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing create-image-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --image=$imageID + run_podman create $imageID + run_podman_testing remove-image-data --key=foo --image=$imageID + run_podman 125 system check + assert "$output" =~ "image data item is missing" "output from 'podman system check' with missing image data" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - storage image data modified" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing create-image-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --image=$imageID + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing modify-image-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --image=$imageID + run_podman 125 system check + assert "$output" =~ "image data item has incorrect" "output from 'podman system check' with modified image data" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check +} + +@test "podman system check - container data missing" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + containerID="$output" + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing create-container-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --container=$containerID + run_podman_testing remove-container-data --key=foo --container=$containerID + run_podman 125 system check + assert "$output" =~ "container data item is missing" "output from 'podman system check' with missing container data" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check + run_podman rmi $imageID +} + +@test "podman system check - container data modified" { + run_podman_testing create-layer + layerID="$output" + make_layer_blob 8 ${PODMAN_TMPDIR}/archive.tar + run_podman_testing populate-layer --layer=$layerID --file=${PODMAN_TMPDIR}/archive.tar + run_podman_testing create-image --layer=$layerID + imageID="$output" + testing_make_image_metadata_for_layer_blobs $imageID ${PODMAN_TMPDIR}/archive.tar + run_podman create $imageID + containerID="$output" + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing create-container-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --container=$containerID + make_random_file ${PODMAN_TMPDIR}/random-data.bin + run_podman_testing modify-container-data --key=foo --file=${PODMAN_TMPDIR}/random-data.bin --container=$containerID + run_podman 125 system check + assert "$output" =~ "container data item has incorrect" "output from 'podman system check' with modified container data" + run_podman 125 system check -r + run_podman 0+w system check -r -f + run_podman system check + run_podman rmi $imageID +} + +function make_layer_blob() { + local tmpdir=$(mktemp -d --tmpdir=${PODMAN_TMPDIR} make_layer_blob.XXXXXX) + local blobfile + local seqargs + for arg in "${@}" ; do + seqargs="${blobfile:+$seqargs $blobfile}" + blobfile="$arg" + done + seqargs="${seqargs:-8}" + local filelist= + for file in $(seq ${seqargs}); do + dd if=/dev/urandom of="$tmpdir/file$file" bs=1 count=$((1024 + $file)) status=none + filelist="$filelist file$file" + done + tar -c --owner=root:0 --group=root:0 -f "$blobfile" -C "$tmpdir" $filelist +} + +function testing_make_image_metadata_for_layer_blobs() { + local tmpdir=$(mktemp -d --tmpdir=${PODMAN_TMPDIR} make_image_metadata.XXXXXX) + local imageID=$1 + shift + echo '{"config":{},"rootfs":{"type":"layers","diff_ids":[' > $tmpdir/config.json + echo '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[' > $tmpdir/manifest + local comma= + for blob in "$@" ; do + local sum=$(sha256sum $blob) + sum=${sum%% *} + local size=$(wc -c $blob) + size=${size%% *} + echo $comma '"sha256:'$sum'"' >> $tmpdir/config.json + echo $comma '{"digest":"sha256:'$sum'","size":'$size',"mediaType":"application/vnd.oci.image.layer.v1.tar"}' >> $tmpdir/manifest + comma=, + done + echo ']}}' >> $tmpdir/config.json + sum=$(sha256sum $tmpdir/config.json) + sum=${sum%% *} + size=$(wc -c $tmpdir/config.json) + size=${size%% *} + echo '],"config":{"digest":"sha256:'$sum'","size":'$size',"mediaType":"application/vnd.oci.image.config.v1+json"}}' >> $tmpdir/manifest + run_podman_testing create-image-data -i $imageID -k sha256:$sum -f $tmpdir/config.json + sum=$(sha256sum $tmpdir/manifest) + sum=${sum%% *} + run_podman_testing create-image-data -i $imageID -k manifest-sha256:$sum -f $tmpdir/manifest + run_podman_testing create-image-data -i $imageID -k manifest -f $tmpdir/manifest +} + +# vim: filetype=sh diff --git a/test/system/helpers.bash b/test/system/helpers.bash index a27ad21f6c..c20961aa76 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -4,6 +4,9 @@ PODMAN=${PODMAN:-podman} QUADLET=${QUADLET:-/usr/libexec/podman/quadlet} +# Podman testing helper used in 331-system-check tests +PODMAN_TESTING=${PODMAN_TESTING:-$(dirname ${BASH_SOURCE})/../../bin/podman-testing} + # crun or runc, unlikely to change. Cache, because it's expensive to determine. PODMAN_RUNTIME= @@ -449,6 +452,14 @@ function run_podman() { fi } +function run_podman_testing() { + printf "\n%s %s %s %s\n" "$(timestamp)" "$_LOG_PROMPT" "$PODMAN_TESTING" "$*" + run $PODMAN_TESTING "$@" + if [[ $status -ne 0 ]]; then + echo "$output" + die "Unexpected error from testing helper, which should always always succeed" + fi +} # Wait for certain output from a container, indicating that it's ready. function wait_for_output { @@ -1178,5 +1189,9 @@ function wait_for_command_output() { die "Timed out waiting for '$cmd' to return '$want'" } +function make_random_file() { + dd if=/dev/urandom of="$1" bs=1 count=${2:-$((${RANDOM} % 8192 + 1024))} status=none +} + # END miscellaneous tools ###############################################################################