diff --git a/.cruft.json b/.cruft.json index afc2fb82..f93d053a 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,13 +1,13 @@ { "template": "https://github.com/projectsyn/commodore-component-template.git", - "commit": "8840f87d25d97ce0d4bfed75d40173caaf4100fc", + "commit": "ba0ed2c8ff147199a748e9ff12cadce32c167fc0", "checkout": "main", "context": { "cookiecutter": { "name": "Rook Ceph", "slug": "rook-ceph", "parameter_key": "rook_ceph", - "test_cases": "defaults openshift4 cephfs", + "test_cases": "defaults openshift4 cephfs rbd-extra-storageclass", "add_lib": "n", "add_pp": "y", "add_golden": "y", diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8ee81acb..a98fc24c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,6 +35,7 @@ jobs: - defaults - openshift4 - cephfs + - rbd-extra-storageclass defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -52,6 +53,7 @@ jobs: - defaults - openshift4 - cephfs + - rbd-extra-storageclass defaults: run: working-directory: ${{ env.COMPONENT_NAME }} diff --git a/Makefile.vars.mk b/Makefile.vars.mk index 9fba8ef1..e2fa6db2 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -57,4 +57,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults -test_instances = tests/defaults.yml tests/openshift4.yml tests/cephfs.yml +test_instances = tests/defaults.yml tests/openshift4.yml tests/cephfs.yml tests/rbd-extra-storageclass.yml diff --git a/component/rbd.libsonnet b/component/rbd.libsonnet index 6acea2ef..3bd28a49 100644 --- a/component/rbd.libsonnet +++ b/component/rbd.libsonnet @@ -29,6 +29,12 @@ local rbd_blockpools = [ local rbd_storageclasses = [ sp.configure_storageclass('rbd', name) for name in std.objectFields(rbd_params) +] + [ + sp.configure_storageclass('rbd', name, suffix=suffix) + for name in std.objectFields(rbd_params) + for suffix in std.objectFields( + std.get(rbd_params[name], 'extra_storage_classes', {}) + ) ]; local rbd_snapclass = [ diff --git a/component/storagepool.libsonnet b/component/storagepool.libsonnet index eedda733..a1d314fc 100644 --- a/component/storagepool.libsonnet +++ b/component/storagepool.libsonnet @@ -20,11 +20,12 @@ local load_storageclass(type) = // return storageclass sc[0]; -local get_sc_config(type, pool) = +local get_sc_config(type, pool, extra_class) = assert std.objectHas(params.ceph_cluster.storage_pools, type); assert std.objectHas( params.ceph_cluster.storage_pools[type], pool ); + // we always use the pool-wide mount options local param_mount_options = com.getValueOrDefault( params.ceph_cluster.storage_pools[type][pool], 'mount_options', @@ -42,22 +43,44 @@ local get_sc_config(type, pool) = for opt in std.objectFields(param_mount_options) ] ); - com.makeMergeable(com.getValueOrDefault( - params.ceph_cluster.storage_pools[type][pool], - 'storage_class_config', - {} - )) + - { + + // get default storage class config for pool + local sc_config = + com.makeMergeable(std.get( + params.ceph_cluster.storage_pools[type][pool], + 'storage_class_config', + {} + )); + + // get extra storage class config for additional storageclasses for pool + local sc_extra_config = + if extra_class != null then + com.makeMergeable(std.get( + std.get( + params.ceph_cluster.storage_pools[type][pool], + 'extra_storage_classes', + {} + ), + extra_class + )) + else + {}; + + // all storage classes for the pool use the default sc config + potential + // extra config. This allows users to only specify the differences to the + // default class for extra classes. + sc_config + sc_extra_config + { mountOptions+: sc_mount_options, }; // subpool used for CephFS -local configure_sc(type, pool, subpool=null) = +local configure_sc(type, pool, subpool=null, suffix=null) = local obj = load_storageclass(type); - local sc_config = get_sc_config(type, pool); + local sc_config = get_sc_config(type, pool, suffix); com.makeMergeable(obj) + sc.storageClass( - '%s-%s-%s' % [ type, pool, params.ceph_cluster.name ] + '%s-%s-%s' % [ type, pool, params.ceph_cluster.name ] + + if suffix != null then '-%s' % suffix else '' ) + sc_config + { diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 3d6ad07d..6a195498 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -234,10 +234,12 @@ IMPORTANT: Storage class device sets added by `cephClusterSpec.storage.storageCl [horizontal] type:: dict keys:: Names of `CephBlockPool` resources -values:: dicts with keys `config` and `mount_options`, and `storage_class_config` +values:: dicts with keys `config` and `mount_options`, `storage_class_config`, and `extra_storage_classes` In this parameter `CephBlockPool` resources are configured. -The component creates exactly one storageclass and volumesnapshotclass per block pool. +The component creates one storageclass and volumesnapshotclass per block pool. + +NOTE: The component will create additional storage classes if field `extra_storage_classes` is provided. By default the parameter holds the following configuration: @@ -263,7 +265,7 @@ This configuration results in * A storage class which creates PVs on this block pool, uses the `ext4` filesystem, supports volume expansion and configures PVs to be mounted with `-o discard`. * A `VolumeSnapshotClass` associated with the storage class -See https://rook.io/docs/rook/v1.9/ceph-pool-crd.html[the Rook.io `CephBlockPool` CRD documentation] for all possible configurations in key `config`. +See https://rook.github.io/docs/rook/v1.14/CRDs/Block-Storage/ceph-block-pool-crd/[the Rook.io `CephBlockPool` CRD documentation] for all possible configurations in key `config`. The values in key `storage_class_config` are merged into the `StorageClass` resource. @@ -275,6 +277,35 @@ Providing a key with value `false` or `null` will result in the key not being ad See the filesystem documentation for the set of supported mount options. For example, see the list of supported mount options for `ext4` in the https://man7.org/linux/man-pages/man5/ext4.5.html#Mount_options_for_ext4[man page]. +==== `storage_pools.rbd..extra_storage_classes` + +[horizontal] +type:: dictionary +default:: `{}` + +.Example +[source,yaml] +---- +storagepool: + extra_storage_classes: + small-files: + parameters: + csi.storage.k8s.io/fstype: ext4 <1> + mkfsOptions: -m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1 -i1024 <2> +---- +<1> When providing `mkfsOptions` it makes sense to explicitly specify the fstype, since mismatched fstype and mkfsOptions will likely result in errors. +Note that provinding `fstype` isn't strictly necessary, if the same fstype is set in `storage_pools.rbd..storage_class_config`. +<2> `-m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1` are the default `mkfsOptions` for ext4 in ceph-csi. + +This parameter allows users to configure additional storage classes for an RBD blockpool. +The component will generate a storage class for each key-value pair in the parameter. + +The key will be used as a suffix for the storage class name. +The resulting storage class name will have the form `rbd---`. + +The component will use the configurations provided in `storage_pools.rbd..mount_options` and `storage_pools.rbd..storage_class_config` as the base for any storage classes defined here. +The provided value will be merged into the base storage class definition. + === `storage_pools.cephfs` [horizontal] diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/apps/rook-ceph.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/apps/rook-ceph.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/00_namespaces.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/00_namespaces.yaml new file mode 100644 index 00000000..74245766 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/00_namespaces.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + openshift.io/node-selector: '' + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: syn-rook-ceph-operator + name: syn-rook-ceph-operator + openshift.io/cluster-monitoring: 'true' + name: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: Namespace +metadata: + annotations: + openshift.io/node-selector: '' + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: syn-rook-ceph-cluster + name: syn-rook-ceph-cluster + openshift.io/cluster-monitoring: 'true' + name: syn-rook-ceph-cluster diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_aggregated_rbac.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_aggregated_rbac.yaml new file mode 100644 index 00000000..c3087cb4 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_aggregated_rbac.yaml @@ -0,0 +1,115 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-view + name: rook-ceph-view + rbac.authorization.k8s.io/aggregate-to-admin: 'true' + rbac.authorization.k8s.io/aggregate-to-edit: 'true' + rbac.authorization.k8s.io/aggregate-to-view: 'true' + name: rook-ceph-view +rules: + - apiGroups: + - ceph.rook.io + resources: + - cephblockpoolradosnamespaces + - cephblockpools + - cephbucketnotifications + - cephbuckettopics + - cephclients + - cephclusters + - cephfilesystemmirrors + - cephfilesystems + - cephfilesystemsubvolumegroups + - cephnfss + - cephobjectrealms + - cephobjectstores + - cephobjectstoreusers + - cephobjectzonegroups + - cephobjectzones + - cephrbdmirrors + verbs: + - get + - list + - watch + - apiGroups: + - objectbucket.io + resources: + - objectbucketclaims + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-edit + name: rook-ceph-edit + rbac.authorization.k8s.io/aggregate-to-admin: 'true' + rbac.authorization.k8s.io/aggregate-to-edit: 'true' + name: rook-ceph-edit +rules: + - apiGroups: + - ceph.rook.io + resources: + - cephblockpoolradosnamespaces + - cephblockpools + - cephbucketnotifications + - cephbuckettopics + - cephclients + - cephclusters + - cephfilesystemmirrors + - cephfilesystems + - cephfilesystemsubvolumegroups + - cephnfss + - cephobjectrealms + - cephobjectstores + - cephobjectstoreusers + - cephobjectzonegroups + - cephobjectzones + - cephrbdmirrors + verbs: + - create + - delete + - deletecollection + - patch + - update + - apiGroups: + - objectbucket.io + resources: + - objectbucketclaims + verbs: + - create + - delete + - deletecollection + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-cluster-reader + name: rook-ceph-cluster-reader + rbac.authorization.k8s.io/aggregate-to-cluster-reader: 'true' + name: rook-ceph-cluster-reader +rules: + - apiGroups: + - objectbucket.io + resources: + - objectbuckets + verbs: + - get + - list + - watch diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/cluster-rbac.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/cluster-rbac.yaml new file mode 100644 index 00000000..53c1f737 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/cluster-rbac.yaml @@ -0,0 +1,341 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-osd + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-purge-osd + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-rgw + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + operator: rook + storage-backend: ceph + name: rook-ceph-default + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-mgr-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-cluster +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-osd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rook-ceph-osd + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - cephclusters + - cephclusters/finalizers + verbs: + - get + - list + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - pods + - services + - pods/log + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - cephclients + - cephclusters + - cephblockpools + - cephfilesystems + - cephnfses + - cephobjectstores + - cephobjectstoreusers + - cephobjectrealms + - cephobjectzonegroups + - cephobjectzones + - cephbuckettopics + - cephbucketnotifications + - cephrbdmirrors + - cephfilesystemmirrors + - cephfilesystemsubvolumegroups + - cephblockpoolradosnamespaces + - cephcosidrivers + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - apps + resources: + - deployments/scale + - deployments + verbs: + - patch + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rook-ceph-purge-osd + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - update + - delete + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cluster-mgmt + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-mgr +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-system + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-system +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-purge-osd + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-purge-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-purge-osd + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrole.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrole.yaml new file mode 100644 index 00000000..17b6fcb2 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrole.yaml @@ -0,0 +1,846 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-system +rules: + - apiGroups: + - '' + resources: + - pods + - pods/log + verbs: + - get + - list + - apiGroups: + - '' + resources: + - pods/exec + verbs: + - create + - apiGroups: + - csiaddons.openshift.io + resources: + - networkfences + verbs: + - create + - get + - update + - delete + - watch + - list + - deletecollection + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-cluster-mgmt +rules: + - apiGroups: + - '' + - apps + - extensions + resources: + - secrets + - pods + - pods/log + - services + - configmaps + - deployments + - daemonsets + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-global +rules: + - apiGroups: + - '' + resources: + - pods + - nodes + - nodes/proxy + - secrets + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + - persistentvolumes + - persistentvolumeclaims + - endpoints + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + - apiGroups: + - ceph.rook.io + resources: + - cephclients + - cephclusters + - cephblockpools + - cephfilesystems + - cephnfses + - cephobjectstores + - cephobjectstoreusers + - cephobjectrealms + - cephobjectzonegroups + - cephobjectzones + - cephbuckettopics + - cephbucketnotifications + - cephrbdmirrors + - cephfilesystemmirrors + - cephfilesystemsubvolumegroups + - cephblockpoolradosnamespaces + - cephcosidrivers + verbs: + - get + - list + - watch + - update + - apiGroups: + - ceph.rook.io + resources: + - cephclients/status + - cephclusters/status + - cephblockpools/status + - cephfilesystems/status + - cephnfses/status + - cephobjectstores/status + - cephobjectstoreusers/status + - cephobjectrealms/status + - cephobjectzonegroups/status + - cephobjectzones/status + - cephbuckettopics/status + - cephbucketnotifications/status + - cephrbdmirrors/status + - cephfilesystemmirrors/status + - cephfilesystemsubvolumegroups/status + - cephblockpoolradosnamespaces/status + verbs: + - update + - apiGroups: + - ceph.rook.io + resources: + - cephclients/finalizers + - cephclusters/finalizers + - cephblockpools/finalizers + - cephfilesystems/finalizers + - cephnfses/finalizers + - cephobjectstores/finalizers + - cephobjectstoreusers/finalizers + - cephobjectrealms/finalizers + - cephobjectzonegroups/finalizers + - cephobjectzones/finalizers + - cephbuckettopics/finalizers + - cephbucketnotifications/finalizers + - cephrbdmirrors/finalizers + - cephfilesystemmirrors/finalizers + - cephfilesystemsubvolumegroups/finalizers + - cephblockpoolradosnamespaces/finalizers + verbs: + - update + - apiGroups: + - policy + - apps + - extensions + resources: + - poddisruptionbudgets + - deployments + - replicasets + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + - apiGroups: + - apps + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - update + - apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-mgr-cluster +rules: + - apiGroups: + - '' + resources: + - configmaps + - nodes + - nodes/proxy + - persistentvolumes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - list + - get + - watch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rook-ceph-mgr-system +rules: + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-object-bucket +rules: + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - apiGroups: + - objectbucket.io + resources: + - objectbucketclaims + verbs: + - list + - watch + - get + - update + - apiGroups: + - objectbucket.io + resources: + - objectbuckets + verbs: + - list + - watch + - get + - create + - update + - delete + - apiGroups: + - objectbucket.io + resources: + - objectbucketclaims/status + - objectbuckets/status + verbs: + - update + - apiGroups: + - objectbucket.io + resources: + - objectbucketclaims/finalizers + - objectbuckets/finalizers + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rook-ceph-osd +rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephfs-csi-nodeplugin +rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephfs-external-provisioner-runner +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - watch + - update + - patch + - create + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - watch + - patch + - update + - create + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotcontents + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotcontents/status + verbs: + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rbd-csi-nodeplugin +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - apiGroups: + - '' + resources: + - serviceaccounts + verbs: + - get + - apiGroups: + - '' + resources: + - serviceaccounts/token + verbs: + - create + - apiGroups: + - '' + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rbd-external-provisioner-runner +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - persistentvolumeclaims/status + verbs: + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - watch + - update + - patch + - create + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - watch + - patch + - update + - create + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotcontents + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - groupsnapshot.storage.k8s.io + resources: + - volumegroupsnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - apiGroups: + - '' + resources: + - serviceaccounts + verbs: + - get + - apiGroups: + - '' + resources: + - serviceaccounts/token + verbs: + - create + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph + app.kubernetes.io/part-of: container-object-storage-interface + name: objectstorage-provisioner-role +rules: + - apiGroups: + - objectstorage.k8s.io + resources: + - buckets + - bucketaccesses + - bucketclaims + - bucketaccessclasses + - buckets/status + - bucketaccesses/status + - bucketclaims/status + - bucketaccessclasses/status + verbs: + - get + - list + - watch + - update + - create + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - '' + resources: + - secrets + - events + verbs: + - get + - delete + - update + - create diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrolebinding.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..b399a970 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/clusterrolebinding.yaml @@ -0,0 +1,121 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-global +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-global +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-object-bucket +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-object-bucket +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rbd-csi-nodeplugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cephfs-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cephfs-external-provisioner-runner +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cephfs-csi-nodeplugin-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cephfs-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rbd-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rbd-external-provisioner-runner +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph + app.kubernetes.io/part-of: container-object-storage-interface + name: objectstorage-provisioner-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: objectstorage-provisioner-role +subjects: + - kind: ServiceAccount + name: objectstorage-provisioner + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/configmap.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/configmap.yaml new file mode 100644 index 00000000..2d7b4530 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/configmap.yaml @@ -0,0 +1,223 @@ +apiVersion: v1 +data: + CSI_CEPHFS_ATTACH_REQUIRED: 'true' + CSI_CEPHFS_FSGROUPPOLICY: File + CSI_CEPHFS_PLUGIN_RESOURCE: | + - name : driver-registrar + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + - name : csi-cephfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + - name : liveness-prometheus + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + CSI_CEPHFS_PROVISIONER_RESOURCE: | + - name : csi-provisioner + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-resizer + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-attacher + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-snapshotter + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-cephfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + - name : liveness-prometheus + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + CSI_DISABLE_HOLDER_PODS: 'true' + CSI_ENABLE_CEPHFS_SNAPSHOTTER: 'true' + CSI_ENABLE_CSIADDONS: 'false' + CSI_ENABLE_ENCRYPTION: 'false' + CSI_ENABLE_HOST_NETWORK: 'true' + CSI_ENABLE_LIVENESS: 'true' + CSI_ENABLE_METADATA: 'false' + CSI_ENABLE_NFS_SNAPSHOTTER: 'true' + CSI_ENABLE_OMAP_GENERATOR: 'false' + CSI_ENABLE_RBD_SNAPSHOTTER: 'true' + CSI_ENABLE_TOPOLOGY: 'false' + CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: 'true' + CSI_FORCE_CEPHFS_KERNEL_CLIENT: 'true' + CSI_GRPC_TIMEOUT_SECONDS: '150' + CSI_NFS_ATTACH_REQUIRED: 'true' + CSI_NFS_FSGROUPPOLICY: File + CSI_NFS_PLUGIN_RESOURCE: | + - name : driver-registrar + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + - name : csi-nfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + CSI_NFS_PROVISIONER_RESOURCE: | + - name : csi-provisioner + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-nfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + - name : csi-attacher + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: 'false' + CSI_PLUGIN_PRIORITY_CLASSNAME: system-node-critical + CSI_PROVISIONER_PRIORITY_CLASSNAME: system-cluster-critical + CSI_PROVISIONER_REPLICAS: '2' + CSI_PROVISIONER_TOLERATIONS: |- + - key: storagenode + operator: Exists + CSI_RBD_ATTACH_REQUIRED: 'true' + CSI_RBD_FSGROUPPOLICY: File + CSI_RBD_PLUGIN_RESOURCE: | + - name : driver-registrar + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + - name : csi-rbdplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + - name : liveness-prometheus + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + CSI_RBD_PROVISIONER_RESOURCE: | + - name : csi-provisioner + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-resizer + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-attacher + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-snapshotter + resource: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 256Mi + - name : csi-rbdplugin + resource: + requests: + memory: 512Mi + limits: + memory: 1Gi + - name : csi-omap-generator + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + - name : liveness-prometheus + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + ROOK_CEPH_ALLOW_LOOP_DEVICES: 'false' + ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: '15' + ROOK_CSIADDONS_IMAGE: quay.io/csiaddons/k8s-sidecar:v0.8.0 + ROOK_CSI_ATTACHER_IMAGE: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 + ROOK_CSI_CEPH_IMAGE: quay.io/cephcsi/cephcsi:v3.11.0 + ROOK_CSI_DISABLE_DRIVER: 'false' + ROOK_CSI_ENABLE_CEPHFS: 'false' + ROOK_CSI_ENABLE_NFS: 'false' + ROOK_CSI_ENABLE_RBD: 'true' + ROOK_CSI_IMAGE_PULL_POLICY: IfNotPresent + ROOK_CSI_PROVISIONER_IMAGE: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 + ROOK_CSI_REGISTRAR_IMAGE: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 + ROOK_CSI_RESIZER_IMAGE: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 + ROOK_CSI_SNAPSHOTTER_IMAGE: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 + ROOK_ENABLE_DISCOVERY_DAEMON: 'false' + ROOK_LOG_LEVEL: INFO + ROOK_OBC_WATCH_OPERATOR_NAMESPACE: 'true' +kind: ConfigMap +metadata: + name: rook-ceph-operator-config + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/deployment.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/deployment.yaml new file mode 100644 index 00000000..15b174b7 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-operator + namespace: syn-rook-ceph-operator +spec: + replicas: 1 + selector: + matchLabels: + app: rook-ceph-operator + strategy: + type: Recreate + template: + metadata: + labels: + app: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + spec: + containers: + - args: + - ceph + - operator + env: + - name: ROOK_CURRENT_NAMESPACE_ONLY + value: 'false' + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: 'true' + - name: ROOK_DISABLE_DEVICE_HOTPLUG + value: 'false' + - name: ROOK_DISCOVER_DEVICES_INTERVAL + value: 60m + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: docker.io/rook/ceph:v1.14.10 + imagePullPolicy: IfNotPresent + name: rook-ceph-operator + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 750m + memory: 512Mi + securityContext: + capabilities: + drop: + - ALL + runAsGroup: 2016 + runAsNonRoot: true + runAsUser: 2016 + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + nodeSelector: + node-role.kubernetes.io/storage: '' + serviceAccountName: rook-ceph-system + tolerations: + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 5 + - key: storagenode + operator: Exists + volumes: + - emptyDir: {} + name: rook-config + - emptyDir: {} + name: default-config-dir diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/resources.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/resources.yaml new file mode 100644 index 00000000..f505f83b --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/resources.yaml @@ -0,0 +1,13778 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephblockpoolradosnamespaces.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPoolRadosNamespace + listKind: CephBlockPoolRadosNamespaceList + plural: cephblockpoolradosnamespaces + singular: cephblockpoolradosnamespace + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - description: Name of the Ceph BlockPool + jsonPath: .spec.blockPoolName + name: BlockPool + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephBlockPoolRadosNamespace represents a Ceph BlockPool Rados + Namespace + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph BlockPool Rados + Namespace + properties: + blockPoolName: + description: |- + BlockPoolName is the name of Ceph BlockPool. Typically it's the name of + the CephBlockPool CR. + type: string + x-kubernetes-validations: + - message: blockPoolName is immutable + rule: self == oldSelf + name: + description: The name of the CephBlockPoolRadosNamespaceSpec namespace. + If not set, the default is the name of the CR. + type: string + x-kubernetes-validations: + - message: name is immutable + rule: self == oldSelf + required: + - blockPoolName + type: object + status: + description: Status represents the status of a CephBlockPool Rados Namespace + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephblockpools.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPool + listKind: CephBlockPoolList + plural: cephblockpools + singular: cephblockpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.info.type + name: Type + type: string + - jsonPath: .status.info.failureDomain + name: FailureDomain + type: string + - jsonPath: .spec.replicated.size + name: Replication + priority: 1 + type: integer + - jsonPath: .spec.erasureCoded.codingChunks + name: EC-CodingChunks + priority: 1 + type: integer + - jsonPath: .spec.erasureCoded.dataChunks + name: EC-DataChunks + priority: 1 + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephBlockPool represents a Ceph Storage Pool + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NamedBlockPoolSpec allows a block pool to be created with a non-default name. + This is more specific than the NamedPoolSpec so we get schema validation on the + allowed pool names that can be specified. + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the + pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) + - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the + snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + name: + description: The desired name of the pool if different from the + CephBlockPool CR name. + enum: + - .rgw.root + - .nfs + - .mgr + type: string + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given + pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set + replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for replicated + pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms + of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of + an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + description: CephBlockPoolStatus represents the mirroring status of + Ceph Storage Pool + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + info: + additionalProperties: + type: string + nullable: true + type: object + mirroringInfo: + description: MirroringInfoSpec is the status of the pool mirroring + properties: + details: + type: string + lastChanged: + type: string + lastChecked: + type: string + mode: + description: Mode is the mirroring mode + type: string + peers: + description: Peers are the list of peer sites connected to that + cluster + items: + description: PeersSpec contains peer details + properties: + client_name: + description: ClientName is the CephX user used to connect + to the peer + type: string + direction: + description: Direction is the peer mirroring direction + type: string + mirror_uuid: + description: MirrorUUID is the mirror UUID + type: string + site_name: + description: SiteName is the current site name + type: string + uuid: + description: UUID is the peer UUID + type: string + type: object + type: array + site_name: + description: SiteName is the current site name + type: string + type: object + mirroringStatus: + description: MirroringStatusSpec is the status of the pool mirroring + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last + changed + type: string + lastChecked: + description: LastChecked is the last time time the status was + checked + type: string + summary: + description: Summary is the mirroring status summary + properties: + daemon_health: + description: DaemonHealth is the health of the mirroring + daemon + type: string + health: + description: Health is the mirroring health + type: string + image_health: + description: ImageHealth is the health of the mirrored image + type: string + states: + description: States is the various state for all mirrored + images + nullable: true + properties: + error: + description: Error is when the mirroring state is errored + type: integer + replaying: + description: Replaying is when the replay of the mirroring + journal is on-going + type: integer + starting_replay: + description: StartingReplay is when the replay of the + mirroring journal starts + type: integer + stopped: + description: Stopped is when the mirroring state is + stopped + type: integer + stopping_replay: + description: StopReplaying is when the replay of the + mirroring journal stops + type: integer + syncing: + description: Syncing is when the image is syncing + type: integer + unknown: + description: Unknown is when the mirroring state is + unknown + type: integer + type: object + type: object + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + snapshotScheduleStatus: + description: SnapshotScheduleStatusSpec is the status of the snapshot + schedule + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last + changed + type: string + lastChecked: + description: LastChecked is the last time time the status was + checked + type: string + snapshotSchedules: + description: SnapshotSchedules is the list of snapshots scheduled + items: + description: SnapshotSchedulesSpec is the list of snapshot + scheduled for images in a pool + properties: + image: + description: Image is the mirrored image + type: string + items: + description: Items is the list schedules times for a given + snapshot + items: + description: SnapshotSchedule is a schedule + properties: + interval: + description: Interval is the interval in which snapshots + will be taken + type: string + start_time: + description: StartTime is the snapshot starting + time + type: string + type: object + type: array + namespace: + description: Namespace is the RADOS namespace the image + is part of + type: string + pool: + description: Pool is the pool name + type: string + type: object + nullable: true + type: array + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephbucketnotifications.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketNotification + listKind: CephBucketNotificationList + plural: cephbucketnotifications + singular: cephbucketnotification + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephBucketNotification represents a Bucket Notifications + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BucketNotificationSpec represent the spec of a Bucket Notification + properties: + events: + description: List of events that should trigger the notification + items: + description: BucketNotificationSpec represent the event type of + the bucket notification + enum: + - s3:ObjectCreated:* + - s3:ObjectCreated:Put + - s3:ObjectCreated:Post + - s3:ObjectCreated:Copy + - s3:ObjectCreated:CompleteMultipartUpload + - s3:ObjectRemoved:* + - s3:ObjectRemoved:Delete + - s3:ObjectRemoved:DeleteMarkerCreated + type: string + type: array + filter: + description: Spec of notification filter + properties: + keyFilters: + description: Filters based on the object's key + items: + description: NotificationKeyFilterRule represent a single + key rule in the Notification Filter spec + properties: + name: + description: Name of the filter - prefix/suffix/regex + enum: + - prefix + - suffix + - regex + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + metadataFilters: + description: Filters based on the object's metadata + items: + description: NotificationFilterRule represent a single rule + in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + tagFilters: + description: Filters based on the object's tags + items: + description: NotificationFilterRule represent a single rule + in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + type: object + topic: + description: The name of the topic associated with this notification + minLength: 1 + type: string + required: + - topic + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephbuckettopics.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketTopic + listKind: CephBucketTopicList + plural: cephbuckettopics + singular: cephbuckettopic + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephBucketTopic represents a Ceph Object Topic for Bucket Notifications + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BucketTopicSpec represent the spec of a Bucket Topic + properties: + endpoint: + description: Contains the endpoint spec of the topic + properties: + amqp: + description: Spec of AMQP endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker/routeable) + enum: + - none + - broker + - routeable + type: string + disableVerifySSL: + description: Indicate whether the server certificate is + validated by the client or not + type: boolean + exchange: + description: Name of the exchange that is used to route + messages based on topics + minLength: 1 + type: string + uri: + description: The URI of the AMQP endpoint to push notification + to + minLength: 1 + type: string + required: + - exchange + - uri + type: object + http: + description: Spec of HTTP endpoint + properties: + disableVerifySSL: + description: Indicate whether the server certificate is + validated by the client or not + type: boolean + sendCloudEvents: + description: 'Send the notifications with the CloudEvents + header: https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md' + type: boolean + uri: + description: The URI of the HTTP endpoint to push notification + to + minLength: 1 + type: string + required: + - uri + type: object + kafka: + description: Spec of Kafka endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker) + enum: + - none + - broker + type: string + disableVerifySSL: + description: Indicate whether the server certificate is + validated by the client or not + type: boolean + uri: + description: The URI of the Kafka endpoint to push notification + to + minLength: 1 + type: string + useSSL: + description: Indicate whether to use SSL when communicating + with the broker + type: boolean + required: + - uri + type: object + type: object + objectStoreName: + description: The name of the object store on which to define the + topic + minLength: 1 + type: string + objectStoreNamespace: + description: The namespace of the object store on which to define + the topic + minLength: 1 + type: string + opaqueData: + description: Data which is sent in each event + type: string + persistent: + description: Indication whether notifications to this endpoint are + persistent or not + type: boolean + required: + - endpoint + - objectStoreName + - objectStoreNamespace + type: object + status: + description: BucketTopicStatus represents the Status of a CephBucketTopic + properties: + ARN: + description: The ARN of the topic generated by the RGW + nullable: true + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephClient represents a Ceph Client + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph Client + properties: + caps: + additionalProperties: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + required: + - caps + type: object + status: + description: Status represents the status of a Ceph Client + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephclusters.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephCluster + listKind: CephClusterList + plural: cephclusters + singular: cephcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Directory used on the K8s nodes + jsonPath: .spec.dataDirHostPath + name: DataDirHostPath + type: string + - description: Number of MONs + jsonPath: .spec.mon.count + name: MonCount + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.phase + name: Phase + type: string + - description: Message + jsonPath: .status.message + name: Message + type: string + - description: Ceph Health + jsonPath: .status.ceph.health + name: Health + type: string + - jsonPath: .spec.external.enable + name: External + type: boolean + - description: Ceph FSID + jsonPath: .status.ceph.fsid + name: FSID + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephCluster is a Ceph storage cluster + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterSpec represents the specification of Ceph Cluster + properties: + annotations: + additionalProperties: + additionalProperties: + type: string + description: Annotations are annotations + type: object + description: The annotations-related configuration to add/set on + each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + cephConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: Ceph Config options + nullable: true + type: object + cephVersion: + description: The version information that instructs Rook to orchestrate + a particular version of Ceph. + nullable: true + properties: + allowUnsupported: + description: Whether to allow unsupported versions (do not set + to true in production) + type: boolean + image: + description: |- + Image is the container image used to launch the ceph daemons, such as quay.io/ceph/ceph: + The full list of images can be found at https://quay.io/repository/ceph/ceph?tab=tags + type: string + imagePullPolicy: + description: |- + ImagePullPolicy describes a policy for if/when to pull a container image + One of Always, Never, IfNotPresent. + enum: + - IfNotPresent + - Always + - Never + - '' + type: string + type: object + cleanupPolicy: + description: |- + Indicates user intent when deleting a cluster; blocks orchestration and should not be set if cluster + deletion is not imminent. + nullable: true + properties: + allowUninstallWithVolumes: + description: AllowUninstallWithVolumes defines whether we can + proceed with the uninstall if they are RBD images still present + type: boolean + confirmation: + description: Confirmation represents the cleanup confirmation + nullable: true + pattern: ^$|^yes-really-destroy-data$ + type: string + sanitizeDisks: + description: SanitizeDisks represents way we sanitize disks + nullable: true + properties: + dataSource: + description: DataSource is the data source to use to sanitize + the disk with + enum: + - zero + - random + type: string + iteration: + description: Iteration is the number of pass to apply the + sanitizing + format: int32 + type: integer + method: + description: Method is the method we use to sanitize disks + enum: + - complete + - quick + type: string + type: object + type: object + continueUpgradeAfterChecksEvenIfNotHealthy: + description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines + if an upgrade should continue even if PGs are not clean + type: boolean + crashCollector: + description: A spec for the crash controller + nullable: true + properties: + daysToRetain: + description: DaysToRetain represents the number of days to retain + crash until they get pruned + type: integer + disable: + description: Disable determines whether we should enable the + crash collector + type: boolean + type: object + csi: + description: CSI Driver Options applied per cluster. + properties: + cephfs: + description: CephFS defines CSI Driver settings for CephFS driver. + properties: + fuseMountOptions: + description: FuseMountOptions defines the mount options + for ceph fuse mounter. + type: string + kernelMountOptions: + description: KernelMountOptions defines the mount options + for kernel mounter. + type: string + type: object + readAffinity: + description: ReadAffinity defines the read affinity settings + for CSI driver. + properties: + crushLocationLabels: + description: |- + CrushLocationLabels defines which node labels to use + as CRUSH location. This should correspond to the values set in + the CRUSH map. + items: + type: string + type: array + enabled: + description: Enables read affinity for CSI driver. + type: boolean + type: object + type: object + dashboard: + description: Dashboard settings + nullable: true + properties: + enabled: + description: Enabled determines whether to enable the dashboard + type: boolean + port: + description: Port is the dashboard webserver port + maximum: 65535 + minimum: 0 + type: integer + prometheusEndpoint: + description: Endpoint for the Prometheus host + type: string + prometheusEndpointSSLVerify: + description: Whether to verify the ssl endpoint for prometheus. + Set to false for a self-signed cert. + type: boolean + ssl: + description: SSL determines whether SSL should be used + type: boolean + urlPrefix: + description: URLPrefix is a prefix for all URLs to use the dashboard + with a reverse proxy + type: string + type: object + dataDirHostPath: + description: The path on the host where config and data can be persisted + pattern: ^/(\S+) + type: string + x-kubernetes-validations: + - message: DataDirHostPath is immutable + rule: self == oldSelf + disruptionManagement: + description: A spec for configuring disruption management. + nullable: true + properties: + machineDisruptionBudgetNamespace: + description: Deprecated. Namespace to look for MDBs by the machineDisruptionBudgetController + type: string + manageMachineDisruptionBudgets: + description: Deprecated. This enables management of machinedisruptionbudgets. + type: boolean + managePodBudgets: + description: This enables management of poddisruptionbudgets + type: boolean + osdMaintenanceTimeout: + description: |- + OSDMaintenanceTimeout sets how many additional minutes the DOWN/OUT interval is for drained failure domains + it only works if managePodBudgets is true. + the default is 30 minutes + format: int64 + type: integer + pgHealthCheckTimeout: + description: |- + PGHealthCheckTimeout is the time (in minutes) that the operator will wait for the placement groups to become + healthy (active+clean) after a drain was completed and OSDs came back up. Rook will continue with the next drain + if the timeout exceeds. It only works if managePodBudgets is true. + No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain. + format: int64 + type: integer + pgHealthyRegex: + description: |- + PgHealthyRegex is the regular expression that is used to determine which PG states should be considered healthy. + The default is `^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep)$` + type: string + type: object + external: + description: |- + Whether the Ceph Cluster is running external to this Kubernetes cluster + mon, mgr, osd, mds, and discover daemons will not be created for external clusters. + nullable: true + properties: + enable: + description: Enable determines whether external mode is enabled + or not + type: boolean + type: object + x-kubernetes-preserve-unknown-fields: true + healthCheck: + description: Internal daemon healthchecks and liveness probe + nullable: true + properties: + daemonHealth: + description: DaemonHealth is the health check for a given daemon + nullable: true + properties: + mon: + description: Monitor represents the health check settings + for the Ceph monitor + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + osd: + description: ObjectStorageDaemon represents the health check + settings for the Ceph OSDs + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + status: + description: Status represents the health check settings + for the Ceph health + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + livenessProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can + be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a + GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + description: LivenessProbe allows changing the livenessProbe + configuration for a given daemon + type: object + startupProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can + be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a + GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + description: StartupProbe allows changing the startupProbe configuration + for a given daemon + type: object + type: object + labels: + additionalProperties: + additionalProperties: + type: string + description: Labels are label for a given daemons + type: object + description: The labels-related configuration to add/set on each + Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + logCollector: + description: Logging represents loggings settings + nullable: true + properties: + enabled: + description: Enabled represents whether the log collector is + enabled + type: boolean + maxLogSize: + anyOf: + - type: integer + - type: string + description: MaxLogSize is the maximum size of the log per ceph + daemons. Must be at least 1M. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + periodicity: + description: Periodicity is the periodicity of the log rotation. + pattern: ^$|^(hourly|daily|weekly|monthly|1h|24h|1d)$ + type: string + type: object + mgr: + description: A spec for mgr related options + nullable: true + properties: + allowMultiplePerNode: + description: AllowMultiplePerNode allows to run multiple managers + on the same node (not recommended) + type: boolean + count: + description: Count is the number of manager daemons to run + maximum: 5 + minimum: 0 + type: integer + modules: + description: Modules is the list of ceph manager modules to + enable/disable + items: + description: Module represents mgr modules that the user wants + to enable or disable + properties: + enabled: + description: Enabled determines whether a module should + be enabled or not + type: boolean + name: + description: Name is the name of the ceph manager module + type: string + settings: + description: Settings to further configure the module + properties: + balancerMode: + description: BalancerMode sets the `balancer` module + with different modes like `upmap`, `crush-compact` + etc + enum: + - '' + - crush-compat + - upmap + - upmap-read + type: string + type: object + type: object + nullable: true + type: array + type: object + mon: + description: A spec for mon related options + nullable: true + properties: + allowMultiplePerNode: + description: AllowMultiplePerNode determines if we can run multiple + monitors on the same node (not recommended) + type: boolean + count: + description: Count is the number of Ceph monitors + maximum: 9 + minimum: 0 + type: integer + failureDomainLabel: + type: string + stretchCluster: + description: StretchCluster is the stretch cluster specification + properties: + failureDomainLabel: + description: 'FailureDomainLabel the failure domain name + (e,g: zone)' + type: string + subFailureDomain: + description: SubFailureDomain is the failure domain within + a zone + type: string + zones: + description: Zones is the list of zones + items: + description: MonZoneSpec represents the specification + of a zone in a Ceph Cluster + properties: + arbiter: + description: Arbiter determines if the zone contains + the arbiter used for stretch cluster mode + type: boolean + name: + description: Name is the name of the zone + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC template + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + nullable: true + type: array + type: object + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC definition + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + zones: + description: Zones are specified when we want to provide zonal + awareness to mons + items: + description: MonZoneSpec represents the specification of a + zone in a Ceph Cluster + properties: + arbiter: + description: Arbiter determines if the zone contains the + arbiter used for stretch cluster mode + type: boolean + name: + description: Name is the name of the zone + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC template + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + type: object + x-kubernetes-validations: + - message: zones must be less than or equal to count + rule: '!has(self.zones) || (has(self.zones) && (size(self.zones) + <= self.count))' + - message: stretchCluster zones must be equal to 3 + rule: '!has(self.stretchCluster) || (has(self.stretchCluster) + && (size(self.stretchCluster.zones) > 0) && (size(self.stretchCluster.zones) + == 3))' + monitoring: + description: Prometheus based Monitoring settings + nullable: true + properties: + enabled: + description: |- + Enabled determines whether to create the prometheus rules for the ceph cluster. If true, the prometheus + types must exist or the creation will fail. Default is false. + type: boolean + externalMgrEndpoints: + description: ExternalMgrEndpoints points to an existing Ceph + prometheus exporter endpoint + items: + description: EndpointAddress is a tuple that describes single + IP address. + properties: + hostname: + description: The Hostname of this endpoint + type: string + ip: + description: |- + The IP of this endpoint. + May not be loopback (127.0.0.0/8 or ::1), link-local (169.254.0.0/16 or fe80::/10), + or link-local multicast (224.0.0.0/24 or ff02::/16). + type: string + nodeName: + description: 'Optional: Node hosting this endpoint. This + can be used to determine endpoints local to a node.' + type: string + targetRef: + description: Reference to object providing the endpoint. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + required: + - ip + type: object + x-kubernetes-map-type: atomic + nullable: true + type: array + externalMgrPrometheusPort: + description: ExternalMgrPrometheusPort Prometheus exporter port + maximum: 65535 + minimum: 0 + type: integer + interval: + description: Interval determines prometheus scrape interval + type: string + metricsDisabled: + description: |- + Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. + If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false. + type: boolean + port: + description: Port is the prometheus server port + maximum: 65535 + minimum: 0 + type: integer + type: object + network: + description: Network related configuration + nullable: true + properties: + addressRanges: + description: |- + AddressRanges specify a list of CIDRs that Rook will apply to Ceph's 'public_network' and/or + 'cluster_network' configurations. This config section may be used for the "host" or "multus" + network providers. + nullable: true + properties: + cluster: + description: Cluster defines a list of CIDRs to use for + Ceph cluster network communication. + items: + description: |- + An IPv4 or IPv6 network CIDR. + + + This naive kubebuilder regex provides immediate feedback for some typos and for a common problem + case where the range spec is forgotten (e.g., /24). Rook does in-depth validation in code. + pattern: ^[0-9a-fA-F:.]{2,}\/[0-9]{1,3}$ + type: string + type: array + public: + description: Public defines a list of CIDRs to use for Ceph + public network communication. + items: + description: |- + An IPv4 or IPv6 network CIDR. + + + This naive kubebuilder regex provides immediate feedback for some typos and for a common problem + case where the range spec is forgotten (e.g., /24). Rook does in-depth validation in code. + pattern: ^[0-9a-fA-F:.]{2,}\/[0-9]{1,3}$ + type: string + type: array + type: object + connections: + description: |- + Settings for network connections such as compression and encryption across the + wire. + nullable: true + properties: + compression: + description: Compression settings for the network connections. + nullable: true + properties: + enabled: + description: |- + Whether to compress the data in transit across the wire. + The default is not set. + type: boolean + type: object + encryption: + description: Encryption settings for the network connections. + nullable: true + properties: + enabled: + description: |- + Whether to encrypt the data in transit across the wire to prevent eavesdropping + the data on the network. The default is not set. Even if encryption is not enabled, + clients still establish a strong initial authentication for the connection + and data integrity is still validated with a crc check. When encryption is enabled, + all communication between clients and Ceph daemons, or between Ceph daemons will + be encrypted. + type: boolean + type: object + requireMsgr2: + description: |- + Whether to require msgr2 (port 3300) even if compression or encryption are not enabled. + If true, the msgr1 port (6789) will be disabled. + Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). + type: boolean + type: object + dualStack: + description: DualStack determines whether Ceph daemons should + listen on both IPv4 and IPv6 + type: boolean + hostNetwork: + description: |- + HostNetwork to enable host network. + If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to + apply the new network settings. + type: boolean + ipFamily: + description: IPFamily is the single stack IPv6 or IPv4 protocol + enum: + - IPv4 + - IPv6 + nullable: true + type: string + multiClusterService: + description: Enable multiClusterService to export the Services + between peer clusters + properties: + clusterID: + description: |- + ClusterID uniquely identifies a cluster. It is used as a prefix to nslookup exported + services. For example: ...svc.clusterset.local + type: string + enabled: + description: |- + Enable multiClusterService to export the mon and OSD services to peer cluster. + Ensure that peer clusters are connected using an MCS API compatible application, + like Globalnet Submariner. + type: boolean + type: object + provider: + description: |- + Provider is what provides network connectivity to the cluster e.g. "host" or "multus". + If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings. + enum: + - '' + - host + - multus + nullable: true + type: string + x-kubernetes-validations: + - message: network provider must be disabled (reverted to + empty string) before a new provider is enabled + rule: self == '' || self == oldSelf + selectors: + additionalProperties: + type: string + description: |- + Selectors define NetworkAttachmentDefinitions to be used for Ceph public and/or cluster + networks when the "multus" network provider is used. This config section is not used for + other network providers. + + + Valid keys are "public" and "cluster". Refer to Ceph networking documentation for more: + https://docs.ceph.com/en/reef/rados/configuration/network-config-ref/ + + + Refer to Multus network annotation documentation for help selecting values: + https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/how-to-use.md#run-pod-with-network-annotation + + + Rook will make a best-effort attempt to automatically detect CIDR address ranges for given + network attachment definitions. Rook's methods are robust but may be imprecise for + sufficiently complicated networks. Rook's auto-detection process obtains a new IP address + lease for each CephCluster reconcile. If Rook fails to detect, incorrectly detects, only + partially detects, or if underlying networks do not support reusing old IP addresses, it is + best to use the 'addressRanges' config section to specify CIDR ranges for the Ceph cluster. + + + As a contrived example, one can use a theoretical Kubernetes-wide network for Ceph client + traffic and a theoretical Rook-only network for Ceph replication traffic as shown: + selectors: + public: "default/cluster-fast-net" + cluster: "rook-ceph/ceph-backend-net" + nullable: true + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + x-kubernetes-validations: + - message: at least one network selector must be specified when + using multus + rule: '!has(self.provider) || (self.provider != ''multus'' || + (self.provider == ''multus'' && size(self.selectors) > 0))' + - message: the legacy hostNetwork setting can only be set if the + network.provider is set to the empty string + rule: '!has(self.hostNetwork) || self.hostNetwork == false || + !has(self.provider) || self.provider == ""' + placement: + additionalProperties: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + priorityClassNames: + additionalProperties: + type: string + description: PriorityClassNames sets priority classes on components + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + removeOSDsIfOutAndSafeToRemove: + description: Remove the OSD that is out and safe to remove only + if this option is true + type: boolean + resources: + additionalProperties: + description: ResourceRequirements describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + description: Resources set resource requests and limits + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + security: + description: Security represents security settings + nullable: true + properties: + keyRotation: + description: KeyRotation defines options for Key Rotation. + nullable: true + properties: + enabled: + default: false + description: Enabled represents whether the key rotation + is enabled. + type: boolean + schedule: + description: Schedule represents the cron schedule for key + rotation. + type: string + type: object + kms: + description: KeyManagementService is the main Key Management + option + nullable: true + properties: + connectionDetails: + additionalProperties: + type: string + description: ConnectionDetails contains the KMS connection + details (address, port etc) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + tokenSecretName: + description: TokenSecretName is the kubernetes secret containing + the KMS token + type: string + type: object + type: object + skipUpgradeChecks: + description: SkipUpgradeChecks defines if an upgrade should be forced + even if one of the check fails + type: boolean + storage: + description: A spec for available storage in the cluster and how + it should be used + nullable: true + properties: + backfillFullRatio: + description: BackfillFullRatio is the ratio at which the cluster + is too full for backfill. Backfill will be disabled if above + this threshold. Default is 0.90. + maximum: 1 + minimum: 0 + nullable: true + type: number + config: + additionalProperties: + type: string + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + deviceFilter: + description: A regular expression to allow more fine-grained + selection of devices on nodes across the cluster + type: string + devicePathFilter: + description: A regular expression to allow more fine-grained + selection of devices with path names + type: string + devices: + description: List of devices to use as storage devices + items: + description: Device represents a disk to use in the cluster + properties: + config: + additionalProperties: + type: string + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + fullpath: + type: string + name: + type: string + type: object + nullable: true + type: array + x-kubernetes-preserve-unknown-fields: true + flappingRestartIntervalHours: + description: |- + FlappingRestartIntervalHours defines the time for which the OSD pods, that failed with zero exit code, will sleep before restarting. + This is needed for OSD flapping where OSD daemons are marked down more than 5 times in 600 seconds by Ceph. + Preventing the OSD pods to restart immediately in such scenarios will prevent Rook from marking OSD as `up` and thus + peering of the PGs mapped to the OSD. + User needs to manually restart the OSD pod if they manage to fix the underlying OSD flapping issue before the restart interval. + The sleep will be disabled if this interval is set to 0. + type: integer + fullRatio: + description: FullRatio is the ratio at which the cluster is + considered full and ceph will stop accepting writes. Default + is 0.95. + maximum: 1 + minimum: 0 + nullable: true + type: number + nearFullRatio: + description: NearFullRatio is the ratio at which the cluster + is considered nearly full and will raise a ceph health warning. + Default is 0.85. + maximum: 1 + minimum: 0 + nullable: true + type: number + nodes: + items: + description: Node is a storage nodes + properties: + config: + additionalProperties: + type: string + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + deviceFilter: + description: A regular expression to allow more fine-grained + selection of devices on nodes across the cluster + type: string + devicePathFilter: + description: A regular expression to allow more fine-grained + selection of devices with path names + type: string + devices: + description: List of devices to use as storage devices + items: + description: Device represents a disk to use in the + cluster + properties: + config: + additionalProperties: + type: string + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + fullpath: + type: string + name: + type: string + type: object + nullable: true + type: array + x-kubernetes-preserve-unknown-fields: true + name: + type: string + resources: + description: ResourceRequirements describes the compute + resource requirements. + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + useAllDevices: + description: Whether to consume all the storage devices + found on a machine + type: boolean + volumeClaimTemplates: + description: PersistentVolumeClaims to use as storage + items: + description: VolumeClaimTemplate is a simplified version + of K8s corev1's PVC. It has no type meta or status. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + type: object + type: array + type: object + nullable: true + type: array + onlyApplyOSDPlacement: + type: boolean + storageClassDeviceSets: + items: + description: StorageClassDeviceSet is a storage class device + set + properties: + config: + additionalProperties: + type: string + description: Provider-specific device configuration + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + count: + description: Count is the number of devices in this set + minimum: 1 + type: integer + encrypted: + description: Whether to encrypt the deviceSet + type: boolean + name: + description: Name is a unique identifier for the set + type: string + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + portable: + description: Portable represents OSD portability across + the hosts + type: boolean + preparePlacement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + resources: + description: ResourceRequirements describes the compute + resource requirements. + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + schedulerName: + description: Scheduler name for OSD pod placement + type: string + tuneDeviceClass: + description: TuneSlowDeviceClass Tune the OSD when running + on a slow Device Class + type: boolean + tuneFastDeviceClass: + description: TuneFastDeviceClass Tune the OSD when running + on a fast Device Class + type: boolean + volumeClaimTemplates: + description: VolumeClaimTemplates is a list of PVC templates + for the underlying storage devices + items: + description: VolumeClaimTemplate is a simplified version + of K8s corev1's PVC. It has no type meta or status. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + type: object + type: array + required: + - count + - name + - volumeClaimTemplates + type: object + nullable: true + type: array + store: + description: OSDStore is the backend storage type used for creating + the OSDs + properties: + type: + description: Type of backend storage to be used while creating + OSDs. If empty, then bluestore will be used + enum: + - bluestore + - bluestore-rdr + type: string + updateStore: + description: |- + UpdateStore updates the backend store for existing OSDs. It destroys each OSD one at a time, cleans up the backing disk + and prepares same OSD on that disk + pattern: ^$|^yes-really-update-store$ + type: string + type: object + useAllDevices: + description: Whether to consume all the storage devices found + on a machine + type: boolean + useAllNodes: + type: boolean + volumeClaimTemplates: + description: PersistentVolumeClaims to use as storage + items: + description: VolumeClaimTemplate is a simplified version of + K8s corev1's PVC. It has no type meta or status. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + spec defines the desired characteristics of a volume requested by a pod author. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + type: object + type: array + type: object + upgradeOSDRequiresHealthyPGs: + description: |- + UpgradeOSDRequiresHealthyPGs defines if OSD upgrade requires PGs are clean. If set to `true` OSD upgrade process won't start until PGs are healthy. + This configuration will be ignored if `skipUpgradeChecks` is `true`. + Default is false. + type: boolean + waitTimeoutForHealthyOSDInMinutes: + description: |- + WaitTimeoutForHealthyOSDInMinutes defines the time the operator would wait before an OSD can be stopped for upgrade or restart. + If the timeout exceeds and OSD is not ok to stop, then the operator would skip upgrade for the current OSD and proceed with the next one + if `continueUpgradeAfterChecksEvenIfNotHealthy` is `false`. If `continueUpgradeAfterChecksEvenIfNotHealthy` is `true`, then operator would + continue with the upgrade of an OSD even if its not ok to stop after the timeout. This timeout won't be applied if `skipUpgradeChecks` is `true`. + The default wait timeout is 10 minutes. + format: int64 + type: integer + type: object + status: + description: ClusterStatus represents the status of a Ceph cluster + nullable: true + properties: + ceph: + description: CephStatus is the details health of a Ceph Cluster + properties: + capacity: + description: Capacity is the capacity information of a Ceph + Cluster + properties: + bytesAvailable: + format: int64 + type: integer + bytesTotal: + format: int64 + type: integer + bytesUsed: + format: int64 + type: integer + lastUpdated: + type: string + type: object + details: + additionalProperties: + description: CephHealthMessage represents the health message + of a Ceph Cluster + properties: + message: + type: string + severity: + type: string + required: + - message + - severity + type: object + type: object + fsid: + type: string + health: + type: string + lastChanged: + type: string + lastChecked: + type: string + previousHealth: + type: string + versions: + description: CephDaemonsVersions show the current ceph version + for different ceph daemons + properties: + cephfs-mirror: + additionalProperties: + type: integer + description: CephFSMirror shows CephFSMirror Ceph version + type: object + mds: + additionalProperties: + type: integer + description: Mds shows Mds Ceph version + type: object + mgr: + additionalProperties: + type: integer + description: Mgr shows Mgr Ceph version + type: object + mon: + additionalProperties: + type: integer + description: Mon shows Mon Ceph version + type: object + osd: + additionalProperties: + type: integer + description: Osd shows Osd Ceph version + type: object + overall: + additionalProperties: + type: integer + description: Overall shows overall Ceph version + type: object + rbd-mirror: + additionalProperties: + type: integer + description: RbdMirror shows RbdMirror Ceph version + type: object + rgw: + additionalProperties: + type: integer + description: Rgw shows Rgw Ceph version + type: object + type: object + type: object + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + message: + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + state: + description: ClusterState represents the state of a Ceph Cluster + type: string + storage: + description: CephStorage represents flavors of Ceph Cluster Storage + properties: + deprecatedOSDs: + additionalProperties: + items: + type: integer + type: array + type: object + deviceClasses: + items: + description: DeviceClasses represents device classes of a + Ceph Cluster + properties: + name: + type: string + type: object + type: array + osd: + description: OSDStatus represents OSD status of the ceph Cluster + properties: + storeType: + additionalProperties: + type: integer + description: StoreType is a mapping between the OSD backend + stores and number of OSDs using these stores + type: object + type: object + type: object + version: + description: ClusterVersion represents the version of a Ceph Cluster + properties: + image: + type: string + version: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephcosidrivers.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephCOSIDriver + listKind: CephCOSIDriverList + plural: cephcosidrivers + shortNames: + - cephcosi + singular: cephcosidriver + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephCOSIDriver represents the CRD for the Ceph COSI Driver + Deployment + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph COSI Driver + properties: + deploymentStrategy: + description: DeploymentStrategy is the strategy to use to deploy + the COSI driver. + enum: + - Never + - Auto + - Always + type: string + image: + description: Image is the container image to run the Ceph COSI driver + type: string + objectProvisionerImage: + description: ObjectProvisionerImage is the container image to run + the COSI driver sidecar + type: string + placement: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + resources: + description: Resources is the resource requirements for the COSI + driver + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephfilesystemmirrors.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephFilesystemMirror + listKind: CephFilesystemMirrorList + plural: cephfilesystemmirrors + singular: cephfilesystemmirror + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephFilesystemMirror is the Ceph Filesystem Mirror object definition + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FilesystemMirroringSpec is the filesystem mirroring specification + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on + each Pod related object. + nullable: true + type: object + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on each + Pod related object. + nullable: true + type: object + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + priorityClassName: + description: PriorityClassName sets priority class on the cephfs-mirror + pods + type: string + resources: + description: The resource requirements for the cephfs-mirror pods + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephfilesystems.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephFilesystem + listKind: CephFilesystemList + plural: cephfilesystems + singular: cephfilesystem + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Number of desired active MDS daemons + jsonPath: .spec.metadataServer.activeCount + name: ActiveMDS + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephFilesystem represents a Ceph Filesystem + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FilesystemSpec represents the spec of a file system + properties: + dataPools: + description: The data pool settings, with optional predefined pool + name. + items: + description: NamedPoolSpec represents the named ceph pool spec + properties: + application: + description: The application name to set on the pool. Only + expected to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of + statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone + if available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or + not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool + or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes + Secret names to add rbd-mirror or cephfs-mirror + peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity + of the snapshot. + type: string + path: + description: Path is the path to snapshot, only + valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + name: + description: Name of the pool + type: string + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable + on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as + a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required + for replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph + in terms of expected consumption of the total cluster + capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or + minute for the health check to run like 60s for + 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + nullable: true + type: array + metadataPool: + description: The metadata pool settings + nullable: true + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if + available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or + image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of + the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on + a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a + string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for + replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in + terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadataServer: + description: The mds pod info + properties: + activeCount: + description: The number of metadata servers that are active. + The remaining servers in the cluster will be in standby mode. + format: int32 + maximum: 50 + minimum: 1 + type: integer + activeStandby: + description: |- + Whether each active MDS instance will have an active standby with a warm metadata cache for faster failover. + If false, standbys will still be available, but will not have a warm metadata cache. + type: boolean + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set + on each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on + each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: ProbeSpec is a wrapper around Probe so it can be + enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + description: PriorityClassName sets priority classes on components + type: string + resources: + description: The resource requirements for the mds pods + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + startupProbe: + description: ProbeSpec is a wrapper around Probe so it can be + enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + required: + - activeCount + type: object + mirroring: + description: The mirroring settings + nullable: true + properties: + enabled: + description: Enabled whether this filesystem is mirrored or + not + type: boolean + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotRetention: + description: |- + Retention is the retention policy for a snapshot schedule + One path has exactly one retention policy. + A policy can however contain multiple count-time period pairs in order to specify complex retention policies + items: + description: SnapshotScheduleRetentionSpec is a retention + policy + properties: + duration: + description: Duration represents the retention duration + for a snapshot + type: string + path: + description: Path is the path to snapshot + type: string + type: object + type: array + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored filesystems + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the + snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + preserveFilesystemOnDelete: + description: Preserve the fs in the cluster on CephFilesystem CR + deletion. Setting this to true automatically implies PreservePoolsOnDelete + is true. + type: boolean + preservePoolsOnDelete: + description: Preserve pools on filesystem deletion + type: boolean + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of + an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - dataPools + - metadataPool + - metadataServer + type: object + status: + description: CephFilesystemStatus represents the status of a Ceph Filesystem + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + info: + additionalProperties: + type: string + description: Use only info and put mirroringStatus in it? + nullable: true + type: object + mirroringStatus: + description: MirroringStatus is the filesystem mirroring status + properties: + daemonsStatus: + description: PoolMirroringStatus is the mirroring status of + a filesystem + items: + description: FilesystemMirrorInfoSpec is the filesystem mirror + status of a given filesystem + properties: + daemon_id: + description: DaemonID is the cephfs-mirror name + type: integer + filesystems: + description: Filesystems is the list of filesystems managed + by a given cephfs-mirror daemon + items: + description: FilesystemsSpec is spec for the mirrored + filesystem + properties: + directory_count: + description: DirectoryCount is the number of directories + in the filesystem + type: integer + filesystem_id: + description: FilesystemID is the filesystem identifier + type: integer + name: + description: Name is name of the filesystem + type: string + peers: + description: Peers represents the mirroring peers + items: + description: FilesystemMirrorInfoPeerSpec is the + specification of a filesystem peer mirror + properties: + remote: + description: Remote are the remote cluster + information + properties: + client_name: + description: ClientName is cephx name + type: string + cluster_name: + description: ClusterName is the name of + the cluster + type: string + fs_name: + description: FsName is the filesystem + name + type: string + type: object + stats: + description: Stats are the stat a peer mirror + properties: + failure_count: + description: FailureCount is the number + of mirroring failure + type: integer + recovery_count: + description: RecoveryCount is the number + of recovery attempted after failures + type: integer + type: object + uuid: + description: UUID is the peer unique identifier + type: string + type: object + type: array + type: object + type: array + type: object + nullable: true + type: array + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last + changed + type: string + lastChecked: + description: LastChecked is the last time time the status was + checked + type: string + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + snapshotScheduleStatus: + description: FilesystemSnapshotScheduleStatusSpec is the status + of the snapshot schedule + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last + changed + type: string + lastChecked: + description: LastChecked is the last time time the status was + checked + type: string + snapshotSchedules: + description: SnapshotSchedules is the list of snapshots scheduled + items: + description: FilesystemSnapshotSchedulesSpec is the list of + snapshot scheduled for images in a pool + properties: + fs: + description: Fs is the name of the Ceph Filesystem + type: string + path: + description: Path is the path on the filesystem + type: string + rel_path: + type: string + retention: + description: FilesystemSnapshotScheduleStatusRetention + is the retention specification for a filesystem snapshot + schedule + properties: + active: + description: Active is whether the scheduled is active + or not + type: boolean + created: + description: Created is when the snapshot schedule + was created + type: string + created_count: + description: CreatedCount is total amount of snapshots + type: integer + first: + description: First is when the first snapshot schedule + was taken + type: string + last: + description: Last is when the last snapshot schedule + was taken + type: string + last_pruned: + description: LastPruned is when the last snapshot + schedule was pruned + type: string + pruned_count: + description: PrunedCount is total amount of pruned + snapshots + type: integer + start: + description: Start is when the snapshot schedule starts + type: string + type: object + schedule: + type: string + subvol: + description: Subvol is the name of the sub volume + type: string + type: object + nullable: true + type: array + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephfilesystemsubvolumegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephFilesystemSubVolumeGroup + listKind: CephFilesystemSubVolumeGroupList + plural: cephfilesystemsubvolumegroups + singular: cephfilesystemsubvolumegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - description: Name of the CephFileSystem + jsonPath: .spec.filesystemName + name: Filesystem + type: string + - jsonPath: .spec.quota + name: Quota + type: string + - jsonPath: .status.info.pinning + name: Pinning + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephFilesystemSubVolumeGroup represents a Ceph Filesystem SubVolumeGroup + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph Filesystem + SubVolumeGroup + properties: + dataPoolName: + description: The data pool name for the Ceph Filesystem subvolume + group layout, if the default CephFS pool is not desired. + type: string + filesystemName: + description: |- + FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of + the CephFilesystem CR. If not coming from the CephFilesystem CR, it can be retrieved from the + list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem + abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes + type: string + x-kubernetes-validations: + - message: filesystemName is immutable + rule: self == oldSelf + name: + description: The name of the subvolume group. If not set, the default + is the name of the subvolumeGroup CR. + type: string + x-kubernetes-validations: + - message: name is immutable + rule: self == oldSelf + pinning: + description: |- + Pinning configuration of CephFilesystemSubVolumeGroup, + reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + only one out of (export, distributed, random) can be set at a time + properties: + distributed: + maximum: 1 + minimum: 0 + nullable: true + type: integer + export: + maximum: 256 + minimum: -1 + nullable: true + type: integer + random: + maximum: 1 + minimum: 0 + nullable: true + type: number + type: object + x-kubernetes-validations: + - message: only one pinning type should be set + rule: (has(self.export) && !has(self.distributed) && !has(self.random)) + || (!has(self.export) && has(self.distributed) && !has(self.random)) + || (!has(self.export) && !has(self.distributed) && has(self.random)) + || (!has(self.export) && !has(self.distributed) && !has(self.random)) + quota: + anyOf: + - type: integer + - type: string + description: Quota size of the Ceph Filesystem subvolume group. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - filesystemName + type: object + status: + description: Status represents the status of a CephFilesystem SubvolumeGroup + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephnfses.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNFS + listKind: CephNFSList + plural: cephnfses + shortNames: + - nfs + singular: cephnfs + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephNFS represents a Ceph NFS + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NFSGaneshaSpec represents the spec of an nfs ganesha server + properties: + rados: + description: RADOS is the Ganesha RADOS specification + nullable: true + properties: + namespace: + description: |- + The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. + This setting is deprecated as it is internally set to the name of the CephNFS. + type: string + pool: + description: |- + The Ceph pool used store the shared configuration for NFS-Ganesha daemons. + This setting is deprecated, as it is internally required to be ".nfs". + type: string + type: object + security: + description: Security allows specifying security configurations + for the NFS cluster + nullable: true + properties: + kerberos: + description: Kerberos configures NFS-Ganesha to secure NFS client + connections with Kerberos. + nullable: true + properties: + configFiles: + description: |- + ConfigFiles defines where the Kerberos configuration should be sourced from. Config files + will be placed into the `/etc/krb5.conf.rook/` directory. + + + If this is left empty, Rook will not add any files. This allows you to manage the files + yourself however you wish. For example, you may build them into your custom Ceph container + image or use the Vault agent injector to securely add the files via annotations on the + CephNFS spec (passed to the NFS server pods). + + + Rook configures Kerberos to log to stderr. We suggest removing logging sections from config + files to avoid consuming unnecessary disk space from logging to files. + properties: + volumeSource: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + type: object + domainName: + description: DomainName should be set to the Kerberos Realm. + type: string + keytabFile: + description: |- + KeytabFile defines where the Kerberos keytab should be sourced from. The keytab file will be + placed into `/etc/krb5.keytab`. If this is left empty, Rook will not add the file. + This allows you to manage the `krb5.keytab` file yourself however you wish. For example, you + may build it into your custom Ceph container image or use the Vault agent injector to + securely add the file via annotations on the CephNFS spec (passed to the NFS server pods). + properties: + volumeSource: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + type: object + principalName: + default: nfs + description: |- + PrincipalName corresponds directly to NFS-Ganesha's NFS_KRB5:PrincipalName config. In + practice, this is the service prefix of the principal name. The default is "nfs". + This value is combined with (a) the namespace and name of the CephNFS (with a hyphen between) + and (b) the Realm configured in the user-provided krb5.conf to determine the full principal + name: /-@. e.g., nfs/rook-ceph-my-nfs@example.net. + See https://github.com/nfs-ganesha/nfs-ganesha/wiki/RPCSEC_GSS for more detail. + type: string + type: object + sssd: + description: |- + SSSD enables integration with System Security Services Daemon (SSSD). SSSD can be used to + provide user ID mapping from a number of sources. See https://sssd.io for more information + about the SSSD project. + nullable: true + properties: + sidecar: + description: Sidecar tells Rook to run SSSD in a sidecar + alongside the NFS-Ganesha server in each NFS pod. + properties: + additionalFiles: + description: |- + AdditionalFiles defines any number of additional files that should be mounted into the SSSD + sidecar. These files may be referenced by the sssd.conf config file. + items: + description: |- + SSSDSidecarAdditionalFile represents the source from where additional files for the the SSSD + configuration should come from and are made available. + properties: + subPath: + description: |- + SubPath defines the sub-path in `/etc/sssd/rook-additional/` where the additional file(s) + will be placed. Each subPath definition must be unique and must not contain ':'. + minLength: 1 + pattern: ^[^:]+$ + type: string + volumeSource: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + required: + - subPath + - volumeSource + type: object + type: array + debugLevel: + description: |- + DebugLevel sets the debug level for SSSD. If unset or set to 0, Rook does nothing. Otherwise, + this may be a value between 1 and 10. See SSSD docs for more info: + https://sssd.io/troubleshooting/basics.html#sssd-debug-logs + maximum: 10 + minimum: 0 + type: integer + image: + description: Image defines the container image that + should be used for the SSSD sidecar. + minLength: 1 + type: string + resources: + description: Resources allow specifying resource requests/limits + on the SSSD sidecar container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + sssdConfigFile: + description: |- + SSSDConfigFile defines where the SSSD configuration should be sourced from. The config file + will be placed into `/etc/sssd/sssd.conf`. If this is left empty, Rook will not add the file. + This allows you to manage the `sssd.conf` file yourself however you wish. For example, you + may build it into your custom Ceph container image or use the Vault agent injector to + securely add the file via annotations on the CephNFS spec (passed to the NFS server pods). + properties: + volumeSource: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + type: object + required: + - image + type: object + type: object + type: object + server: + description: Server is the Ganesha Server specification + properties: + active: + description: The number of active Ganesha servers + type: integer + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set + on each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + hostNetwork: + description: Whether host networking is enabled for the Ganesha + server. If not set, the network settings from the cluster + CR will be applied. + nullable: true + type: boolean + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on + each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: |- + A liveness-probe to verify that Ganesha server has valid run-time state. + If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + logLevel: + description: LogLevel set logging level + type: string + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + description: PriorityClassName sets the priority class on the + pods + type: string + resources: + description: Resources set resource requests and limits + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - active + type: object + required: + - server + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephobjectrealms.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectRealm + listKind: CephObjectRealmList + plural: cephobjectrealms + singular: cephobjectrealm + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephObjectRealm represents a Ceph Object Store Gateway Realm + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObjectRealmSpec represent the spec of an ObjectRealm + nullable: true + properties: + pull: + description: PullSpec represents the pulling specification of a + Ceph Object Storage Gateway Realm + properties: + endpoint: + pattern: ^https*:// + type: string + type: object + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephobjectstores.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStore + listKind: CephObjectStoreList + plural: cephobjectstores + singular: cephobjectstore + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.info.endpoint + name: Endpoint + type: string + - jsonPath: .status.info.secureEndpoint + name: SecureEndpoint + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephObjectStore represents a Ceph Object Store Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObjectStoreSpec represent the spec of a pool + properties: + allowUsersInNamespaces: + description: |- + The list of allowed namespaces in addition to the object store namespace + where ceph object store users may be created. Specify "*" to allow all + namespaces, otherwise list individual namespaces that are to be allowed. + This is useful for applications that need object store credentials + to be created in their own namespace, where neither OBCs nor COSI + is being used to create buckets. The default is empty. + items: + type: string + type: array + dataPool: + description: The data pool settings + nullable: true + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if + available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or + image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of + the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on + a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a + string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for + replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in + terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gateway: + description: The rgw pod info + nullable: true + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set + on each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + caBundleRef: + description: The name of the secret that stores custom ca-bundle + with root and intermediate certificates. + nullable: true + type: string + dashboardEnabled: + description: Whether rgw dashboard is enabled for the rgw daemon. + If not set, the rgw dashboard will be enabled. + nullable: true + type: boolean + x-kubernetes-preserve-unknown-fields: true + disableMultisiteSyncTraffic: + description: |- + DisableMultisiteSyncTraffic, when true, prevents this object store's gateways from + transmitting multisite replication data. Note that this value does not affect whether + gateways receive multisite replication traffic: see ObjectZone.spec.customEndpoints for that. + If false or unset, this object store's gateways will be able to transmit multisite + replication data. + type: boolean + externalRgwEndpoints: + description: |- + ExternalRgwEndpoints points to external RGW endpoint(s). Multiple endpoints can be given, but + for stability of ObjectBucketClaims, we highly recommend that users give only a single + external RGW endpoint that is a load balancer that sends requests to the multiple RGWs. + items: + description: |- + EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of + Kubernetes's v1.EndpointAddress. + properties: + hostname: + description: The DNS-addressable Hostname of this endpoint. + This field will be preferred over IP if both are given. + type: string + ip: + description: The IP of this endpoint. As a legacy behavior, + this supports being given a DNS-addressable hostname + as well. + type: string + type: object + x-kubernetes-map-type: atomic + nullable: true + type: array + hostNetwork: + description: Whether host networking is enabled for the rgw + daemon. If not set, the network settings from the cluster + CR will be applied. + nullable: true + type: boolean + x-kubernetes-preserve-unknown-fields: true + instances: + description: The number of pods in the rgw replicaset. + format: int32 + nullable: true + type: integer + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on + each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + port: + description: The port the rgw service will be listening on (http) + format: int32 + type: integer + priorityClassName: + description: PriorityClassName sets priority classes on the + rgw pods + type: string + resources: + description: The resource requirements for the rgw pods + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + securePort: + description: The port the rgw service will be listening on (https) + format: int32 + maximum: 65535 + minimum: 0 + nullable: true + type: integer + service: + description: The configuration related to add/set on each rgw + service. + nullable: true + properties: + annotations: + additionalProperties: + type: string + description: |- + The annotations-related configuration to add/set on each rgw service. + nullable + optional + type: object + type: object + sslCertificateRef: + description: The name of the secret that stores the ssl certificate + for secure rgw connections + nullable: true + type: string + type: object + healthCheck: + description: The RGW health probes + nullable: true + properties: + readinessProbe: + description: ProbeSpec is a wrapper around Probe so it can be + enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + startupProbe: + description: ProbeSpec is a wrapper around Probe so it can be + enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable + or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + type: object + hosting: + description: Hosting settings for the object store + properties: + dnsNames: + description: |- + A list of DNS names in which bucket can be accessed via virtual host path. These names need to valid according RFC-1123. + Each domain requires wildcard support like ingress loadbalancer. + Do not include the wildcard itself in the list of hostnames (e.g. use "mystore.example.com" instead of "*.mystore.example.com"). + Add all hostnames including user-created Kubernetes Service endpoints to the list. + CephObjectStore Service Endpoints and CephObjectZone customEndpoints are automatically added to the list. + The feature is supported only for Ceph v18 and later versions. + items: + type: string + type: array + type: object + metadataPool: + description: The metadata pool settings + nullable: true + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if + available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or + image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of + the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on + a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a + string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for + replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in + terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + preservePoolsOnDelete: + description: Preserve pools on object store deletion + type: boolean + security: + description: Security represents security settings + nullable: true + properties: + keyRotation: + description: KeyRotation defines options for Key Rotation. + nullable: true + properties: + enabled: + default: false + description: Enabled represents whether the key rotation + is enabled. + type: boolean + schedule: + description: Schedule represents the cron schedule for key + rotation. + type: string + type: object + kms: + description: KeyManagementService is the main Key Management + option + nullable: true + properties: + connectionDetails: + additionalProperties: + type: string + description: ConnectionDetails contains the KMS connection + details (address, port etc) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + tokenSecretName: + description: TokenSecretName is the kubernetes secret containing + the KMS token + type: string + type: object + s3: + description: The settings for supporting AWS-SSE:S3 with RGW + nullable: true + properties: + connectionDetails: + additionalProperties: + type: string + description: ConnectionDetails contains the KMS connection + details (address, port etc) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + tokenSecretName: + description: TokenSecretName is the kubernetes secret containing + the KMS token + type: string + type: object + type: object + sharedPools: + description: The pool information when configuring RADOS namespaces + in existing pools. + nullable: true + properties: + dataPoolName: + description: The data pool used for creating RADOS namespaces + in the object store + type: string + x-kubernetes-validations: + - message: object store shared data pool is immutable + rule: self == oldSelf + metadataPoolName: + description: The metadata pool used for creating RADOS namespaces + in the object store + type: string + x-kubernetes-validations: + - message: object store shared metadata pool is immutable + rule: self == oldSelf + preserveRadosNamespaceDataOnDelete: + description: Whether the RADOS namespaces should be preserved + on deletion of the object store + type: boolean + required: + - dataPoolName + - metadataPoolName + type: object + zone: + description: The multisite info + nullable: true + properties: + name: + description: RGW Zone the Object Store is in + type: string + required: + - name + type: object + type: object + status: + description: ObjectStoreStatus represents the status of a Ceph Object + Store resource + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + endpoints: + properties: + insecure: + items: + type: string + nullable: true + type: array + secure: + items: + type: string + nullable: true + type: array + type: object + info: + additionalProperties: + type: string + nullable: true + type: object + message: + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephobjectstoreusers.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStoreUser + listKind: CephObjectStoreUserList + plural: cephobjectstoreusers + shortNames: + - rcou + - objectuser + singular: cephobjectstoreuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephObjectStoreUser represents a Ceph Object Store Gateway + User + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObjectStoreUserSpec represent the spec of an Objectstoreuser + properties: + capabilities: + description: Additional admin-level capabilities for the Ceph object + store user + nullable: true + properties: + amz-cache: + description: Add capabilities for user to send request to RGW + Cache API header. Documented in https://docs.ceph.com/en/quincy/radosgw/rgw-cache/#cache-api + enum: + - '*' + - read + - write + - read, write + type: string + bilog: + description: Add capabilities for user to change bucket index + logging. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + bucket: + description: Admin capabilities to read/write Ceph object store + buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + buckets: + description: Admin capabilities to read/write Ceph object store + buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + datalog: + description: Add capabilities for user to change data logging. + Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + info: + description: Admin capabilities to read/write information about + the user. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + mdlog: + description: Add capabilities for user to change metadata logging. + Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + metadata: + description: Admin capabilities to read/write Ceph object store + metadata. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + oidc-provider: + description: Add capabilities for user to change oidc provider. + Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + ratelimit: + description: Add capabilities for user to set rate limiter for + user and bucket. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + roles: + description: Admin capabilities to read/write roles for user. + Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + usage: + description: Admin capabilities to read/write Ceph object store + usage. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + user: + description: Admin capabilities to read/write Ceph object store + users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + user-policy: + description: Add capabilities for user to change user policies. + Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + users: + description: Admin capabilities to read/write Ceph object store + users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + zone: + description: Admin capabilities to read/write Ceph object store + zones. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + type: object + clusterNamespace: + description: The namespace where the parent CephCluster and CephObjectStore + are found + type: string + displayName: + description: The display name for the ceph users + type: string + quotas: + description: ObjectUserQuotaSpec can be used to set quotas for the + object store user to limit their usage. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/?#quota-management) + for more + nullable: true + properties: + maxBuckets: + description: Maximum bucket limit for the ceph user + nullable: true + type: integer + maxObjects: + description: Maximum number of objects across all the user's + buckets + format: int64 + nullable: true + type: integer + maxSize: + anyOf: + - type: integer + - type: string + description: |- + Maximum size limit of all objects across all the user's buckets + See https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity for more info. + nullable: true + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + store: + description: The store the user will be created in + type: string + type: object + status: + description: ObjectStoreUserStatus represents the status Ceph Object + Store Gateway User + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephobjectzonegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZoneGroup + listKind: CephObjectZoneGroupList + plural: cephobjectzonegroups + singular: cephobjectzonegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephObjectZoneGroup represents a Ceph Object Store Gateway + Zone Group + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObjectZoneGroupSpec represent the spec of an ObjectZoneGroup + properties: + realm: + description: The display name for the ceph users + type: string + required: + - realm + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephobjectzones.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZone + listKind: CephObjectZoneList + plural: cephobjectzones + singular: cephobjectzone + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephObjectZone represents a Ceph Object Store Gateway Zone + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObjectZoneSpec represent the spec of an ObjectZone + properties: + customEndpoints: + description: |- + If this zone cannot be accessed from other peer Ceph clusters via the ClusterIP Service + endpoint created by Rook, you must set this to the externally reachable endpoint(s). You may + include the port in the definition. For example: "https://my-object-store.my-domain.net:443". + In many cases, you should set this to the endpoint of the ingress resource that makes the + CephObjectStore associated with this CephObjectStoreZone reachable to peer clusters. + The list can have one or more endpoints pointing to different RGW servers in the zone. + + + If a CephObjectStore endpoint is omitted from this list, that object store's gateways will + not receive multisite replication data + (see CephObjectStore.spec.gateway.disableMultisiteSyncTraffic). + items: + type: string + nullable: true + type: array + dataPool: + description: The data pool settings + nullable: true + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if + available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or + image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of + the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on + a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a + string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for + replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in + terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadataPool: + description: The metadata pool settings + nullable: true + properties: + application: + description: The application name to set on the pool. Only expected + to be set for rgw pools. + type: string + compressionMode: + description: |- + DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" + The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) + Do NOT set a default value for kubebuilder as this will override the Parameters + enum: + - none + - passive + - aggressive + - force + - '' + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the + pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use + in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: |- + Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: |- + Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). + The number of chunks required to recover an object when any single OSD is lost is the same + as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if + available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or + image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret + names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot + for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot + scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of + the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid + for CephFS + type: string + startTime: + description: StartTime indicates when to start the + snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on + a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: |- + MaxBytes represents the quota in bytes + Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a + string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier + settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance + tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance + tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica + in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you + to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated + storage pool, including the object itself (required for + replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure + domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in + terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check + of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute + for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + preservePoolsOnDelete: + default: true + description: Preserve pools on object zone deletion + type: boolean + sharedPools: + description: The pool information when configuring RADOS namespaces + in existing pools. + nullable: true + properties: + dataPoolName: + description: The data pool used for creating RADOS namespaces + in the object store + type: string + x-kubernetes-validations: + - message: object store shared data pool is immutable + rule: self == oldSelf + metadataPoolName: + description: The metadata pool used for creating RADOS namespaces + in the object store + type: string + x-kubernetes-validations: + - message: object store shared metadata pool is immutable + rule: self == oldSelf + preserveRadosNamespaceDataOnDelete: + description: Whether the RADOS namespaces should be preserved + on deletion of the object store + type: boolean + required: + - dataPoolName + - metadataPoolName + type: object + zoneGroup: + description: The display name for the ceph users + type: string + required: + - dataPool + - metadataPool + - zoneGroup + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + helm.sh/resource-policy: keep + name: cephrbdmirrors.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephRBDMirror + listKind: CephRBDMirrorList + plural: cephrbdmirrors + singular: cephrbdmirror + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephRBDMirror represents a Ceph RBD Mirror + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: RBDMirroringSpec represents the specification of an RBD + mirror daemon + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on + each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + count: + description: Count represents the number of rbd mirror instance + to run + minimum: 1 + type: integer + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on each + Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names + to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + placement: + nullable: true + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + description: PriorityClassName sets priority class on the rbd mirror + pods + type: string + resources: + description: The resource requirements for the rbd mirror pods + nullable: true + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - count + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph + Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + helm.sh/resource-policy: keep + name: objectbucketclaims.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucketClaim + listKind: ObjectBucketClaimList + plural: objectbucketclaims + shortNames: + - obc + - obcs + singular: objectbucketclaim + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + additionalConfig: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + bucketName: + type: string + generateBucketName: + type: string + objectBucketName: + type: string + storageClassName: + type: string + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + helm.sh/resource-policy: keep + name: objectbuckets.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucket + listKind: ObjectBucketList + plural: objectbuckets + shortNames: + - ob + - obs + singular: objectbucket + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + additionalState: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + authentication: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + nullable: true + type: object + claimRef: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + endpoint: + nullable: true + properties: + additionalConfig: + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + bucketHost: + type: string + bucketName: + type: string + bucketPort: + format: int32 + type: integer + region: + type: string + subRegion: + type: string + type: object + reclaimPolicy: + type: string + storageClassName: + type: string + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/role.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/role.yaml new file mode 100644 index 00000000..878702d5 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/role.yaml @@ -0,0 +1,100 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-system + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + - extensions + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - delete + - apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - get + - create + - delete + - apiGroups: + - multicluster.x-k8s.io + resources: + - serviceexports + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cephfs-external-provisioner-cfg + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rbd-external-provisioner-cfg + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/rolebinding.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/rolebinding.yaml new file mode 100644 index 00000000..4ba93cea --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/rolebinding.yaml @@ -0,0 +1,48 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-system + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cephfs-external-provisioner-cfg +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rbd-external-provisioner-cfg +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/securityContextConstraints.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/securityContextConstraints.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/serviceaccount.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/serviceaccount.yaml new file mode 100644 index 00000000..2ee9c0ac --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_rook_ceph_helmchart/rook-ceph/templates/serviceaccount.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/created-by: helm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: rook-ceph-operator + helm.sh/chart: rook-ceph-v1.14.10 + operator: rook + storage-backend: ceph + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-plugin-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-provisioner-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-plugin-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-provisioner-sa + namespace: syn-rook-ceph-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph + app.kubernetes.io/part-of: container-object-storage-interface + name: objectstorage-provisioner + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/02_openshift_sccs.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/02_openshift_sccs.yaml new file mode 100644 index 00000000..520bf410 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/02_openshift_sccs.yaml @@ -0,0 +1,81 @@ +allowHostDirVolumePlugin: true +allowHostIPC: true +allowHostNetwork: true +allowHostPID: false +allowHostPorts: true +allowPrivilegedContainer: true +allowedCapabilities: + - MKNOD +apiVersion: security.openshift.io/v1 +defaultAddCapabilities: [] +fsGroup: + type: MustRunAs +kind: SecurityContextConstraints +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph + name: rook-ceph +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: + - All +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: + - system:serviceaccount:syn-rook-ceph-operator:rook-ceph-system + - system:serviceaccount:syn-rook-ceph-cluster:default + - system:serviceaccount:syn-rook-ceph-cluster:rook-ceph-default + - system:serviceaccount:syn-rook-ceph-cluster:rook-ceph-mgr + - system:serviceaccount:syn-rook-ceph-cluster:rook-ceph-osd +volumes: + - configMap + - downwardAPI + - emptyDir + - hostPath + - persistentVolumeClaim + - projected + - secret +--- +allowHostDirVolumePlugin: true +allowHostIPC: true +allowHostNetwork: true +allowHostPID: true +allowHostPorts: true +allowPrivilegedContainer: true +allowedCapabilities: + - SYS_ADMIN +apiVersion: security.openshift.io/v1 +fsGroup: + type: RunAsAny +kind: SecurityContextConstraints +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-csi + name: rook-ceph-csi +priority: null +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +supplementalGroups: + type: RunAsAny +users: + - system:serviceaccount:syn-rook-ceph-operator:rook-csi-rbd-plugin-sa + - system:serviceaccount:syn-rook-ceph-operator:rook-csi-rbd-provisioner-sa + - system:serviceaccount:syn-rook-ceph-operator:rook-csi-cephfs-plugin-sa + - system:serviceaccount:syn-rook-ceph-operator:rook-csi-cephfs-provisioner-sa + - system:serviceaccount:syn-rook-ceph-operator:default +volumes: + - configMap + - emptyDir + - hostPath + - projected diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/03_rbac_fixes.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/03_rbac_fixes.yaml new file mode 100644 index 00000000..e4a50b73 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/03_rbac_fixes.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: syn-rook-ceph-cephfs-provisioner-fix + name: syn-rook-ceph-cephfs-provisioner-fix + name: syn-rook-ceph-cephfs-provisioner-fix +rules: + - apiGroups: + - '' + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: syn-rook-ceph-cephfs-provisioner-fix + name: syn-rook-ceph-cephfs-provisioner-fix + name: syn-rook-ceph-cephfs-provisioner-fix +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: syn-rook-ceph-cephfs-provisioner-fix +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: syn-rook-ceph-operator diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_cluster.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_cluster.yaml new file mode 100644 index 00000000..7d254121 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_cluster.yaml @@ -0,0 +1,84 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cluster + name: cluster + name: cluster + namespace: syn-rook-ceph-cluster +spec: + cephVersion: + allowUnsupported: false + image: quay.io/ceph/ceph:v18.2.4 + dataDirHostPath: /var/lib/rook + disruptionManagement: + managePodBudgets: true + osdMaintenanceTimeout: 30 + healthCheck: + daemonHealth: + mon: + disabled: false + interval: 45s + timeout: 600s + mon: + allowMultiplePerNode: false + count: 3 + monitoring: + enabled: true + network: + provider: host + placement: + all: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/storage + operator: Exists + tolerations: + - key: storagenode + operator: Exists + resources: + mgr: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + mon: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 250m + memory: 2Gi + osd: + limits: + cpu: '2' + memory: 5Gi + requests: + cpu: '2' + memory: 5Gi + storage: + storageClassDeviceSets: + - count: 3 + encrypted: true + name: cluster + placement: {} + portable: false + tuneFastDeviceClass: false + volumeClaimTemplates: + - spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: localblock + volumeMode: Block + useAllDevices: false + useAllNodes: false diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_configoverride.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_configoverride.yaml new file mode 100644 index 00000000..44e1dd5d --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_configoverride.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +data: + config: | + [global] + mon_data_avail_warn = 15 + mon_osd_backfillfull_ratio = 0.8 + mon_osd_full_ratio = 0.85 + mon_osd_nearfull_ratio = 0.75 +kind: ConfigMap +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-config-override + name: rook-config-override + name: rook-config-override + namespace: syn-rook-ceph-cluster diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_rbac.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_rbac.yaml new file mode 100644 index 00000000..8e7e818f --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_rbac.yaml @@ -0,0 +1,376 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-cluster +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-default + name: rook-ceph-default + name: rook-ceph-default + namespace: syn-rook-ceph-cluster +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-mgr + name: rook-ceph-mgr + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-osd + name: rook-ceph-osd + name: rook-ceph-osd + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - '' + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-metrics + name: rook-ceph-metrics + name: rook-ceph-metrics + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - '' + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-mgr + name: rook-ceph-mgr + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - '' + resources: + - pods + - services + - pods/log + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-monitoring + name: rook-ceph-monitoring + name: rook-ceph-monitoring + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-osd + name: rook-ceph-osd + name: rook-ceph-osd + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - cephclusters + - cephclusters/finalizers + verbs: + - get + - list + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-metrics + name: rook-ceph-metrics + name: rook-ceph-metrics + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-metrics +subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-cluster-mgmt + name: rook-ceph-cluster-mgmt + name: rook-ceph-cluster-mgmt + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-osd + name: rook-ceph-osd + name: rook-ceph-osd + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-mgr + name: rook-ceph-mgr + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-mgr +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-mgr-system-cluster + name: rook-ceph-mgr-system-cluster + name: rook-ceph-mgr-system-cluster + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-system +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-monitoring + name: rook-ceph-monitoring + name: rook-ceph-monitoring + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-monitoring +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: syn-rook-ceph-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-mgr-cluster-cluster + name: rook-ceph-mgr-cluster-cluster + name: rook-ceph-mgr-cluster-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-cluster +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-osd-cluster + name: rook-ceph-osd-cluster + name: rook-ceph-osd-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: syn-rook-ceph-cluster diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_toolbox.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_toolbox.yaml new file mode 100644 index 00000000..b0fc7308 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/10_cephcluster_toolbox.yaml @@ -0,0 +1,145 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-tools + name: rook-ceph-tools + name: rook-ceph-tools + namespace: syn-rook-ceph-cluster +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-tools + name: rook-ceph-tools + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-tools + name: rook-ceph-tools + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/storage + operator: Exists + containers: + - command: + - /bin/bash + - -c + - | + # Replicate the script from toolbox.sh inline so the ceph image + # can be run directly, instead of requiring the rook toolbox + CEPH_CONFIG="/etc/ceph/ceph.conf" + MON_CONFIG="/etc/rook/mon-endpoints" + KEYRING_FILE="/etc/ceph/keyring" + + # create a ceph config file in its default location so ceph/rados tools can be used + # without specifying any arguments + write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + cat < ${CEPH_CONFIG} + [global] + mon_host = ${mon_endpoints} + + [client.admin] + keyring = ${KEYRING_FILE} + EOF + } + + # watch the endpoints config file and update if the mon endpoints ever change + watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done + } + + # read the secret from an env var (for backward compatibility), or from the secret file + ceph_secret=${ROOK_CEPH_SECRET} + if [[ "$ceph_secret" == "" ]]; then + ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring) + fi + + # create the keyring file + cat < ${KEYRING_FILE} + [${ROOK_CEPH_USERNAME}] + key = ${ceph_secret} + EOF + + # write the initial config file + write_endpoints + + # continuously update the mon endpoints if they fail over + watch_endpoints + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + key: ceph-username + name: rook-ceph-mon + image: docker.io/rook/ceph:v1.14.10 + imagePullPolicy: IfNotPresent + name: rook-ceph-tools + securityContext: {} + tty: true + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + - mountPath: /etc/rook + name: mon-endpoint-volume + - mountPath: /var/lib/rook-ceph-mon + name: ceph-admin-secret + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: rook-ceph-default + tolerations: + - key: storagenode + operator: Exists + volumes: + - name: ceph-admin-secret + secret: + items: + - key: ceph-secret + path: secret.keyring + optional: false + secretName: rook-ceph-mon + - configMap: + items: + - key: data + path: mon-endpoints + name: rook-ceph-mon-endpoints + name: mon-endpoint-volume + - emptyDir: {} + name: ceph-config diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/20_storagepools.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/20_storagepools.yaml new file mode 100644 index 00000000..6fbe174a --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/20_storagepools.yaml @@ -0,0 +1,16 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: storagepool + name: storagepool + name: storagepool + namespace: syn-rook-ceph-cluster +spec: + failureDomain: host + replicated: + requireSafeReplicaSize: true + size: 3 diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_snapshotclasses.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_snapshotclasses.yaml new file mode 100644 index 00000000..0f3a5d81 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_snapshotclasses.yaml @@ -0,0 +1,14 @@ +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: syn-rook-ceph-operator.rbd.csi.ceph.com +kind: VolumeSnapshotClass +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-rbd-cluster + name: rook-ceph-rbd-cluster +parameters: + clusterID: syn-rook-ceph-cluster + csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/snapshotter-secret-namespace: syn-rook-ceph-cluster diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_storageclasses.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_storageclasses.yaml new file mode 100644 index 00000000..e52b049d --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/30_storageclasses.yaml @@ -0,0 +1,56 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rbd-storagepool-cluster + name: rbd-storagepool-cluster + name: rbd-storagepool-cluster +mountOptions: + - discard +parameters: + clusterID: syn-rook-ceph-cluster + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: syn-rook-ceph-cluster + csi.storage.k8s.io/fstype: ext4 + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: syn-rook-ceph-cluster + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: syn-rook-ceph-cluster + imageFeatures: layering + imageFormat: '2' + pool: storagepool +provisioner: syn-rook-ceph-operator.rbd.csi.ceph.com +reclaimPolicy: Delete +--- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rbd-storagepool-cluster-small-files + name: rbd-storagepool-cluster-small-files + name: rbd-storagepool-cluster-small-files +mountOptions: + - discard +parameters: + clusterID: syn-rook-ceph-cluster + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: syn-rook-ceph-cluster + csi.storage.k8s.io/fstype: ext4 + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: syn-rook-ceph-cluster + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: syn-rook-ceph-cluster + imageFeatures: layering + imageFormat: '2' + mkfsOptions: -m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1 -i1024 + pool: storagepool +provisioner: syn-rook-ceph-operator.rbd.csi.ceph.com +reclaimPolicy: Delete diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_alertrules.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_alertrules.yaml new file mode 100644 index 00000000..2d37f332 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_alertrules.yaml @@ -0,0 +1,875 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: syn-prometheus-ceph-rules + prometheus: rook-prometheus + role: alert-rules + name: syn-prometheus-ceph-rules + namespace: syn-rook-ceph-cluster +spec: + groups: + - name: cluster health + rules: + - alert: SYN_CephHealthError + annotations: + description: The cluster state has been HEALTH_ERROR for more than 5 minutes. + Please check 'ceph health detail' for more information. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephHealthError.html + summary: Ceph is in the ERROR state + expr: ceph_health_status == 2 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.2.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephHealthWarning + annotations: + description: The cluster state has been HEALTH_WARN for more than 15 minutes. + Please check 'ceph health detail' for more information. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephHealthWarning.html + summary: Ceph is in the WARNING state + expr: ceph_health_status == 1 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: mon + rules: + - alert: SYN_CephMonDownQuorumAtRisk + annotations: + description: '{{ $min := query "floor(count(ceph_mon_metadata) / 2) + + 1" | first | value }}Quorum requires a majority of monitors (x {{ $min + }}) to be active. Without quorum the cluster will become inoperable, + affecting all services and connected clients. The following monitors + are down: {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) + group_left(hostname) (ceph_mon_metadata * 0)" }} - {{ .Labels.ceph_daemon + }} on {{ .Labels.hostname }} {{- end }}' + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephMonDownQuorumAtRisk.html + summary: Monitor quorum is at risk + expr: | + ( + (ceph_health_detail{name="MON_DOWN"} == 1) * on() ( + count(ceph_mon_quorum_status == 1) == bool (floor(count(ceph_mon_metadata) / 2) + 1) + ) + ) == 1 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.3.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephMonDown + annotations: + description: | + {{ $down := query "count(ceph_mon_quorum_status == 0)" | first | value }}{{ $s := "" }}{{ if gt $down 1.0 }}{{ $s = "s" }}{{ end }}You have {{ $down }} monitor{{ $s }} down. Quorum is still intact, but the loss of an additional monitor will make your cluster inoperable. The following monitors are down: {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) group_left(hostname) (ceph_mon_metadata * 0)" }} - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephMonDown.html + summary: One or more monitors down + expr: | + count(ceph_mon_quorum_status == 0) <= (count(ceph_mon_metadata) - floor(count(ceph_mon_metadata) / 2) + 1) + for: 30s + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: osd + rules: + - alert: SYN_CephOSDDown + annotations: + description: | + {{ $num := query "count(ceph_osd_up == 0)" | first | value }}{{ $s := "" }}{{ if gt $num 1.0 }}{{ $s = "s" }}{{ end }}{{ $num }} OSD{{ $s }} down for over 5mins. The following OSD{{ $s }} {{ if eq $s "" }}is{{ else }}are{{ end }} down: {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0"}} - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-down + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephOSDDown.html + summary: An OSD has been marked down + expr: ceph_health_detail{name="OSD_DOWN"} == 1 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.4.2 + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephOSDFull + annotations: + description: An OSD has reached the FULL threshold. Writes to pools that + share the affected OSD will be blocked. Use 'ceph health detail' and + 'ceph osd df' to identify the problem. To resolve, add capacity to the + affected OSD's failure domain, restore down/out OSDs, or delete unwanted + data. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-full + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephOSDFull.html + summary: OSD full, writes blocked + expr: ceph_health_detail{name="OSD_FULL"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.4.6 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephOSDFlapping + annotations: + description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was + marked down and back up {{ $value | humanize }} times once a minute + for 5 minutes. This may indicate a network issue (latency, packet loss, + MTU mismatch) on the cluster network, or the public network if no cluster + network is deployed. Check the network stats on the listed host(s). + documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephOSDFlapping.html + summary: Network issues are causing OSDs to flap (mark each other down) + expr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) + * 60 > 1 + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.4.4 + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: mds + rules: + - alert: SYN_CephFilesystemDamaged + annotations: + description: Filesystem metadata has been corrupted. Data may be inaccessible. + Analyze metrics from the MDS daemon admin socket, or escalate to support. + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephFilesystemDamaged.html + summary: CephFS filesystem is damaged. + expr: ceph_health_detail{name="MDS_DAMAGE"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.5.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephFilesystemOffline + annotations: + description: All MDS ranks are unavailable. The MDS daemons managing metadata + are down, rendering the filesystem offline. + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-all-down + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephFilesystemOffline.html + summary: CephFS filesystem is offline + expr: ceph_health_detail{name="MDS_ALL_DOWN"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.5.3 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephFilesystemDegraded + annotations: + description: One or more metadata daemons (MDS ranks) are failed or in + a damaged state. At best the filesystem is partially available, at worst + the filesystem is completely unusable. + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-degraded + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephFilesystemDegraded.html + summary: CephFS filesystem is degraded + expr: ceph_health_detail{name="FS_DEGRADED"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.5.4 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephFilesystemFailureNoStandby + annotations: + description: An MDS daemon has failed, leaving only one active rank and + no available standby. Investigate the cause of the failure or add a + standby MDS. + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-with-failed-mds + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephFilesystemFailureNoStandby.html + summary: MDS daemon failed, no further standby available + expr: ceph_health_detail{name="FS_WITH_FAILED_MDS"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.5.5 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephFilesystemReadOnly + annotations: + description: The filesystem has switched to READ ONLY due to an unexpected + error when writing to the metadata pool. Either analyze the output from + the MDS daemon admin socket, or escalate to support. + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephFilesystemReadOnly.html + summary: CephFS filesystem in read only mode due to write error(s) + expr: ceph_health_detail{name="MDS_HEALTH_READ_ONLY"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.5.2 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: mgr + rules: + - alert: SYN_CephMgrModuleCrash + annotations: + description: One or more mgr modules have crashed and have yet to be acknowledged + by an administrator. A crashed module may impact functionality within + the cluster. Use the 'ceph crash' command to determine which module + has failed, and archive it to acknowledge the failure. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#recent-mgr-module-crash + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephMgrModuleCrash.html + summary: A manager module has recently crashed + expr: ceph_health_detail{name="RECENT_MGR_MODULE_CRASH"} == 1 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.6.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephMgrPrometheusModuleInactive + annotations: + description: The mgr/prometheus module at {{ $labels.instance }} is unreachable. + This could mean that the module has been disabled or the mgr daemon + itself is down. Without the mgr/prometheus module metrics and alerts + will no longer function. Open a shell to an admin node or toolbox pod + and use 'ceph -s' to to determine whether the mgr is active. If the + mgr is not active, restart it, otherwise you can determine module status + with 'ceph mgr module ls'. If it is not listed as enabled, enable it + with 'ceph mgr module enable prometheus'. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephMgrPrometheusModuleInactive.html + summary: The mgr/prometheus module is not available + expr: up{job="ceph"} == 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.6.2 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: pgs + rules: + - alert: SYN_CephPGsInactive + annotations: + description: '{{ $value }} PGs have been inactive for more than 5 minutes + in pool {{ $labels.name }}. Inactive placement groups are not able to + serve read/write requests.' + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPGsInactive.html + summary: One or more placement groups are inactive + expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total + - ceph_pg_active) > 0 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.7.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephPGsDamaged + annotations: + description: During data consistency checks (scrub), at least one PG has + been flagged as being damaged or inconsistent. Check to see which PG + is affected, and attempt a manual repair if necessary. To list problematic + placement groups, use 'rados list-inconsistent-pg '. To repair + PGs use the 'ceph pg repair ' command. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-damaged + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPGsDamaged.html + summary: Placement group damaged, manual intervention needed + expr: ceph_health_detail{name=~"PG_DAMAGED|OSD_SCRUB_ERRORS"} == 1 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.7.4 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephPGRecoveryAtRisk + annotations: + description: Data redundancy is at risk since one or more OSDs are at + or above the 'full' threshold. Add more capacity to the cluster, restore + down/out OSDs, or delete unwanted data. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-recovery-full + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPGRecoveryAtRisk.html + summary: OSDs are too full for recovery + expr: ceph_health_detail{name="PG_RECOVERY_FULL"} == 1 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.7.5 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephPGUnavailableBlockingIO + annotations: + description: Data availability is reduced, impacting the cluster's ability + to service I/O. One or more placement groups (PGs) are in a state that + blocks I/O. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-availability + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPGUnavailableBlockingIO.html + summary: PG is unavailable, blocking I/O + expr: ((ceph_health_detail{name="PG_AVAILABILITY"} == 1) - scalar(ceph_health_detail{name="OSD_DOWN"})) + == 1 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.7.3 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephPGBackfillAtRisk + annotations: + description: Data redundancy may be at risk due to lack of free space + within the cluster. One or more OSDs have reached the 'backfillfull' + threshold. Add more capacity, or delete unwanted data. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-backfill-full + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPGBackfillAtRisk.html + summary: Backfill operations are blocked due to lack of free space + expr: ceph_health_detail{name="PG_BACKFILL_FULL"} == 1 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.7.6 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: nodes + rules: + - alert: SYN_CephNodeRootFilesystemFull + annotations: + description: 'Root volume is dangerously full: {{ $value | humanize }}% + free.' + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephNodeRootFilesystemFull.html + summary: Root filesystem is dangerously full + expr: node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} + * 100 < 5 + for: 5m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.8.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephNodeNetworkBondDegraded + annotations: + description: Bond {{ $labels.master }} is degraded on Node {{ $labels.instance + }}. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephNodeNetworkBondDegraded.html + summary: Degraded Bond on Node {{ $labels.instance }} + expr: | + node_bonding_slaves - node_bonding_active != 0 + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: pools + rules: + - alert: SYN_CephPoolFull + annotations: + description: A pool has reached its MAX quota, or OSDs supporting the + pool have reached the FULL threshold. Until this is resolved, writes + to the pool will be blocked. Pool Breakdown (top 5) {{- range query + "topk(5, sort_desc(ceph_pool_percent_used * on(pool_id) group_right + ceph_pool_metadata))" }} - {{ .Labels.name }} at {{ .Value }}% {{- end + }} Increase the pool's quota, or add capacity to the cluster first then + increase the pool's quota (e.g. ceph osd pool set quota + max_bytes ) + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pool-full + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephPoolFull.html + summary: Pool is full - writes are blocked + expr: ceph_health_detail{name="POOL_FULL"} > 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.9.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: healthchecks + rules: + - alert: SYN_CephDaemonSlowOps + annotations: + description: '{{ $labels.ceph_daemon }} operations are taking too long + to process (complaint time exceeded)' + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#slow-ops + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephDaemonSlowOps.html + summary: '{{ $labels.ceph_daemon }} operations are slow to complete' + expr: ceph_daemon_health_metrics{type="SLOW_OPS"} > 0 + for: 30s + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: hardware + rules: + - alert: SYN_HardwareStorageError + annotations: + description: Some storage devices are in error. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwareStorageError.html + summary: Storage devices error(s) detected + expr: ceph_health_detail{name="HARDWARE_STORAGE"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_HardwareMemoryError + annotations: + description: DIMM error(s) detected. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwareMemoryError.html + summary: DIMM error(s) detected + expr: ceph_health_detail{name="HARDWARE_MEMORY"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.2 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_HardwareProcessorError + annotations: + description: Processor error(s) detected. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwareProcessorError.html + summary: Processor error(s) detected + expr: ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.3 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_HardwareNetworkError + annotations: + description: Network error(s) detected. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwareNetworkError.html + summary: Network error(s) detected + expr: ceph_health_detail{name="HARDWARE_NETWORK"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.4 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_HardwarePowerError + annotations: + description: Power supply error(s) detected. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwarePowerError.html + summary: Power supply error(s) detected + expr: ceph_health_detail{name="HARDWARE_POWER"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.5 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_HardwareFanError + annotations: + description: Fan error(s) detected. Check `ceph health detail`. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/HardwareFanError.html + summary: Fan error(s) detected + expr: ceph_health_detail{name="HARDWARE_FANS"} > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.13.6 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: PrometheusServer + rules: + - alert: SYN_PrometheusJobMissing + annotations: + description: The prometheus job that scrapes from Ceph MGR is no longer + defined, this will effectively mean you'll have no metrics or alerts + for the cluster. Please review the job definitions in the prometheus.yml + file of the prometheus instance. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/PrometheusJobMissing.html + summary: The scrape job for Ceph MGR is missing from Prometheus + expr: absent(up{job="rook-ceph-mgr"}) + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.12.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_PrometheusJobExporterMissing + annotations: + description: The prometheus job that scrapes from Ceph Exporter is no + longer defined, this will effectively mean you'll have no metrics or + alerts for the cluster. Please review the job definitions in the prometheus.yml + file of the prometheus instance. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/PrometheusJobExporterMissing.html + summary: The scrape job for Ceph Exporter is missing from Prometheus + expr: sum(absent(up{job="rook-ceph-exporter"})) and sum(ceph_osd_metadata{ceph_version=~"^ceph + version (1[89]|[2-9][0-9]).*"}) > 0 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.12.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: rados + rules: + - alert: SYN_CephObjectMissing + annotations: + description: The latest version of a RADOS object can not be found, even + though all OSDs are up. I/O requests for this object from clients will + block (hang). Resolving this issue may require the object to be rolled + back to a prior version manually, and manually verified. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#object-unfound + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephObjectMissing.html + summary: Object(s) marked UNFOUND + expr: (ceph_health_detail{name="OBJECT_UNFOUND"} == 1) * on() (count(ceph_osd_up + == 1) == bool count(ceph_osd_metadata)) == 1 + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.10.1 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: generic + rules: + - alert: SYN_CephDaemonCrash + annotations: + description: One or more daemons have crashed recently, and need to be + acknowledged. This notification ensures that software crashes do not + go unseen. To acknowledge a crash, use the 'ceph crash archive ' + command. + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks/#recent-crash + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephDaemonCrash.html + summary: One or more Ceph daemons have crashed, and are pending acknowledgement + expr: ceph_health_detail{name="RECENT_CRASH"} == 1 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.1.2 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: rbdmirror + rules: + - alert: SYN_CephRBDMirrorImagesPerDaemonHigh + annotations: + description: Number of image replications per daemon is not supposed to + go beyond threshold 100 + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephRBDMirrorImagesPerDaemonHigh.html + summary: Number of image replications are now above 100 + expr: sum by (ceph_daemon, namespace) (ceph_rbd_mirror_snapshot_image_snapshots) + > 100 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.10.2 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephRBDMirrorImagesNotInSync + annotations: + description: Both local and remote RBD mirror images should be in sync. + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephRBDMirrorImagesNotInSync.html + summary: Some of the RBD mirror images are not in sync with the remote + counter parts. + expr: sum by (ceph_daemon, image, namespace, pool) (topk by (ceph_daemon, + image, namespace, pool) (1, ceph_rbd_mirror_snapshot_image_local_timestamp) + - topk by (ceph_daemon, image, namespace, pool) (1, ceph_rbd_mirror_snapshot_image_remote_timestamp)) + != 0 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.10.3 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephRBDMirrorImagesNotInSyncVeryHigh + annotations: + description: More than 10% of the images have synchronization problems + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephRBDMirrorImagesNotInSyncVeryHigh.html + summary: Number of unsynchronized images are very high. + expr: count by (ceph_daemon) ((topk by (ceph_daemon, image, namespace, pool) + (1, ceph_rbd_mirror_snapshot_image_local_timestamp) - topk by (ceph_daemon, + image, namespace, pool) (1, ceph_rbd_mirror_snapshot_image_remote_timestamp)) + != 0) > (sum by (ceph_daemon) (ceph_rbd_mirror_snapshot_snapshots)*.1) + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.10.4 + severity: critical + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_CephRBDMirrorImageTransferBandwidthHigh + annotations: + description: Detected a heavy increase in bandwidth for rbd replications + (over 80%) in the last 30 min. This might not be a problem, but it is + good to review the number of images being replicated simultaneously + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/CephRBDMirrorImageTransferBandwidthHigh.html + summary: The replication network usage has been increased over 80% in + the last 30 minutes. Review the number of images being replicated. This + alert will be cleaned automatically after 30 minutes + expr: rate(ceph_rbd_mirror_journal_replay_bytes[30m]) > 0.80 + for: 1m + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.10.5 + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: nvmeof + rules: + - alert: SYN_NVMeoFSubsystemNamespaceLimit + annotations: + description: Subsystems have a max namespace limit defined at creation + time. This alert means that no more namespaces can be added to {{ $labels.nqn + }} + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFSubsystemNamespaceLimit.html + summary: '{{ $labels.nqn }} subsystem has reached its maximum number of + namespaces ' + expr: (count by(nqn) (ceph_nvmeof_subsystem_namespace_metadata)) >= ceph_nvmeof_subsystem_namespace_limit + for: 1m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFTooManyGateways + annotations: + description: You may create many gateways, but 4 is the tested limit + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFTooManyGateways.html + summary: 'Max supported gateways exceeded ' + expr: count(ceph_nvmeof_gateway_info) > 4.00 + for: 1m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFMaxGatewayGroupSize + annotations: + description: You may create many gateways in a gateway group, but 2 is + the tested limit + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFMaxGatewayGroupSize.html + summary: 'Max gateways within a gateway group ({{ $labels.group }}) exceeded ' + expr: count by(group) (ceph_nvmeof_gateway_info) > 2.00 + for: 1m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFSingleGatewayGroup + annotations: + description: Although a single member gateway group is valid, it should + only be used for test purposes + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFSingleGatewayGroup.html + summary: 'The gateway group {{ $labels.group }} consists of a single gateway + - HA is not possible ' + expr: count by(group) (ceph_nvmeof_gateway_info) == 1 + for: 5m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFHighGatewayCPU + annotations: + description: Typically, high CPU may indicate degraded performance. Consider + increasing the number of reactor cores + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFHighGatewayCPU.html + summary: 'CPU used by {{ $labels.instance }} NVMe-oF Gateway is high ' + expr: label_replace(avg by(instance) (rate(ceph_nvmeof_reactor_seconds_total{mode="busy"}[1m])),"instance","$1","instance","(.*):.*") + > 80.00 + for: 10m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFGatewayOpenSecurity + annotations: + description: It is good practice to ensure subsystems use host security + to reduce the risk of unexpected data loss + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFGatewayOpenSecurity.html + summary: 'Subsystem {{ $labels.nqn }} has been defined without host level + security ' + expr: ceph_nvmeof_subsystem_metadata{allow_any_host="yes"} + for: 5m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFTooManySubsystems + annotations: + description: Although you may continue to create subsystems in {{ $labels.gateway_host + }}, the configuration may not be supported + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFTooManySubsystems.html + summary: 'The number of subsystems defined to the gateway exceeds supported + values ' + expr: count by(gateway_host) (label_replace(ceph_nvmeof_subsystem_metadata,"gateway_host","$1","instance","(.*):.*")) + > 16.00 + for: 1m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFVersionMismatch + annotations: + description: This may indicate an issue with deployment. Check cephadm + logs + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFVersionMismatch.html + summary: 'The cluster has different NVMe-oF gateway releases active ' + expr: count(count by(version) (ceph_nvmeof_gateway_info)) > 1 + for: 1h + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFHighClientCount + annotations: + description: The supported limit for clients connecting to a subsystem + is 32 + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFHighClientCount.html + summary: 'The number of clients connected to {{ $labels.nqn }} is too + high ' + expr: ceph_nvmeof_subsystem_host_count > 32.00 + for: 1m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFHighHostCPU + annotations: + description: High CPU on a gateway host can lead to CPU contention and + performance degradation + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFHighHostCPU.html + summary: 'The CPU is high ({{ $value }}%) on NVMeoF Gateway host ({{ $labels.host + }}) ' + expr: 100-((100*(avg by(host) (label_replace(rate(node_cpu_seconds_total{mode="idle"}[5m]),"host","$1","instance","(.*):.*")) + * on(host) group_right label_replace(ceph_nvmeof_gateway_info,"host","$1","instance","(.*):.*")))) + >= 80.00 + for: 10m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFInterfaceDown + annotations: + description: A NIC used by one or more subsystems is in a down state + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFInterfaceDown.html + summary: 'Network interface {{ $labels.device }} is down ' + expr: ceph_nvmeof_subsystem_listener_iface_info{operstate="down"} + for: 30s + labels: + oid: 1.3.6.1.4.1.50495.1.2.1.14.1 + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFInterfaceDuplex + annotations: + description: Until this is resolved, performance from the gateway will + be degraded + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFInterfaceDuplex.html + summary: 'Network interface {{ $labels.device }} is not running in full + duplex mode ' + expr: ceph_nvmeof_subsystem_listener_iface_info{duplex!="full"} + for: 30s + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFHighReadLatency + annotations: + description: High latencies may indicate a constraint within the cluster + e.g. CPU, network. Please investigate + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFHighReadLatency.html + summary: The average read latency over the last 5 mins has reached 10 + ms or more on {{ $labels.gateway }} + expr: label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_read_seconds_total[1m]) + / rate(ceph_nvmeof_bdev_reads_completed_total[1m])))),"gateway","$1","instance","(.*):.*") + > 0.01 + for: 5m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - alert: SYN_NVMeoFHighWriteLatency + annotations: + description: High latencies may indicate a constraint within the cluster + e.g. CPU, network. Please investigate + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/NVMeoFHighWriteLatency.html + summary: The average write latency over the last 5 mins has reached 20 + ms or more on {{ $labels.gateway }} + expr: label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_write_seconds_total[5m]) + / rate(ceph_nvmeof_bdev_writes_completed_total[5m])))),"gateway","$1","instance","(.*):.*") + > 0.02 + for: 5m + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + type: ceph_default + - name: syn-rook-ceph-additional.rules + rules: + - alert: RookCephOperatorScaledDown + annotations: + runbook_url: https://hub.syn.tools/rook-ceph/runbooks/RookCephOperatorScaledDown.html + summary: rook-ceph operator scaled to 0 for more than 1 hour. + expr: kube_deployment_spec_replicas{deployment="rook-ceph-operator", namespace="syn-rook-ceph-operator"} + == 0 + for: 1h + labels: + severity: warning + syn: 'true' + syn_component: rook-ceph + - expr: sum(ceph_mon_num_sessions{}) + record: ceph_mon_num_sessions:sum + - expr: count(ceph_mon_quorum_status{}) + record: ceph_mon_quorum_status:count + - expr: avg(ceph_osd_apply_latency_ms{}) + record: ceph_osd_apply_latency_ms:avg + - expr: avg(ceph_osd_commit_latency_ms{}) + record: ceph_osd_commit_latency_ms:avg + - expr: sum(ceph_osd_numpg{}) + record: ceph_osd_numpg:sum + - expr: sum(rate(ceph_osd_op_r{}[5m])) + record: ceph_osd_op_r:rate5m + - expr: avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) + >= 0) + record: ceph_osd_op_r_latency:avg5m + - expr: sum(rate(ceph_osd_op_r_out_bytes{}[5m])) + record: ceph_osd_op_r_out_bytes:rate5m + - expr: sum(ceph_osd_op_r_out_bytes{}) + record: ceph_osd_op_r_out_bytes:sum + - expr: sum(rate(ceph_osd_op_w{}[5m])) + record: ceph_osd_op_w:rate5m + - expr: sum(rate(ceph_osd_op_w_in_bytes{}[5m])) + record: ceph_osd_op_w_in_bytes:rate5m + - expr: sum(ceph_osd_op_w_in_bytes{}) + record: ceph_osd_op_w_in_bytes:sum + - expr: avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) + >= 0) + record: ceph_osd_op_w_latency:avg5m + - expr: sum(ceph_pool_objects{}) + record: ceph_pool_objects:sum diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_csi_driver_metrics.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_csi_driver_metrics.yaml new file mode 100644 index 00000000..79a44e75 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/40_csi_driver_metrics.yaml @@ -0,0 +1,64 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-metrics + name: rook-ceph-metrics + name: rook-ceph-metrics + namespace: syn-rook-ceph-operator +rules: + - apiGroups: + - '' + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: rook-ceph-metrics + name: rook-ceph-metrics + name: rook-ceph-metrics + namespace: syn-rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-metrics +subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: csi-metrics + team: rook + name: csi-metrics + namespace: syn-rook-ceph-operator +spec: + endpoints: + - interval: 5s + path: /metrics + port: csi-http-metrics + namespaceSelector: + matchNames: + - syn-rook-ceph-operator + selector: + matchLabels: + app: csi-metrics diff --git a/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/99_cleanup.yaml b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/99_cleanup.yaml new file mode 100644 index 00000000..132b7409 --- /dev/null +++ b/tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/99_cleanup.yaml @@ -0,0 +1,106 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cleanup-alertrules + name: cleanup-alertrules + name: cleanup-alertrules + namespace: syn-rook-ceph-cluster +rules: + - apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - delete +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cleanup-alertrules + name: cleanup-alertrules + name: cleanup-alertrules + namespace: syn-rook-ceph-cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cleanup-alertrules + name: cleanup-alertrules + name: cleanup-alertrules + namespace: syn-rook-ceph-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cleanup-alertrules +subjects: + - kind: ServiceAccount + name: cleanup-alertrules + namespace: syn-rook-ceph-cluster +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cleanup-alertrules + name: cleanup-alertrules + name: cleanup-alertrules + namespace: syn-rook-ceph-cluster +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app.kubernetes.io/component: rook-ceph + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: cleanup-alertrules + name: cleanup-alertrules + spec: + containers: + - args: + - -n + - syn-rook-ceph-cluster + - delete + - --ignore-not-found + - prometheusrules.monitoring.coreos.com + - prometheus-ceph-v16-rules + command: + - kubectl + env: + - name: HOME + value: /home + image: docker.io/bitnami/kubectl:1.28.13@sha256:d2637001a8d5c4b61986029be790d0f03827870022954259dd376cdcca9f9fc8 + imagePullPolicy: IfNotPresent + name: cleanup-alertrules + ports: [] + stdin: false + tty: false + volumeMounts: + - mountPath: /home + name: home + workingDir: /home + imagePullSecrets: [] + initContainers: [] + restartPolicy: OnFailure + serviceAccountName: cleanup-alertrules + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: home diff --git a/tests/rbd-extra-storageclass.yml b/tests/rbd-extra-storageclass.yml new file mode 100644 index 00000000..1c1c40e0 --- /dev/null +++ b/tests/rbd-extra-storageclass.yml @@ -0,0 +1,32 @@ +applications: + - openshift4-monitoring + +parameters: + facts: + distribution: openshift4 + kapitan: + dependencies: + - type: https + source: https://raw.githubusercontent.com/projectsyn/component-storageclass/v1.0.0/lib/storageclass.libsonnet + output_path: vendor/lib/storageclass.libsonnet + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v3.5.0/lib/openshift4-monitoring-alert-patching.libsonnet + output_path: vendor/lib/alert-patching.libsonnet + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v3.5.0/lib/openshift4-monitoring-prom.libsonnet + output_path: vendor/lib/prom.libsonnet + + storageclass: + defaults: {} + defaultClass: "" + + rook_ceph: + ceph_cluster: + storage_pools: + rbd: + storagepool: + extra_storage_classes: + small-files: + parameters: + csi.storage.k8s.io/fstype: ext4 + mkfsOptions: -m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1 -i1024