Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs: Rework some blueprints to separate DB tools image and kanister-tools image #3284

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+177 −43
Open

docs: Rework some blueprints to separate DB tools image and kanister-tools image #3284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
819316a
docs: Rework some blueprints to separate DB tools image and kanister-…
hairyhum Dec 10, 2024
ffec870
Merge branch 'master' into blueprints-rework-db
hairyhum Dec 13, 2024
b30dd9f
Use 666 instead of 777 for pipe file access
hairyhum Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 44 additions & 11 deletions examples/mongodb/blueprint-v2/mongo-blueprint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ actions:
# `kopiaOutput` is the name provided to kando using `--output-name` flag
kopiaSnapshot: "{{ .Phases.takeConsistentBackup.Output.kopiaOutput }}"
phases:
- func: KubeTask
- func: MultiContainerRun
name: takeConsistentBackup
objects:
mongosecret:
Expand All @@ -20,8 +20,13 @@ actions:
namespace: "{{ .StatefulSet.Namespace }}"
args:
namespace: "{{ .StatefulSet.Namespace }}"
image: ghcr.io/kanisterio/mongodb:0.112.0
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: bitnami/mongodb:7.0-debian-12
backgroundCommand:
- bash
- -o
- errexit
Expand All @@ -32,15 +37,27 @@ actions:
host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local'
dbPassword='{{ index .Phases.takeConsistentBackup.Secrets.mongosecret.Data "mongodb-root-password" | toString }}'
dump_cmd="mongodump --oplog --gzip --archive --host ${host} -u root -p ${dbPassword}"
${dump_cmd} > /tmp/data

outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
outputCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
backup_file_path='rs_backup.gz'
${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -

restore:
inputArtifactNames:
# The kopia snapshot info created in backup phase can be used here
# Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot`
- mongoBackup
phases:
- func: KubeTask
- func: MultiContainerRun
name: pullFromStore
objects:
mongosecret:
Expand All @@ -49,8 +66,26 @@ actions:
namespace: "{{ .StatefulSet.Namespace }}"
args:
namespace: "{{ .StatefulSet.Namespace }}"
image: ghcr.io/kanisterio/mongodb:0.112.0
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
backgroundCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
backup_file_path='rs_backup.gz'
kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data

outputImage: bitnami/mongodb:7.0-debian-12
outputCommand:
- bash
- -o
- errexit
Expand All @@ -61,9 +96,7 @@ actions:
host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local'
dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "mongodb-root-password" | toString }}'
restore_cmd="mongorestore --gzip --archive --oplogReplay --drop --host ${host} -u root -p ${dbPassword}"
backup_file_path='rs_backup.gz'
kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd}
cat /tmp/data | ${restore_cmd}
delete:
inputArtifactNames:
# The kopia snapshot info created in backup phase can be used here
Expand All @@ -74,7 +107,7 @@ actions:
name: deleteFromStore
args:
namespace: "{{ .Namespace.Name }}"
image: ghcr.io/kanisterio/mongodb:0.112.0
image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
command:
- bash
- -o
Expand Down
54 changes: 44 additions & 10 deletions examples/mssql/blueprint-v2/mssql-blueprint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,47 +8,69 @@ actions:
mssqlCloudDump:
kopiaSnapshot: "{{ .Phases.dumpToObjectStore.Output.kopiaOutput }}"
phases:
- func: KubeTask
- func: MultiContainerRun
name: dumpToObjectStore
objects:
mssql:
kind: Secret
name: '{{ index .Object.metadata.labels "app" }}'
namespace: '{{ .Deployment.Namespace }}'
args:
image: ghcr.io/kanisterio/mssql-tools:0.112.0
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: ghcr.io/kanisterio/mssql-tools:0.112.0
backgroundCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
kopiaPath="backup.tar.gz"
root_password="{{ index .Phases.dumpToObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}"
root_uname="sa"
mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}"
server_name="{{ index .Deployment.Name }}.{{index .Deployment.Namespace}}.svc.cluster.local"
databases=$(/opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "SET NOCOUNT ON; SELECT name FROM sys.databases WHERE name NOT IN ('master','model','msdb','tempdb')" -b -s "," -h -1)
for database in $databases; do /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "backup database $database to disk = '/tmp/backup/$database.bak' with format;"; done
kubectl cp ${mssql_pod}:/tmp/backup /tmp/backup
tar zcvf - -C /tmp/ backup | kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" -
tar zcvf - -C /tmp/ backup > /tmp/data
kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup

outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
outputCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
kopiaPath="backup.tar.gz"
cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" -

restore:
inputArtifactNames:
- mssqlCloudDump
phases:
- func: KubeTask
- func: MultiContainerRun
name: restoreFromObjectStore
objects:
mssql:
kind: Secret
name: '{{ index .Object.metadata.labels "app" }}'
namespace: '{{ .Deployment.Namespace }}'
args:
image: ghcr.io/kanisterio/mssql-tools:0.112.0
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
backgroundCommand:
- bash
- -o
- errexit
Expand All @@ -58,23 +80,35 @@ actions:
- |
kopiaPath="backup.tar.gz"
kopia_snap='{{ .ArtifactsIn.mssqlCloudDump.KopiaSnapshot }}'
kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - > /tmp/data

outputImage: ghcr.io/kanisterio/mssql-tools:0.112.0
outputCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
root_password="{{ index .Phases.restoreFromObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}"
root_uname="sa"
mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}"
server_name="{{ index .Deployment.Name }}.{{ index .Deployment.Namespace }}.svc.cluster.local"
kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - | tar zxvf - -C /tmp/
cat /tmp/data | tar zxvf - -C /tmp/
kubectl cp /tmp/backup ${mssql_pod}:/tmp/backup
backup_files=$(ls /tmp/backup)
for script in $backup_files; do database="$(cut -d'.' -f1 <<<"$script")"; /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "restore database $database from disk = '/tmp/backup/$script' with replace"; done
kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup

delete:
inputArtifactNames:
- mssqlCloudDump
phases:
- func: KubeTask
name: deleteFromBlobStore
args:
image: ghcr.io/kanisterio/mssql-tools:0.112.0
image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
command:
- bash
- -o
Expand Down
55 changes: 44 additions & 11 deletions examples/mysql/blueprint-v2/mysql-blueprint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,45 +11,67 @@ actions:
# `kopiaOutput` is the name provided to kando using `--output-name` flag
kopiaSnapshot: "{{ .Phases.dumpToStore.Output.kopiaOutput }}"
phases:
- func: KubeTask
- func: MultiContainerRun
name: dumpToStore
objects:
mysqlSecret:
kind: Secret
name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}'
namespace: '{{ .StatefulSet.Namespace }}'
args:
image: ghcr.io/kanisterio/mysql-sidecar:0.112.0
namespace: "{{ .StatefulSet.Namespace }}"
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: mysql:8
backgroundCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
backup_file_path="dump.sql"
root_password="{{ index .Phases.dumpToStore.Secrets.mysqlSecret.Data "mysql-root-password" | toString }}"
dump_cmd="mysqldump --column-statistics=0 -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }} --single-transaction --all-databases"
${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
${dump_cmd} > /tmp/data

outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
outputCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
backup_file_path="dump.sql"
cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -

restore:
inputArtifactNames:
# The kopia snapshot info created in backup phase can be used here
# Use the `--kopia-snapshot` flag in kando to pass in `mysqlBackup.KopiaSnapshot`
- mysqlBackup
phases:
- func: KubeTask
- func: MultiContainerRun
name: restoreFromStore
objects:
mysqlSecret:
kind: Secret
name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}'
namespace: '{{ .StatefulSet.Namespace }}'
args:
image: ghcr.io/kanisterio/mysql-sidecar:0.112.0
namespace: "{{ .StatefulSet.Namespace }}"
command:
sharedVolumeMedium: Memory

initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 666 /tmp/data"]

backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
backgroundCommand:
- bash
- -o
- errexit
Expand All @@ -59,9 +81,20 @@ actions:
- |
backup_file_path="dump.sql"
kopia_snap='{{ .ArtifactsIn.mysqlBackup.KopiaSnapshot }}'
kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data

outputImage: mysql:8
outputCommand:
- bash
- -o
- errexit
- -o
- pipefail
- -c
- |
root_password="{{ index .Phases.restoreFromStore.Secrets.mysqlSecret.Data "mysql-root-password" | toString }}"
restore_cmd="mysql -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }}"
kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd}
cat /tmp/data | mysql -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }}

delete:
inputArtifactNames:
# The kopia snapshot info created in backup phase can be used here
Expand All @@ -71,7 +104,7 @@ actions:
- func: KubeTask
name: deleteFromStore
args:
image: ghcr.io/kanisterio/mysql-sidecar:0.112.0
image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}'
namespace: "{{ .Namespace.Name }}"
command:
- bash
Expand Down
Loading
Loading