diff --git a/README.md b/README.md index 0ca0477..77f48dc 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Table of Contents: | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------- | | **[Serverless Jobs Hello World](jobs/terraform-hello-world/README.md)**
An example of building a container image and running it as a Serverless Job using Terraform. | N/A | [Terraform]-[Console] | | **[Serverless MLOps](jobs/ml-ops/README.md)**
An example of running a Serverless Machine Leaning workflow. | Python | [Terraform]-[Console]-[CLI] | +| **[Auto Snapshot Instances](jobs/instances-snapshot/README.md)**
Use Serverless Jobs to create snapshots of your instances | Go | [Console] | ### 💬 Messaging and Queueing diff --git a/jobs/instances-snapshot/Dockerfile b/jobs/instances-snapshot/Dockerfile new file mode 100644 index 0000000..4df584d --- /dev/null +++ b/jobs/instances-snapshot/Dockerfile @@ -0,0 +1,16 @@ +# Using apline/golang image +FROM golang:1.22-alpine + +# Set destination for COPY +WORKDIR /app + +# Copy required files +COPY go.mod ./ +COPY go.sum ./ +COPY *.go ./ + +# Build the executable +RUN go build -o /jobs-snapshot + +# Run the executable +CMD [ "/jobs-snapshot" ] diff --git a/jobs/instances-snapshot/README.md b/jobs/instances-snapshot/README.md new file mode 100644 index 0000000..cc645a3 --- /dev/null +++ b/jobs/instances-snapshot/README.md @@ -0,0 +1,72 @@ +# Serverless Jobs for automatic Instance snapshot + +This project shows how it's possible to automate tasks using Serverless Jobs. + +This example is very simple, it generates snapshots of your desired Instance. + +# Set-up + +## Requirements + +- Scaleway Account +- Docker daemon running to build the image +- Container registry namespace created, for this example we assume that your namespace name is `jobs-snapshot`: [doc here](https://www.scaleway.com/en/docs/containers/container-registry/how-to/create-namespace/) + +## Step 1 : Build and push to Container registry + +Serverless Jobs, like Serverless Containers (which are suited for HTTP applications), works +with Containers. So first, using your terminal reach this folder and run the following commands: + +```shell +# First command is to login to container registry, you can find it in Scaleway console +docker login rg.fr-par.scw.cloud/jobs-snapshot -u nologin --password-stdin <<< "$SCW_SECRET_KEY" + +# Here we build the image to push +docker build -t rg.fr-par.scw.cloud/jobs-snapshot/jobs-snapshot:v1 . + +# Push the image online to be used on Serverless Jobs +docker push rg.fr-par.scw.cloud/jobs-snapshot/jobs-snapshot:v1 +``` + +Note about Serverless Containers versus Serverless Jobs. As we do not expose a web server and we do not +require features such as auto-scaling, Serverless Jobs are perfect for this use case. + +To check if everyting is ok, on the Console you can verify if your tag is present in Container Registry. + +## Step 2: Creating the Job Definition + +On Scaleway Console on the following link you can create a new Job Definition: https://console.scaleway.com/serverless-jobs/jobs/create?region=fr-par + +1. On Container image, select the image you created in the step before. +1. You can set the image name to something clear like `jobs-snapshot` too. +1. For the region you can select the one you prefer :) +1. Regarding the resources you can keep the default values, this job is fast and do not require specific compute power or memory. +1. To schedule your job for example every night at 2am, you can set the cron to `0 2 * * *`. +1. Important: advanced option, you need to set the following environment variables: + +- `INSTANCE_ID`: grab the instance ID you want to create snapshots from +- `INSTANCE_ZONE`: you need to give the ZONE of you instance, like `fr-par-2` +- `SCW_ACCESS_KEY`: your access key +- `SCW_SECRET_KEY`: your secret key +- `SCW_DEFAULT_ORGANIZATION_ID`: your organzation ID + +1. Click "create job" + +## Step 3: Run the job + +On your created Job Definition, just click the button "Run Job" and within seconds it should be successful. + +# Possible improvements + +You can exercice by adding the following features: + +- Instead of managing a single instance, make it account wide +- Add disk backups +- Add alerts if something goes wrong + +# Additional content + +- [Jobs Documentation](https://www.scaleway.com/en/docs/serverless/jobs/how-to/create-job-from-scaleway-registry/) +- [Other methods to deploy Jobs](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/deploy-job/) +- [Secret key / access key doc](https://www.scaleway.com/en/docs/identity-and-access-management/iam/how-to/create-api-keys/) +- [CRON schedule help](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/cron-schedules/) diff --git a/jobs/instances-snapshot/go.mod b/jobs/instances-snapshot/go.mod new file mode 100644 index 0000000..fcee6c0 --- /dev/null +++ b/jobs/instances-snapshot/go.mod @@ -0,0 +1,7 @@ +module github.com/scaleway/serverless-examples/jobs/instances-snapshot + +go 1.22.2 + +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 + +require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/jobs/instances-snapshot/go.sum b/jobs/instances-snapshot/go.sum new file mode 100644 index 0000000..b5ff887 --- /dev/null +++ b/jobs/instances-snapshot/go.sum @@ -0,0 +1,8 @@ +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/jobs/instances-snapshot/main.go b/jobs/instances-snapshot/main.go new file mode 100644 index 0000000..20bacbb --- /dev/null +++ b/jobs/instances-snapshot/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "fmt" + "os" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +func main() { + fmt.Println("creating snapshot of instance...") + + // Create a Scaleway client with credentials from environment variables. + client, err := scw.NewClient( + // Get your organization ID at https://console.scaleway.com/organization/settings + scw.WithDefaultOrganizationID(os.Getenv("SCW_DEFAULT_ORGANIZATION_ID")), + + // Get your credentials at https://console.scaleway.com/iam/api-keys + scw.WithAuth(os.Getenv("SCW_ACCESS_KEY"), os.Getenv("SCW_SECRET_KEY")), + + // Get more about our availability zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ + scw.WithDefaultRegion(scw.RegionFrPar), + ) + if err != nil { + panic(err) + } + + // Create SDK objects for Scaleway Instance product + instanceAPI := instance.NewAPI(client) + + if err := createSnapshots(instanceAPI); err != nil { + panic(err) + } +} + +func createSnapshots(instanceAPI *instance.API) error { + gotInstance, err := instanceAPI.GetServer(&instance.GetServerRequest{ + ServerID: os.Getenv("INSTANCE_ID"), + Zone: scw.Zone(os.Getenv("INSTANCE_ZONE")), + }) + if err != nil { + return fmt.Errorf("error while getting instance %w", err) + } + + for _, volume := range gotInstance.Server.Volumes { + snapshotResp, err := instanceAPI.CreateSnapshot(&instance.CreateSnapshotRequest{ + Name: volume.Name + RandomString(4), + VolumeID: &volume.ID, + VolumeType: instance.SnapshotVolumeTypeBSSD, + Zone: scw.Zone(os.Getenv("INSTANCE_ZONE")), + }) + if err != nil { + return fmt.Errorf("error while creating snapshopt %w", err) + } + fmt.Println("created snapshot ", snapshotResp.Snapshot.ID) + } + + return nil +} + +func init() { + if os.Getenv("SCW_DEFAULT_ORGANIZATION_ID") == "" { + panic("missing SCW_DEFAULT_ORGANIZATION_ID") + } + + if os.Getenv("SCW_ACCESS_KEY") == "" { + panic("missing SCW_ACCESS_KEY") + } + + if os.Getenv("SCW_SECRET_KEY") == "" { + panic("missing SCW_SECRET_KEY") + } + + if os.Getenv("INSTANCE_ID") == "" { + panic("missing INSTANCE_ID") + } + + if os.Getenv("INSTANCE_ZONE") == "" { + panic("missing INSTANCE_ZONE") + } +} diff --git a/jobs/instances-snapshot/string.go b/jobs/instances-snapshot/string.go new file mode 100644 index 0000000..ec19790 --- /dev/null +++ b/jobs/instances-snapshot/string.go @@ -0,0 +1,21 @@ +package main + +import "crypto/rand" + +// RandomString is used to generate a random string containing upper and lower characters +// + number, of size n. +func RandomString(n int) string { + const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + bytes := make([]byte, n) + + if _, err := rand.Read(bytes); err != nil { + panic(err) + } + + for i, b := range bytes { + bytes[i] = letters[b%byte(len(letters))] + } + + return string(bytes) +}