[feat] add large-messages project

README.md

@@ -80,6 +80,7 @@ Table of Contents:
 | **[Kong API Gateway](projects/kong-api-gateway/README.md)** <br/> Deploying a Kong Gateway on containers to provide routing to functions. | CaaS & FaaS | Python   | [Serverless Framework] |
 | **[Serverless Gateway](https://github.com/scaleway/serverless-gateway)** <br/> Our serverless gateway for functions and containers.       | API Gateway | Python   | [Python API Framework] |
 | **[Monitoring Glaciers](projects/blogpost-glacier/README.md)** <br/> A project to monitor glaciers and the impact of global warming.      | S3 & RDB    | Golang   | [Serverless Framework] |
+| **[Manage large message](projects/large-messages/README.md)** <br/> An example of infrastructure to manage large messages.      | PaaS & S3    | Python   | [Terraform] |
 
 [Serverless Framework]: https://github.com/scaleway/serverless-scaleway-functions
 [Terraform]: https://registry.terraform.io/providers/scaleway/scaleway/latest/docs

projects/large-messages/README.md

@@ -0,0 +1,34 @@
+# Serverless large messages architecture
+
+This repository contains the source code for the this tutorial: [Create a serverless architecture that manage large messages, with Scaleway Messaging and Queuing NATS, Serverless Functions and Object Storage.](https://github.com/scaleway/docs-content/blob/int-add-mnq-tuto/tutorials/large-messages/index.mdx)
+
+## Requirements
+
+This example assumes that you are familiar with:
+
+- how messaging and queuing works. You can find out more in the [Scaleway official documentation](https://www.scaleway.com/en/docs/serverless/messaging/quickstart/)
+- how serverless functions work. If needed, you can check [this page](https://www.scaleway.com/en/docs/serverless/functions/quickstart/).
+- how S3 object storage works, and especially how to create a bucket and upload files within a bucket. Please refer to Scaleway's documentation [here](https://www.scaleway.com/en/docs/storage/object/quickstart/).
+
+## Context
+
+This example shows how to handle large messages using NATS, Object Storage and Serverless Functions. 
+
+## Setup
+
+Once you have cloned this repository you can run:
+
+```console
+terraform init
+terraform plan
+terraform apply
+```
+
+## Running
+
+To run the example, you can execute:
+```console
+./upload_img.sh path/to/your/image
+```
+Once the script has finished, you should see your image and a pdf with the same name in the bucket you've created.
+

projects/large-messages/function/handler/large_messages.py

@@ -0,0 +1,48 @@
+import os
+import boto3
+from botocore.exceptions import ClientError
+import img2pdf
+from PIL import Image
+
+endpoint_url = os.getenv("ENDPOINT_URL")
+bucket_region = os.getenv("BUCKET_REGION")
+bucket_name = os.getenv("BUCKET_NAME")
+access_key_id = os.getenv("ACCESS_KEY_ID")
+secret_access_key = os.getenv("SECRET_ACCESS_KEY")
+
+
+def convert_img_to_pdf(img_path, pdf_path):
+    image = Image.open(img_path)
+    pdf_bytes = img2pdf.convert(image.filename)
+    file = open(pdf_path, "wb")
+    file.write(pdf_bytes)
+    image.close()
+    file.close()
+    print("Successfully made pdf file")
+
+
+def handle(event, context):
+    input_file = event["body"]
+    output_file = os.path.splitext(input_file)[0] + ".pdf"
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=endpoint_url,
+        region_name=bucket_region,
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+    )
+
+    try:
+        s3.download_file(bucket_name, input_file, input_file)
+        print("Object " + input_file + " downloaded")
+        convert_img_to_pdf(input_file, output_file)
+        s3.upload_file(output_file, bucket_name, output_file)
+        print("Object " + input_file + " uploaded")
+    except ClientError as e:
+        print(e)
+        return {
+            "body": {"message": e.response["Error"]["Message"]},
+            "statusCode": e.response["Error"]["Code"],
+        }
+
+    return {"body": {"message": "Converted in PDF sucessfully"}, "statusCode": 200}

projects/large-messages/function/requirements.txt

@@ -0,0 +1,2 @@
+boto3
+img2pdf

projects/large-messages/main.tf

@@ -0,0 +1,136 @@
+terraform {
+  required_providers {
+    scaleway = {
+      source = "scaleway/scaleway"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+    archive = {
+      source = "hashicorp/archive"
+    }
+  }
+  required_version = ">= 0.13"
+}
+
+variable "scw_access_key_id" {
+  type      = string
+  sensitive = true
+}
+
+variable "scw_secret_access_key" {
+  type      = string
+  sensitive = true
+}
+
+provider "scaleway" {
+  zone = "fr-par-1"
+}
+
+// Object Bucket
+
+resource "random_id" "bucket" {
+  byte_length = 8
+}
+
+resource "scaleway_object_bucket" "large_messages" {
+  name = "large-messages-${random_id.bucket.hex}"
+}
+
+resource "scaleway_object_bucket_acl" "large_messages" {
+  bucket = scaleway_object_bucket.large_messages.id
+  acl    = "private"
+}
+
+output "bucket_name" {
+  value       = scaleway_object_bucket.large_messages.name
+  description = "Bucket name to use with the producer script"
+}
+
+// MNQ Nats
+
+resource "scaleway_mnq_nats_account" "large_messages" {
+  name = "nats-acc-large-messages"
+}
+
+resource "scaleway_mnq_nats_credentials" "large_messages" {
+  name       = "nats-large-messages-creds"
+  account_id = scaleway_mnq_nats_account.large_messages.id
+}
+
+resource "local_file" "nats_credential" {
+  content         = scaleway_mnq_nats_credentials.large_messages.file
+  filename        = "large-messages.creds"
+  file_permission = 644
+}
+
+output "nats_url" {
+  value       = scaleway_mnq_nats_account.large_messages.endpoint
+  description = "NATS url to use with the producer script"
+}
+
+// Function
+
+resource "null_resource" "install_dependencies" {
+  provisioner "local-exec" {
+    command = <<-EOT
+      cd function
+      [ -d "./function/package" ] && rm -rf ./package
+      PYTHON_VERSION=3.11
+      docker run --rm -v $(pwd):/home/app/function --workdir /home/app/function rg.fr-par.scw.cloud/scwfunctionsruntimes-public/python-dep:$PYTHON_VERSION \
+        pip3 install --upgrade -r requirements.txt --no-cache-dir --target ./package
+      cd ..
+    EOT
+  }
+
+  triggers = {
+    hash = filesha256("./function/handler/large_messages.py")
+  }
+}
+
+data "archive_file" "function_zip" {
+  type        = "zip"
+  source_dir  = "./function"
+  output_path = "./function.zip"
+
+  depends_on = [null_resource.install_dependencies]
+}
+
+resource "scaleway_function_namespace" "large_messages" {
+  name        = "large-messages-function"
+  description = "Large messages namespace"
+}
+
+resource "scaleway_function" "large_messages" {
+  namespace_id = scaleway_function_namespace.large_messages.id
+  runtime      = "python311"
+  handler      = "handler/large_messages.handle"
+  privacy      = "private"
+  zip_file     = "function.zip"
+  zip_hash     = data.archive_file.function_zip.output_sha256
+  deploy       = true
+  memory_limit = "2048"
+  environment_variables = {
+    ENDPOINT_URL  = scaleway_object_bucket.large_messages.api_endpoint
+    BUCKET_REGION = scaleway_object_bucket.large_messages.region
+    BUCKET_NAME   = scaleway_object_bucket.large_messages.name
+  }
+  secret_environment_variables = {
+    ACCESS_KEY_ID     = var.scw_access_key_id
+    SECRET_ACCESS_KEY = var.scw_secret_access_key
+  }
+
+  depends_on = [data.archive_file.function_zip]
+}
+
+resource "scaleway_function_trigger" "large_messages" {
+  function_id = scaleway_function.large_messages.id
+  name        = "large-messages-trigger"
+  nats {
+    account_id = scaleway_mnq_nats_account.large_messages.id
+    subject    = "large-messages"
+  }
+}

projects/large-messages/upload_img.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+SCW_BUCKET=$(terraform output bucket_name)
+SCW_NATS_URL=$(terraform output nats_url)
+
+# Nats context creation and selection
+nats context save large-messages --server=$SCW_NATS_URL --creds=./large-messages.creds
+nats context select large-messages
+
+# Upload file to S3
+aws s3 cp $1 s3://$SCW_BUCKET
+
+# Send the name of the file in NATS
+nats pub large-messages $(basename $1)