test: add e2e tests

biome.json

@@ -1,11 +1,15 @@
 {
 	"$schema": "https://biomejs.dev/schemas/1.5.3/schema.json",
+	"vcs": {
+		"clientKind": "git",
+		"enabled": true,
+		"useIgnoreFile": true
+	},
 	"organizeImports": {
 		"enabled": true
 	},
 	"linter": {
 		"enabled": true,
-		"ignore": ["dist", "node_modules"],
 		"rules": {
 			"recommended": true,
 			"style": {

examples/base64/package.json

@@ -1,20 +1,20 @@
 {
-  "name": "middy-store-example-base64",
-  "private": true,
-  "type": "module",
-  "main": "handler.ts",
-  "scripts": {
-    "start": "tsx handler"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@middy/core": "^5.2.3",
-    "middy-store": "workspace:*",
-    "middy-store-base64": "workspace:*"
-  },
-  "devDependencies": {
-    "@types/aws-lambda": "^8.10.133"
-  }
-}
+	"name": "middy-store-example-base64",
+	"private": true,
+	"type": "module",
+	"main": "handler.ts",
+	"scripts": {
+		"start": "tsx handler"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"@middy/core": "^5.2.3",
+		"middy-store": "workspace:*",
+		"middy-store-base64": "workspace:*"
+	},
+	"devDependencies": {
+		"@types/aws-lambda": "^8.10.133"
+	}
+}

examples/base64/tsconfig.json

@@ -1,4 +1,4 @@
 {
-  "extends": "@tsconfig/node18/tsconfig.json",
-  "compilerOptions": {},
-}
+	"extends": "@tsconfig/node18/tsconfig.json",
+	"compilerOptions": {}
+}

examples/s3/index.ts

@@ -1,4 +1,4 @@
-import { randomBytes, randomUUID } from "crypto";
+import { randomBytes, randomUUID } from "node:crypto";
 import middy from "@middy/core";
 import { Handler } from "aws-lambda";
 import { middyStore } from "middy-store";

examples/s3/package.json

@@ -1,19 +1,19 @@
 {
-  "name": "middy-store-example-s3",
-  "version": "1.0.0",
-  "description": "",
-  "private": true,
-  "type": "module",
-  "main": "index.ts",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@middy/core": "^5.2.3",
-    "middy-store": "workspace:*",
-    "middy-store-s3": "workspace:*"
-  }
-}
+	"name": "middy-store-example-s3",
+	"version": "1.0.0",
+	"description": "",
+	"private": true,
+	"type": "module",
+	"main": "index.ts",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"@middy/core": "^5.2.3",
+		"middy-store": "workspace:*",
+		"middy-store-s3": "workspace:*"
+	}
+}

examples/s3/tsconfig.json

@@ -1,4 +1,4 @@
 {
-  "extends": "@tsconfig/node18/tsconfig.json",
-  "compilerOptions": {},
-}
+	"extends": "@tsconfig/node18/tsconfig.json",
+	"compilerOptions": {}
+}

examples/stepfunctions/functions/multiple.ts

@@ -1,6 +1,6 @@
-import { randomBytes, randomUUID } from "crypto";
+import { randomBytes, randomUUID } from "node:crypto";
 import middy from "@middy/core";
-import { Handler } from "aws-lambda";
+import type { Handler } from "aws-lambda";
 import { loadInput, storeOutput } from "middy-store";
 import { S3Store } from "middy-store-s3";
 

examples/stepfunctions/functions/single.ts

@@ -1,7 +1,7 @@
-import { randomBytes, randomUUID } from "crypto";
+import { randomBytes, randomUUID } from "node:crypto";
 import middy from "@middy/core";
-import { Handler } from "aws-lambda";
-import { loadInput, storeOutput } from "middy-store-";
+import type { Handler } from "aws-lambda";
+import { loadInput, storeOutput } from "middy-store";
 import { S3Store } from "middy-store-s3";
 
 const MAX_OUTPUT_SIZE_KB = 256; // https://docs.aws.amazon.com/step-functions/latest/dg/limits.html

examples/stepfunctions/package.json

@@ -1,27 +1,27 @@
 {
-  "name": "middy-store-example-stepfunctions",
-  "version": "1.0.0",
-  "description": "",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "deploy": "pnpm sls deploy"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@middy/core": "^5.2.3",
-    "middy-store": "workspace:*",
-    "middy-store-s3": "workspace:*"
-  },
-  "devDependencies": {
-    "esbuild": "^0.20.1",
-    "serverless": "^3.38.0",
-    "serverless-esbuild": "^1.51.0",
-    "serverless-step-functions": "^3.19.0",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.3.3"
-  }
-}
+	"name": "middy-store-example-stepfunctions",
+	"version": "1.0.0",
+	"description": "",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1",
+		"deploy": "pnpm sls deploy"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"@middy/core": "^5.2.3",
+		"middy-store": "workspace:*",
+		"middy-store-s3": "workspace:*"
+	},
+	"devDependencies": {
+		"esbuild": "^0.20.1",
+		"serverless": "^3.38.0",
+		"serverless-esbuild": "^1.51.0",
+		"serverless-step-functions": "^3.19.0",
+		"ts-node": "^10.9.2",
+		"typescript": "^5.3.3"
+	}
+}

examples/stepfunctions/tsconfig.json

@@ -1,4 +1,4 @@
 {
-  "extends": "@tsconfig/node18/tsconfig.json",
-  "compilerOptions": {},
-}
+	"extends": "@tsconfig/node18/tsconfig.json",
+	"compilerOptions": {}
+}

examples/stepfunctions_map/functions/single.ts

@@ -0,0 +1,116 @@
+import { randomBytes, randomUUID } from "node:crypto";
+import middy from "@middy/core";
+import type { Handler } from "aws-lambda";
+import { middyStore } from "middy-store";
+import { S3Store } from "middy-store-s3";
+
+const MAX_OUTPUT_SIZE_KB = 256; // https://docs.aws.amazon.com/step-functions/latest/dg/limits.html
+
+const store = new S3Store({
+	bucket: process.env.PAYLOAD_BUCKET!,
+	key: randomUUID(),
+});
+
+type Payload = {
+	content: string;
+};
+
+type GeneratePayloadInput = any;
+
+type GeneratePayloadOutput = {
+	sizeInKb: number;
+	payload: Payload;
+	iterator: Array<number>;
+};
+
+const SIZE = 100 * 1024 * 1024; // 100 MB
+const ITERATIONS = 50;
+
+const generatePayloads: Handler<
+	GeneratePayloadInput,
+	GeneratePayloadOutput
+> = async (input, context, callback) => {
+	console.log("Generating payload", { input });
+
+	// Generate a large payload
+	const content = randomBytes(SIZE).toString("base64");
+
+	const output = {
+		...input,
+		sizeInKb: Buffer.byteLength(content, "utf8") / 1024,
+		payload: {
+			content,
+		},
+		iterator: Array.from({ length: ITERATIONS }, (_, i) => i),
+	};
+
+	console.log(`Generated payload: ${output.sizeInKb} KB`);
+
+	return output;
+};
+
+export const generatePayloadsHandler = middy()
+	.use(
+		middyStore({
+			// logger: console.log,
+			stores: [
+				new S3Store({
+					// logger: console.log,
+					region: "us-east-1",
+					bucket: process.env.PAYLOAD_BUCKET!,
+					// key: makeEtlKey(),
+					format: {
+						type: "url",
+						format: "s3-region-path",
+					},
+				}),
+			],
+			read: true,
+			write: {
+				selector: "payload",
+			},
+		}),
+	)
+	.handler(generatePayloads);
+
+type MapPayloadIteratorInput = GeneratePayloadOutput & {
+	index: number;
+};
+const mapPayloadIterator: Handler<MapPayloadIteratorInput> = async (
+	input,
+	context,
+): Promise<undefined> => {
+	console.log(`Iterator index${input.index}`);
+
+	const payload = input.payload;
+
+	console.log(
+		`Actual payload size: ${Buffer.byteLength(payload.content, "utf8")} KB`,
+	);
+
+	return undefined;
+};
+
+export const mapPayloadIteratorHandler = middy()
+	.use(
+		middyStore({
+			// logger: console.log,
+			stores: [
+				new S3Store({
+					// logger: console.log,
+					region: "us-east-1",
+					bucket: process.env.PAYLOAD_BUCKET!,
+					// key: makeEtlKey(),
+					format: {
+						type: "url",
+						format: "s3-region-path",
+					},
+				}),
+			],
+			read: true,
+			write: {
+				selector: "payload",
+			},
+		}),
+	)
+	.handler(mapPayloadIterator);

examples/stepfunctions_map/package.json

@@ -0,0 +1,27 @@
+{
+	"name": "middy-store-example-stepfunctions",
+	"version": "1.0.0",
+	"description": "",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1",
+		"deploy": "pnpm sls deploy"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"@middy/core": "^5.2.3",
+		"middy-store": "workspace:*",
+		"middy-store-s3": "workspace:*"
+	},
+	"devDependencies": {
+		"esbuild": "^0.20.1",
+		"serverless": "^3.38.0",
+		"serverless-esbuild": "^1.51.0",
+		"serverless-step-functions": "^3.19.0",
+		"ts-node": "^10.9.2",
+		"typescript": "^5.3.3"
+	}
+}