PIPE-155 Add pipelines prefix option (#7510)

cloudflare · Dec 12, 2024 · 004af53 · 004af53
1 parent 3bc0f28
commit 004af53
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 12 deletions.
diff --git a/.changeset/clean-peas-exist.md b/.changeset/clean-peas-exist.md
@@ -0,0 +1,5 @@
+---
+"wrangler": patch
+---
+
+Add file prefix option to wrangler pipelines commands
diff --git a/packages/wrangler/src/__tests__/pipelines.test.ts b/packages/wrangler/src/__tests__/pipelines.test.ts
@@ -16,24 +16,29 @@ describe("pipelines", () => {
 	runInTempDir();
 
 	const samplePipeline = {
-		currentVersion: 1,
 		id: "0001",
+		version: 1,
 		name: "my-pipeline",
 		metadata: {},
 		source: [
 			{
 				type: "binding",
 				format: "json",
 			},
+			{
+				type: "http",
+				format: "json",
+				authentication: false,
+			},
 		],
 		transforms: [],
 		destination: {
-			type: "json",
+			type: "r2",
+			format: "json",
 			batch: {},
 			compression: {
 				type: "none",
 			},
-			format: "json",
 			path: {
 				bucket: "bucket",
 			},
@@ -365,10 +370,12 @@ describe("pipelines", () => {
 				                           Default: No transformation worker  [string]
 				      --compression        Sets the compression format of output files
 				                           Default: gzip  [string] [choices: \\"none\\", \\"gzip\\", \\"deflate\\"]
-				      --filepath           The path to store files in the destination bucket
+				      --prefix             Optional base path to store files in the destination bucket
+				                           Default: (none)  [string]
+				      --filepath           The path to store partitioned files in the destination bucket
 				                           Default: event_date=\${date}/hr=\${hr}  [string]
-				      --filename           The name of the file in the bucket. Must contain \\"\${slug}\\". File extension is optional
-				                           Default: \${slug}-\${hr}.json  [string]
+				      --filename           The name of each unique file in the bucket. Must contain \\"\${slug}\\". File extension is optional
+				                           Default: \${slug}\${extension}  [string]
 				      --binding            Enable Worker binding to this pipeline  [boolean] [default: true]
 				      --http               Enable HTTPS endpoint to send data to this pipeline  [boolean] [default: true]
 				      --authentication     Require authentication (Cloudflare API Token) to send data to the HTTPS endpoint  [boolean] [default: false]
@@ -465,24 +472,29 @@ describe("pipelines", () => {
 			expect(std.out).toMatchInlineSnapshot(`
 				"Retrieving config for pipeline \\"foo\\".
 				{
-				  \\"currentVersion\\": 1,
 				  \\"id\\": \\"0001\\",
+				  \\"version\\": 1,
 				  \\"name\\": \\"my-pipeline\\",
 				  \\"metadata\\": {},
 				  \\"source\\": [
 				    {
 				      \\"type\\": \\"binding\\",
 				      \\"format\\": \\"json\\"
+				    },
+				    {
+				      \\"type\\": \\"http\\",
+				      \\"format\\": \\"json\\",
+				      \\"authentication\\": false
 				    }
 				  ],
 				  \\"transforms\\": [],
 				  \\"destination\\": {
-				    \\"type\\": \\"json\\",
+				    \\"type\\": \\"r2\\",
+				    \\"format\\": \\"json\\",
 				    \\"batch\\": {},
 				    \\"compression\\": {
 				      \\"type\\": \\"none\\"
 				    },
-				    \\"format\\": \\"json\\",
 				    \\"path\\": {
 				      \\"bucket\\": \\"bucket\\"
 				    }

diff --git a/packages/wrangler/src/pipelines/client.ts b/packages/wrangler/src/pipelines/client.ts
@@ -44,6 +44,7 @@ export type PipelineUserConfig = {
 		};
 		path: {
 			bucket: string;
+			prefix?: string;
 			filepath?: string;
 			filename?: string;
 		};
@@ -58,7 +59,7 @@ export type PipelineUserConfig = {
 // Pipeline from v4 API
 export type Pipeline = Omit<PipelineUserConfig, "destination"> & {
 	id: string;
-	currentVersion: number;
+	version: number;
 	endpoint: string;
 	destination: Omit<PipelineUserConfig["destination"], "credentials"> & {
 		credentials?: PipelineUserConfig["destination"]["credentials"];

diff --git a/packages/wrangler/src/pipelines/index.ts b/packages/wrangler/src/pipelines/index.ts
@@ -129,15 +129,21 @@ function addCreateAndUpdateOptions(yargs: Argv<CommonYargsOptions>) {
 			choices: ["none", "gzip", "deflate"],
 			demandOption: false,
 		})
+		.option("prefix", {
+			describe:
+				"Optional base path to store files in the destination bucket \nDefault: (none)",
+			type: "string",
+			demandOption: false,
+		})
 		.option("filepath", {
 			describe:
-				"The path to store files in the destination bucket \nDefault: event_date=${date}/hr=${hr}",
+				"The path to store partitioned files in the destination bucket \nDefault: event_date=${date}/hr=${hr}",
 			type: "string",
 			demandOption: false,
 		})
 		.option("filename", {
 			describe:
-				'The name of the file in the bucket. Must contain "${slug}". File extension is optional \nDefault: ${slug}-${hr}.json',
+				'The name of each unique file in the bucket. Must contain "${slug}". File extension is optional \nDefault: ${slug}${extension}',
 			type: "string",
 			demandOption: false,
 		})
@@ -275,6 +281,9 @@ export function pipelines(pipelineYargs: CommonYargsArgv) {
 					pipelineConfig.transforms.push(parseTransform(args.transform));
 				}
 
+				if (args.prefix) {
+					pipelineConfig.destination.path.prefix = args.prefix;
+				}
 				if (args.filepath) {
 					pipelineConfig.destination.path.filepath = args.filepath;
 				}
@@ -464,6 +473,9 @@ export function pipelines(pipelineYargs: CommonYargsArgv) {
 					pipelineConfig.transforms.push(parseTransform(args.transform));
 				}
 
+				if (args.prefix) {
+					pipelineConfig.destination.path.prefix = args.prefix;
+				}
 				if (args.filepath) {
 					pipelineConfig.destination.path.filepath = args.filepath;
 				}