From 004af53928ba96060c0d644fc8a98e7a3a5e6957 Mon Sep 17 00:00:00 2001 From: Oliver Yu <387030+oliy@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:40:38 -0600 Subject: [PATCH] PIPE-155 Add pipelines prefix option (#7510) --- .changeset/clean-peas-exist.md | 5 ++++ .../wrangler/src/__tests__/pipelines.test.ts | 30 +++++++++++++------ packages/wrangler/src/pipelines/client.ts | 3 +- packages/wrangler/src/pipelines/index.ts | 16 ++++++++-- 4 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 .changeset/clean-peas-exist.md diff --git a/.changeset/clean-peas-exist.md b/.changeset/clean-peas-exist.md new file mode 100644 index 000000000000..675dce1b772a --- /dev/null +++ b/.changeset/clean-peas-exist.md @@ -0,0 +1,5 @@ +--- +"wrangler": patch +--- + +Add file prefix option to wrangler pipelines commands diff --git a/packages/wrangler/src/__tests__/pipelines.test.ts b/packages/wrangler/src/__tests__/pipelines.test.ts index 324850cdf288..7b62ee44ca45 100644 --- a/packages/wrangler/src/__tests__/pipelines.test.ts +++ b/packages/wrangler/src/__tests__/pipelines.test.ts @@ -16,8 +16,8 @@ describe("pipelines", () => { runInTempDir(); const samplePipeline = { - currentVersion: 1, id: "0001", + version: 1, name: "my-pipeline", metadata: {}, source: [ @@ -25,15 +25,20 @@ describe("pipelines", () => { type: "binding", format: "json", }, + { + type: "http", + format: "json", + authentication: false, + }, ], transforms: [], destination: { - type: "json", + type: "r2", + format: "json", batch: {}, compression: { type: "none", }, - format: "json", path: { bucket: "bucket", }, @@ -365,10 +370,12 @@ describe("pipelines", () => { Default: No transformation worker [string] --compression Sets the compression format of output files Default: gzip [string] [choices: \\"none\\", \\"gzip\\", \\"deflate\\"] - --filepath The path to store files in the destination bucket + --prefix Optional base path to store files in the destination bucket + Default: (none) [string] + --filepath The path to store partitioned files in the destination bucket Default: event_date=\${date}/hr=\${hr} [string] - --filename The name of the file in the bucket. Must contain \\"\${slug}\\". File extension is optional - Default: \${slug}-\${hr}.json [string] + --filename The name of each unique file in the bucket. Must contain \\"\${slug}\\". File extension is optional + Default: \${slug}\${extension} [string] --binding Enable Worker binding to this pipeline [boolean] [default: true] --http Enable HTTPS endpoint to send data to this pipeline [boolean] [default: true] --authentication Require authentication (Cloudflare API Token) to send data to the HTTPS endpoint [boolean] [default: false] @@ -465,24 +472,29 @@ describe("pipelines", () => { expect(std.out).toMatchInlineSnapshot(` "Retrieving config for pipeline \\"foo\\". { - \\"currentVersion\\": 1, \\"id\\": \\"0001\\", + \\"version\\": 1, \\"name\\": \\"my-pipeline\\", \\"metadata\\": {}, \\"source\\": [ { \\"type\\": \\"binding\\", \\"format\\": \\"json\\" + }, + { + \\"type\\": \\"http\\", + \\"format\\": \\"json\\", + \\"authentication\\": false } ], \\"transforms\\": [], \\"destination\\": { - \\"type\\": \\"json\\", + \\"type\\": \\"r2\\", + \\"format\\": \\"json\\", \\"batch\\": {}, \\"compression\\": { \\"type\\": \\"none\\" }, - \\"format\\": \\"json\\", \\"path\\": { \\"bucket\\": \\"bucket\\" } diff --git a/packages/wrangler/src/pipelines/client.ts b/packages/wrangler/src/pipelines/client.ts index 8c391cd38e9a..e6a728abc71b 100644 --- a/packages/wrangler/src/pipelines/client.ts +++ b/packages/wrangler/src/pipelines/client.ts @@ -44,6 +44,7 @@ export type PipelineUserConfig = { }; path: { bucket: string; + prefix?: string; filepath?: string; filename?: string; }; @@ -58,7 +59,7 @@ export type PipelineUserConfig = { // Pipeline from v4 API export type Pipeline = Omit & { id: string; - currentVersion: number; + version: number; endpoint: string; destination: Omit & { credentials?: PipelineUserConfig["destination"]["credentials"]; diff --git a/packages/wrangler/src/pipelines/index.ts b/packages/wrangler/src/pipelines/index.ts index b1cf9a221134..c7ba0e353568 100644 --- a/packages/wrangler/src/pipelines/index.ts +++ b/packages/wrangler/src/pipelines/index.ts @@ -129,15 +129,21 @@ function addCreateAndUpdateOptions(yargs: Argv) { choices: ["none", "gzip", "deflate"], demandOption: false, }) + .option("prefix", { + describe: + "Optional base path to store files in the destination bucket \nDefault: (none)", + type: "string", + demandOption: false, + }) .option("filepath", { describe: - "The path to store files in the destination bucket \nDefault: event_date=${date}/hr=${hr}", + "The path to store partitioned files in the destination bucket \nDefault: event_date=${date}/hr=${hr}", type: "string", demandOption: false, }) .option("filename", { describe: - 'The name of the file in the bucket. Must contain "${slug}". File extension is optional \nDefault: ${slug}-${hr}.json', + 'The name of each unique file in the bucket. Must contain "${slug}". File extension is optional \nDefault: ${slug}${extension}', type: "string", demandOption: false, }) @@ -275,6 +281,9 @@ export function pipelines(pipelineYargs: CommonYargsArgv) { pipelineConfig.transforms.push(parseTransform(args.transform)); } + if (args.prefix) { + pipelineConfig.destination.path.prefix = args.prefix; + } if (args.filepath) { pipelineConfig.destination.path.filepath = args.filepath; } @@ -464,6 +473,9 @@ export function pipelines(pipelineYargs: CommonYargsArgv) { pipelineConfig.transforms.push(parseTransform(args.transform)); } + if (args.prefix) { + pipelineConfig.destination.path.prefix = args.prefix; + } if (args.filepath) { pipelineConfig.destination.path.filepath = args.filepath; }