From 3ab16487ae1a4f2e5e54667704d17eb9c10f53bb Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Tue, 29 Oct 2024 14:51:21 +0100 Subject: [PATCH 1/2] BB-616 Provide commands for lifecycle listing --- bin/backbeat-cli.js | 168 ++++++++++++++++++++++++++++++++++++ docs/backbeat-cli.md | 200 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100755 bin/backbeat-cli.js create mode 100644 docs/backbeat-cli.md diff --git a/bin/backbeat-cli.js b/bin/backbeat-cli.js new file mode 100755 index 000000000..e51b11790 --- /dev/null +++ b/bin/backbeat-cli.js @@ -0,0 +1,168 @@ +#!/usr/bin/env node + +const program = require('commander'); +const BackbeatClient = require('../lib/clients/BackbeatClient.js'); + +const pkg = require('../package.json'); +const werelogs = require('werelogs'); + +const config = require('../conf/Config'); + +const log = new werelogs.Logger('Backbeat:CLI'); + +werelogs.configure({ + level: config.log.logLevel, + dump: config.log.dumpLevel, +}); + +function createClient() { + return new BackbeatClient({ + endpoint: process.env.BACKBEAT_ENDPOINT || 'http://127.0.0.1:8000', + region: process.env.AWS_REGION || 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID || 'accessKey1', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || 'verySecretKey1', + }); +} + +program + .version(pkg.version) + .description('CLI for interacting with the Backbeat service'); + +program + .command('list-lifecycle-currents') + .description('List current lifecycle objects in a bucket') + .option('-b, --bucket ', 'Name of the bucket') + .option('-d, --before-date ', 'Limit to keys modified before this date (YYYY-MM-DD)') + .option('-e, --excluded-data-store-name ', 'Exclude specific data store name') + .option('-enc, --encoding-type ', 'Encoding type (e.g., url)', 'url') + .option('-m, --marker ', 'Marker for pagination') + .option('-k, --max-keys ', 'Maximum number of keys to return', parseInt) + .option('-p, --prefix ', 'Filter keys by prefix') + .action(options => { + if (!options.bucket) { + log.error('Error: --bucket option is required.', { options }); + process.exit(1); + } + + const client = createClient(); + + const params = { + Bucket: options.bucket, + BeforeDate: options.beforeDate, + ExcludedDataStoreName: options.excludedDataStoreName, + EncodingType: options.encodingType, + Marker: options.marker, + MaxKeys: options.maxKeys, + Prefix: options.prefix, + }; + + // Remove undefined parameters + Object.keys(params).forEach( + key => params[key] === undefined && delete params[key] + ); + + client.listLifecycleCurrents(params, (err, data) => { + if (err) { + log.error('Error listing lifecycle currents', { error: err }); + process.exit(1); + } else { + console.log(data); + } + }); + }); + +program + .command('list-lifecycle-noncurrents') + .description('List non-current lifecycle objects in a bucket') + .option('-b, --bucket ', 'Name of the bucket') + .option('-d, --before-date ', 'Limit to keys modified before this date (YYYY-MM-DD)') + .option('-e, --excluded-data-store-name ', 'Exclude specific data store name') + .option('-enc, --encoding-type ', 'Encoding type (e.g., url)', 'url') + .option('-km, --key-marker ', 'Key marker for pagination') + .option('-vm, --version-id-marker ', 'Version ID marker for pagination') + .option('-k, --max-keys ', 'Maximum number of keys to return', parseInt) + .option('-p, --prefix ', 'Filter keys by prefix') + .action(options => { + if (!options.bucket) { + log.error('Error: --bucket option is required.', { options }); + process.exit(1); + } + + const client = createClient(); + + const params = { + Bucket: options.bucket, + BeforeDate: options.beforeDate, + ExcludedDataStoreName: options.excludedDataStoreName, + EncodingType: options.encodingType, + KeyMarker: options.keyMarker, + VersionIdMarker: options.versionIdMarker, + MaxKeys: options.maxKeys, + Prefix: options.prefix, + }; + + // Remove undefined parameters + Object.keys(params).forEach( + key => params[key] === undefined && delete params[key] + ); + + client.listLifecycleNonCurrents(params, (err, data) => { + if (err) { + log.error('Error listing lifecycle non currents', { error: err }); + process.exit(1); + } else { + console.log(data); + } + }); + }); + +program + .command('list-lifecycle-orphans') + .description('List orphan lifecycle objects in a bucket') + .option('-b, --bucket ', 'Name of the bucket') + .option('-d, --before-date ', 'Limit to keys modified before this date (YYYY-MM-DD)') + .option('-e, --excluded-data-store-name ', 'Exclude specific data store name') + .option('-enc, --encoding-type ', 'Encoding type (e.g., url)', 'url') + .option('-m, --marker ', 'Marker for pagination') + .option('-k, --max-keys ', 'Maximum number of keys to return', parseInt) + .option('-p, --prefix ', 'Filter keys by prefix') + .action(options => { + if (!options.bucket) { + log.error('Error: --bucket option is required.', { options }); + process.exit(1); + } + + const client = createClient(); + + const params = { + Bucket: options.bucket, + BeforeDate: options.beforeDate, + ExcludedDataStoreName: options.excludedDataStoreName, + EncodingType: options.encodingType, + Marker: options.marker, + MaxKeys: options.maxKeys, + Prefix: options.prefix, + }; + + // Remove undefined parameters + Object.keys(params).forEach( + key => params[key] === undefined && delete params[key] + ); + + client.listLifecycleOrphans(params, (err, data) => { + if (err) { + log.error('Error listing lifecycle orphans', { error: err }); + process.exit(1); + } else { + console.log(data); + } + }); + }); + +// Parse the command-line arguments +program.parse(process.argv); + +// If no command is provided, display help +if (!process.argv.slice(2).length) { + program.outputHelp(); +} diff --git a/docs/backbeat-cli.md b/docs/backbeat-cli.md new file mode 100644 index 000000000..eade1e1ce --- /dev/null +++ b/docs/backbeat-cli.md @@ -0,0 +1,200 @@ +# Backbeat CLI documentation + +Welcome to the **Backbeat CLI** documentation. This guide provides detailed instructions on installing, configuring, and using the Backbeat Command Line Interface (CLI) to interact with the Backbeat service. + +## Table of Contents + +- [Introduction](#introduction) +- [Installation](#installation) +- [Configuration](#configuration) +- [Usage](#usage) + - [Available commands](#available-commands) + - [list-lifecycle-currents](#list-lifecycle-currents) + - [list-lifecycle-noncurrents](#list-lifecycle-noncurrents) + - [list-lifecycle-orphans](#list-lifecycle-orphans) +- [Examples](#examples) +- [Help](#help) + + +## Introduction + +The **Backbeat CLI** is a command-line tool designed to facilitate interactions with the Backbeat service. It allows users to list lifecycle objects within buckets, providing functionalities to manage and monitor lifecycle states effectively. + +## Installation + +To install and set up the Backbeat CLI, follow these steps: + +1. **Clone the repository** + +```bash +git clone http://github.com/scality/backbeat +``` + +1. ** TODO: REMOVE BEFORE MERGING** + +```bash +git checkout improvement/BB-616/bb-cli +``` + +2. **Go to the project directory** + +```bash +cd backbeat +``` + +3. **Install dependencies** + +Ensure you have Node.js installed (version 16 or higher is recommended). + +```bash +npm install +``` + +4. **Create an alias for easy access** + +```bash +alias backbeat-cli='node bin/backbeat.cli.js' +``` + +## Configuration + +Before using the Backbeat CLI, configure the necessary environment variables to establish a connection with the Backbeat service. + +### Environment variables + +- `BACKBEAT_ENDPOINT`: The endpoint URL for the Backbeat service. + *Default*: `http://127.0.0.1:8000` + +- `AWS_REGION`: The AWS region to use. + *Default*: `us-east-1` + +- `AWS_ACCESS_KEY_ID`: Your AWS access key ID. + *Default*: `accessKey1` + +- `AWS_SECRET_ACCESS_KEY`: Your AWS secret access key. + *Default*: `verySecretKey1` + +You can set these variables in your shell or include them in a configuration file as needed. + +```bash +export BACKBEAT_ENDPOINT='http://your-backbeat-endpoint.com' +export AWS_REGION='your-region' +export AWS_ACCESS_KEY_ID='your-access-key-id' +export AWS_SECRET_ACCESS_KEY='your-secret-access-key' +``` + +## Usage + +After installation and configuration, you can use the Backbeat CLI to execute various commands. The general syntax is: + +```bash +backbeat-cli [options] +``` + +### Available commands + +#### `list-lifecycle-currents` + +**Description**: List current lifecycle objects in a specified bucket. + +**Options**: + +- `-b, --bucket `: **(Required)** Name of the bucket. +- `-d, --before-date `: Limit to keys modified before this date (`YYYY-MM-DD`). +- `-e, --excluded-data-store-name `: Exclude specific data store name. +- `-enc, --encoding-type `: Encoding type (e.g., `url`). + *Default*: `url` +- `-m, --marker `: Marker for pagination. +- `-k, --max-keys `: Maximum number of keys to return. +- `-p, --prefix `: Filter keys by prefix. + +**Usage example**: + +```bash +backbeat-cli list-lifecycle-currents --bucket my-bucket --before-date 2024-10-26 +``` + +#### `list-lifecycle-noncurrents` + +**Description**: List non-current lifecycle objects in a specified bucket. + +**Options**: + +- `-b, --bucket `: **(Required)** Name of the bucket. +- `-d, --before-date `: Limit to keys modified before this date (`YYYY-MM-DD`). +- `-e, --excluded-data-store-name `: Exclude specific data store name. +- `-enc, --encoding-type `: Encoding type (e.g., `url`). + *Default*: `url` +- `-km, --key-marker `: Key marker for pagination. +- `-vm, --version-id-marker `: Version ID marker for pagination. +- `-k, --max-keys `: Maximum number of keys to return. +- `-p, --prefix `: Filter keys by prefix. + +**Usage example**: + +```bash +backbeat-cli list-lifecycle-noncurrents --bucket my-bucket --prefix logs/ +``` + +#### `list-lifecycle-orphans` + +**Description**: List orphan lifecycle objects in a specified bucket. + +**Options**: + +- `-b, --bucket `: **(Required)** Name of the bucket. +- `-d, --before-date `: Limit to keys modified before this date (`YYYY-MM-DD`). +- `-e, --excluded-data-store-name `: Exclude specific data store name. +- `-enc, --encoding-type `: Encoding type (e.g., `url`). + *Default*: `url` +- `-m, --marker `: Marker for pagination. +- `-k, --max-keys `: Maximum number of keys to return. +- `-p, --prefix `: Filter keys by prefix. + +**Usage example**: + +```bash +backbeat-cli list-lifecycle-orphans --bucket my-bucket --max-keys 50 +``` + +## Examples + +### Example 1: List current lifecycle objects + +Lists current lifecycle objects in the source bucket that were modified before October 26, 2024: + +```bash +backbeat-cli list-lifecycle-currents --bucket source --before-date 2024-10-26 +``` + +### Example 2: List non-current lifecycle objects with prefix + +Lists non-current lifecycle objects in my-bucket with keys starting with images/: + +```bash +backbeat-cli list-lifecycle-noncurrents --bucket my-bucket --prefix images/ +``` + +### Example 3: List orphan lifecycle objects with pagination + +Lists orphan lifecycle objects in the archive bucket, starting from marker abc123, limiting the output to 100 keys: + +```bash +backbeat-cli list-lifecycle-orphans --bucket archive --marker abc123 --max-keys 100 +``` + +## Help + +To view the help information for the Backbeat CLI and its commands, use the `--help` flag: + +```bash +backbeat-cli --help +``` + +For help on a specific command, append --help after the command: + +```bash +backbeat-cli list-lifecycle-currents --help +``` + +This will display detailed information about the command's usage, options, and examples. From c3b212b45a6cb2fc372d2824829f85f7e4816ca9 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Wed, 13 Nov 2024 11:19:20 +0100 Subject: [PATCH 2/2] BB-616 use the backbeatClient with the same settings as in production --- bin/backbeat-cli.js | 41 +++++++++++++++++++++++++++++++++++------ docs/backbeat-cli.md | 6 +----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/bin/backbeat-cli.js b/bin/backbeat-cli.js index e51b11790..dffd5a4e8 100755 --- a/bin/backbeat-cli.js +++ b/bin/backbeat-cli.js @@ -1,7 +1,10 @@ #!/usr/bin/env node const program = require('commander'); -const BackbeatClient = require('../lib/clients/BackbeatClient.js'); +const url = require('url'); +const https = require('https'); +const http = require('http'); +const { createBackbeatClient } = require('../lib/clients/utils.js'); const pkg = require('../package.json'); const werelogs = require('werelogs'); @@ -15,12 +18,38 @@ werelogs.configure({ dump: config.log.dumpLevel, }); +function parseEndpoint(endpoint) { + try { + const parsedUrl = new url.URL(endpoint); + const transport = parsedUrl.protocol.slice(0, -1); + const host = parsedUrl.hostname; + const port = parsedUrl.port; + + return { transport, host, port }; + } catch (error) { + throw new Error(`Invalid endpoint URL: ${endpoint}`); + } +} + function createClient() { - return new BackbeatClient({ - endpoint: process.env.BACKBEAT_ENDPOINT || 'http://127.0.0.1:8000', - region: process.env.AWS_REGION || 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID || 'accessKey1', - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || 'verySecretKey1', + const { transport, host, port } = parseEndpoint(process.env.BACKBEAT_ENDPOINT || 'http://127.0.0.1:8000'); + + let agent; + if (transport === 'https') { + agent = new https.Agent({ keepAlive: true }); + } else { + agent = new http.Agent({ keepAlive: true }); + } + + return createBackbeatClient({ + transport, + host, + port, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID || 'accessKey1', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || 'verySecretKey1', + }, + agent, }); } diff --git a/docs/backbeat-cli.md b/docs/backbeat-cli.md index eade1e1ce..041ce2c77 100644 --- a/docs/backbeat-cli.md +++ b/docs/backbeat-cli.md @@ -53,7 +53,7 @@ npm install 4. **Create an alias for easy access** ```bash -alias backbeat-cli='node bin/backbeat.cli.js' +alias backbeat-cli='node bin/backbeat-cli.js' ``` ## Configuration @@ -65,9 +65,6 @@ Before using the Backbeat CLI, configure the necessary environment variables to - `BACKBEAT_ENDPOINT`: The endpoint URL for the Backbeat service. *Default*: `http://127.0.0.1:8000` -- `AWS_REGION`: The AWS region to use. - *Default*: `us-east-1` - - `AWS_ACCESS_KEY_ID`: Your AWS access key ID. *Default*: `accessKey1` @@ -78,7 +75,6 @@ You can set these variables in your shell or include them in a configuration fil ```bash export BACKBEAT_ENDPOINT='http://your-backbeat-endpoint.com' -export AWS_REGION='your-region' export AWS_ACCESS_KEY_ID='your-access-key-id' export AWS_SECRET_ACCESS_KEY='your-secret-access-key' ```