Merge pull request #7258 from Neon-White/handle-invalid-azure-metadat…

…a-keys Handle invalid Azure object metadata keys
noobaa · Apr 17, 2023 · f4c47f1 · f4c47f1
2 parents 50a6761 + 5a6f245
commit f4c47f1
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 12 deletions.
diff --git a/src/endpoint/s3/ops/s3_put_object.js b/src/endpoint/s3/ops/s3_put_object.js
@@ -49,7 +49,8 @@ async function put_object(req, res) {
         tagging,
         tagging_copy: s3_utils.is_copy_tagging_directive(req),
         encryption,
-        lock_settings
+        lock_settings,
+        azure_invalid_md_header: req.headers['azure-metadata-handling'] || undefined
     });
 
     if (reply.version_id && reply.version_id !== 'null') {

diff --git a/src/sdk/namespace_blob.js b/src/sdk/namespace_blob.js
@@ -12,7 +12,9 @@ const azure_storage = require('../util/new_azure_storage_wrap');
 const stream_utils = require('../util/stream_utils');
 const s3_utils = require('../endpoint/s3/s3_utils');
 const schema_utils = require('../util/schema_utils');
-const valid_attr_regex = /^[A-Za-z_][A-Za-z0-9_]+$/;
+const invalid_azure_md_regex = /[^a-zA-Z0-9_]/g;
+const XATTR_RENAME_PREFIX = 'rename_';
+const XATTR_RENAME_KEY_PREFIX = 'rename_key_';
 
 /**
  * @implements {nb.Namespace}
@@ -409,16 +411,38 @@ class NamespaceBlob {
         return { obj_id };
     }
 
+    // Adhere to azcopy's ResolveInvalidKey for handling of invalid metadata keys
+    _rename_invalid_md_key(xattr, md_key) {
+        const valid_key = md_key.replace(invalid_azure_md_regex, '_');
+        xattr[XATTR_RENAME_PREFIX + valid_key] = xattr[md_key];
+        xattr[XATTR_RENAME_KEY_PREFIX + valid_key] = md_key;
+        delete xattr[md_key];
+    }
+
+    _exclude_invalid_md_key(xattr, md_key) {
+            delete xattr[md_key];
+    }
+
     _check_valid_xattr(params) {
-        // This md validation check is a part of namespace blob because but Azure Blob 
-        // accepts C# identifiers only but S3 accepts other xattr too.
-        const is_invalid_attr = ([key, val]) => !valid_attr_regex.test(key);
-        const invalid_attr = Object.entries(params.xattr).find(is_invalid_attr);
-        if (invalid_attr) {
-            const err = new Error('InvalidMetadata: metadata keys are invalid.');
-            err.rpc_code = 'INVALID_REQUEST';
-            throw err;
-        }
+        const modified_xattr = _.cloneDeep(params.xattr);
+        for (const md_key of Object.keys(params.xattr)) {
+            invalid_azure_md_regex.lastIndex = 0;
+            if (invalid_azure_md_regex.test(md_key)) {
+                switch (params.azure_invalid_md_header) {
+                    case 'ExcludeIfInvalid':
+                        this._exclude_invalid_md_key(modified_xattr, md_key);
+                        break;
+                    case 'FailIfInvalid':
+                        throw new Error('Object metadata key ' + md_key + ' is invalid in Azure and the FailIfInvalid header value was provided, thus the upload has failed');
+                    case 'RenameIfInvalid':
+                        this._rename_invalid_md_key(modified_xattr, md_key);
+                        break;
+                    default:
+                        this._rename_invalid_md_key(modified_xattr, md_key);
+                    }
+                }
+            }
+        params.xattr = modified_xattr;
     }
 
     async upload_multipart(params, object_sdk) {
@@ -656,6 +680,19 @@ class NamespaceBlob {
         const xattr = _.extend(obj.metadata, {
             'noobaa-namespace-blob-container': this.container,
         });
+        const modified_xattr = _.cloneDeep(xattr);
+        // Restore invalid metadata keys
+        for (const md_key of Object.keys(xattr)) {
+            if (md_key.startsWith(XATTR_RENAME_KEY_PREFIX)) {
+                const restored_key = md_key.substring(11);
+                if (_.has(modified_xattr, XATTR_RENAME_PREFIX + restored_key)) {
+                    modified_xattr[modified_xattr[md_key]] = modified_xattr[XATTR_RENAME_PREFIX + restored_key];
+                    delete modified_xattr[md_key];
+                    delete modified_xattr[XATTR_RENAME_PREFIX + restored_key];
+                }
+            }
+        }
+
         return {
             obj_id: blob_etag,
             bucket,
@@ -664,7 +701,7 @@ class NamespaceBlob {
             etag,
             create_time: flat_obj.lastModified,
             content_type: flat_obj.contentType,
-            xattr
+            xattr: modified_xattr
         };
     }
 

diff --git a/src/util/http_utils.js b/src/util/http_utils.js
@@ -525,6 +525,11 @@ function check_headers(req, options) {
         throw new options.ErrorClass(options.error_bad_request);
     }
 
+    if (req.headers['azure-metadata-handling'] && !_.includes(['ExcludeIfInvalid', 'FailIfInvalid', 'RenameIfInvalid'], req.headers['azure-metadata-handling'])) {
+        throw new options.ErrorClass(options.error_bad_request);
+    }
+
+
     if (req.method === 'POST' || req.method === 'PUT') parse_content_length(req, options);
 
     const content_md5_b64 = req.headers['content-md5'];