From 70ec77dc1db9da41c6e18fdb51c6264ac4162729 Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Wed, 6 Sep 2023 09:29:33 -0700 Subject: [PATCH] `note.add` handler and tests. md5 server supports GET path?chunk=N to retrieve the binary payload for each chunk. --- .../lib/notecard_binary/NotecardBinary.h | 397 ++++++++++++------ .../card.binary/test/test_card_binary.cpp | 44 +- test/hitl/scripts/md5srv.bats | 24 ++ test/hitl/scripts/md5srv.py | 113 +++-- test/hitl/scripts/test_binary_generators.h | 60 --- 5 files changed, 408 insertions(+), 230 deletions(-) delete mode 100644 test/hitl/scripts/test_binary_generators.h diff --git a/test/hitl/card.binary/lib/notecard_binary/NotecardBinary.h b/test/hitl/card.binary/lib/notecard_binary/NotecardBinary.h index 64c2ee82..a61bbacc 100644 --- a/test/hitl/card.binary/lib/notecard_binary/NotecardBinary.h +++ b/test/hitl/card.binary/lib/notecard_binary/NotecardBinary.h @@ -135,7 +135,7 @@ class AbstractBinaryGenerator : public BinaryGenerator /** * @brief Maintains a view of another generator. - * + * */ class BinaryGeneratorView : public AbstractBinaryGenerator { BinaryGenerator& _image; @@ -330,22 +330,22 @@ class NotecardBinary } /** - * @brief This struct is passed to the transfer callback after each iteration of the Notecard's + * @brief This struct is passed to the transfer callback after each iteration of the Notecard's * binary buffer being filled to the requested capacity. */ struct TransferDetails { - const char* name; // a name for this transfer - size_t transferred; // bytes transferred so far (not including this transfer) - size_t total; // the total number of bytes to be transferred - size_t currentTransferIndex; // The index of this transfer, 0-based. - size_t currentTransferSize; // the current transfer size (the combined size of all card.binary.pot requests) - uint8_t* currentTransferData; // the current transfer data (non-const so we can mess with it to validate failure.) + const char* name; // a name for this transfer + size_t transferred; // bytes transferred so far (not including this transfer) + size_t total; // the total number of bytes to be transferred + size_t currentTransferIndex; // The index (i.e. chunk) of this transfer, 0-based. + size_t currentTransferSize; // the current transfer size (the combined size of all card.binary.pot requests) + uint8_t* currentTransferData; // the current transfer data (non-const so we can mess with it to validate failure.) const uint8_t* currentTransferMD5; // MD5 of the current transfer data BinaryGenerator& currentTransferImage; // the current image for this transfer bool isComplete; // true when this is the last transfer BinaryGenerator& totalImage; // the complete image - const uint8_t* totalMD5; // MD5 hash for the complete transfer. set only when isComplete is true. - size_t totalChunks; + const uint8_t* totalMD5; // MD5 hash for the complete transfer. valid only when isComplete is true. + size_t totalChunks; // the total number of chunks sent. Valid only when isComplete is true. }; using transfer_cb_t = std::function; @@ -371,7 +371,7 @@ class NotecardBinary /** * @brief The size of each chunk of data sent to the notecard. */ - size_t chunkSize; + size_t chunkSize; /** * @brief A work buffer used to hold each encoded chunk. */ @@ -380,18 +380,25 @@ class NotecardBinary * @brief The size of the work buffer */ size_t bufferLength; - + /** * @brief The maximum amount of data to put in the Notecard's binary buffer. */ size_t maxBinarySize; - + /** * @brief When true, validate the data using `card.binary.get`. - * */ bool validate; + /** + * @brief The chunk sizes to use for retrieving the binary via card.binary.get. + * Default is to use maxCardBinary. + */ + const int32_t* validateChunkSizes; + size_t validateChunkSizesCount; + + /** * @brief A transfer callback. Once the data has been (optionally) validated, invoke the callback to * process the binary on the notecard further. (E.g. via note.add or web.get) @@ -400,7 +407,7 @@ class NotecardBinary }; bool transferBinary(const BinaryTransfer& tx) { - return _transferBinary(tx.imageName, tx.image, tx.chunkSize, tx.buffer, tx.bufferLength, tx.maxBinarySize, tx.validate, tx.transfer_cb); + return _transferBinary(tx.imageName, tx.image, tx.chunkSize, tx.buffer, tx.bufferLength, tx.maxBinarySize, tx.validate, tx.validateChunkSizes, tx.validateChunkSizesCount, tx.transfer_cb); } /** @@ -417,11 +424,14 @@ class NotecardBinary * clearing the buffer. * @param validate When true `card.binary.get` is used to validate the buffer contents. Note: This has the * side-effect of clearing the buffer. + * @param validateChunkSizes The chunk sizes to use when validating the binary buffer using `card.binary.get`. + * @param validateChunkSizesCount The number of chunk sizes in the validateChunkSizes array. When 0, the entire buffer is validated + * with one call to `card.binary.get`. * @param transfer_cb Callback function that is invoked after every successful complete transfer to the Notecard. - * @return true - * @return false + * @return true + * @return false */ - bool _transferBinary(const char* imageName, BinaryGenerator& image, size_t chunkSize, uint8_t* buffer, size_t bufferLength, size_t maxBinarySize, bool validate, transfer_cb_t transfer_cb) + bool _transferBinary(const char* imageName, BinaryGenerator& image, size_t chunkSize, uint8_t* buffer, size_t bufferLength, size_t maxBinarySize, bool validate, const int32_t* validateChunkSizes, size_t validateChunkSizesCount, transfer_cb_t transfer_cb) { const size_t totalSize = image.length(); if (totalSize==0) { @@ -511,15 +521,30 @@ class NotecardBinary tx.totalChunks = tx.currentTransferIndex+1; } - if (validate && !validateBinaryReceived(tx, buffer, bufferLength)) { - return false; + if (validate) { + int32_t defaultSizes[1] = { 0 }; + if (!validateChunkSizesCount || !validateChunkSizes) { + validateChunkSizesCount = 1; + validateChunkSizes = defaultSizes; + } + for (size_t idx=0; idx=300) { + notecard.logDebugf("web.get result was %d\n", result); + } + else if (!body) { + notecard.logDebugf("web.get body is not present\n"); + } + else if (cobs) { + notecard.logDebugf("web.get unexpected response property 'cobs' is present\n"); + } + else { + size_t expectedChunks = chunked ? tx.totalChunks : 1; + size_t actualLength = JGetNumber(body, "length"); + size_t actualChunks = JGetNumber(body, "chunks"); + const char* actualMD5String = JGetString(body, "md5"); + char expectedMD5String[NOTE_MD5_HASH_STRING_SIZE]; + NoteMD5HashToString((uint8_t*)tx.totalMD5, expectedMD5String, sizeof(expectedMD5String)); + if (!actualMD5String) actualMD5String = "not a string"; + if (strcmp(actualMD5String, expectedMD5String)) { + notecard.logDebugf("web.get MD5 actual!=expected: %s!=%s\n", actualMD5String, expectedMD5String); + } + else if (actualLength!=tx.total) { + notecard.logDebugf("web.get total length: actual!=expected: %d!=%d\n", actualLength, tx.total); + } + else if (actualChunks!=expectedChunks) { + notecard.logDebugf("web.get total chunks: actual!=expected: %d!=%d\n", actualChunks, expectedChunks); + } + else { + notecard.logDebugf("web.get response validated: md5=%s, chunks=%d, length=%d.\n", actualMD5String, actualChunks, actualLength); + success = true; + } + } + } + } + // delete the file on the server + if (!success || tx.isComplete) { + // clean up the server files + if (J *req = notecard.newRequest("web.delete")) { + JAddStringToObject(req, "route", alias); + if (name) { + JAddStringToObject(req, "name", name); + } + J* rsp = notecard.requestAndResponse(req); + if (rsp) { + int result = JGetNumber(rsp, "result"); + if (result != 200) { + notecard.logDebugf("HTTP status %d trying to delete %s\n", result, name); + success = false; + } + JDelete(rsp); + } + } + } + return success; + } +}; + /** * @brief Posts data to the md5 server endpoint and validates the expected length and md5 against * the response received. Can operated in chunked mode, or unchunked mode (from the perspective * of the endpoint.) - * + * * In Chunked mode, the endpoint receives each filled card.binary buffer as it arrives. * In Unchunked mode, the endpoint receives just one payload containing the entire image. - * + * * After the data has been sent, a `web.get` request is used to retrieve the total length and md5 * for the entire image, and the number of chunks received. */ -class WebPostHandler { - const char* alias; +class WebPostHandler : public FileHandler { const char* name; const char* content; bool chunked; // When true send each chunk directly to the endpoint. When false, have notehub combine the chunks and send all at once. @@ -681,25 +804,25 @@ class WebPostHandler { public: /** * @brief Construct a new Web Post Handler object - * - * @param alias - * @param name - * @param content + * + * @param alias + * @param name + * @param content * @param chunked When true, each completed binary transfer is sent to the server. When false * the transfers are combined and sent to the server in one hit. * @param verify In chunked mode, verify the md5 of all chunks received by the server. */ WebPostHandler(const char* alias, const char* name, const char* content, bool chunked, bool webPut) - : alias(alias), name(name), content(content), chunked(chunked), webPut(webPut) {} + : FileHandler(alias), name(name), content(content), chunked(chunked), webPut(webPut) {} /** * @brief Sends a web.post request with the binary buffer as content. In chunked mode, * each request is sent to the endpoint immediately. In non-chunked mode, the requests * are assembled by notehub and sent as a single request to the endpoint. - * - * @param tx - * @return true - * @return false + * + * @param tx + * @return true + * @return false */ bool handleTransfer(const NotecardBinary::TransferDetails& tx) { @@ -819,73 +942,7 @@ class WebPostHandler { // to verify all chunks, the md5 server must be started with the `--save` flag if (success && tx.isComplete) { - success = false; - // get the total md5 for all chunks - J *req = notecard.newRequest("web.get"); - JAddStringToObject(req, "route", alias); - if (name) { - JAddStringToObject(req, "name", name); - } - J* rsp = notecard.requestAndResponse(req); - if (rsp) { - int result = JGetNumber(rsp, "result"); - J* body = JGetObject(rsp, "body"); - int cobs = JGetNumber(rsp, "cobs"); - const char* err = JGetString(rsp, "err"); - if (err && err[0]) { - notecard.logDebugf("web.get failed with error: %s\n", err); - } - else if (result<200 || result>=300) { - notecard.logDebugf("web.get result was %d\n", result); - } - else if (!body) { - notecard.logDebugf("web.get body is not present\n"); - } - else if (cobs) { - notecard.logDebugf("web.get unexpected response property 'cobs' is present\n"); - } - else { - size_t expectedChunks = chunked ? tx.totalChunks : 1; - size_t actualLength = JGetNumber(body, "length"); - size_t actualChunks = JGetNumber(body, "chunks"); - const char* actualMD5String = JGetString(body, "md5"); - char expectedMD5String[NOTE_MD5_HASH_STRING_SIZE]; - NoteMD5HashToString((uint8_t*)tx.totalMD5, expectedMD5String, sizeof(expectedMD5String)); - if (!actualMD5String) actualMD5String = "not a string"; - if (strcmp(actualMD5String, expectedMD5String)) { - notecard.logDebugf("web.get MD5 actual!=expected: %s!=%s\n", actualMD5String, expectedMD5String); - } - else if (actualLength!=tx.total) { - notecard.logDebugf("web.get total length: actual!=expected: %d!=%d\n", actualLength, tx.total); - } - else if (actualChunks!=expectedChunks) { - notecard.logDebugf("web.get total chunks: actual!=expected: %d!=%d\n", actualChunks, expectedChunks); - } - else { - notecard.logDebugf("web.get response validated: md5=%d, chunks=%d, length=%d.\n", actualMD5String, actualChunks, actualLength); - success = true; - } - } - } - } - // delete the file on the server - if (!success || tx.isComplete) { - // clean up the server files - if (J *req = notecard.newRequest("web.delete")) { - JAddStringToObject(req, "route", alias); - if (name) { - JAddStringToObject(req, "name", name); - } - J* rsp = notecard.requestAndResponse(req); - if (rsp) { - int result = JGetNumber(rsp, "result"); - if (result != 200) { - notecard.logDebugf("HTTP status %d trying to delete %s\n", result, name); - success = false; - } - JDelete(rsp); - } - } + success = validateReceivedContent(tx, name, chunked); } } return success; @@ -898,6 +955,82 @@ class WebPostHandler { } }; +class NoteAddHandler : public FileHandler { + const char* name; +public: + /** + * @brief Construct a new NoteAdd Handler object + * @param alias Alias of the route that directs to the MD5 server. + * @param name The name of the file being transferred. + */ + NoteAddHandler(const char* alias, const char* name) + : FileHandler(alias), name(name) {} + + /** + * @brief Sends a note.add request with the binary buffer as the payload. + * + * @param tx + * @return true + * @return false + */ + bool handleTransfer(const NotecardBinary::TransferDetails& tx) + { + bool success = false; + bool chunked = (tx.currentTransferSize!=tx.total); // only one chunk equal to the total + + if (!NotecardBinary::waitForNotecardConnected(NOT_CONNECTED_TIMEOUT)) { + notecard.logDebug("Cannot perform note.add request, Notecard not connected."); + return false; + } + + if (J* req = notecard.newRequest("note.add")) { + JAddStringToObject(req, "file", "cardbinary.qo"); + JAddBoolToObject(req, "binary", true); + JAddBoolToObject(req, "live", true); + JAddBoolToObject(req, "sync", true); + J* body = JAddObjectToObject(req, "body"); + JAddStringToObject(body, "name", name); + + char md5String[NOTE_MD5_HASH_STRING_SIZE]; + NoteMD5HashToString((uint8_t*)tx.currentTransferMD5, md5String, sizeof(md5String)); + + JAddStringToObject(body, "md5", md5String); + JAddNumberToObject(body, "length", tx.currentTransferImage.length()); + JAddNumberToObject(body, "offset", tx.transferred); + JAddNumberToObject(body, "total", tx.total); + if (chunked) { + JAddNumberToObject(body, "chunk", tx.currentTransferIndex); + } + J* rsp = notecard.requestAndResponse(req); + if (rsp) { + const char* err = JGetString(rsp, "err"); + if (err && err[0]) { + notecard.logDebugf("note.add error: %s\n", err); + } + else { + success = true; + } + JDelete(rsp); + } + else { + notecard.logDebugf("no response from Notecard\n"); + } + } + else { + notecard.logDebugf("Could not allocate request\n"); + } + + if (success && tx.isComplete) { + success = validateReceivedContent(tx, name, chunked); + } + return success; + } + + NotecardBinary::transfer_cb_t transfer_callback() { + using namespace std::placeholders; + return std::bind(&NoteAddHandler::handleTransfer, this, _1); + } +}; extern BinaryImage small_binary; extern BufferBinaryGenerator small_image; \ No newline at end of file diff --git a/test/hitl/card.binary/test/test_card_binary.cpp b/test/hitl/card.binary/test/test_card_binary.cpp index c73e2ac9..39eb8554 100644 --- a/test/hitl/card.binary/test/test_card_binary.cpp +++ b/test/hitl/card.binary/test/test_card_binary.cpp @@ -151,6 +151,8 @@ void binaryTransferTest(const char* name, BinaryGenerator& generator, NotecardIn WebPostHandler webPostChunkedHandler(NOTEHUB_PROXY_ROUTE_ALIAS, tx.imageName, MIME_APPLICATION_OCTETSTREAM, true, false); WebPostHandler webPutHandler(NOTEHUB_PROXY_ROUTE_ALIAS, tx.imageName, MIME_APPLICATION_OCTETSTREAM, false, true); WebPostHandler webPutChunkedHandler(NOTEHUB_PROXY_ROUTE_ALIAS, tx.imageName, MIME_APPLICATION_OCTETSTREAM, true, true); + NoteAddHandler noteAddHandler(NOTEHUB_PROXY_ROUTE_ALIAS, tx.imageName); + // NoteAddHandler noteAddHandler; switch (testArgs._handler) { case BinaryTestArgs::NONE: @@ -162,6 +164,9 @@ void binaryTransferTest(const char* name, BinaryGenerator& generator, NotecardIn case BinaryTestArgs::WEB_POST_UNCHUNKED: tx.transfer_cb = webPostHandler.transfer_callback(); break; + case BinaryTestArgs::NOTE_ADD: + tx.transfer_cb = noteAddHandler.transfer_callback(); + break; default: TEST_FAIL_MESSAGE("Unknown transfer handler"); return; @@ -179,8 +184,9 @@ const BinaryTestArgs webpost_chunked = BinaryTestArgs().handler(BinaryTestArgs:: const BinaryTestArgs note_add = BinaryTestArgs().handler(BinaryTestArgs::NOTE_ADD); const BinaryTestArgs validate; -// negative sizes validate based on the size of the card.binary. -const int validate_chunk_sizes[6] = { 255, 256, 1234, 4096, -1 /* binary size -1 */, 0 /* binary size */}; +// negative sizes validate based on the size of the card.binary minus the absolute value. So 0 means +// validate using a buffer exactly large enough to hold the entire transferred binary. +const int validate_chunk_sizes[9] = { 255, 256, 1234, 4095, 4096, -3, -2, -1 /* binary size -1 */, 0 /* binary size */}; const BinaryTestArgs validate_in_chunks = BinaryTestArgs().validateChunkSizes(validate_chunk_sizes); @@ -198,7 +204,7 @@ const BinaryTestArgs validate_in_chunks = BinaryTestArgs().validateChunkSizes(va // testArgs: additional test arguments // This global avoids any capture on the lambda so it remains a regular C functions. -BinaryTestArgs* currentTestArgs; +const BinaryTestArgs* currentTestArgs; #define RUN_SIZE(imagename, image, iface, ifacename, size, sizename, testArgs, ...) { \ currentTestArgs = &testArgs; \ @@ -364,6 +370,20 @@ TEST(test_max_length_aux_serial) RUN_AUX_SERIAL_BAUDRATE(imagename, image, 160*1024, 160k, baudrate, __VA_ARGS__); +#define RUN_SMOKE_TESTS(interface, interfacename) \ + { /* validate the received data */ \ + RUN_SIZE(Random_2345, BuildRandom, NOTECARD_IF_I2C, i2c, 50*1026, 50k, validate_in_chunks, 2345); \ + } \ + { /* maximum card.binary size is 10k, transferring 50k total, with chunked web.post on each binary buffer */ \ + BinaryTestArgs max10k_webpost = BinaryTestArgs().maxCardBinary(10*1023).handler(BinaryTestArgs::WEB_POST_UNCHUNKED); \ + RUN_SIZE(Random_1234, BuildRandom, NOTECARD_IF_I2C, i2c, 50*1026, 50k, max10k_webpost, 1234); \ + } \ + { /* maximum card.binary size is 10k, transferring 50k total, with chunked web.post on each binary buffer */ \ + BinaryTestArgs max10k_webpost_chunked = BinaryTestArgs().maxCardBinary(10*1023).handler(BinaryTestArgs::WEB_POST_CHUNKED); \ + RUN_SIZE(Random_1234, BuildRandom, NOTECARD_IF_I2C, i2c, 50*1026, 50k, max10k_webpost_chunked, 1234); \ + } \ + + /** * @brief Runs the card.binary test suite. * @@ -375,17 +395,17 @@ void testsuite_card_binary() // initialize max_binary_length for the max_length tests RUN_TEST(test_get_max_binary_length); - bool smoke_tests = true; + bool smoke_tests = false; + + { + BinaryTestArgs max10k_note_add = BinaryTestArgs().maxCardBinary(1*1023).handler(BinaryTestArgs::NOTE_ADD); + RUN_SIZE(Random_1234, BuildRandom, NOTECARD_IF_I2C, i2c, 5*1026, 5k, max10k_note_add, 1234); + } if (smoke_tests) { - { // maximum card.binary size is 10k, transferring 50k total, with chunked web.post on each binary buffer - BinaryTestArgs max10k_webpost = BinaryTestArgs().maxCardBinary(10*1023).handler(BinaryTestArgs::WEB_POST_UNCHUNKED); - RUN_SIZE(Random_1234, BuildRandom, NOTECARD_IF_I2C, i2c, 50*1026, 50k, max10k_webpost, 1234); - } - { // maximum card.binary size is 10k, transferring 50k total, with chunked web.post on each binary buffer - BinaryTestArgs max10k_webpost_chunked = BinaryTestArgs().maxCardBinary(10*1023).handler(BinaryTestArgs::WEB_POST_CHUNKED); - RUN_SIZE(Random_1234, BuildRandom, NOTECARD_IF_I2C, i2c, 50*1026, 50k, max10k_webpost_chunked, 1234); - } + RUN_SMOKE_TESTS(NOTECARD_IF_I2C, i2c); + RUN_SMOKE_TESTS(NOTECARD_IF_SERIAL, serial); + RUN_SMOKE_TESTS(NOTECARD_IF_AUX_SERIAL, auxserial); } // RUN_AUX_SERIAL_ALL_BAUDRATES(all_sevens, AllSevens, TINY_SIZE, TINY_SIZE_NAME); diff --git a/test/hitl/scripts/md5srv.bats b/test/hitl/scripts/md5srv.bats index d1f0eb55..8c011397 100644 --- a/test/hitl/scripts/md5srv.bats +++ b/test/hitl/scripts/md5srv.bats @@ -11,6 +11,7 @@ export clean_test_dir=true bats_require_minimum_version 1.10.0 +# NB: on OSX, export BATS_LIB_PATH=/usr/local/lib bats_load_library "bats-support" bats_load_library "bats-assert" @@ -189,4 +190,27 @@ function assert_json() { assert_failure assert_output --partial "already exists" assert_json code 409 +} + +@test "Can post a note which is decoded and the payload saved to the server" { + startMD5Server + waitForMD5ServerWithTimeout + run curl -s -X POST $md5url/addnote?note=1 -d "@note.json" -H "X-Access-Token: $token" -H "Content-Type: application/json" + stopMD5Server + assert_success + assert_json md5 f54095bd47b357cbd57ace4b4da76eaa + assert_json length 10230 + assert [ -e $test_dir/Random_1234/payload.bin ] +} + +@test "Can post a chunk and then retrieve it" { + startMD5Server + waitForMD5ServerWithTimeout + content=abcdef + curl -s -X POST $md5url/getchunk?chunk=0 -d "$content" -H "X-Access-Token: $token" + run curl -s -X GET $md5url/getchunk?chunk=0 -d "$content" -H "X-Access-Token: $token" + stopMD5Server + assert_success + assert_output "${content}" + assert [ -e $test_dir/getchunk/payload00000.bin ] } \ No newline at end of file diff --git a/test/hitl/scripts/md5srv.py b/test/hitl/scripts/md5srv.py index a5c24b2c..a1690a1c 100755 --- a/test/hitl/scripts/md5srv.py +++ b/test/hitl/scripts/md5srv.py @@ -3,6 +3,7 @@ import hashlib import os import json +import base64 import argparse from functools import cached_property from http.server import BaseHTTPRequestHandler, HTTPServer @@ -30,8 +31,18 @@ def __init__(self, status_code, message, detail=None): class WebRequestHandler(BaseHTTPRequestHandler): """ - Writes the body of post requests out to a file. - + GET: retrieves the total length and md5 of all chunks in a directory denoted by the URL path. The URL path + is interpreted relative to the serving directory. + If the query parameter "note" is present, the request body is parsed as JSON and the payload saved to + a file using the `chunk` event body property. + POST/PUT: saves the body to a file, with an optional `chunk` query parameter for writing one or more chunks. + If no chunk number is given, the file `payload.bin` is written to the directory. Otherwise payloadXXXXX.bin is written + where XXXXX is the zero-padded chunk number. + Returns a 409 (Conflict) error if the file already exists. + + DELETE: deletes a directory containing one or more chunks. + + Paths are fully sanitized. """ def __init__(self, args, *others) -> None: @@ -58,9 +69,6 @@ def post_data(self): def form_data(self): return dict(parse_qsl(self.post_data.decode("utf-8"))) - def do_HEAD(self): - self.send_response(200, "Ok") - def do_GET(self): self.do(self.get_md5) @@ -68,25 +76,28 @@ def do_DELETE(self): self.do(self.delete_files) def do_POST(self): - self.do(self.write_file) + self.do(self.write_file_or_note) def do_PUT(self): - self.do(self.write_file) + self.do(self.write_file_or_note) - def send_status(self, code: int, message: str, detail: str | dict, headers: dict = {}): + def send_status(self, code: int, message: str, detail: str | dict | bytes, headers: dict = {}): """ Send a status code with a message and optional detail, which is returned as JSON in the response body. """ self.send_response(code, message) for header in headers.items(): self.send_header(*header) - self.send_header("Content-Type", "application/json") + self.send_header("Content-Type", "application/json" if type(detail) is not bytes else "application/octet-stream") self.end_headers() detail = detail or message if detail is not None: - key = "err" if code > 400 else "text" if not type(detail) is dict else None - json_str = json.dumps(detail) - reply_body = json_str if key is None else f'{{"{key}":{json_str},"code":{code}}}' - reply_body = reply_body + '\n' - self.wfile.write(reply_body.encode('utf-8')) + if type(detail) is bytes: + self.wfile.write(detail) + else: + key = "err" if code > 400 else "text" if not type(detail) is dict else None + json_str = json.dumps(detail) + reply_body = json_str if key is None else f'{{"{key}":{json_str},"code":{code}}}' + reply_body = reply_body + '\n' + self.wfile.write(reply_body.encode('utf-8')) def do(self, handler): """ Handle a request. If an exception is thrown, raises an internal server error. """ @@ -111,6 +122,19 @@ def get_md5(self): if not os.path.exists(dirname): raise HTTPException(404, "Not Found", f"Directory {self.url.path} not found.") + chunk = self.query_data.get('chunk') + headers = {} + if chunk is None: + response = self.md5_for_directory(dirname) + else: + response = self.chunk_content(dirname, chunk) + headers = {"Content-Length": len(response)} + self.send_status(200, "ok", response, headers) + + def chunk_content(self, dirname, chunk): + return self._read_file(dirname, chunk) + + def md5_for_directory(self, dirname): files = os.listdir(dirname) files.sort() @@ -129,7 +153,7 @@ def get_md5(self): md5str = md5context.hexdigest() response = {"md5": md5str, "length": length, "chunks": len(files)} - self.send_status(200, "Ok", response) + return response def delete_files(self): dirname = self.validate_url_path(self.url.path) @@ -138,35 +162,72 @@ def delete_files(self): shutil.rmtree(dirname) self.send_status(200, "Ok", f"Deleted {self.url.path}") + def write_file_or_note(self): + if self.query_data.get('note') is not None: + self.do(self.write_note) + else: + self.do(self.write_file) + + def write_note(self): + content_type = self.headers['Content-Type'] + if content_type != 'application/json': + raise HTTPException(400, f"Unsupported content type: {content_type}") + data = self.post_data + jsondata = json.loads(data) + body = jsondata['body'] # non-optional keys + name = body['name'] + length = body['length'] + md5 = body['md5'] + # todo - could check the offset + payload = base64.b64decode(jsondata['payload']) + chunk = body.get('chunk', None) # optional + self._write_file(name, chunk, length, payload, md5) + def write_file(self): + length = int(self.headers['Content-Length']) + if not length: + raise HTTPException(400, "Request body is empty.") + chunk = self.query_data.get("chunk", None) + return self._write_file(self.url.path, chunk, length, self.post_data, None) + + def _read_file(self, path, chunk): + filename = self._chunk_file(path, chunk) + if not os.path.exists(filename): + raise HTTPException(404, 'Not found') + with open(filename, 'rb') as output_file: + data = output_file.read() + return data + + def _chunk_file(self, dirname, chunk): + chunk_index = None if chunk is None else int(chunk) + filename = os.path.join(dirname, f"payload{chunk_index:05d}.bin" if chunk_index is not None else "payload.bin") + self.validate_path(filename) + return filename + + def _write_file(self, path, chunk, length, data, md5): """ Write the request body to a file and returns the MD5 """ - dirname = self.validate_url_path(self.url.path) + dirname = self.validate_url_path(path) if self.args.save: if not os.path.exists(dirname): log_sensitive(f"Creating directory {dirname}") - log(f"creating directory {self.url.path}") + log(f"creating directory {path}") os.makedirs(dirname) - chunk = self.query_data.get("chunk", None) - chunkIndex = 0 if not chunk else int(chunk) - filename = os.path.join(dirname, f"payload{chunkIndex:05d}.bin" if chunk else "payload.bin") - self.validate_path(filename) + filename = self._chunk_file(dirname, chunk) if os.path.exists(filename): raise HTTPException(409, "Conflict", f"Chunk {chunk} already exists for {self.url.path}" if chunk else f"File {self.url.path} already exists.") - length = int(self.headers['Content-Length']) - if not length: - raise HTTPException(400, "Request body is empty.") - data = self.post_data if len(data) != length: raise HTTPException(400, "Invalid content length.", - f"Payload length does not equal given Content-Length: {len(data)}!={length}") + f"Payload length does not equal given length {len(data)}!={length}") if self.args.save: with open(filename, 'wb') as output_file: output_file.write(data) md5str = hashlib.md5(data).hexdigest() + if md5 and md5str != md5: + raise HTTPException(400, f"MD5 mismatch. actual {md5str}!=expected {md5}") response = {"md5": md5str, "length": length} content_type = self.headers['Content-Type'] if content_type is not None: diff --git a/test/hitl/scripts/test_binary_generators.h b/test/hitl/scripts/test_binary_generators.h deleted file mode 100644 index 1d2f90bd..00000000 --- a/test/hitl/scripts/test_binary_generators.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once -#include "NotecardBinary.h" - -/** - * @brief Unity and PlatformIO support generation of multiple distinct binaries. - * However, I didn't feel it necessary to produce another binary for these tests. - */ -void testsuite_binary_generators(); - - -struct BuildAllSameValue { - uint8_t buf[128]; - BufferBinaryGenerator chunk; - RepeatedBinaryGenerator image; - - BuildAllSameValue(size_t length, uint8_t value) - : chunk(buf, sizeof(buf)), image(chunk, length) - { - memset(buf, value, sizeof(buf)); - } - - BinaryGenerator& operator ()() - { - return image; - } -}; - -struct AllZeros : BuildAllSameValue { - AllZeros(size_t length) : BuildAllSameValue(length, 0) {} -}; - -struct AllSevens : BuildAllSameValue { - AllSevens(size_t length) : BuildAllSameValue(length, 7) {} -}; - -struct BuildFromBinaryImage { - BufferBinaryGenerator image; - - BuildFromBinaryImage(size_t length, const BinaryImage& binary) - : image(binary.data, binary.length) {} - - BinaryGenerator& operator ()() - { - return image; - } -}; - -struct SmallBinaryImage : BuildFromBinaryImage { - SmallBinaryImage(size_t size) : BuildFromBinaryImage(size, small_binary) {} -}; - -struct BuildRandom { - RandomBinaryGenerator image; - BuildRandom(size_t size, int seed) : image(size, seed) {} - BinaryGenerator& operator ()() - { - return image; - } -}; -