diff --git a/.github/workflows/notecard-binary-tests.yml b/.github/workflows/notecard-binary-tests.yml index 480dd2c9..6a8cb90a 100644 --- a/.github/workflows/notecard-binary-tests.yml +++ b/.github/workflows/notecard-binary-tests.yml @@ -12,12 +12,12 @@ permissions: checks: write jobs: - md5srv-test: - uses: ./.github/workflows/md5srv-tests.yml + # md5srv-test: + # uses: ./.github/workflows/md5srv-tests.yml notecard-binary-test: # needs: md5srv-test - runs-on: [self-hosted, swan, notecard, stlink, notecard-serial, md5srv, notehub-client] + runs-on: ubuntu-latest defaults: run: shell: bash @@ -35,27 +35,25 @@ jobs: DELETE_NOTEHUB_ROUTES: true # CREATE_NOTEHUB_ROUTES set to false to use the already created routes on notehub CREATE_NOTEHUB_ROUTES: true - # START_MD5SRV set to false to skip starting the MD5 server. There should be one - # already running locally with MD5SRV_PORT/ADDRESS/TOKEN set correspondingly. - START_MD5SRV: true - # START_TUNNELMOLE: set to false to skip starting tunnel mole. - START_TUNNELMOLE: true - # When neither tunneling solution is used (because they're already instantiated outside of the workflow) - # be sure to set MD5SRV_URL in the environment steps: + - name: Connect to Tailscale + uses: tailscale/github-action@v2 + with: + # TODO: Add these to note-c repo. + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_CLIENT_SECRET }} + tags: tag:ci - name: Checkout note-c repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Generate MD5 Server Token run: | [ -n "$MD5SRV_TOKEN" ] || echo "MD5SRV_TOKEN=`uuidgen`" >> $GITHUB_ENV - # propagate the environment variable so that it's available in the `env` context - echo "MD5SRV_PORT=$MD5SRV_PORT" >> $GITHUB_ENV - - name: Check env vars - run: | - . scripts/check_runner_config.sh - echo NOTEHUB_PROXY_ROUTE_ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS + # - name: Check env vars + # run: | + # . scripts/check_runner_config.sh + # echo NOTEHUB_PROXY_ROUTE_ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS - name: Install PlatformIO dependencies run: | @@ -75,18 +73,16 @@ jobs: run: | mkdir md5srv-files ./scripts/run_md5srv.sh - - - name: Start tunnelmole - if: env.START_TUNNELMOLE!='false' - run: | - rm -f tmole.log - ./scripts/run_tmole.sh + sleep 2 + echo $(cat md5srv.log) - name: Check MD5 server is available run: | # the request will return a 401 from md5srv, but that's expected without the access token # Curl still returns success because it could contact the server - code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL` + MD5SRV_URL=$(ip addr show tailscale0 | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) + curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL:8080 + code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL:8080` if [ "$code" -ge "500" ]; then echo "5xx error ($code) from tunnel." exit 1 @@ -94,114 +90,114 @@ jobs: echo "MD5 server is available." fi - - name: Create Notehub accesss token - if: env.CREATE_NOTEHUB_ROUTES!='false' - run: | - curl -f -X POST \ - -L 'https://${{env.NOTEHUB}}/oauth2/token' \ - -H 'content-type: application/x-www-form-urlencoded' \ - -d grant_type=client_credentials \ - -d client_id=$NOTEHUB_CLIENT_ID \ - -d client_secret=$NOTEHUB_CLIENT_SECRET | \ - { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; } - - - name: Create Notehub HTTP route - if: env.CREATE_NOTEHUB_ROUTES!='false' - run: | - # ?note=1 instructs the MD5 server to process the content as an event, extracting the path - # from the event body. - route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \ - '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'` - echo "Notehub HTTP route request: $route_req" - route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \ - -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-` - echo "Notehub HTTP route: $route" - route_uid=`echo $route | jq -r .uid` - if [ -n "$route_uid" ]; then - echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV - else - echo "Failed to create or parse Notehub HTTP route." - exit 1 - fi - - - name: Create Notehub proxy route - run: | - ALIAS="$NOTEHUB_PROXY_ROUTE_ALIAS" - route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \ - '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \ - | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \ - -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-` - echo "Notehub proxy route: $route" - route_uid=`echo "$route" | jq -r .uid` - if [ -n $route_uid ]; then - echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV - echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV - else - echo "Failed to create or parse Notehub proxy route." - exit 1 - fi - - - name: Build and upload test firmware and run tests - run: | - source venv/bin/activate - export PLATFORMIO_BUILD_FLAGS="'-D NOTEHUB_PROXY_ROUTE_ALIAS=\"$NOTEHUB_PROXY_ROUTE_ALIAS\"' '-D PRODUCT_UID=\"$NOTEHUB_PRODUCT_UID\"'" - echo "build flags $PLATFORMIO_BUILD_FLAGS" - cd $PIO_PROJECT_DIR - platformio test -v -e debug \ - --json-output-path test.json \ - --junit-output-path test.xml - - - name: Publish test report - uses: mikepenz/action-junit-report@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: env.GITHUB_TOKEN && (success() || failure()) # always run even if the previous step fails - with: - report_paths: '**/test/hitl/card.binary/test*.xml' - check_name: Notecard Binary HIL Tests - require_tests: true - - - name: Cleanup Notehub proxy route - if: always() - run: | - if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_PROXY_ROUTE_UID" ]; then - echo "Deleting Notehub proxy route." - curl -f -s -X DELETE \ - -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \ - -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" - fi - - - name: Cleanup Notehub HTTP route - if: always() - run: | - if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_HTTP_ROUTE_UID" ]; then - echo "Deleting Notehub HTTP route." - curl -f -s -X DELETE \ - -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \ - -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" - fi - - - - name: Cleanup tmole - if: always() - run: | - if [ -n "$TMOLE_PID" ]; then - echo "Stopping tmole." - kill $TMOLE_PID - else - echo "tmole not running (TMOLE_PID is empty)." - fi - - - name: Cleanup MD5 server - if: always() - run: | - if [ -d md5srv-files ]; then - echo "Deleting md5srv-files directory." - rm -rf md5srv-files - fi - if [ -n "$MD5SRV_PID" ]; then - echo "Stopping MD5 server." - kill $MD5SRV_PID - else - echo "MD5 server not running (MD5SRV_PID is empty)." - fi + # - name: Create Notehub accesss token + # if: env.CREATE_NOTEHUB_ROUTES!='false' + # run: | + # curl -f -X POST \ + # -L 'https://${{env.NOTEHUB}}/oauth2/token' \ + # -H 'content-type: application/x-www-form-urlencoded' \ + # -d grant_type=client_credentials \ + # -d client_id=$NOTEHUB_CLIENT_ID \ + # -d client_secret=$NOTEHUB_CLIENT_SECRET | \ + # { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; } + + # - name: Create Notehub HTTP route + # if: env.CREATE_NOTEHUB_ROUTES!='false' + # run: | + # # ?note=1 instructs the MD5 server to process the content as an event, extracting the path + # # from the event body. + # route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \ + # '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'` + # echo "Notehub HTTP route request: $route_req" + # route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \ + # -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-` + # echo "Notehub HTTP route: $route" + # route_uid=`echo $route | jq -r .uid` + # if [ -n "$route_uid" ]; then + # echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV + # else + # echo "Failed to create or parse Notehub HTTP route." + # exit 1 + # fi + + # - name: Create Notehub proxy route + # run: | + # ALIAS="$NOTEHUB_PROXY_ROUTE_ALIAS" + # route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \ + # '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \ + # | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \ + # -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-` + # echo "Notehub proxy route: $route" + # route_uid=`echo "$route" | jq -r .uid` + # if [ -n $route_uid ]; then + # echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV + # echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV + # else + # echo "Failed to create or parse Notehub proxy route." + # exit 1 + # fi + + # - name: Build and upload test firmware and run tests + # run: | + # source venv/bin/activate + # export PLATFORMIO_BUILD_FLAGS="'-D NOTEHUB_PROXY_ROUTE_ALIAS=\"$NOTEHUB_PROXY_ROUTE_ALIAS\"' '-D PRODUCT_UID=\"$NOTEHUB_PRODUCT_UID\"'" + # echo "build flags $PLATFORMIO_BUILD_FLAGS" + # cd $PIO_PROJECT_DIR + # platformio test -v -e debug \ + # --json-output-path test.json \ + # --junit-output-path test.xml + + # - name: Publish test report + # uses: mikepenz/action-junit-report@v3 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # if: env.GITHUB_TOKEN && (success() || failure()) # always run even if the previous step fails + # with: + # report_paths: '**/test/hitl/card.binary/test*.xml' + # check_name: Notecard Binary HIL Tests + # require_tests: true + + # - name: Cleanup Notehub proxy route + # if: always() + # run: | + # if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_PROXY_ROUTE_UID" ]; then + # echo "Deleting Notehub proxy route." + # curl -f -s -X DELETE \ + # -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \ + # -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" + # fi + + # - name: Cleanup Notehub HTTP route + # if: always() + # run: | + # if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_HTTP_ROUTE_UID" ]; then + # echo "Deleting Notehub HTTP route." + # curl -f -s -X DELETE \ + # -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \ + # -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" + # fi + + + # - name: Cleanup tmole + # if: always() + # run: | + # if [ -n "$TMOLE_PID" ]; then + # echo "Stopping tmole." + # kill $TMOLE_PID + # else + # echo "tmole not running (TMOLE_PID is empty)." + # fi + + # - name: Cleanup MD5 server + # if: always() + # run: | + # if [ -d md5srv-files ]; then + # echo "Deleting md5srv-files directory." + # rm -rf md5srv-files + # fi + # if [ -n "$MD5SRV_PID" ]; then + # echo "Stopping MD5 server." + # kill $MD5SRV_PID + # else + # echo "MD5 server not running (MD5SRV_PID is empty)." + # fi diff --git a/scripts/run_md5srv.sh b/scripts/run_md5srv.sh index 4e30380c..c848b874 100755 --- a/scripts/run_md5srv.sh +++ b/scripts/run_md5srv.sh @@ -1,4 +1,14 @@ . venv/bin/activate -python3 ./test/hitl/scripts/md5srv.py --dir md5srv-files --save > md5srv.log 2>&1 & + +ADDR=$(ip addr show tailscale0 | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) +if [[ -z "$ADDR" ]]; then + echo "Couldn't find tailscale0 IP address." + exit 1 +fi + +python3 ./test/hitl/scripts/md5srv.py \ + --dir md5srv-files \ + --address $ADDR \ + --save > md5srv.log 2>&1 & MD5SRV_PID=$! echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV diff --git a/test/hitl/scripts/md5srv.py b/test/hitl/scripts/md5srv.py index 43340d4a..d348b15f 100755 --- a/test/hitl/scripts/md5srv.py +++ b/test/hitl/scripts/md5srv.py @@ -23,6 +23,7 @@ def log_sensitive(s: str): class HTTPException(Exception): + def __init__(self, status_code, message, detail=None): self.status_code = status_code self.message = message @@ -81,19 +82,26 @@ def do_POST(self): def do_PUT(self): self.do(self.write_file_or_note) - def send_status(self, code: int, message: str, detail: str | dict | bytes, headers: dict = {}): + def send_status(self, + code: int, + message: str, + detail: str | dict | bytes, + headers: dict = {}): """ Send a status code with a message and optional detail, which is returned as JSON in the response body. """ self.send_response(code, message) for header in headers.items(): self.send_header(*header) - self.send_header("Content-Type", "application/json" if type(detail) is not bytes else "application/octet-stream") + self.send_header( + "Content-Type", "application/json" + if type(detail) is not bytes else "application/octet-stream") self.end_headers() detail = detail or message if detail is not None: if type(detail) is bytes: self.wfile.write(detail) else: - key = "err" if code > 400 else "text" if not type(detail) is dict else None + key = "err" if code > 400 else "text" if not type( + detail) is dict else None json_str = json.dumps(detail) reply_body = json_str if key is None else f'{{"{key}":{json_str},"code":{code}}}' reply_body = reply_body + '\n' @@ -115,12 +123,15 @@ def validate_token(self): requested = self.headers['X-Access-Token'] or '' token = self.args.token or '' if token and requested != token: - raise HTTPException(403, "Forbidden") if requested else HTTPException(401, "Unauthorized") + raise HTTPException(403, + "Forbidden") if requested else HTTPException( + 401, "Unauthorized") def get_md5(self): dirname = self.validate_url_path(self.url.path) if not os.path.exists(dirname): - raise HTTPException(404, "Not Found", f"Directory {self.url.path} not found.") + raise HTTPException(404, "Not Found", + f"Directory {self.url.path} not found.") chunk = self.query_data.get('chunk') headers = {} @@ -144,7 +155,9 @@ def md5_for_directory(self, dirname): abs_filename = os.path.join(dirname, filename) self.validate_path(abs_filename) if not os.path.isfile(abs_filename): - raise HTTPException(403, "Not a file.", f"{os.path.join(self.url.path,filename)} is not a file") + raise HTTPException( + 403, "Not a file.", + f"{os.path.join(self.url.path,filename)} is not a file") with open(abs_filename, 'rb') as file: data = file.read() @@ -171,14 +184,15 @@ def write_file_or_note(self): def write_note(self): content_type = self.headers['Content-Type'] if content_type != 'application/json': - raise HTTPException(400, f"Unsupported content type: {content_type}") + raise HTTPException(400, + f"Unsupported content type: {content_type}") data = self.post_data try: event = json.loads(data) except json.decoder.JSONDecodeError as e: print(f"JSON decode error: {data} {e}") raise e - body = event['body'] # non-optional keys + body = event['body'] # non-optional keys name = body['name'] length = body['length'] md5 = body['md5'] @@ -186,8 +200,10 @@ def write_note(self): payload = base64.b64decode(event['payload']) payload_length = event.get('payload_length') if payload_length is not None and payload_length != length: - raise HTTPException(400, "Payload length mismatch", f"payload_length {payload_length}!=length {length}") - chunk = body.get('chunk') # optional + raise HTTPException( + 400, "Payload length mismatch", + f"payload_length {payload_length}!=length {length}") + chunk = body.get('chunk') # optional self._write_file(name, chunk, length, payload, md5) def write_file(self): @@ -195,7 +211,8 @@ def write_file(self): if not length: raise HTTPException(400, "Request body is empty.") chunk = self.query_data.get("chunk", None) - return self._write_file(self.url.path, chunk, length, self.post_data, None) + return self._write_file(self.url.path, chunk, length, self.post_data, + None) def _read_file(self, path, chunk): filename = self._chunk_file(path, chunk) @@ -207,7 +224,9 @@ def _read_file(self, path, chunk): def _chunk_file(self, dirname, chunk): chunk_index = None if chunk is None else int(chunk) - filename = os.path.join(dirname, f"payload{chunk_index:05d}.bin" if chunk_index is not None else "payload.bin") + filename = os.path.join( + dirname, f"payload{chunk_index:05d}.bin" + if chunk_index is not None else "payload.bin") self.validate_path(filename) return filename @@ -222,19 +241,24 @@ def _write_file(self, path, chunk, length, data, md5): filename = self._chunk_file(dirname, chunk) if os.path.exists(filename): - raise HTTPException(409, "Conflict", f"Chunk {chunk} already exists for {self.url.path}" - if chunk else f"File {self.url.path} already exists.") + raise HTTPException( + 409, "Conflict", + f"Chunk {chunk} already exists for {self.url.path}" + if chunk else f"File {self.url.path} already exists.") if len(data) != length: - raise HTTPException(400, "Invalid content length.", - f"Payload length does not equal given length {len(data)}!={length}") + raise HTTPException( + 400, "Invalid content length.", + f"Payload length does not equal given length {len(data)}!={length}" + ) if self.args.save: with open(filename, 'wb') as output_file: output_file.write(data) md5str = hashlib.md5(data).hexdigest() if md5 and md5str != md5: - raise HTTPException(400, f"MD5 mismatch. actual {md5str}!=expected {md5}") + raise HTTPException( + 400, f"MD5 mismatch. actual {md5str}!=expected {md5}") response = {"md5": md5str, "length": length} content_type = self.headers['Content-Type'] if content_type is not None: @@ -242,7 +266,8 @@ def _write_file(self, path, chunk, length, data, md5): self.send_status(200, "Ok", response) def validate_url_path(self, urlpath): - return self.validate_path(os.path.join(self.args.directory, urlpath.lstrip('/'))) + return self.validate_path( + os.path.join(self.args.directory, urlpath.lstrip('/'))) def validate_path(self, path): abs_path = os.path.realpath(os.path.abspath(path)) @@ -252,14 +277,12 @@ def validate_path(self, path): return abs_path def dump_request(self) -> str: - return json.dumps( - { - "path": self.url.path, - "query_data": self.query_data, - "post_data": self.post_data.decode("utf-8"), - "form_data": self.form_data - } - ) + return json.dumps({ + "path": self.url.path, + "query_data": self.query_data, + "post_data": self.post_data.decode("utf-8"), + "form_data": self.form_data + }) def main(args): @@ -279,25 +302,26 @@ def build_request_handler(*request_args): if __name__ == "__main__": - port = os.environ.get("MD5SRV_PORT") address = os.environ.get("MD5SRV_ADDRESS") token = os.environ.get("MD5SRV_TOKEN") parser = argparse.ArgumentParser( - description='Run a simple webserver to save and validate web.post requests.') - parser.add_argument( - '--port', - default=port or "8080", - required=False, - help='The TCP port to bind to.') + description= + 'Run a simple webserver to save and validate web.post requests.') + parser.add_argument('--port', + default="8080", + required=False, + help='The TCP port to bind to.') parser.add_argument('--address', default=address or "0.0.0.0", required=False, help='The IP address to bind to.') - parser.add_argument('--dir', - dest="directory", - required=False, - help='The working directory. All files are stored and retrieved from here.') + parser.add_argument( + '--dir', + dest="directory", + required=False, + help= + 'The working directory. All files are stored and retrieved from here.') # parser.add_argument('--timeout', # default=5, # required=False, @@ -307,12 +331,14 @@ def build_request_handler(*request_args): required=False, action='store_true', help='Save the content received to the filesystem.') - parser.add_argument('--token', - default=None, - required=False, - help='The authorization token required in X-Access-Token header.') + parser.add_argument( + '--token', + default=None, + required=False, + help='The authorization token required in X-Access-Token header.') args = parser.parse_args() args.token = args.token or token - args.directory = os.getcwd() if not args.directory else os.path.abspath(args.directory) + args.directory = os.getcwd() if not args.directory else os.path.abspath( + args.directory) main(args)