Merge pull request #50 from CybercentreCanada/update/tests

Update/tests [dev]

.vscode/settings.json

@@ -0,0 +1,38 @@
+{
+    "editor.codeActionsOnSave": {
+        "source.organizeImports": true,
+    },
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        120
+    ],
+    "editor.tabSize": 4,
+    "editor.wordWrap": "wordWrapColumn",
+    "editor.wordWrapColumn": 120,
+    "files.insertFinalNewline": true,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "isort.args": [
+        "-l",
+        "120",
+        "--profile=black",
+        // "--src=${workspaceFolder}"
+    ],
+    "python.formatting.autopep8Args": [
+        "--max-line-length",
+        "120",
+        "--experimental"
+    ],
+    "python.formatting.provider": "autopep8",
+    "python.formatting.blackArgs": [
+        "--line-length=120"
+    ],
+    "python.linting.enabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.flake8Args": [
+        "--max-line-length=120",
+        //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373
+        "--ignore=E203,W503"
+    ],
+    "python.linting.pylintEnabled": false,
+}

CONTRIBUTING.md

@@ -5,7 +5,7 @@ This guide covers the basics of how to contribute to the Assemblyline project.
 Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
 
 ## Tell us want you want to build/fix
-Before you start coding anything you should connect with the [Assemblyline community](https://groups.google.com/d/forum/cse-cst-assemblyline) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision off the system.
+Before you start coding anything you should connect with the Assemblyline community via the [Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the [central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
 
 ## Git workflow
 
@@ -18,8 +18,8 @@ Before you start coding anything you should connect with the [Assemblyline commu
 #### Transfer your service repo
 If you've worked on a new service that you want to be included in the default service selection you'll have to transfer the repo into our control.
 
-#### You are not allow to merge:
+#### You are not allowed to merge:
 
 Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories.
 
-We check for new pull requests every day and will merge them in once they have been approved by someone in our team.
+We check for new pull requests every day and will merge them in once they have been approved by someone in our team.

assemblyline_service_client/task_handler.py

@@ -8,11 +8,10 @@
 import tempfile
 import time
 from json import JSONDecodeError
-from typing import Optional
+from typing import Any, Optional
 
 import requests
 import yaml
-
 from assemblyline.common.digests import get_sha256_for_file
 from assemblyline.common.str_utils import StringTable
 from assemblyline.odm.messages.task import Task as ServiceTask
@@ -79,7 +78,7 @@ def __init__(self, shutdown_timeout=SHUTDOWN_SECONDS_LIMIT, api_host=None, api_k
 
         self.log.setLevel(LOG_LEVEL)
 
-    def _path(self, prefix, *args):
+    def _path(self, prefix, *args) -> str:
         """
         Calculate the API path using the prefix as shown:
             /api/v1/<prefix>/[arg1/[arg2/[...]]][?k1=v1[...]]
@@ -168,6 +167,7 @@ def update_service_manifest(self, data):
     def cleanup_working_directory(self, folder_path):
         for file in os.listdir(folder_path):
             file_path = os.path.join(folder_path, file)
+            # if file_path not in [self.task_fifo_path, self.done_fifo_path]:
             if file_path != self.task_fifo_path or file_path != self.done_fifo_path:
                 try:
                     if os.path.isfile(file_path):
@@ -177,7 +177,7 @@ def cleanup_working_directory(self, folder_path):
                 except Exception:
                     pass
 
-    def request_with_retries(self, method: str, url: str, get_api_response=True, max_retry=None, **kwargs):
+    def request_with_retries(self, method: str, url: str, get_api_response=True, max_retry=None, **kwargs) -> Optional[Any]:
         if 'headers' in kwargs:
             self.session.headers.update(kwargs['headers'])
             kwargs.pop('headers')
@@ -280,7 +280,7 @@ def try_run(self):
                     self.task_fifo = None
                     self.done_fifo = None
                     if self.running:
-                        self.log.error(f"[{self.task.sid}] One of the pipe to the service is broken. "
+                        self.log.error(f"[{self.task.sid}] One of the pipes to the service is broken. "
                                        f"Marking task as failed recoverable...")
                     self.status = STATUSES.ERROR_FOUND
 
@@ -382,7 +382,7 @@ def download_file(self, sha256, sid) -> Optional[str]:
 
                 # If the file retrieved is different from what we requested, report the error
                 if received_file_sha256 != sha256:
-                    self.log.error(f"[{sid}] Downloaded ({received_file_sha256}) doesn't match requested ({sha256})"
+                    self.log.error(f"[{sid}] Downloaded ({received_file_sha256}) doesn't match requested ({sha256}). "
                                    "Reporting task error to service server.")
                     self.status = STATUSES.ERROR_FOUND
                     return

pipelines/azure-test.yaml

@@ -31,7 +31,7 @@ jobs:
           sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
           sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U pip cython setuptools
           sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
-          sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt
+          sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r tests/requirements.txt
         displayName: Setup environment
       - script: python -m pytest -rsx -vv
         displayName: Test

tests/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+requests-mock