Merge branch 'master' into add_tag_efficiency

.gitignore

@@ -83,6 +83,3 @@ venv.bak/
 
 # Cython debug symbols
 cython_debug/
-
-# vscode workspace
-.vscode/

.vscode/settings.json

@@ -0,0 +1,32 @@
+{
+    "editor.codeActionsOnSave": {
+        "source.organizeImports": true,
+    },
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        120
+    ],
+    "editor.tabSize": 4,
+    "editor.wordWrap": "wordWrapColumn",
+    "editor.wordWrapColumn": 120,
+    "files.insertFinalNewline": true,
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "isort.args": [
+        "-l",
+        "120",
+        "--profile=black",
+        // "--src=${workspaceFolder}"
+    ],
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "black-formatter.args": [
+        "--line-length=120"
+    ],
+    "flake8.args": [
+        "--max-line-length=120",
+        //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373
+        "--ignore=E203,W503"
+    ],
+}

CONTRIBUTING.md

@@ -5,7 +5,7 @@ This guide covers the basics of how to contribute to the Assemblyline project.
 Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
 
 ## Tell us want you want to build/fix
-Before you start coding anything you should connect with the [Assemblyline community](https://groups.google.com/d/forum/cse-cst-assemblyline) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision off the system.
+Before you start coding anything you should connect with the Assemblyline community via the [Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the [central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
 
 ## Git workflow
 
@@ -18,8 +18,8 @@ Before you start coding anything you should connect with the [Assemblyline commu
 #### Transfer your service repo
 If you've worked on a new service that you want to be included in the default service selection you'll have to transfer the repo into our control.
 
-#### You are not allow to merge:
+#### You are not allowed to merge:
 
 Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories.
 
-We check for new pull requests every day and will merge them in once they have been approved by someone in our team.
+We check for new pull requests every day and will merge them in once they have been approved by someone in our team.

assemblyline_extra_feature_service/extra_feature.py

@@ -1,6 +1,7 @@
 from pprint import pformat
+
 from assemblyline_v4_service.common.base import ServiceBase
-from assemblyline_v4_service.common.result import Result, ResultSection, BODY_FORMAT
+from assemblyline_v4_service.common.result import BODY_FORMAT, Result, ResultSection
 
 
 class ExtraFeature(ServiceBase):

assemblyline_extra_feature_service/test/test_extrafeatures.py

@@ -1,8 +1,8 @@
 import os
-import pytest
 
+import pytest
 from assemblyline.common.importing import load_module_by_path
-from assemblyline_v4_service.testing.helper import TestHelper
+from assemblyline_service_utilities.testing.helper import TestHelper
 
 # Force manifest location
 os.environ['SERVICE_MANIFEST_PATH'] = os.path.join(os.path.dirname(__file__), "..", "service_manifest.yml")

assemblyline_result_sample_service/result_sample.py

@@ -4,18 +4,38 @@
 import tempfile
 
 from assemblyline.common import forge
-from assemblyline.common.attack_map import software_map, attack_map, group_map, revoke_map
+from assemblyline.common.attack_map import attack_map, group_map, revoke_map, software_map
 from assemblyline.common.dict_utils import flatten
 from assemblyline.common.hexdump import hexdump
-from assemblyline_v4_service.common.base import ServiceBase
-from assemblyline_v4_service.common.result import DividerSectionBody, GraphSectionBody, KVSectionBody, ProcessItem, \
-    ResultGraphSection, ResultImageSection, ResultJSONSection, ResultKeyValueSection, ResultMemoryDumpSection, \
-    ResultMultiSection, ResultProcessTreeSection, ResultSection, BODY_FORMAT, Heuristic, ResultTextSection, \
-    ResultURLSection, ResultTableSection, TableRow, TextSectionBody, Result, ResultOrderedKeyValueSection, \
-    ResultTimelineSection
 
 # DO NOT IMPORT IN YOUR SERVICE. These are just for creating randomized results.
-from assemblyline.odm.randomizer import get_random_phrase, get_random_ip, get_random_host, get_random_tags
+from assemblyline.odm.randomizer import get_random_host, get_random_ip, get_random_phrase, get_random_tags
+from assemblyline_v4_service.common.base import ServiceBase
+from assemblyline_v4_service.common.result import (
+    BODY_FORMAT,
+    DividerSectionBody,
+    GraphSectionBody,
+    Heuristic,
+    KVSectionBody,
+    ProcessItem,
+    Result,
+    ResultGraphSection,
+    ResultImageSection,
+    ResultJSONSection,
+    ResultKeyValueSection,
+    ResultMemoryDumpSection,
+    ResultMultiSection,
+    ResultOrderedKeyValueSection,
+    ResultProcessTreeSection,
+    ResultSection,
+    ResultTableSection,
+    ResultTextSection,
+    ResultTimelineSection,
+    ResultURLSection,
+    TableRow,
+    TextSectionBody,
+)
+
 # DO NOT LIST BODY FORMATS LIKE THIS. This is again for the data randomizer.
 FORMAT_LIST = [BODY_FORMAT.TEXT, BODY_FORMAT.MEMORY_DUMP]
 
@@ -323,6 +343,10 @@ def execute(self, request):
                         "an_int": 103,
                     },
                 }}))
+            # Optional: Set custom column ordering for table.
+            # Column order is automatically inferred/updated on `ResultTableSection.add_row()`.
+            # Passing an empty list ([]) to `ResultTableSection.set_column_order()` will display the columns in alphabetical order
+            table_section.set_column_order(['a_str', 'a_bool', 'an_int', 'extra_column_there', 'nested_key_value_pair'])
             result.add_section(table_section)
 
             # ==================================================================

assemblyline_result_sample_service/test/test_resultsample.py

@@ -1,8 +1,8 @@
 import os
-import pytest
 
+import pytest
 from assemblyline.common.importing import load_module_by_path
-from assemblyline_v4_service.testing.helper import TestHelper
+from assemblyline_service_utilities.testing.helper import TestHelper
 
 # Force manifest location
 os.environ['SERVICE_MANIFEST_PATH'] = os.path.join(os.path.dirname(__file__), "..", "service_manifest.yml")

assemblyline_v4_service/common/api.py

@@ -1,10 +1,11 @@
 import os
-import requests
 import time
 import traceback
+from io import StringIO
 
+import requests
 from assemblyline_core.safelist_client import SafelistClient
-from io import StringIO
+
 DEFAULT_SERVICE_SERVER = "http://localhost:5003"
 DEFAULT_AUTH_KEY = "ThisIsARandomAuthKey...ChangeMe!"
 DEVELOPMENT_MODE = False