Merge pull request #317 from CybercentreCanada/filestore-tests

Testing and bugfixes (dev)

assemblyline/common/net_static.py

@@ -1,5 +1,5 @@
 # from https://data.iana.org/TLD/tlds-alpha-by-domain.txt
-# Version 2018051800, Last Updated Fri May 18 07:07:02 2018 UTC
+# Version 2021073000, Last Updated Fri Jul 30 07:07:01 2021 UTC
 TLDS_ALPHA_BY_DOMAIN = [
     "AAA",
     "AARP",
@@ -51,6 +51,7 @@
     "ALSACE",
     "ALSTOM",
     "AM",
+    "AMAZON",
     "AMERICANEXPRESS",
     "AMERICANFAMILY",
     "AMEX",
@@ -300,6 +301,7 @@
     "COUPON",
     "COUPONS",
     "COURSES",
+    "CPA",
     "CR",
     "CREDIT",
     "CREDITCARD",
@@ -489,6 +491,7 @@
     "GAMES",
     "GAP",
     "GARDEN",
+    "GAY",
     "GB",
     "GBIZ",
     "GD",
@@ -748,6 +751,7 @@
     "LIXIL",
     "LK",
     "LLC",
+    "LLP",
     "LOAN",
     "LOANS",
     "LOCKER",
@@ -1156,6 +1160,7 @@
     "SONG",
     "SONY",
     "SOY",
+    "SPA",
     "SPACE",
     "SPIEGEL",
     "SPORT",
@@ -1164,6 +1169,7 @@
     "SR",
     "SRL",
     "SRT",
+    "SS",
     "ST",
     "STADA",
     "STAPLES",
@@ -1383,6 +1389,7 @@
     "XN--45BR5CYL",
     "XN--45BRJ9C",
     "XN--45Q11C",
+    "XN--4DBRK0CE",
     "XN--4GBRIM",
     "XN--54B7FTA0CC",
     "XN--55QW42G",
@@ -1408,6 +1415,7 @@
     "XN--C1AVG",
     "XN--C2BR7G",
     "XN--CCK2B3B",
+    "XN--CCKWCXETD",
     "XN--CG4BKI",
     "XN--CLCHC0EA0B2G2A9GCD",
     "XN--CZR694B",
@@ -1444,6 +1452,7 @@
     "XN--J1AEF",
     "XN--J1AMH",
     "XN--J6W193G",
+    "XN--JLQ480N2RG",
     "XN--JLQ61U9W7B",
     "XN--JVR189M",
     "XN--KCRX77D1X4A",
@@ -1460,13 +1469,15 @@
     "XN--MGBAAKC7DVF",
     "XN--MGBAAM7A8H",
     "XN--MGBAB2BD",
+    "XN--MGBAH1A3HJKRD",
     "XN--MGBAI9AZGQP6J",
     "XN--MGBAYH7GPA",
     "XN--MGBB9FBPOB",
     "XN--MGBBH1A",
     "XN--MGBBH1A71E",
     "XN--MGBC0A9AZCG",
     "XN--MGBCA7DZDO",
+    "XN--MGBCPQ6GPA1A",
     "XN--MGBERP4A5D4AR",
     "XN--MGBGU82A",
     "XN--MGBI4ECEXP",
@@ -1492,8 +1503,10 @@
     "XN--PBT977C",
     "XN--PGBS0DH",
     "XN--PSSY2U",
+    "XN--Q7CE6A",
     "XN--Q9JYB4C",
     "XN--QCKA1PMC",
+    "XN--QXA6A",
     "XN--QXAM",
     "XN--RHQV96G",
     "XN--ROVU88B",

assemblyline/filestore/transport/base.py

@@ -72,19 +72,6 @@ def upload(self, src_path: str, dst_path: str):
         """
         raise TransportException("Not Implemented")
 
-    def upload_batch(self, local_remote_tuples):
-        """
-        Upload multiple files specified by list of (local, remote) tuples.
-        Transports that can optimize batch file transfers should write a custom upload_batch.
-        """
-        failed_tuples = []
-        for (src_path, dst_path) in local_remote_tuples:
-            try:
-                self.upload(src_path, dst_path)
-            except Exception as e:
-                failed_tuples.append((src_path, dst_path, str(e)))
-        return failed_tuples
-
     # Buffer based functions
     def get(self, path: str) -> bytes:
         """

assemblyline/filestore/transport/ftp.py

@@ -194,10 +194,6 @@ def upload(self, src_path: str, dst_path: str):
         self.ftp.rename(temppath, finalpath)
         assert (self.exists(dst_path))
 
-    @reconnect_retry_on_fail
-    def upload_batch(self, local_remote_tuples):
-        return super(TransportFTP, self).upload_batch(local_remote_tuples)
-
     # Buffer based functions
     @reconnect_retry_on_fail
     def get(self, path) -> bytes:

assemblyline/filestore/transport/local.py

@@ -43,13 +43,13 @@ def exists(self, path):
         path = self.normalize(path)
         return os.path.exists(path)
 
-    def getmtime(self, path):
-        path = self.normalize(path)
-
-        try:
-            return os.path.getmtime(path)
-        except OSError:
-            return 0
+    # def getmtime(self, path):
+    #     path = self.normalize(path)
+    #
+    #     try:
+    #         return os.path.getmtime(path)
+    #     except OSError:
+    #         return 0
 
     def makedirs(self, path):
         path = self.normalize(path)

assemblyline/filestore/transport/sftp.py

@@ -134,10 +134,6 @@ def upload(self, src_path, dst_path):
         self.sftp.rename(temppath, finalpath)
         assert (self.exists(dst_path))
 
-    @reconnect_retry_on_fail
-    def upload_batch(self, local_remote_tuples):
-        return super(TransportSFTP, self).upload_batch(local_remote_tuples)
-
     # Buffer based functions
     @reconnect_retry_on_fail
     def get(self, path):

assemblyline/remote/datatypes/exporting_counter.py

@@ -164,12 +164,12 @@ def export_metrics_once(name, schema, metrics, host=None, counter_type=None, con
         if metric in counter_schema:
             counts[metric] += value
         elif metric in timer_schema:
-            counts[name + ".c"] += 1
-            counts[name + ".t"] += value
+            counts[metric + ".c"] += 1
+            counts[metric + ".t"] += value
         else:
             raise ValueError(f"{metric} is not an accepted counter")
 
-    counts['type'] = counter_type
+    counts['type'] = counter_type or name
     counts['name'] = name
     counts['host'] = host
 

pipelines/config.yml

@@ -11,5 +11,7 @@ core:
     persistent:
       host: localhost
       port: 6379
+  metrics:
+    export_interval: 1
 datastore:
   hosts: ["localhost:9200"]

setup.py

@@ -95,6 +95,7 @@
             'pytest-cov',
             'retrying',
             'pytest-mock',
+            'pyftpdlib',
         ]
     },
     package_data={

test/key.pem

@@ -0,0 +1,31 @@
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANALE6wPhlNJ+aLt
+AX4HL43lI16CDs/9MOlX2a8/4Bp+cXv7sdPPJ6MTbqvrhpnA14cVgHl1LRu2OGxm
+etV7MwRQbJZ5NtVMLjKrU9wcYod7B9ZKUel4Jgbjk0CtO2txYbi9gNOkWaxwcmNF
+BfKIsaGOQkB5vzK7mnf7dO9ALYE/AgMBAAECgYA9rrsTbbru4OUCGHEz05+W25RE
+Bh2sLy6cUK67Fh403L56+yI7YZUn9a//iyJqXdHJPGfOGx7Xs4xBH5VVzGRQXo7i
+t6HOsB/oDwOTt5JKImJ+0JY6cn2MhWbsNY+oPJppe7CRoUKURHZY61+WDi8zT1mR
+Qrfo3jDgg6cX3zZwcQJBAP/wy8S2LN24okziCfssyF3WHb1Pkvc0/ITQle2+gQTZ
+YyF1H+2xGJOF3/wi19sE2bQuXigg0Ou+lyR1z3cFnRcCQQDQF2+AjB2mFrPqZ9Md
+qnP4GUrKT574CsHy5G0OniHSFrauKRCBjEwm4RXRm9lfs/RWA81/s7RTFWCJUq9m
+hmYZAkEAtK2PnAGjMK7b3Hyh4TAfDqdN/UvEi0FbloMNpHUc7YhtQ7xEWu7vU41p
+rrwGN/Z3nYwyKg/ojNPSLQoB+Jr85wJAZjPcc8pdlYF5BBvSOLPLGYNylELe1PyT
+nXRLi+5mtgSp3IgWr0n07POH/9cHwFVmIAjmGV5tppDNRSTzOOuxoQJBAKBKAMJm
+a64VkrqR1xkm9PYeUbNV8X28USnsPkw4I2shHHmwMwj+Vyo10IC0XtDto7ZrVAM9
+v5XYnKwRopUnj9c=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICWjCCAcOgAwIBAgIUQJONlWz9w+fbJgb/CmPv7Mj5wT0wDQYJKoZIhvcNAQEL
+BQAwPzELMAkGA1UEBhMCQ0ExEDAOBgNVBAgMB09udGFyaW8xDzANBgNVBAcMBk90
+dGF3YTENMAsGA1UECgwEQ0NDUzAeFw0yMTA3MjkxNzU1MzBaFw0zMTA3MjcxNzU1
+MzBaMD8xCzAJBgNVBAYTAkNBMRAwDgYDVQQIDAdPbnRhcmlvMQ8wDQYDVQQHDAZP
+dHRhd2ExDTALBgNVBAoMBENDQ1MwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGB
+ANALE6wPhlNJ+aLtAX4HL43lI16CDs/9MOlX2a8/4Bp+cXv7sdPPJ6MTbqvrhpnA
+14cVgHl1LRu2OGxmetV7MwRQbJZ5NtVMLjKrU9wcYod7B9ZKUel4Jgbjk0CtO2tx
+Ybi9gNOkWaxwcmNFBfKIsaGOQkB5vzK7mnf7dO9ALYE/AgMBAAGjUzBRMB0GA1Ud
+DgQWBBTpHO34t3bWXUt0+eR9M/7KiGnEnzAfBgNVHSMEGDAWgBTpHO34t3bWXUt0
++eR9M/7KiGnEnzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBAChY
+fK7P81aqFQeWimgKD1AE/uzVToHCEcOryUl7VrQkHjToFyzeuXcUF/+n4pjyss8r
+mLmZolYrwuQ95UpEsNc0j/uVODFPxztjQYwi25UZS4YUSCxgufulanuaWIm4TdEs
+Mxt9/sQFrE0FZ6xivB27BiKEqmP+Q8g7yeZYOS4w
+-----END CERTIFICATE-----

test/requirements.txt

@@ -2,4 +2,6 @@ pytest
 pytest-cov
 retrying
 codecov
-pytest-mock
+pytest-mock
+pyftpdlib
+pyopenssl

test/test_common.py

@@ -1,6 +1,9 @@
 
 import hashlib
+import io
 import os
+import zipfile
+
 import pytest
 import random
 import re
@@ -25,6 +28,7 @@
 from assemblyline.common.isotime import now_as_iso, iso_to_epoch, epoch_to_local, local_to_epoch, epoch_to_iso, now, \
     now_as_local
 from assemblyline.common.iprange import is_ip_reserved, is_ip_private
+from assemblyline.common.memory_zip import InMemoryZip
 from assemblyline.common.security import get_random_password, get_password_hash, verify_password
 from assemblyline.common.str_utils import safe_str, translate_str
 from assemblyline.common.uid import get_random_id, get_id_from_data, TINY, SHORT, MEDIUM, LONG
@@ -432,3 +436,14 @@ def test_uid():
     for c_id in [rid, id_test, id_test_l, id_test_m, id_test_s, id_test_t]:
         for x in c_id:
             assert x in BASE62_ALPHABET
+
+
+def test_mem_zip():
+    obj = InMemoryZip()
+    obj.append('a.txt', 'abc abc')
+    obj.append('b.txt', '11111111')
+
+    buffer = io.BytesIO(obj.read())
+    reader = zipfile.ZipFile(buffer)
+    assert reader.read('a.txt') == b'abc abc'
+    assert reader.read('b.txt') == b'11111111'