[Security] remove ujson package (CVE-2022-31116, CVE-2022-31117, CVE-2021-45958)

build-scripts/ubuntu-2004/build-3rd-parties.sh

@@ -126,4 +126,3 @@ build_from_pypi semver 2.13.0
 build_from_pypi sha3
 build_from_pypi six
 build_from_pypi sortedcontainers 1.5.7
-build_from_pypi ujson 1.33

build-scripts/ubuntu-2004/prepare-package.sh

@@ -29,7 +29,6 @@ if [ "$distro_packages" = "debian-packages" ]; then
   # Update the package names to match the versions that are pre-installed on the os.
   echo -e "\nAdapt the dependencies for the Canonical archive"
   #### ToDo adjust packages for the Cannonical archive for Ubuntu 20.04 (focal)
-  # sed -i "s~ujson==1.33~ujson==1.33-1build1~" setup.py
   # sed -i "s~prompt_toolkit==0.57~prompt_toolkit==0.57-1~" setup.py
   # sed -i "s~msgpack-python==0.4.6~msgpack==0.4.6-1build1~" setup.py
 elif [ "$distro_packages" = "python-packages" ]; then

common/serializers/json_serializer.py

@@ -2,45 +2,27 @@
 
 
 import base64
+import json
 from typing import Dict
 
 from common.serializers.mapping_serializer import MappingSerializer
 
-try:
-    import ujson as json
-    from ujson import encode as uencode
 
-    # Older versions of ujson's encode do not support `sort_keys`, if that
-    # is the case default to using json
-    uencode({'xx': '123', 'aa': 90}, sort_keys=True)
+class OrderedJsonEncoder(json.JSONEncoder):
+    def __init__(self, *args, **kwargs):
+        kwargs['ensure_ascii'] = False
+        kwargs['sort_keys'] = True
+        kwargs['separators'] = (',', ':')
+        super().__init__(*args, **kwargs)
 
-    class UJsonEncoder:
-        @staticmethod
-        def encode(o):
-            if isinstance(o, (bytes, bytearray)):
-                return '"{}"'.format(base64.b64encode(o).decode("utf-8"))
-            else:
-                return uencode(o, sort_keys=True)
+    def encode(self, o):
+        if isinstance(o, (bytes, bytearray)):
+            return '"{}"'.format(base64.b64encode(o).decode("utf-8"))
+        else:
+            return super().encode(o)
 
-    JsonEncoder = UJsonEncoder()
 
-except (ImportError, TypeError):
-    import json
-
-    class OrderedJsonEncoder(json.JSONEncoder):
-        def __init__(self, *args, **kwargs):
-            kwargs['ensure_ascii'] = False
-            kwargs['sort_keys'] = True
-            kwargs['separators'] = (',', ':')
-            super().__init__(*args, **kwargs)
-
-        def encode(self, o):
-            if isinstance(o, (bytes, bytearray)):
-                return '"{}"'.format(base64.b64encode(o).decode("utf-8"))
-            else:
-                return super().encode(o)
-
-    JsonEncoder = OrderedJsonEncoder()
+JsonEncoder = OrderedJsonEncoder()
 
 
 class JsonSerializer(MappingSerializer):

common/serializers/signing_serializer.py

@@ -90,6 +90,3 @@ def serialize(self, obj, level=0, objname=None, topLevelKeysToIgnore=None,
             return res
 
         return res.encode('utf-8')
-
-        # topLevelKeysToIgnore = topLevelKeysToIgnore or []
-        # return ujson.dumps({k:obj[k] for k in obj.keys() if k not in topLevelKeysToIgnore}, sort_keys=True)

dev-setup/ubuntu/ubuntu-2004/SetupVMTest.txt

@@ -82,7 +82,6 @@
             sortedcontainers==1.5.7 \
             timeout-decorator==0.5.0 \
             toml==0.10.2 \
-            ujson==1.33 \
             wcwidth==0.2.5 \
             wheel==0.34.2 \
             zipp==1.2.0

ledger/test/helper.py

@@ -108,7 +108,6 @@ def create_ledger_rocksdb_storage(txn_serializer, hash_serializer, tempdir, init
 
 
 def create_ledger_chunked_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
-    chunk_creator = None
     db_name = 'transactions'
     if isinstance(txn_serializer, MsgPackSerializer):
         # TODO: fix chunk_creator support
@@ -119,6 +118,8 @@ def chunk_creator(name):
                                                   isLineNoKey=True,
                                                   storeContentHash=False,
                                                   ensureDurability=False)
+    else:
+        chunk_creator = None
     store = ChunkedFileStore(tempdir,
                              db_name,
                              isLineNoKey=True,

plenum/common/channel.py

@@ -156,7 +156,7 @@ def _process_sync(self, msg: Any):
         # This is done so that messages can include additional metadata
         # isinstance is not used here because it returns true for NamedTuple
         # as well.
-        if type(msg) != tuple:
+        if not isinstance(type(msg), tuple):
             msg = (msg,)
         handler = self._find_handler(msg[0])
         if handler is None:

plenum/recorder/recorder.py

@@ -1,14 +1,10 @@
 import os
 import time
+import json
 from typing import Callable
 
 from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys
 
-try:
-    import ujson as json
-except ImportError:
-    import json
-
 
 class Recorder:
     INCOMING_FLAG = 0

plenum/server/plugin/stats_consumer/plugin_firebase_stats_consumer.py

@@ -6,11 +6,11 @@
 from plenum.common.types import EVENT_PERIODIC_STATS_THROUGHPUT, \
     EVENT_NODE_STARTED, EVENT_REQ_ORDERED, EVENT_PERIODIC_STATS_LATENCIES, \
     PLUGIN_TYPE_STATS_CONSUMER, EVENT_VIEW_CHANGE, EVENT_PERIODIC_STATS_NODES, \
-    EVENT_PERIODIC_STATS_TOTAL_REQUESTS, EVENT_PERIODIC_STATS_NODE_INFO,\
+    EVENT_PERIODIC_STATS_TOTAL_REQUESTS, EVENT_PERIODIC_STATS_NODE_INFO, \
     EVENT_PERIODIC_STATS_SYSTEM_PERFORMANCE_INFO
 from stp_core.common.log import getlogger
 from plenum.config import STATS_SERVER_IP, STATS_SERVER_PORT, STATS_SERVER_MESSAGE_BUFFER_MAX_SIZE
-from plenum.server.plugin.stats_consumer.stats_publisher import StatsPublisher,\
+from plenum.server.plugin.stats_consumer.stats_publisher import StatsPublisher, \
     Topic
 from plenum.server.plugin_loader import HasDynamicallyImportedModules
 from plenum.server.stats_consumer import StatsConsumer

plenum/test/recorder/test_recorder.py

@@ -1,14 +1,10 @@
 import random
 import time
+import json
 from collections import OrderedDict
 
 from plenum.common.util import randomString
 
-try:
-    import ujson as json
-except ImportError:
-    import json
-
 import pytest
 
 from plenum.recorder.recorder import Recorder

scripts/test_zmq/test_zmq/zstack.py

@@ -1,10 +1,5 @@
 from test_zmq.authenticator import MultiZapAuthenticator
-
-try:
-    import ujson as json
-except ImportError:
-    import json
-
+import json
 import os
 import shutil
 import sys

setup.py

@@ -131,8 +131,6 @@ def run(self):
                         'six',
                         ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118)
                         'sortedcontainers==1.5.7',
-                        ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118)
-                        'ujson==1.33',
                         ],
 
     setup_requires=['pytest-runner==5.3.0'],

stp_zmq/zstack.py

@@ -1,4 +1,5 @@
 import inspect
+import json
 
 from plenum.common.constants import OP_FIELD_NAME, BATCH
 from plenum.common.metrics_collector import NullMetricsCollector
@@ -8,10 +9,6 @@
 from stp_core.common.constants import CONNECTION_PREFIX, ZMQ_NETWORK_PROTOCOL
 from stp_zmq.client_message_provider import ClientMessageProvider
 
-try:
-    import ujson as json
-except ImportError:
-    import json
 
 import os
 import shutil