[Dependencies] Remove pickle5 backport (ray-project#40338)

Now that we remove Python 3.7 support, we don't need the pickle5 backport any more :)

.gitignore

@@ -2,7 +2,6 @@
 *test-output.xml
 /bazel-*
 /python/ray/core/**/*
-/python/ray/pickle5_files/
 /python/ray/thirdparty_files/
 /python/ray/_private/runtime_env/agent/thirdparty_files/
 /python/ray/pyarrow_files/

ci/ci.sh

@@ -208,9 +208,7 @@ prepare_docker() {
 
 # For running Serve tests on Windows.
 test_serve() {
-  local pathsep=":" args=()
   if [ "${OSTYPE}" = msys ]; then
-    pathsep=";"
     args+=(
       python/ray/serve/...
       -python/ray/serve/tests:test_cross_language # Ray java not built on Windows yet.
@@ -229,14 +227,9 @@ test_serve() {
     BUILDKITE_PARALLEL_JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-'1'}
     test_shard_selection=$(python ./ci/ray_ci/bazel_sharding.py --exclude_manual --index "${BUILDKITE_PARALLEL_JOB}" --count "${BUILDKITE_PARALLEL_JOB_COUNT}" "${args[@]}")
 
-    # TODO(mehrdadn): We set PYTHONPATH here to let Python find our pickle5 under pip install -e.
-    # It's unclear to me if this should be necessary, but this is to make tests run for now.
-    # Check why this issue doesn't arise on Linux/Mac.
-    # Ideally importing ray.cloudpickle should import pickle5 automatically.
     # shellcheck disable=SC2046,SC2086
     bazel test --config=ci \
       --build_tests_only $(./ci/run/bazel_export_options) \
-      --test_env=PYTHONPATH="${PYTHONPATH-}${pathsep}${WORKSPACE_DIR}/python/ray/pickle5_files" \
       --test_env=CI="1" \
       --test_env=RAY_CI_POST_WHEEL_TESTS="1" \
       --test_env=USERPROFILE="${USERPROFILE}" \
@@ -248,9 +241,7 @@ test_serve() {
 
 # For running Python tests on Windows (excluding Serve).
 test_python() {
-  local pathsep=":" args=()
   if [ "${OSTYPE}" = msys ]; then
-    pathsep=";"
     args+=(
       python/ray/tests/...
       python/ray/train:test_windows
@@ -289,14 +280,9 @@ test_python() {
     BUILDKITE_PARALLEL_JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-'1'}
     test_shard_selection=$(python ./ci/ray_ci/bazel_sharding.py --exclude_manual --index "${BUILDKITE_PARALLEL_JOB}" --count "${BUILDKITE_PARALLEL_JOB_COUNT}" "${args[@]}")
 
-    # TODO(mehrdadn): We set PYTHONPATH here to let Python find our pickle5 under pip install -e.
-    # It's unclear to me if this should be necessary, but this is to make tests run for now.
-    # Check why this issue doesn't arise on Linux/Mac.
-    # Ideally importing ray.cloudpickle should import pickle5 automatically.
     # shellcheck disable=SC2046,SC2086
     bazel test --config=ci \
       --build_tests_only $(./ci/run/bazel_export_options) \
-      --test_env=PYTHONPATH="${PYTHONPATH-}${pathsep}${WORKSPACE_DIR}/python/ray/pickle5_files" \
       --test_env=CI="1" \
       --test_env=RAY_CI_POST_WHEEL_TESTS="1" \
       --test_env=USERPROFILE="${USERPROFILE}" \

ci/env/install-minimal.sh

@@ -32,7 +32,6 @@ MINIMAL_INSTALL=1 PYTHON=${PYTHON_VERSION} "${WORKSPACE_DIR}/ci/env/install-depe
 
 # Re-install Ray wheels
 rm -rf "${WORKSPACE_DIR}/python/ray/thirdparty_files"
-rm -rf "${WORKSPACE_DIR}/python/ray/pickle5_files"
 eval "${WORKSPACE_DIR}/ci/ci.sh build"
 
 # Install test requirements

python/ray/__init__.py

@@ -44,19 +44,6 @@ def _configure_system():
                 "previous versions may leak memory."
             )
 
-    # MUST add pickle5 to the import path because it will be imported by some
-    # raylet modules.
-    #
-    # When running Python version < 3.8, Ray needs to use pickle5 instead of
-    # Python's built-in pickle. Add the directory containing pickle5 to the
-    # Python path so that we find the pickle5 version packaged with Ray and
-    # not a pre-existing pickle5.
-    if sys.version_info < (3, 8):
-        pickle5_path = os.path.join(
-            os.path.abspath(os.path.dirname(__file__)), "pickle5_files"
-        )
-        sys.path.insert(0, pickle5_path)
-
     # Importing psutil & setproctitle. Must be before ray._raylet is
     # initialized.
     thirdparty_files = os.path.join(

python/ray/dag/py_obj_scanner.py

@@ -1,15 +1,7 @@
 import io
-import sys
 from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union
 
-# For python < 3.8 we need to explicitly use pickle5 to support protocol 5
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle  # noqa: F401
-    except ImportError:
-        import pickle  # noqa: F401
-else:
-    import pickle  # noqa: F401
+import pickle  # noqa: F401
 
 import ray
 from ray.dag.base import DAGNodeBase

python/ray/util/client/client_pickler.py

@@ -22,7 +22,6 @@
 """
 
 import io
-import sys
 
 from typing import NamedTuple
 from typing import Any
@@ -41,13 +40,7 @@
 from ray.util.client.common import InProgressSentinel
 import ray.core.generated.ray_client_pb2 as ray_client_pb2
 
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle  # noqa: F401
-    except ImportError:
-        import pickle  # noqa: F401
-else:
-    import pickle  # noqa: F401
+import pickle  # noqa: F401
 
 
 # NOTE(barakmich): These PickleStubs are really close to

python/requirements/ml/data-test-requirements.txt

@@ -1,7 +1,6 @@
 # Used by CI for datasets tests.
 # https://github.com/ray-project/ray/pull/29448#discussion_r1006256498
 
-pickle5; python_version < '3.8'
 python-snappy
 tensorflow-datasets
 datasets

python/setup.py

@@ -1,6 +1,5 @@
 import argparse
 import errno
-import glob
 import io
 import logging
 import os
@@ -10,13 +9,10 @@
 import shutil
 import subprocess
 import sys
-import tarfile
-import tempfile
 import urllib.error
 import urllib.parse
 import urllib.request
 import warnings
-import zipfile
 from enum import Enum
 from itertools import chain
 
@@ -41,14 +37,12 @@
 BAZEL_ARGS = os.getenv("BAZEL_ARGS")
 BAZEL_LIMIT_CPUS = os.getenv("BAZEL_LIMIT_CPUS")
 
-PICKLE5_SUBDIR = os.path.join("ray", "pickle5_files")
 THIRDPARTY_SUBDIR = os.path.join("ray", "thirdparty_files")
 RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR = os.path.join(
     "ray", "_private", "runtime_env", "agent", "thirdparty_files"
 )
 
 CLEANABLE_SUBDIRS = [
-    PICKLE5_SUBDIR,
     THIRDPARTY_SUBDIR,
     RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR,
 ]
@@ -65,11 +59,6 @@
 # https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
 pyd_suffix = ".pyd" if sys.platform == "win32" else ".so"
 
-pickle5_url = (
-    "https://github.com/pitrou/pickle5-backport/archive/"
-    "e6117502435aba2901585cc6c692fb9582545f08.tar.gz"
-)
-
 
 def find_version(*filepath):
     # Extract version information from filepath
@@ -409,30 +398,6 @@ def download(url):
     return result
 
 
-# Installs pickle5-backport into the local subdirectory.
-def download_pickle5(pickle5_dir):
-    pickle5_file = urllib.parse.unquote(urllib.parse.urlparse(pickle5_url).path)
-    pickle5_name = re.sub("\\.tar\\.gz$", ".tgz", pickle5_file, flags=re.I)
-    url_path_parts = os.path.splitext(pickle5_name)[0].split("/")
-    (project, commit) = (url_path_parts[2], url_path_parts[4])
-    pickle5_archive = download(pickle5_url)
-    with tempfile.TemporaryDirectory() as work_dir:
-        tf = tarfile.open(None, "r", io.BytesIO(pickle5_archive))
-        try:
-            tf.extractall(work_dir)
-        finally:
-            tf.close()
-        src_dir = os.path.join(work_dir, project + "-" + commit)
-        args = [sys.executable, "setup.py", "-q", "bdist_wheel"]
-        subprocess.check_call(args, cwd=src_dir)
-        for wheel in glob.glob(os.path.join(src_dir, "dist", "*.whl")):
-            wzf = zipfile.ZipFile(wheel, "r")
-            try:
-                wzf.extractall(pickle5_dir)
-            finally:
-                wzf.close()
-
-
 def patch_isdir():
     """
     Python on Windows is having hard times at telling if a symlink is
@@ -555,19 +520,6 @@ def build(build_python, build_java, build_cpp):
             ).format(name="BAZEL_SH")
             raise RuntimeError(msg)
 
-    # Check if the current Python already has pickle5 (either comes with newer
-    # Python versions, or has been installed by us before).
-    pickle5 = None
-    if sys.version_info >= (3, 8, 2):
-        import pickle as pickle5
-    else:
-        try:
-            import pickle5
-        except ImportError:
-            pass
-    if not pickle5:
-        download_pickle5(os.path.join(ROOT_DIR, PICKLE5_SUBDIR))
-
     # Note: We are passing in sys.executable so that we use the same
     # version of Python to build packages inside the build.sh script. Note
     # that certain flags will not be passed along such as --user or sudo.
@@ -702,12 +654,6 @@ def pip_run(build_ext):
 
     if setup_spec.type == SetupType.RAY:
         setup_spec.files_to_include += ray_files
-        # We also need to install pickle5 along with Ray, so make sure that the
-        # relevant non-Python pickle5 files get copied.
-        pickle5_dir = os.path.join(ROOT_DIR, PICKLE5_SUBDIR)
-        setup_spec.files_to_include += walk_directory(
-            os.path.join(pickle5_dir, "pickle5")
-        )
 
         thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
         setup_spec.files_to_include += walk_directory(thirdparty_dir)