U24 support for deploy-agent (#1653)

* (trivial) ws fix

* (trivial) imports fix

* Run on py38 and py312

Update build steps for py38 and py312.

This also drops support for py36

* Bump dependencies

Bump the dependencies to support py38 and py312.

Support for py36 is dropped

* Migrate boto to boto3

The `boto` dependency is no longer supported on py312. Switch to boto3

.github/workflows/python.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.12]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

deploy-agent/deployd/common/config.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import getpass
 import logging
 import os
 import json
@@ -188,7 +189,6 @@ def get_log_directory(self) -> str:
         return self.get_var("log_directory", "/tmp/deployd/logs")
 
     def get_user_role(self) -> str:
-        import getpass
         return self.get_var("user_role", getpass.getuser())
 
     def get_restful_service_url(self) -> str:

deploy-agent/deployd/common/stats.py

@@ -19,10 +19,10 @@
 import socket
 import json
 import os
-import requests 
+import requests
 
 if IS_PINTEREST:
-    from pinstatsd.statsd import sc, sc_v2
+    from pinstatsd.statsd import sc, sc_v2, statsd_context_timer
 else:
     class sc:
         @staticmethod
@@ -61,7 +61,6 @@ def timing(*args, **kwargs) -> None:
 
 def create_stats_timer(name, sample_rate=1.0, tags=None):
     if IS_PINTEREST:
-        from pinstatsd.statsd import statsd_context_timer
         timer = statsd_context_timer(entry_name=name,
                                      sample_rate=sample_rate,
                                      tags=tags,
@@ -108,9 +107,9 @@ def create_sc_gauge(name, value, sample_rate=1.0, tags=None) -> None:
     else:
         return
 
-def send_statsboard_metric(name, value, tags=None) -> None: 
+def send_statsboard_metric(name, value, tags=None) -> None:
     tags['host'] = socket.gethostname()
-    tags_params = [f"{tag}={tags[tag]}" for tag in tags] 
+    tags_params = [f"{tag}={tags[tag]}" for tag in tags]
     tags_str = ",".join(tags_params)
     url = (
         f"{STATSBOARD_URL}put/"
@@ -121,7 +120,7 @@ def send_statsboard_metric(name, value, tags=None) -> None:
     resp = requests.put(url)
     if resp.status_code == 200:
         log.info("Successfully send the metric to statsboard")
-    
+
 class MetricCacheConfigurationError(ValueError):
     """ Raised when MetricCache has missing configuration """
     def __init__(self, name, value) -> None:

deploy-agent/deployd/download/download_helper_factory.py

@@ -15,8 +15,8 @@
 from logging import Logger, getLogger
 from typing import Optional
 
+import boto3
 from future.moves.urllib.parse import urlparse
-from boto.s3.connection import S3Connection
 
 from deployd.download.download_helper import DownloadHelper
 from deployd.download.s3_download_helper import S3DownloadHelper
@@ -38,8 +38,8 @@ def gen_downloader(url, config) -> Optional[DownloadHelper]:
             if aws_access_key_id is None or aws_secret_access_key is None:
                 log.error("aws access key id and secret access key not found")
                 return None
-            aws_conn = S3Connection(aws_access_key_id, aws_secret_access_key, True)
-            return S3DownloadHelper(local_full_fn=url, aws_connection=aws_conn, url=None, config=config)
+            s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+            return S3DownloadHelper(local_full_fn=url, s3_client=s3_client, url=None, config=config)
         elif url_parse.scheme == 'file':
             return LocalDownloadHelper(url=url)
         else:

deploy-agent/deployd/download/s3_download_helper.py

@@ -16,7 +16,9 @@
 import logging
 import re
 
-from boto.s3.connection import S3Connection
+import boto3
+import botocore
+
 from deployd.common.config import Config
 from deployd.common.status_code import Status
 from deployd.download.download_helper import DownloadHelper, DOWNLOAD_VALIDATE_METRICS
@@ -27,16 +29,16 @@
 
 class S3DownloadHelper(DownloadHelper):
 
-    def __init__(self, local_full_fn, aws_connection=None, url=None, config=None) -> None:
+    def __init__(self, local_full_fn, s3_client=None, url=None, config=None) -> None:
         super(S3DownloadHelper, self).__init__(local_full_fn)
         self._s3_matcher = "^s3://(?P<BUCKET>[a-zA-Z0-9\-_]+)/(?P<KEY>[a-zA-Z0-9\-_/\.]+)/?"
         self._config = config if config else Config()
-        if aws_connection:
-            self._aws_connection = aws_connection
+        if s3_client:
+            self._s3_client = s3_client
         else:
             aws_access_key_id = self._config.get_aws_access_key()
             aws_secret_access_key = self._config.get_aws_access_secret()
-            self._aws_connection = S3Connection(aws_access_key_id, aws_secret_access_key, True)
+            self._s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
 
         if url:
             self._url = url
@@ -52,13 +54,13 @@ def download(self, local_full_fn) -> int:
             return Status.FAILED
 
         try:
-            filekey = self._aws_connection.get_bucket(self._bucket_name).get_key(self._key)
-            if filekey is None:
+            object_metadata = self._get_object_metadata()
+            if object_metadata is None:
                 log.error("s3 key {} not found".format(self._key))
                 return Status.FAILED
 
-            filekey.get_contents_to_filename(local_full_fn)
-            etag = filekey.etag
+            self._s3_client.download_file(self._bucket_name, self._key, local_full_fn)
+            etag = object_metadata["ETag"]
             if "-" not in etag:
                 if etag.startswith('"') and etag.endswith('"'):
                     etag = etag[1:-1]
@@ -76,6 +78,20 @@ def download(self, local_full_fn) -> int:
             log.error("Failed to get package from s3: {}".format(traceback.format_exc()))
             return Status.FAILED
 
+    def _get_object_metadata(self):
+        """
+        Return the object metadata at the specified key, or None if not found
+        """
+        try:
+            return self._s3_client.head_object(Bucket=self._bucket_name, Key=self._key)
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "404":
+                # The key does not exist.
+                return None
+            else:
+              # Something else has gone wrong.
+              raise
+
     def validate_source(self) -> bool:
         allow_list = self._config.get_s3_download_allow_list()
         tags = {'type': 's3', 'url': self._url, 'bucket' : self._bucket_name}
@@ -85,4 +101,4 @@ def validate_source(self) -> bool:
             return True
         else:
             log.error(f"{self._bucket_name} is not in the allow list: {allow_list}.")
-            return False
+            return False

deploy-agent/requirements.txt

@@ -1,6 +1,7 @@
 -e .
 
-PyYAML==5.3.1
+PyYAML==5.3.1; python_version < '3.12'
+PyYAML==6.0.1; python_version >= '3.12'
 zipp==1.2.0
 configparser==4.0.2
 python-daemon==2.0.6

deploy-agent/requirements_test.txt

@@ -3,8 +3,8 @@
 # tests
 tox==1.6.1
 pytest-cov
-pytest==6.2.2; python_version >= '3'
-pytest==4.6.11; python_version < '3'
+pytest==6.2.2; python_version < '3.12'
+pytest==8.2.2; python_version >= '3.12'
 mock==1.0.1
 flake8==2.5.4
 pep8

deploy-agent/setup.py

@@ -24,12 +24,12 @@
                    'deploy-stager = deployd.staging.stager:main']
 
 install_requires = [
-    "requests==2.20.0",
-    "gevent>=1.0.2,<=1.2.2; python_version < '3'",
-    "gevent>=1.0.2,<=1.5.0; python_version < '3.8'",
-    "gevent==20.12.0; python_version >= '3.8'",
+    "requests==2.29.0; python_version < '3.12'",
+    "requests==2.32.3; python_version >= '3.12'",
+    "gevent==20.12.0; python_version < '3.12'",
+    "gevent==24.2.1; python_version >= '3.12'",
     "lockfile==0.10.2",
-    "boto>=2.39.0",
+    "boto3==1.34.134",
     "python-daemon==2.0.6",
     "future==0.18.2"
 ]

deploy-agent/tests/unit/deploy/download/test_download_helper.py

@@ -3,9 +3,9 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#  
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-#    
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,23 +35,23 @@ def setUpClass(cls):
 
         target = os.path.join(builds_dir, 'mock.txt')
         cls.target = target
-        cls.aws_conn = mock.Mock()
-        aws_filekey = cls.aws_conn.get_bucket.return_value.get_key.return_value
+        cls.s3_client = mock.MagicMock()
+        cls.s3_client.head_object.return_value.__getitem__.return_value = "f7673f4693aab49e3f8e643bc54cb70a"
 
-        def get_contents_to_filename(fn):
+        def download_file(bucket, key, fn):
             with open(fn, 'w') as file:
                 file.write("hello mock\n")
-        aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename)
-        aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a"
+        cls.s3_client.download_file = mock.Mock(side_effect=download_file)
+
 
     def test_download_s3(self):
-        downloader = S3DownloadHelper(self.target, self.aws_conn, self.url)
+        downloader = S3DownloadHelper(self.target, self.s3_client, self.url)
         downloader.download(self.target)
-        self.aws_conn.get_bucket.assert_called_once_with("pinterest-builds")
-        self.aws_conn.get_bucket.return_value.get_key.assert_called_once_with("teletraan/mock.txt")
-        self.aws_conn.get_bucket.return_value.get_key.return_value\
-            .get_contents_to_filename\
-            .assert_called_once_with(self.target)
+        self.s3_client.head_object.assert_called_once_with(Bucket="pinterest-builds", Key="teletraan/mock.txt")
+        self.s3_client.download_file\
+            .assert_called_once_with("pinterest-builds", "teletraan/mock.txt", self.target)
+        self.s3_client.head_object.return_value.__getitem__\
+            .assert_called_once_with("ETag")
 
     @classmethod
     def tearDownClass(cls):

deploy-agent/tests/unit/deploy/download/test_s3_download_helper.py

@@ -28,7 +28,7 @@ def setUp(self, mock_aws_key, mock_aws_secret):
         mock_aws_key.return_value = "test_key"
         mock_aws_secret.return_value= "test_secret"
         self.config = Config()
-        self.downloader = S3DownloadHelper(local_full_fn='', aws_connection=None, url="s3://bucket1/key1", config=self.config)
+        self.downloader = S3DownloadHelper(local_full_fn='', s3_client=None, url="s3://bucket1/key1", config=self.config)
 
     @mock.patch('deployd.common.config.Config.get_s3_download_allow_list')
     def test_validate_url_with_allow_list(self, mock_get_s3_download_allow_list):

deploy-agent/thirdparty/requirements.txt

@@ -1,11 +1,13 @@
-PyYAML==5.3.1
+PyYAML==5.3.1; python_version < '3.12'
+PyYAML==6.0.1; python_version >= '3.12'
 argparse==1.2.1
 configparser==4.0.2
 lockfile==0.10.2
 pinlogger==0.53.0
 pinstatsd==1.4.6
 python-daemon==2.3.2
-requests==2.27.1
+requests==2.29.0; python_version < '3.12'
+requests==2.32.3; python_version >= '3.12'
 setuptools==59.0.1
 strictyaml==1.6.1
-zipp==1.2.0
+zipp==1.2.0

deploy-agent/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=py36,py38
+envlist=py38,py312
 recreate=True
 
 [testenv]
@@ -13,5 +13,5 @@ commands = ruff {posargs}
 
 [gh-actions]
 python =
-    3.6: py36
     3.8: py38
+    3.12: py312