Merge branch 'release/2.2.0'

Release v2.2.0

.vscode/settings.json

@@ -13,7 +13,7 @@
   ],
   "[python]": {
     "editor.codeActionsOnSave": {
-      "source.organizeImports": true
+      "source.organizeImports": "explicit"
     }
   },
   "cSpell.words": [

.vscode/tasks.json

@@ -99,7 +99,7 @@
     {
       "label": "Clean Cache/tmp files",
       "type": "shell",
-      "command": "rm -rf ./.mypy_cache/ ./.pytest_cache/ ./coverage.xml ./.coverage",
+      "command": "rm -rf ./.mypy_cache/ ./.pytest_cache/ ./dist/ ./coverage.xml ./.coverage README.txt",
       "problemMatcher": [],
       "group": {
         "kind": "none"

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 2.2.0 (2024-12-13)
+
+* Bugfix (local,openssh,paramiko): Remove IP/hostname from command path ([Andrea Dainese](https://github.com/dainok), [John Hollowell](https://github.com/jhollowe))
+* Addition (https): Allow passing certificate for TLS verification ([gdelaneync](https://github.com/gdelaneync))
+* Bugfix (local,openssh,paramiko): Prevent a returned task ID (UPID) from throwing an error ([Adam Dorsey](https://github.com/asdorsey), [John Hollowell](https://github.com/jhollowe))
+* Improvement (local,openssh,paramiko): Attempt to encode binary payloads as UTF-8 before sending/JSON-ing ([Adam Dorsey](https://github.com/asdorsey))
+
 ## 2.1.0 (2024-08-10)
 
 * Improvement (docs): Update Readme with updated example ([Rob Wolinski](https://github.com/trekie86))
@@ -7,7 +14,7 @@
 * Bugfix (https): Fix BytesWarning when logging response status/content ([Walter Doekes](https://github.com/wdoekes))
 * Improvement (meta): Update devcontainer to modern unified schema ([John Hollowell](https://github.com/jhollowe))
 * Improvement (meta): Add 3.12 to CI matrix, remove 3.7 testing ([John Hollowell](https://github.com/jhollowe))
-* Improvement (all): Fix improper spliting of non-exec QEMU commands ([John Hollowell](https://github.com/jhollowe))
+* Improvement (all): Fix improper splitting of non-exec QEMU commands ([John Hollowell](https://github.com/jhollowe))
 
 ## 2.0.1 (2022-12-19)
 
@@ -42,7 +49,7 @@
 * Improvement (command_base): Refactor code to have a unified CLI backend base for `openssh`, `ssh_paramiko`, and `local` backends ([Markus Reiter](https://github.com/reitermarkus))
 * Improvement (https): Support IPv6 addresses ([Daviddcc](https://github.com/dcasier))
 * Improvement: Move CI to GitHub actions from Travis.ci ([John Hollowell](https://github.com/jhollowe))
-* Improvement: Cleanup documentaiton and move to dedicated site ([John Hollowell](https://github.com/jhollowe))
+* Improvement: Cleanup documentation and move to dedicated site ([John Hollowell](https://github.com/jhollowe))
 * Improvement: Add `pre-commit` hooks for formatting and linting and format all code ([John Hollowell](https://github.com/jhollowe))
 
 ## 1.2.0 (2021-10-07)
@@ -77,7 +84,7 @@
 * Improvement (https): Added option to specify port in hostname parameter ([pvanagtmaal](https://github.com/pvanagtmaal))
 * Improvement: Added stderr to the Response content ([Jérôme Schneider](https://github.com/merinos))
 * Bugfix (ssh_paramiko): Paramiko python3: stdout and stderr must be a str not bytes ([Jérôme Schneider](https://github.com/merinos))
-* New lxc example in docu ([Geert Stappers](https://github.com/stappersg))
+* New lxc example in documentation ([Geert Stappers](https://github.com/stappersg))
 
 ## 1.0.2 (2017-12-02)
 * Tarball repackaged with tests

proxmoxer/__init__.py

@@ -1,6 +1,6 @@
 __author__ = "Oleg Butovich"
-__copyright__ = "(c) Oleg Butovich 2013-2017"
-__version__ = "2.1.0"
+__copyright__ = "(c) Oleg Butovich 2013-2024"
+__version__ = "2.2.0"
 __license__ = "MIT"
 
 from .core import *  # noqa

proxmoxer/backends/command_base.py

@@ -88,7 +88,12 @@ def request(self, method, url, data=None, params=None, headers=None):
 
         command = [f"{self.service}sh", cmd, url]
         # convert the options dict into a 2-tuple with the key formatted as a flag
-        option_pairs = [(f"-{k}", str(v)) for k, v in chain(data.items(), params.items())]
+        option_pairs = []
+        for k, v in chain(data.items(), params.items()):
+            try:
+                option_pairs.append((f"-{k}", str(v, "utf-8")))
+            except TypeError:
+                option_pairs.append((f"-{k}", str(v)))
         # add back in all the command arguments as their own pairs
         if data_command is not None:
             if isinstance(data_command, list):
@@ -111,15 +116,22 @@ def is_http_status_string(s):
             return re.match(r"\d\d\d [a-zA-Z]", str(s))
 
         if stderr:
-            # sometimes contains extra text like 'trying to acquire lock...OK'
-            status_code = next(
-                (
-                    int(line.split()[0])
-                    for line in stderr.splitlines()
-                    if is_http_status_string(line)
-                ),
-                500,
+            # assume if we got a task ID that the request was successful
+            task_id_pattern = re.compile(
+                r"UPID:[\w-]+:[0-9a-fA-F]{8}:[0-9a-fA-F]{8}:[0-9a-fA-F]{8}:\w+:[\w\._-]+:[\w\.@_-]+:\w*"
             )
+            if task_id_pattern.search(str(stdout)) or task_id_pattern.search(str(stderr)):
+                status_code = 200
+            else:
+                # sometimes contains extra text like 'trying to acquire lock...OK'
+                status_code = next(
+                    (
+                        int(line.split()[0])
+                        for line in stderr.splitlines()
+                        if is_http_status_string(line)
+                    ),
+                    500,
+                )
         else:
             status_code = 200
         if stdout:
@@ -147,13 +159,13 @@ def loads_errors(self, response):
 class CommandBaseBackend:
     def __init__(self):
         self.session = None
-        self.target = ""
+        self.target = None
 
     def get_session(self):
         return self.session
 
     def get_base_url(self):
-        return self.target
+        return ""
 
     def get_serializer(self):
         return JsonSimpleSerializer()

proxmoxer/backends/https.py

@@ -42,10 +42,11 @@ def get_cookies(self):
     def get_tokens(self):
         return None, None
 
-    def __init__(self, timeout=5, service="PVE", verify_ssl=False):
+    def __init__(self, timeout=5, service="PVE", verify_ssl=False, cert=None):
         self.timeout = timeout
         self.service = service
         self.verify_ssl = verify_ssl
+        self.cert = cert
 
 
 class ProxmoxHTTPAuth(ProxmoxHTTPAuthBase):
@@ -75,6 +76,7 @@ def _get_new_tokens(self, password=None, otp=None):
             verify=self.verify_ssl,
             timeout=self.timeout,
             data=data,
+            cert=self.cert,
         ).json()["data"]
         if response_data is None:
             raise AuthenticationError(
@@ -125,6 +127,7 @@ def __call__(self, req):
             SERVICES[self.service]["token_separator"],
             self.token_value,
         )
+        req.cert = self.cert
         return req
 
 
@@ -263,7 +266,9 @@ def __init__(
         token_value=None,
         path_prefix=None,
         service="PVE",
+        cert=None,
     ):
+        self.cert = cert
         host_port = ""
         if len(host.split(":")) > 2:  # IPv6
             if host.startswith("["):
@@ -296,6 +301,7 @@ def __init__(
                 verify_ssl=verify_ssl,
                 timeout=timeout,
                 service=service,
+                cert=self.cert,
             )
         elif password is not None:
             if "password" not in SERVICES[service]["supported_https_auths"]:
@@ -309,12 +315,14 @@ def __init__(
                 verify_ssl=verify_ssl,
                 timeout=timeout,
                 service=service,
+                cert=self.cert,
             )
         else:
             config_failure("No valid authentication credentials were supplied")
 
     def get_session(self):
         session = ProxmoxHttpSession()
+        session.cert = self.cert
         session.auth = self.auth
         # cookies are taken from the auth
         session.headers["Connection"] = "keep-alive"

proxmoxer/core.py

@@ -217,7 +217,8 @@ def __init__(self, host=None, backend="https", service="PVE", **kwargs):
         }
 
     def __repr__(self):
-        return f"ProxmoxAPI ({self._backend_name} backend for {self._store['base_url']})"
+        dest = getattr(self._backend, "target", self._store.get("base_url"))
+        return f"ProxmoxAPI ({self._backend_name} backend for {dest})"
 
     def get_tokens(self):
         """Return the auth and csrf tokens.

setup.py

@@ -44,6 +44,8 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python",
         "Topic :: Software Development :: Libraries :: Python Modules",

tests/test_command_base.py

@@ -56,6 +56,23 @@ def test_request_basic(self, mock_exec):
             "json",
         ]
 
+    def test_request_task(self, mock_exec_task):
+        resp = self._session.request("GET", self.base_url + "/stdout")
+
+        assert resp.status_code == 200
+        assert (
+            resp.content == "UPID:node:003094EA:095F1EFE:63E88772:download:file.iso:root@pam:done"
+        )
+
+        resp_stderr = self._session.request("GET", self.base_url + "/stderr")
+
+        assert resp_stderr.status_code == 200
+        assert (
+            resp_stderr.content
+            == "UPID:node:003094EA:095F1EFE:63E88772:download:file.iso:root@pam:done"
+        )
+        # assert False  # DEBUG
+
     def test_request_error(self, mock_exec_err):
         resp = self._session.request(
             "GET", self.base_url + "/fake/echo", data={"thing": "403 Unauthorized"}
@@ -105,6 +122,22 @@ def test_request_data(self, mock_exec):
             "json",
         ]
 
+    def test_request_bytes_data(self, mock_exec):
+        resp = self._session.request(
+            "GET", self.base_url + "/fake/echo", data={"key": b"bytes-value"}
+        )
+
+        assert resp.status_code == 200
+        assert resp.content == [
+            "pvesh",
+            "get",
+            self.base_url + "/fake/echo",
+            "-key",
+            "bytes-value",
+            "--output-format",
+            "json",
+        ]
+
     def test_request_qemu_exec(self, mock_exec):
         resp = self._session.request(
             "POST",
@@ -213,7 +246,7 @@ def test_init(self):
         b = command_base.CommandBaseBackend()
 
         assert b.session is None
-        assert b.target == ""
+        assert b.target is None
 
     def test_get_session(self):
         assert self.backend.get_session() == self.sess
@@ -242,6 +275,15 @@ def _exec_err(_, cmd):
     return None, "\n".join(cmd)
 
 
+@classmethod
+def _exec_task(_, cmd):
+    upid = "UPID:node:003094EA:095F1EFE:63E88772:download:file.iso:root@pam:done"
+    if "stderr" in cmd[2]:
+        return None, upid
+    else:
+        return upid, None
+
+
 @classmethod
 def upload_file_obj_echo(_, file_obj, remote_path):
     return file_obj, remote_path
@@ -261,6 +303,12 @@ def mock_exec():
         yield
 
 
+@pytest.fixture
+def mock_exec_task():
+    with mock.patch.object(command_base.CommandBaseSession, "_exec", _exec_task):
+        yield
+
+
 @pytest.fixture
 def mock_exec_err():
     with mock.patch.object(command_base.CommandBaseSession, "_exec", _exec_err):

tests/test_core.py

@@ -336,6 +336,40 @@ def test_get_tokens_local(self):
         assert ticket is None
         assert csrf is None
 
+    def test_init_with_cert(self):
+        prox = core.ProxmoxAPI(
+            "host",
+            token_name="name",
+            token_value="value",
+            service="pVe",
+            backend="hTtPs",
+            cert="somepem",
+        )
+
+        assert isinstance(prox, core.ProxmoxAPI)
+        assert isinstance(prox, core.ProxmoxResource)
+        assert isinstance(prox._backend, https.Backend)
+        assert prox._backend.auth.service == "PVE"
+        assert prox._backend.cert == "somepem"
+        assert prox._store["session"].cert == "somepem"
+
+    def test_init_with_cert_key(self):
+        prox = core.ProxmoxAPI(
+            "host",
+            token_name="name",
+            token_value="value",
+            service="pVe",
+            backend="hTtPs",
+            cert=("somepem", "somekey"),
+        )
+
+        assert isinstance(prox, core.ProxmoxAPI)
+        assert isinstance(prox, core.ProxmoxResource)
+        assert isinstance(prox._backend, https.Backend)
+        assert prox._backend.auth.service == "PVE"
+        assert prox._backend.cert == ("somepem", "somekey")
+        assert prox._store["session"].cert == ("somepem", "somekey")
+
 
 class MockSession:
     def request(self, method, url, data=None, params=None):

tests/test_https.py

@@ -276,6 +276,7 @@ class TestProxmoxHttpSession:
     def test_request_basic(self, mock_pve):
         resp = self._session.request("GET", self.base_url + "/fake/echo")
         content = resp.json()
+        assert self._session.cert is None
 
         assert content["method"] == "GET"
         assert content["url"] == self.base_url + "/fake/echo"