Skip to content

Commit

Permalink
Add switchover to requests_toolbelt for large file
Browse files Browse the repository at this point in the history
When requests handles files, it reads the entire file into memory and sends in a single payload.
Currently the python SSL tooling does not support single payloads larger than 2^31-1 bytes (https://bugs.python.org/issue42853#msg384566)
This means that files ~ >= 2GiB cannot be sent using requests

This commit allows a switchover to the streaming multipart payload provided by requests_toolbelt (if installed)
  • Loading branch information
jhollowe committed Oct 2, 2021
1 parent 2f9c81e commit eb74f65
Showing 1 changed file with 36 additions and 5 deletions.
41 changes: 36 additions & 5 deletions proxmoxer/backends/https.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@


import json
import os
import sys
import time
import logging
Expand Down Expand Up @@ -179,17 +180,34 @@ def request(self, method, url, params=None, data=None, headers=None, cookies=Non
if (not c) and a:
cookies = a.get_cookies()

#filter out streams
# filter out streams
files = files or {}
data = data or {}
largeFiles = {}
for k, v in data.copy().items():
if is_file(v):
files[k] = v
if getFileSize(v) > 2147483647:
largeFiles[k] = v
else:
files[k] = v
del data[k]

headers = None
if not files and serializer:
headers = {"content-type": 'application/x-www-form-urlencoded'}
# if there are any large file, send all data and files using streaming multipart encoding
if len(largeFiles) > 0 or True:
try:
from requests_toolbelt import MultipartEncoder
encoder = MultipartEncoder(
fields={**data, **files, **largeFiles})
data = encoder
files = None
headers = {'Content-Type': encoder.content_type}
except ImportError:
logger.error(
"Unable to upload files larger than 2 GiB, Install 'requests_toolbelt' to add support")
return None

if not (files or largeFiles) and serializer:
headers = {"Content-Type": 'application/x-www-form-urlencoded'}

return super(ProxmoxHttpSession, self).request(method, url, params, data, headers, cookies, files, auth,
timeout, allow_redirects, proxies, hooks, stream, verify, cert)
Expand Down Expand Up @@ -235,3 +253,16 @@ def get_serializer(self):
def get_tokens(self):
"""Return the in-use auth and csrf tokens if using user/password auth."""
return self.auth.get_tokens()


def getFileSize(fileObj):
# store existing file cursor location
startingCursor = fileObj.tell()

# get size
size = fileObj.seek(0, os.SEEK_END)

# reset cursor
fileObj.seek(startingCursor)

return size

0 comments on commit eb74f65

Please sign in to comment.