Refactor decoding format, escape unconvertable

I've had underlying concern for a while now that universally using utf-8
isn't correct because it doesn't respect locale and this serves as
fairly strong confirmation. From a little bit of digging, checking seems
as simple as calling `locale.getencoding()`, but the documentation
claims that this is ANSI for windows (meaning changing it would affect
the majority of users). As a result, I've refactored out all the string
decoding to use a common variable and set it to display an escaped
version of any unconvertable characters, but am leaving the format as
utf-8 until I have further information.

See #324

videohelpersuite/load_video_nodes.py

@@ -13,7 +13,8 @@
 from comfy.k_diffusion.utils import FolderOfImages
 from .logger import logger
 from .utils import BIGMAX, DIMMAX, calculate_file_hash, get_sorted_dir_files_from_directory,\
-        lazy_get_audio, hash_path, validate_path, strip_path, try_download_video, is_url, imageOrLatent, ffmpeg_path
+        lazy_get_audio, hash_path, validate_path, strip_path, try_download_video,  \
+        is_url, imageOrLatent, ffmpeg_path, ENCODE_ARGS
 
 
 video_extensions = ['webm', 'mp4', 'mkv', 'gif', 'mov']
@@ -146,8 +147,8 @@ def ffmpeg_frame_generator(video, force_rate, frame_load_cap, start_time,
                                     stderr=subprocess.PIPE,check=True)
     except subprocess.CalledProcessError as e:
         raise Exception("An error occurred in the ffmepg subprocess:\n" \
-                + e.stderr.decode("utf-8"))
-    lines = dummy_res.stderr.decode("utf-8")
+                + e.stderr.decode(*ENCODE_ARGS))
+    lines = dummy_res.stderr.decode(*ENCODE_ARGS)
     for line in lines.split('\n'):
         match = re.search(", ([1-9]|\\d{2,})x(\\d+).*, ([\\d\\.]+) fps", line)
         if match is not None:
@@ -226,7 +227,7 @@ def ffmpeg_frame_generator(video, force_rate, frame_load_cap, start_time,
                     current_offset = 0
     except BrokenPipeError as e:
         raise Exception("An error occured in the ffmpeg subprocess:\n" \
-                + proc.stderr.read().decode("utf-8"))
+                + proc.stderr.read().decode(*ENCODE_ARGS))
     if meta_batch is not None:
         meta_batch.inputs.pop(unique_id)
         meta_batch.has_closed_inputs = True

videohelpersuite/nodes.py

@@ -20,7 +20,9 @@
 from .load_video_nodes import LoadVideoUpload, LoadVideoPath, LoadVideoFFmpegUpload, LoadVideoFFmpegPath, LoadImagePath
 from .load_images_nodes import LoadImagesFromDirectoryUpload, LoadImagesFromDirectoryPath
 from .batched_nodes import VAEEncodeBatched, VAEDecodeBatched
-from .utils import ffmpeg_path, get_audio, hash_path, validate_path, requeue_workflow, gifski_path, calculate_file_hash, strip_path, try_download_video, is_url, imageOrLatent, BIGMAX, merge_filter_args
+from .utils import ffmpeg_path, get_audio, hash_path, validate_path, requeue_workflow, \
+        gifski_path, calculate_file_hash, strip_path, try_download_video, is_url, \
+        imageOrLatent, BIGMAX, merge_filter_args, ENCODE_ARGS
 from comfy.utils import ProgressBar
 
 folder_paths.folder_names_and_paths["VHS_video_formats"] = (
@@ -139,9 +141,9 @@ def ffmpeg_process(args, video_format, video_metadata, file_path, env):
                 #and seems to never occur concurrent to the metadata issue
                 if os.path.exists(file_path):
                     raise Exception("An error occurred in the ffmpeg subprocess:\n" \
-                            + err.decode("utf-8"))
+                            + err.decode(*ENCODE_ARGS))
                 #Res was not set
-                print(err.decode("utf-8"), end="", file=sys.stderr)
+                print(err.decode(*ENCODE_ARGS), end="", file=sys.stderr)
                 logger.warn("An error occurred when saving with metadata")
     if res != b'':
         with subprocess.Popen(args + [file_path], stderr=subprocess.PIPE,
@@ -157,10 +159,10 @@ def ffmpeg_process(args, video_format, video_metadata, file_path, env):
             except BrokenPipeError as e:
                 res = proc.stderr.read()
                 raise Exception("An error occurred in the ffmpeg subprocess:\n" \
-                        + res.decode("utf-8"))
+                        + res.decode(*ENCODE_ARGS))
     yield total_frames_output
     if len(res) > 0:
-        print(res.decode("utf-8"), end="", file=sys.stderr)
+        print(res.decode(*ENCODE_ARGS), end="", file=sys.stderr)
 
 def gifski_process(args, video_format, file_path, env):
     frame_data = yield
@@ -186,14 +188,14 @@ def gifski_process(args, video_format, file_path, env):
                 resgs = procgs.stderr.read()
                 raise Exception("An error occurred while creating gifski output\n" \
                         + "Make sure you are using gifski --version >=1.32.0\nffmpeg: " \
-                        + resff.decode("utf-8") + '\ngifski: ' + resgs.decode("utf-8"))
+                        + resff.decode(*ENCODE_ARGS) + '\ngifski: ' + resgs.decode(*ENCODE_ARGS))
     if len(resff) > 0:
-        print(resff.decode("utf-8"), end="", file=sys.stderr)
+        print(resff.decode(*ENCODE_ARGS), end="", file=sys.stderr)
     if len(resgs) > 0:
-        print(resgs.decode("utf-8"), end="", file=sys.stderr)
+        print(resgs.decode(*ENCODE_ARGS), end="", file=sys.stderr)
     #should always be empty as the quiet flag is passed
     if len(outgs) > 0:
-        print(outgs.decode("utf-8"))
+        print(outgs.decode(*ENCODE_ARGS))
 
 def to_pingpong(inp):
     if not hasattr(inp, "__getitem__"):
@@ -474,7 +476,7 @@ def pad(image):
                                    capture_output=True, check=True)
                 except subprocess.CalledProcessError as e:
                     raise Exception("An error occurred in the ffmpeg prepass:\n" \
-                            + e.stderr.decode("utf-8"))
+                            + e.stderr.decode(*ENCODE_ARGS))
             if "inputs_main_pass" in video_format:
                 args = args[:13] + video_format['inputs_main_pass'] + args[13:]
 
@@ -550,9 +552,9 @@ def pad(image):
                                          env=env, capture_output=True, check=True)
                 except subprocess.CalledProcessError as e:
                     raise Exception("An error occured in the ffmpeg subprocess:\n" \
-                            + e.stderr.decode("utf-8"))
+                            + e.stderr.decode(*ENCODE_ARGS))
                 if res.stderr:
-                    print(res.stderr.decode("utf-8"), end="", file=sys.stderr)
+                    print(res.stderr.decode(*ENCODE_ARGS), end="", file=sys.stderr)
                 output_files.append(output_file_with_audio_path)
                 #Return this file with audio to the webui.
                 #It will be muted unless opened or saved with right click
@@ -672,9 +674,9 @@ def convert_audio(self, audio):
                                  capture_output=True, check=True)
         except subprocess.CalledProcessError as e:
             raise Exception("An error occured in the ffmpeg subprocess:\n" \
-                    + e.stderr.decode("utf-8"))
+                    + e.stderr.decode(*ENCODE_ARGS))
         if res.stderr:
-            print(res.stderr.decode("utf-8"), end="", file=sys.stderr)
+            print(res.stderr.decode(*ENCODE_ARGS), end="", file=sys.stderr)
         return (lambda: res.stdout,)
 
 class VHSAudioToAudio:
@@ -699,8 +701,8 @@ def convert_audio(self, vhs_audio):
             audio = torch.frombuffer(bytearray(res.stdout), dtype=torch.float32)
         except subprocess.CalledProcessError as e:
             raise Exception("An error occured in the ffmpeg subprocess:\n" \
-                    + e.stderr.decode("utf-8"))
-        match = re.search(', (\\d+) Hz, (\\w+), ',res.stderr.decode('utf-8'))
+                    + e.stderr.decode(*ENCODE_ARGS))
+        match = re.search(', (\\d+) Hz, (\\w+), ',res.stderr.decode(*ENCODE_ARGS))
         if match:
             ar = int(match.group(1))
             #NOTE: Just throwing an error for other channel types right now

videohelpersuite/server.py

@@ -4,7 +4,8 @@
 import time
 import subprocess
 import re
-from .utils import is_url, get_sorted_dir_files_from_directory, ffmpeg_path, validate_sequence, is_safe_path, strip_path, try_download_video
+from .utils import is_url, get_sorted_dir_files_from_directory, ffmpeg_path, \
+        validate_sequence, is_safe_path, strip_path, try_download_video, ENCODE_ARGS
 from comfy.k_diffusion.utils import FolderOfImages
 
 web = server.web
@@ -78,15 +79,15 @@ async def view_video(request):
     try:
         res = subprocess.run([ffmpeg_path] + in_args + ['-t', '0', '-f', 'null', '-'],
                              capture_output=True, check=True)
-        match = re.search(': Video: (\\w+) .+, (\\d+) fps,', res.stderr.decode('utf-8'))
+        match = re.search(': Video: (\\w+) .+, (\\d+) fps,', res.stderr.decode(*ENCODE_ARGS))
         if match:
             base_fps = float(match.group(2))
             if match.group(1) == 'vp9':
                 #force libvpx for transparency
                 in_args = ['-c:v', 'libvpx-vp9'] + in_args
     except subprocess.CalledProcessError as e:
         print("An error occurred in the ffmpeg prepass:\n" \
-                + e.stderr.decode("utf-8"))
+                + e.stderr.decode(*ENCODE_ARGS))
     vfilters = []
     target_rate = float(query.get('force_rate', 0)) or base_fps
     modified_rate = target_rate / float(query.get('select_every_nth',1))

videohelpersuite/utils.py

@@ -18,10 +18,12 @@
 
 DIMMAX = 8192
 
+ENCODE_ARGS = ("utf-8", 'backslashreplace')
+
 def ffmpeg_suitability(path):
     try:
         version = subprocess.run([path, "-version"], check=True,
-                                 capture_output=True).stdout.decode("utf-8")
+                                 capture_output=True).stdout.decode(*ENCODE_ARGS)
     except:
         return 0
     score = 0
@@ -98,10 +100,10 @@ def try_download_video(url):
                               "-P", folder_paths.get_temp_directory(), url],
                              capture_output=True, check=True)
         #strip newline
-        file = res.stdout.decode('utf-8')[:-1]
+        file = res.stdout.decode(*ENCODE_ARGS)[:-1]
     except subprocess.CalledProcessError as e:
         raise Exception("An error occurred in the yt-dl process:\n" \
-                + e.stderr.decode("utf-8"))
+                + e.stderr.decode(*ENCODE_ARGS))
         file = None
     download_history[url] = file
     return file
@@ -198,10 +200,10 @@ def get_audio(file, start_time=0, duration=0):
         res =  subprocess.run(args + ["-f", "f32le", "-"],
                               capture_output=True, check=True)
         audio = torch.frombuffer(bytearray(res.stdout), dtype=torch.float32)
-        match = re.search(', (\\d+) Hz, (\\w+), ',res.stderr.decode('utf-8'))
+        match = re.search(', (\\d+) Hz, (\\w+), ',res.stderr.decode(*ENCODE_ARGS))
     except subprocess.CalledProcessError as e:
         raise Exception(f"VHS failed to extract audio from {file}:\n" \
-                + e.stderr.decode("utf-8"))
+                + e.stderr.decode(*ENCODE_ARGS))
     if match:
         ar = int(match.group(1))
         #NOTE: Just throwing an error for other channel types right now