From dd2ffab206af4c98aaca7060fe892144a98e3bce Mon Sep 17 00:00:00 2001
From: JensWendt <92271654+JensWendt@users.noreply.github.com>
Date: Wed, 6 Sep 2023 22:33:42 +0200
Subject: [PATCH 1/2] Solution for delimiter reading issue utilizing dynamic
 checkpoints

---
 omero/annotation_scripts/KeyVal_from_csv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py
index 2c30b21b7..506937c33 100644
--- a/omero/annotation_scripts/KeyVal_from_csv.py
+++ b/omero/annotation_scripts/KeyVal_from_csv.py
@@ -34,6 +34,7 @@
 import sys
 import csv
 import copy
+from math import floor
 
 from omero.util.populate_roi import DownloadingOriginalFileProvider
 
@@ -158,28 +159,30 @@ def keyval_from_csv(conn, script_params):
         # Needs omero-py 5.9.1 or later
         temp_name = temp_file.name
         with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle:
+            file_length = len(file_handle.read(-1))
             try:
                 delimiter = csv.Sniffer().sniff(
-                    file_handle.read(500), ",;\t").delimiter
+                    file_handle.read(floor(file_length/4)), ",;\t").delimiter
                 print("Using delimiter: ", delimiter,
-                      " after reading 500 characters")
+                    f" after reading {floor(file_length/4)} characters")
             except Exception:
                 file_handle.seek(0)
                 try:
                     delimiter = csv.Sniffer().sniff(
-                        file_handle.read(1000), ",;\t").delimiter
+                        file_handle.read(floor(file_length/2)), ",;\t").delimiter
                     print("Using delimiter: ", delimiter,
-                          " after reading 1000 characters")
+                        f" after reading {floor(file_length/2)} characters")
                 except Exception:
                     file_handle.seek(0)
                     try:
                         delimiter = csv.Sniffer().sniff(
-                            file_handle.read(2000), ";,\t").delimiter
+                            file_handle.read(floor(file_length*0.75)), ",;\t").delimiter
                         print("Using delimiter: ", delimiter,
-                              " after reading 2000 characters")
+                        f" after reading {floor(file_length*0.75)} characters")
                     except Exception:
                         print("Failed to sniff delimiter, using ','")
                         delimiter = ","
+
             # reset to start and read whole file...
             file_handle.seek(0)
             data = list(csv.reader(file_handle, delimiter=delimiter))

From 8738ad1bef73ee2faf2d0d60756972e21f16a6cf Mon Sep 17 00:00:00 2001
From: JensWendt <92271654+JensWendt@users.noreply.github.com>
Date: Thu, 7 Sep 2023 13:49:42 +0200
Subject: [PATCH 2/2] resolving Flake8 issues and implements better method to
 get file_length

---
 omero/annotation_scripts/KeyVal_from_csv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py
index 506937c33..1bd8f5797 100644
--- a/omero/annotation_scripts/KeyVal_from_csv.py
+++ b/omero/annotation_scripts/KeyVal_from_csv.py
@@ -158,27 +158,30 @@ def keyval_from_csv(conn, script_params):
         temp_file = provider.get_original_file_data(original_file)
         # Needs omero-py 5.9.1 or later
         temp_name = temp_file.name
+        file_length = original_file.size.val
         with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle:
-            file_length = len(file_handle.read(-1))
             try:
                 delimiter = csv.Sniffer().sniff(
                     file_handle.read(floor(file_length/4)), ",;\t").delimiter
                 print("Using delimiter: ", delimiter,
-                    f" after reading {floor(file_length/4)} characters")
+                      f" after reading {floor(file_length/4)} characters")
             except Exception:
                 file_handle.seek(0)
                 try:
                     delimiter = csv.Sniffer().sniff(
-                        file_handle.read(floor(file_length/2)), ",;\t").delimiter
+                        file_handle.read(floor(file_length/2)),
+                        ",;\t").delimiter
                     print("Using delimiter: ", delimiter,
-                        f" after reading {floor(file_length/2)} characters")
+                          f"after reading {floor(file_length/2)} characters")
                 except Exception:
                     file_handle.seek(0)
                     try:
                         delimiter = csv.Sniffer().sniff(
-                            file_handle.read(floor(file_length*0.75)), ",;\t").delimiter
+                            file_handle.read(floor(file_length*0.75)),
+                            ",;\t").delimiter
                         print("Using delimiter: ", delimiter,
-                        f" after reading {floor(file_length*0.75)} characters")
+                              f" after reading {floor(file_length*0.75)}"
+                              " characters")
                     except Exception:
                         print("Failed to sniff delimiter, using ','")
                         delimiter = ","