Changed prepare samples to use Numpy array for X data (#65)

* Changed prepare samples to use np array for X The previous method, with a list, took too much memory. With the old method, using 20 GB worth of 1080p images, memory usage would exceed my computer's 16 GB. With the updated code it does not exceed 1 GB. Kind of solved this issue: #2 * Update utils.py Co-authored-by: Kevin Hughes <[email protected]> * Update with suggestions Co-authored-by: Kevin Hughes <[email protected]>
kevinhughes27 · Jun 4, 2021 · 6ab288d · 6ab288d
1 parent 1b10c9a
commit 6ab288d
Showing 1 changed file with 11 additions and 4 deletions.
diff --git a/utils.py b/utils.py
@@ -203,10 +203,17 @@ def viewer(sample):
 def prepare(samples):
     print("Preparing data")
 
-    X = []
     y = []
-
+    
     for sample in samples:
+        image_files = load_imgs(sample)
+        num_samples += len(image_files)
+
+    X = np.empty(shape=(num_samples,Sample.IMG_H,Sample.IMG_W,3),dtype=np.uint8)
+    print(f"There are {num_samples} samples")
+
+    idx = 0
+for idx, sample in enumerate(samples):
         print(sample)
 
         # load sample
@@ -219,10 +226,10 @@ def prepare(samples):
         for image_file in image_files:
             image = imread(image_file)
             vec = resize_image(image)
-            X.append(vec)
+            X[idx] = vec
+
 
     print("Saving to file...")
-    X = np.asarray(X)
     y = np.concatenate(y)
 
     np.save("data/X", X)