rename iso scripts, some docs

ami_scripts/iso_transcoder.py → ami_scripts/iso_transcoder_makemkv.py

@@ -7,6 +7,76 @@
 import tempfile
 import sys
 import os
+from pymediainfo import MediaInfo
+from collections import defaultdict
+
+# Predefined categories for classification
+CATEGORIES = {
+    "NTSC DVD SD (D1 Resolution)": {
+        "Width": 720,
+        "Height": 480,
+        "DisplayAspectRatio": "1.333",
+        "PixelAspectRatio": "0.889",
+        "FrameRate": "29.970",
+    },
+    "NTSC DVD Widescreen": {
+        "Width": 720,
+        "Height": 480,
+        "DisplayAspectRatio": "1.777",
+        "PixelAspectRatio": "1.185",
+        "FrameRate": "29.970",
+    },
+    "NTSC DVD SD (4SIF Resolution)": {
+        "Width": 704,
+        "Height": 480,
+        "DisplayAspectRatio": "1.333",
+        "PixelAspectRatio": "0.909",
+        "FrameRate": "29.970",
+    },
+    "NTSC DVD (SIF Resolution)": {
+        "Width": 352,
+        "Height": 240,
+        "DisplayAspectRatio": "1.339",
+        "PixelAspectRatio": "0.913",
+        "FrameRate": "29.970",
+    },
+    "PAL DVD SD (D1 Resolution)": {
+        "Width": 720,
+        "Height": 576,
+        "DisplayAspectRatio": "1.333",
+        "PixelAspectRatio": "1.067",
+        "FrameRate": "25.000",
+    },
+    "PAL DVD Widescreen": {
+        "Width": 720,
+        "Height": 576,
+        "DisplayAspectRatio": "1.778",
+        "PixelAspectRatio": "1.422",
+        "FrameRate": "25.000",
+    },
+    "PAL DVD (CIF Resolution)": {
+        "Width": 352,
+        "Height": 288,
+        "DisplayAspectRatio": "1.333",
+        "PixelAspectRatio": "1.092",
+        "FrameRate": "25.000",
+    },
+    "PAL DVD Half-D1 Resolution": {
+        "Width": 352,
+        "Height": 576,
+        "DisplayAspectRatio": "1.333",
+        "PixelAspectRatio": "2.182",
+        "FrameRate": "25.000",
+    },
+    "PAL DVD Half-D1 Resolution Widescreen": {
+        "Width": 352,
+        "Height": 576,
+        "DisplayAspectRatio": "1.778",
+        "PixelAspectRatio": "2.909",
+        "FrameRate": "25.000",
+    },
+}
+
 
 # Check for colorama installation
 try:
@@ -18,6 +88,7 @@
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 
+
 def verify_makemkvcon_installation():
     try:
         result = subprocess.run(["makemkvcon", "-r", "info", "disc:9999"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
@@ -59,24 +130,26 @@ def build_ffmpeg_command(input_file, output_file):
     ]
     return ffmpeg_command
 
-def concatenate_mkvs(mkv_files):
+
+def process_iso_with_makemkv(iso_path, output_directory):
+    logging.info(f"Processing ISO {iso_path} with MakeMKV")
+    output_path = Path(tempfile.mkdtemp())  # Temporary directory for MKVs
+
+    makemkv_command = ["makemkvcon", "mkv", f"iso:{iso_path}", "all", str(output_path)]
     try:
-        with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as tmp_mkv_file:
-            mkvmerge_command = ["mkvmerge", "-o", tmp_mkv_file.name]
-            for mkv_file in mkv_files[:-1]:
-                mkvmerge_command.extend([str(mkv_file), "+"])
-            mkvmerge_command.append(str(mkv_files[-1]))
-            subprocess.run(mkvmerge_command, check=True)
-            return tmp_mkv_file.name
+        subprocess.run(makemkv_command, check=True)
+        logging.info(f"MKV files created from {iso_path} in {output_path}")
+        return output_path
     except subprocess.CalledProcessError:
-        logging.error("MKVMerge failed, could not concatenate MKVs.")
+        logging.error(f"MakeMKV processing failed for {iso_path}")
         return None
 
+
 def transcode_mkv_files(mkv_directory, iso_basename, output_directory, force_concat):
     mkv_files = sorted(mkv_directory.glob("*.mkv"))
     if not mkv_files:
         logging.error(f"No MKV files found in {mkv_directory}")
-        return
+        return False  # Indicate failure
 
     if force_concat:
         logging.info("Concatenating MKV files using mkvmerge...")
@@ -87,12 +160,15 @@ def transcode_mkv_files(mkv_directory, iso_basename, output_directory, force_con
             ffmpeg_command = build_ffmpeg_command(concatenated_mkv, output_file)
             try:
                 subprocess.run(ffmpeg_command, check=True)
+                return True  # Indicate success
             except subprocess.CalledProcessError:
                 logging.error(f"Transcoding failed for concatenated MKV of {iso_basename}")
+                return False
             finally:
                 Path(concatenated_mkv).unlink()  # Clean up concatenated temp MKV file
         else:
             logging.error(f"Concatenation failed for {iso_basename}, skipping transcoding.")
+            return False
     else:
         for idx, mkv_file in enumerate(mkv_files, start=1):
             output_file = output_directory / f"{iso_basename}f01r{str(idx).zfill(2)}_sc.mp4" if len(mkv_files) > 1 else output_directory / f"{iso_basename}_sc.mp4"
@@ -103,6 +179,9 @@ def transcode_mkv_files(mkv_directory, iso_basename, output_directory, force_con
                 subprocess.run(ffmpeg_command, check=True)
             except subprocess.CalledProcessError:
                 logging.error(f"Transcoding failed for {mkv_file}")
+                return False  # Indicate failure
+    return True  # Indicate success if all files processed successfully
+
 
 def verify_transcoding(iso_paths, output_directory):
     total_isos = len(iso_paths)
@@ -132,6 +211,67 @@ def verify_transcoding(iso_paths, output_directory):
         for iso in failed_isos:
             print(f" - {iso}")
 
+
+def extract_video_properties(file_path):
+    """Extract video properties using pymediainfo."""
+    media_info = MediaInfo.parse(file_path)
+    for track in media_info.tracks:
+        if track.track_type == "Video":
+            return {
+                "Width": int(track.width),
+                "Height": int(track.height),
+                "DisplayAspectRatio": track.display_aspect_ratio,
+                "PixelAspectRatio": track.pixel_aspect_ratio,
+                "FrameRate": track.frame_rate,
+            }
+    return None
+
+
+def classify_mp4(mp4_files):
+    """Classify MP4 files based on predefined categories."""
+    classification_counts = defaultdict(int)
+    outliers = []
+
+    for file in mp4_files:
+        properties = extract_video_properties(file)
+        if not properties:
+            outliers.append(file)
+            continue
+
+        classified = False
+        for category, criteria in CATEGORIES.items():
+            if all(str(properties[key]) == str(value) for key, value in criteria.items()):
+                classification_counts[category] += 1
+                classified = True
+                break
+
+        if not classified:
+            outliers.append(file)
+
+    return classification_counts, outliers
+
+
+def summarize_classifications(classification_counts, outliers):
+    """Print classification summary and outliers."""
+    print("\nClassification Summary:")
+    for category, count in classification_counts.items():
+        print(f"- {category}: {count} MP4(s)")
+
+    print("\nOutliers:")
+    if outliers:
+        for outlier in outliers:
+            print(f"- {outlier}")
+    else:
+        print("None")
+
+
+def post_process_check(output_directory):
+    """Run MediaInfo classification as a post-process check."""
+    mp4_files = list(Path(output_directory).glob("*.mp4"))
+    classification_counts, outliers = classify_mp4(mp4_files)
+    summarize_classifications(classification_counts, outliers)
+
+
 def main():
     parser = argparse.ArgumentParser(description='Transcode MKV files created from ISO images to H.264 MP4s.')
     parser.add_argument('-i', '--input', dest='i', required=True, help='Path to the input directory with ISO files')
@@ -153,8 +293,11 @@ def main():
         mkv_output_dir = process_iso_with_makemkv(iso_file, output_directory)
 
         if mkv_output_dir:
-            transcode_mkv_files(mkv_output_dir, iso_basename, output_directory, args.force)
-            processed_iso_paths.append(iso_file)
+            transcode_success = transcode_mkv_files(mkv_output_dir, iso_basename, output_directory, args.force)
+            if transcode_success:
+                processed_iso_paths.append(iso_file)
+            else:
+                logging.error(f"Skipping verification for {iso_file} due to transcoding failure.")
             # Clean up temporary MKV directory
             for mkv_file in mkv_output_dir.glob("*.mkv"):
                 mkv_file.unlink()
@@ -165,5 +308,8 @@ def main():
     # Verify transcoding results and print a summary
     verify_transcoding(processed_iso_paths, output_directory)
 
+    # Run post-process classification
+    post_process_check(output_directory)
+
 if __name__ == "__main__":
-    main()
+    main()

docs/pages/preservationLab/opticalMedia/dvdProcessing/iso-creation.md

@@ -7,10 +7,11 @@ grand_parent: Optical Media
 
 ---
 
-
 # ISO Creation
 {: .no_toc }
 
+This page provides a comprehensive guide for creating ISO images from DVDs as part of a preservation workflow. It outlines the tools and methods used to ensure the best possible recovery of data from discs, ranging from routine processes for handling minor damage to advanced techniques for recovering data from heavily damaged or non-standard DVDs.
+
 ## Table of contents
 {: .no_toc .text-delta }
 
@@ -21,7 +22,11 @@ grand_parent: Optical Media
 
 ### 1. ISO Creation
 
-DVD preservation presents unique challenges due to the potential for physical degradation and various forms of copy protection. Our approach leverages a combination of tools, with each stage representing an escalation in complexity and recovery capability.
+DVD preservation presents unique challenges due to potential for physical degradation and various forms of copy protection. Our approach leverages a combination of tools, with each stage representing an escalation in complexity and recovery capability.
+
+#### **Preliminary Step: Physical Inspection and Cleaning**
+
+Before any digital processing, inspect the DVD for scratches, dirt, or other physical damage. If issues are present, use the RTI/ELM USA DVD Resurfacing Machine to clean and repair the disc. This fully automatic machine offers four cleaning settings, with level 4 being the most intensive and effective for problematic discs.
 
 #### **Primary Approach: MakeMKV with Plextor Disc Drive**
 
@@ -39,9 +44,24 @@ We start with MakeMKV, which is particularly effective at handling commercial DV
 - Limited for non-video content
 - No partial/incremental backup functionality, unlike `ddrescue`
 
-#### **Secondary Approach: ddrescue with Plextor, ASUS, or LG Drives**
+#### **Secondary Approach: ISOBuster with Nimbie on Windows**
+
+If MakeMKV cannot produce a usable ISO, our next attempt is with ISOBuster and a Nimbie autoloader on a Windows system. ISOBuster excels at reading non-standard formats and recovering files from corrupted file systems.
+
+##### **ISOBuster Strengths**
+- Handles non-standard formats effectively
+- Extracts specific files even from damaged file systems
+- Provides detailed error reporting
+- Useful for selective file recovery rather than full disc images
+
+##### **ISOBuster Weaknesses**
+- Requires a paid license
+- Less automated than other tools
+- Limited handling of copy-protected discs
+
+#### **Final Approach: ddrescue with Plextor, ASUS, or LG Drives**
 
-If MakeMKV cannot produce a usable ISO, we use `ddrescue`, either with the same Plextor drive or an alternative (ASUS or LG). `ddrescue` provides a lower-level read with detailed logging for interrupted copies, making it ideal for more challenging discs.
+If both MakeMKV and ISOBuster fail to create a usable ISO, we use `ddrescue`, either with the same Plextor drive or an alternative (ASUS or LG). `ddrescue` provides a lower-level read with detailed logging for interrupted copies, making it ideal for more challenging discs.
 
 ##### **ddrescue Strengths**
 - Performs multiple passes with different strategies
@@ -55,28 +75,13 @@ If MakeMKV cannot produce a usable ISO, we use `ddrescue`, either with the same
 - Playability is not guaranteed even if data recovery is more complete
 - Can be significantly slower than MakeMKV
 
-#### **Final Approach: ISOBuster with Nimbie on Windows**
-
-If both MakeMKV and `ddrescue` fail to create a usable ISO, our last attempt is with ISOBuster and a Nimbie autoloader on a Windows system. ISOBuster excels at reading non-standard formats and recovering files from corrupted file systems.
-
-##### **ISOBuster Strengths**
-- Handles non-standard formats effectively
-- Extracts specific files even from damaged file systems
-- Provides detailed error reporting
-- Useful for selective file recovery rather than full disc images
-
-##### **ISOBuster Weaknesses**
-- Requires a paid license
-- Less automated than other tools
-- Limited handling of copy-protected discs
-
-In summary, we prioritize MakeMKV for its speed, copy protection handling, and high success rate with partially damaged discs. `ddrescue` serves as our fallback for low-level recovery, and ISOBuster is our final approach for non-standard formats or heavily corrupted files.
+In summary, we prioritize MakeMKV for its speed, copy protection handling, and high success rate with partially damaged discs. ISOBuster serves as our fallback for non-standard formats or corrupted file systems, while `ddrescue` is reserved for the most challenging cases requiring low-level recovery.
 
 ---
 
 ### 2. ISO Creation Tools: Installation Notes and Workflow Quirks
 
-This section outlines the installation requirements and any quirks for using MakeMKV, `ddrescue`, and ISOBuster.
+This section outlines the installation requirements and any quirks for using MakeMKV, ISOBuster, and `ddrescue`.
 
 ---
 
@@ -94,7 +99,7 @@ This installs both the GUI and CLI (`makemkvcon`). Settings modified in the GUI
 To organize DVD Title Sets by Source ID, adjust the output filename template in the MakeMKV GUI:
 1. Go to `Preferences` → `General` and turn on Expert Mode.
 2. Go to `Advanced`.
-2. Set the filename template to:
+3. Set the filename template to:
 
    ```
    {NAME1}{_s:SN}{-:CMNT1}{-:DT}{title:+DFLT}{_t:N2}
@@ -103,6 +108,23 @@ By adding `{_s:SN}` to MakeMKV's default output filename template, the resulting
 
 ---
 
+#### **ISOBuster**
+
+ISOBuster + Nimbie requires a Windows system with restricted admin privileges.
+
+1. Connect the Nimbie to the Windows computer.
+2. Right-click on the disc drive in `My Computer` and select "Inspect Drive with ISOBuster" (Admin credentials will be required).
+
+**ISO Creation Steps in ISOBuster:**  
+1. Open ISOBuster; the contents of the DVD drive will display in the main window.
+2. Right-click the drive or volume you want to create the ISO from, then select "Create ISO Image File...".
+3. In the dialog:
+   - Set output location and filename for the ISO.
+   - Choose options such as including hidden files or creating a full raw image.
+4. Click "Start" to begin the ISO creation.
+
+---
+
 #### **ddrescue**
 
 `ddrescue` is particularly sensitive to whether the disc is mounted. To use it, follow these steps:
@@ -126,29 +148,13 @@ By adding `{_s:SN}` to MakeMKV's default output filename template, the resulting
 
 ---
 
-#### **ISOBuster**
-
-ISOBuster + Nimbie requires a Windows system with restricted admin privileges.
-
-1. Connect the Nimbie to the Windows computer.
-2. Right-click on the disc drive in `My Computer` and select "Inspect Drive with ISOBuster" (Admin credentials will be required).
-
-**ISO Creation Steps in ISOBuster:**  
-1. Open ISOBuster; the contents of the DVD drive will display in the main window.
-2. Right-click the drive or volume you want to create the ISO from, then select "Create ISO Image File...".
-3. In the dialog:
-   - Set output location and filename for the ISO.
-   - Choose options such as including hidden files or creating a full raw image.
-4. Click "Start" to begin the ISO creation.
-
----
-
 ### Summary of Approaches
 
 | Approach           | Tool       | Strengths                                          | Weaknesses                                         |
 |--------------------|------------|----------------------------------------------------|----------------------------------------------------|
+| Preliminary        | RTI/ELM USA DVD Resurfacing Machine | Effective cleaning and repair of physical disc issues | Requires physical equipment and maintenance        |
 | Primary            | MakeMKV    | Fast, good with copy protection, playable results  | Limited on non-video content, skips unreadable sections |
-| Secondary          | ddrescue   | Low-level recovery, detailed logs, multiple passes | Slow, lacks copy protection handling               |
-| Tertiary (final)   | ISOBuster  | Non-standard format handling, file-specific recovery | Manual, limited copy protection handling           |
+| Secondary          | ISOBuster  | Non-standard format handling, file-specific recovery | Manual, limited copy protection handling           |
+| Tertiary (final)   | ddrescue   | Low-level recovery, detailed logs, multiple passes | Slow, lacks copy protection handling               |
 
 ---