From 29c8080178a3db1c63dc0b729ec3501c503cc5e7 Mon Sep 17 00:00:00 2001
From: "Chen, Bohan" <bohan.chen@intel.com>
Date: Fri, 1 Dec 2023 09:49:38 +0800
Subject: [PATCH] [Encode] Fix some coverity issues exposed in encode

Fix some remaining encode coverity issues: 1 Unchecked return value; 1 Resource leak; 2 Modulo by zero

Signed-off-by: Chen, Bohan <bohan.chen@intel.com>
---
 encode/av1encode.c  |  4 ++--
 encode/avcenc.c     |  7 ++++++-
 encode/hevcencode.c | 14 +++++++++++++-
 encode/vp9enc.c     |  7 ++++++-
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/encode/av1encode.c b/encode/av1encode.c
index 5a7fc16..dd2d9ed 100644
--- a/encode/av1encode.c
+++ b/encode/av1encode.c
@@ -2900,9 +2900,9 @@ static int calc_PSNR(double *psnr)
             srcyuv_ptr = mmap(0, fourM, PROT_READ, MAP_SHARED, fileno(srcyuv_fp), i);
             recyuv_ptr = mmap(0, fourM, PROT_READ, MAP_SHARED, fileno(recyuv_fp), i);
             if ((srcyuv_ptr == MAP_FAILED) || (recyuv_ptr == MAP_FAILED)) {
-                if (srcyuv_ptr) 
+                if (srcyuv_ptr != MAP_FAILED) 
                     munmap(srcyuv_ptr, fourM);
-                if (recyuv_ptr) 
+                if (recyuv_ptr != MAP_FAILED) 
                     munmap(recyuv_ptr, fourM);
                 printf("Failed to mmap YUV files\n");
                 return 1;
diff --git a/encode/avcenc.c b/encode/avcenc.c
index a41fc3f..6ab8a46 100644
--- a/encode/avcenc.c
+++ b/encode/avcenc.c
@@ -2117,7 +2117,12 @@ int main(int argc, char *argv[])
     file_size = ftello(yuv_fp);
     frame_size = picture_width * picture_height + ((picture_width * picture_height) >> 1) ;
 
-    if ((file_size < frame_size) || ((frame_size != 0) && (file_size % frame_size))) {
+    if (frame_size == 0) {
+        fclose(yuv_fp);
+        printf("Frame size is not correct\n");
+        return -1;
+    }
+    if ((file_size < frame_size) || (file_size % frame_size)) {
         fclose(yuv_fp);
         printf("The YUV file's size is not correct\n");
         return -1;
diff --git a/encode/hevcencode.c b/encode/hevcencode.c
index dc38ccc..d2ecd37 100644
--- a/encode/hevcencode.c
+++ b/encode/hevcencode.c
@@ -47,6 +47,13 @@
         exit(1);                                                        \
     }
 
+#define CHECK_CONDITION(cond)                                                \
+    if(!(cond))                                                              \
+    {                                                                        \
+        fprintf(stderr, "Unexpected condition: %s:%d\n", __func__, __LINE__);\
+        exit(1);                                                             \
+    }
+
 #include "loadsurface.h"
 
 #define NAL_REF_IDC_NONE        0
@@ -1920,7 +1927,8 @@ static int process_cmdline(int argc, char *argv[])
         else {
             struct stat tmp;
 
-            fstat(fileno(srcyuv_fp), &tmp);
+            int ret = fstat(fileno(srcyuv_fp), &tmp);
+            CHECK_CONDITION(ret == 0);
             srcyuv_frames = tmp.st_size / (frame_width * frame_height * 1.5);
             printf("Source YUV file %s with %llu frames\n", srcyuv_fn, srcyuv_frames);
 
@@ -3277,6 +3285,10 @@ static int calc_PSNR(double *psnr)
             srcyuv_ptr = mmap(0, fourM, PROT_READ, MAP_SHARED, fileno(srcyuv_fp), i);
             recyuv_ptr = mmap(0, fourM, PROT_READ, MAP_SHARED, fileno(recyuv_fp), i);
             if ((srcyuv_ptr == MAP_FAILED) || (recyuv_ptr == MAP_FAILED)) {
+                if (srcyuv_ptr != MAP_FAILED) 
+                    munmap(srcyuv_ptr, fourM);
+                if (recyuv_ptr != MAP_FAILED) 
+                    munmap(recyuv_ptr, fourM);
                 printf("Failed to mmap YUV files\n");
                 return 1;
             }
diff --git a/encode/vp9enc.c b/encode/vp9enc.c
index 7da95e1..14d4609 100644
--- a/encode/vp9enc.c
+++ b/encode/vp9enc.c
@@ -1541,7 +1541,12 @@ main(int argc, char *argv[])
     file_size = ftello(yuv_fp);
     frame_size = picture_width * picture_height + ((picture_width * picture_height) >> 1) ;
 
-    if ((file_size < frame_size) || ((frame_size != 0) && (file_size % frame_size))) {
+    if (frame_size == 0) {
+        fclose(yuv_fp);
+        printf("Frame size is not correct\n");
+        return -1;
+    }
+    if ((file_size < frame_size) || (file_size % frame_size)) {
         fclose(yuv_fp);
         printf("The YUV file's size is not correct\n");
         return -1;