Merge pull request #159 from jpwhite4/shared_perf

Performance Improvements for shared_jobs script
ubccr · Mar 29, 2019 · 70098c8 · 70098c8
2 parents fd3ef5b + 488bac4
commit 70098c8
Showing 1 changed file with 17 additions and 3 deletions.
diff --git a/background_scripts/supremm_sharedjobs.php b/background_scripts/supremm_sharedjobs.php
@@ -202,9 +202,26 @@ function shared_jobs($resource_id, $start, $end)
             $jbu->execute(array($key));
         }
         $logger->debug("Processed " . count($sharedjobs) . " jobs for host " . $host['name']);
+
+        if ($jobpeercount > 250000) {
+            fclose($jobpeersfile);
+            batch_update($jobpeersfname, $jobpeercount);
+            $jobpeersfile = fopen($jobpeersfname, 'wb');
+            $jobpeercount = 0;
+        }
     }
 
     fclose($jobpeersfile);
+    batch_update($jobpeersfname, $jobpeercount);
+
+    unlink($jobpeersfname);
+    $db->handle()->exec('DROP TABLE IF EXISTS `modw_supremm`.`job_tmp`');
+}
+
+function batch_update($jobpeersfname, $jobpeercount)
+{
+    global $logger;
+    global $db;
 
     if ($jobpeercount > 0) {
         $logger->info('Batch update ' . $jobpeercount . ' job peers');
@@ -222,9 +239,6 @@ function shared_jobs($resource_id, $start, $end)
     } else {
         $logger->info('Skipping batch update - no job peers found');
     }
-
-    unlink($jobpeersfname);
-    $db->handle()->exec('DROP TABLE IF EXISTS `modw_supremm`.`job_tmp`');
 }
 
 /**