From ae74d1141556a517766b05c2b9f139e4517484bb Mon Sep 17 00:00:00 2001 From: Danny Terweij Date: Fri, 21 Feb 2014 23:39:31 +0100 Subject: [PATCH] Improvement of Scavenge part 2. - Changing scavenge time immediately affected - Refactored master cron - Added more logging in both master cron methods (lowmemflag on/off) - Code Cleanups regarding master cron and scavenge code --- kloxo/bin/common/clearsession.php | 29 +++-- kloxo/bin/common/function.sh | 51 +++++--- kloxo/bin/scavenge.php | 9 +- kloxo/bin/sisinfoc.php | 69 +++++----- kloxo/httpdocs/htmllib/lib/lxserverlib.php | 118 +++++++++++------- .../htmllib/phplib/lib/generallib.php | 28 ++++- kloxo/sbin/kloxo-wrapper.sh | 23 +++- kloxo/sbin/kloxo.php | 45 +++++-- kloxo/src/kloxo.c | 45 ++++--- 9 files changed, 274 insertions(+), 143 deletions(-) diff --git a/kloxo/bin/common/clearsession.php b/kloxo/bin/common/clearsession.php index 15b72e34a7..ae6abc9402 100644 --- a/kloxo/bin/common/clearsession.php +++ b/kloxo/bin/common/clearsession.php @@ -1,5 +1,22 @@ . +// include_once 'htmllib/lib/displayinclude.php'; function clearsession_main() @@ -34,13 +51,3 @@ function clearsession_main() } clearsession_main(); - -// I do not want to wait for 600 secs in debug mode :) -// Altho why is there a sleep at all.... -if ($sgbl->dbg >= 0) { -dprint("Sleeping for 10 seconds....\n"); -sleep(10); -} else { -dprint("Sleeping for 600 seconds....\n"); -sleep(600); -} diff --git a/kloxo/bin/common/function.sh b/kloxo/bin/common/function.sh index 717b1de99e..c9f67aac66 100644 --- a/kloxo/bin/common/function.sh +++ b/kloxo/bin/common/function.sh @@ -1,17 +1,30 @@ #!/bin/sh - -# issue #623 - +# Kloxo, Hosting Control Panel +# +# Copyright (C) 2000-2009 LxLabs +# Copyright (C) 2009-2014 LxCenter +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This file is loaded from kloxo-wrapper.sh +# __path_php_path="/usr/local/lxlabs/ext/php/php"; -__path_program_root="/usr/local/lxlabs/$progname/"; -__path_slave_db="/usr/local/lxlabs/$progname/etc/conf/slave-db.db"; - -# __path_server_path="../sbin/$progname.php"; -# __path_server_exe="../sbin/$progname.exe"; -# __path_low_memory_file="../etc/flag/lowmem.flag"; +__path_program_root="/usr/local/lxlabs/kloxo"; +__path_slave_db="/usr/local/lxlabs/kloxo/etc/conf/slave-db.db"; -__path_server_path="/usr/local/lxlabs/kloxo/sbin/$progname.php"; -__path_server_exe="/usr/local/lxlabs/kloxo/sbin/$progname.exe"; +__path_server_path="/usr/local/lxlabs/kloxo/sbin/kloxo.php"; +__path_server_exe="/usr/local/lxlabs/kloxo/sbin/kloxo.exe"; __path_low_memory_file="/usr/local/lxlabs/kloxo/etc/flag/lowmem.flag"; kill_and_save_pid() { @@ -22,14 +35,14 @@ kill_and_save_pid() { } save_pid() { - echo $$ > "$__path_program_root/pid/$name.pid"; + echo $$ > "$__path_program_root/pid/kloxo.pid"; } kill_pid() { name=$1 - pid=`cat $__path_program_root/pid/$name.pid`; + pid=`cat $__path_program_root/pid/kloxo.pid`; kill $pid 2>/dev/null - usleep 10000 + usleep 1000 kill -9 $pid 2>/dev/null } @@ -47,10 +60,12 @@ wrapper_main() { while : ; do if [ -f $__path_low_memory_file ] ; then - /bin/cp $__path_server_exe.core $__path_server_exe - chmod 755 $__path_server_exe - $__path_server_exe $string >/dev/null 2>&1 - else + /bin/cp $__path_server_exe.core $__path_server_exe; + chmod 755 $__path_server_exe; + echo "Starting Kloxo core (lowmem)" + $__path_server_exe $string >/dev/null 2>&1; + else + echo "Starting Kloxo core" $__path_php_path $__path_server_path $string >/dev/null 2>&1; fi sleep 10; diff --git a/kloxo/bin/scavenge.php b/kloxo/bin/scavenge.php index 9777ef68db..c31bf6fe46 100644 --- a/kloxo/bin/scavenge.php +++ b/kloxo/bin/scavenge.php @@ -28,6 +28,9 @@ function scavenge_main() { global $gbl, $sgbl, $login, $ghtml; + log_scavenge("Fix log dir"); + passthru("$sgbl->__path_php_path ../bin/common/fixlogdir.php"); + log_scavenge("### Starting Scavenge"); initProgramlib('admin'); uploadStatsLxCenter(); @@ -60,9 +63,6 @@ function scavenge_main() system("service lighttpd restart"); } - log_scavenge("Fix log dir"); - passthru("$sgbl->__path_php_path ../bin/common/fixlogdir.php"); - log_scavenge("InstallApp update"); passthru("$sgbl->__path_php_path ../bin/installapp-update.phps"); @@ -82,6 +82,9 @@ function scavenge_main() auto_update(); log_scavenge("### End Scavenge"); + + // Wait at least 60 seconds before ending the scavenge + sleep(60); } exit_if_secondary_master(); diff --git a/kloxo/bin/sisinfoc.php b/kloxo/bin/sisinfoc.php index 1e14aaf3d0..76783f2715 100644 --- a/kloxo/bin/sisinfoc.php +++ b/kloxo/bin/sisinfoc.php @@ -1,48 +1,57 @@ -. +// include_once "htmllib/lib/include.php"; include_once "htmllib/lib/lxguardincludelib.php"; + + +function collect_traffic() +{ + $flfile = "__path_program_etc/last_sisinfoc"; + $ret = lfile_get_unserialize($flfile); + $ret['time'] = time(); + lfile_put_serialize($flfile, $ret); +} + $global_dontlogshell = true; + exit_if_secondary_master(); -exit_if_another_instance_running(); -debug_for_backend(); +exit_if_another_instance_running(); +debug_for_backend(); watchdog__sync::watchRun(); - if ($sgbl->is_this_master()) { - $gbl->is_master = true; - initProgram('admin'); - run_mail_to_ticket(); + $gbl->is_master = true; + initProgram('admin'); + run_mail_to_ticket(); } - monitor_load(); -collect_traffic(); -lxguard_main(); -add_to_log("/var/log/kloxo/smtp.log"); -add_to_log("/var/log/kloxo/courier"); +collect_traffic(); +lxguard_main(); -function collect_traffic() -{ - $flfile = "__path_program_etc/last_sisinfoc"; - $ret = lfile_get_unserialize($flfile); - $interval = 20 * 60; - //$interval = 2; - if ((time() - $ret['time']) < $interval) { - //return; - } - $oldtime = $ret['time']; - //if (!$oldtime) { $oldtime = time() - 20 * 60 ; } - if (!$oldtime) { $oldtime = time() - 5 * 60 ; } - $newtime = time(); - $ret['time'] = time(); - lfile_put_serialize($flfile, $ret); +add_to_log("/var/log/kloxo/smtp.log"); - //mailtraffic:generateGraph($oldtime, $newtime); - //webtraffic::generateGraph($oldtime, $newtime); -} +add_to_log("/var/log/kloxo/courier"); diff --git a/kloxo/httpdocs/htmllib/lib/lxserverlib.php b/kloxo/httpdocs/htmllib/lib/lxserverlib.php index 6db7213b73..f45963370c 100644 --- a/kloxo/httpdocs/htmllib/lib/lxserverlib.php +++ b/kloxo/httpdocs/htmllib/lib/lxserverlib.php @@ -1,16 +1,32 @@ -. +// +// This file is running when lowmem flag is disabled +// function lxserver_main() { global $gbl, $sgbl, $login, $ghtml; global $argv, $argc; - // Set time limit to indefinite execution - if ($argv[1] === 'slave') { $login = new Client(null, null, 'slave'); - //Initthisdef uses the db to load the drivers. NO longer callable in slave. - //$login->initThisDef(); $gbl->is_slave = true; $gbl->is_master = false; $rmt = unserialize(lfile_get_contents("__path_slave_db")); @@ -28,17 +44,11 @@ function lxserver_main() } $login->cttype = 'admin'; - //set_error_handler("lx_error_handler"); - //set_exception_handler("lx_exception_handler"); - - set_time_limit (0); - if (WindowsOs()) { - some_server_windows(); - } else { - some_server(); - } - + // Set php script execution timer to unlimited + set_time_limit(0); + // Start internal socket for remote + some_server(); } @@ -47,7 +57,6 @@ function lxserver_main() function do_server_stuff() { global $gbl, $sgbl, $login, $ghtml; - //dprint("in Do server stuff\n"); if (if_demo()) { return; @@ -57,13 +66,24 @@ function do_server_stuff() timed_execution(); if ($sgbl->is_this_master()) { $schour = null; - $schour = $login->getObject('general')->generalmisc_b->scavengehour; - $scminute = $login->getObject('general')->generalmisc_b->scavengeminute; - //dprint("Cron exec $schour, $scminute\n"); + $scminute = null; + + $timefile = "../etc/conf/scavenge_time.conf"; + + if (lxfile_exists($timefile)) { + + $readvalue = file_get_contents($timefile); + $readvalue = explode(" ", $readvalue); + $schour = $readvalue['0']; + $scminute = $readvalue['1']; + + } + + log_log("cron_exec", "Initialize Scavenge Cronjob"); if ($schour) { cron_exec($schour, $scminute, "exec_scavenge"); } else { - cron_exec("3", "57", "exec_scavenge"); + cron_exec("03", "35", "exec_scavenge"); } } } catch (exception $e) { @@ -77,24 +97,34 @@ function cron_exec($hour, $minute, $func) { static $localvar; - //dprint("in Cron exec\n"); - //dprintr($localvar); + $time = mktime($hour, $minute); + $now = time(); + + $nowH = date("H"); + $nowM = date("i"); - $time = mktime($hour, $minute , 0, date('n'), date('j'), date("Y")); - $now = time(); + if ($func === "exec_scavenge") { + $niceNameFunc = "Scavenge"; + } else { + $niceNameFunc = $func; + } + + log_log("cron_exec", "Cron $niceNameFunc starts at ($hour:$minute)"); + log_log("cron_exec", "Time now is ($nowH:$nowM)"); if (isset($localvar[$func]) && $localvar[$func]) { - //dprint("Already execed \n"); if ($now > $time + 2 * 60) { - $localvar[$func] = false; - } - return ; - + log_log("cron_exec", "Cron timing: $niceNameFunc finished, back to normal state."); + $localvar[$func] = false; + } else { + log_log("cron_exec", "Cron timing: $niceNameFunc is running."); + return ; + } } - if ($now > $time && $now < $time + 2* 60) { - $localvar[$func] = true; - log_log("cron_exec", "Execing $func"); + if ($hour === $nowH && $minute === $nowM) { + $localvar[$func] = true; + log_log("cron_exec", "Starting $niceNameFunc"); $func(); } } @@ -104,8 +134,7 @@ function timed_exec($time, $func) $v = "global_v$func"; global $$v; $ct = time(); - if (($ct - $$v) >= $time * 30 ) { - //dprint("Executing at $ct {$$v} rd time $func\n"); + if (($ct - $$v) >= $time * 30 ) { $$v = $ct; $func(); } @@ -113,9 +142,8 @@ function timed_exec($time, $func) function exec_scavenge() { - // TODO: Not used function - global $gbl, $sgbl, $login, $ghtml; - dprint("Execing collect quota\n"); + global $gbl, $sgbl, $login, $ghtml; + $olddir = getcwd(); lchdir("__path_program_htmlbase"); exec_with_all_closed("$sgbl->__path_php_path ../bin/scavenge.php"); @@ -128,7 +156,7 @@ function checkRestart() if (if_demo()) { return; } - + log_log("cron_exec","Check service restarts...\n"); $res = lscandir_without_dot("__path_program_etc/.restart"); if ($res === false) { @@ -143,19 +171,21 @@ function checkRestart() } lunlink("__path_program_etc/.restart/$r"); dprint("Restarting $cmd\n"); - // THe 3,4 etc are the tcp ports of this program, and it should be closed, else some programs will grab it. - //exec("/etc/init.d/$cmd restart /dev/null 2>&1 3exception = null; } catch (exception $e) { - dprint("Coaught Execption: " . $e->getMessage()); + dprint("Caught Exception: " . $e->getMessage()); $res = new Remote(); $res->ret = -1; $res->exception = $e; diff --git a/kloxo/httpdocs/htmllib/phplib/lib/generallib.php b/kloxo/httpdocs/htmllib/phplib/lib/generallib.php index eb6eb21351..1017d639a3 100644 --- a/kloxo/httpdocs/htmllib/phplib/lib/generallib.php +++ b/kloxo/httpdocs/htmllib/phplib/lib/generallib.php @@ -1,5 +1,22 @@ -. +// @@ -179,7 +196,12 @@ function isSync() { return false; } function updateScavengeTime($param) { - $ret = lfile_put_contents("../etc/conf/scavenge_time.conf", "{$param['generalmisc_b-scavengehour']} {$param['generalmisc_b-scavengeminute']}"); + global $gbl, $sgbl, $login, $ghtml; + // Add leading zero's. + $padH = str_pad($param['generalmisc_b-scavengehour'], 2, '0', STR_PAD_LEFT); + $padM = str_pad($param['generalmisc_b-scavengeminute'], 2, '0', STR_PAD_LEFT); + + $ret = lfile_put_contents("../etc/conf/scavenge_time.conf", "{$padH} {$padM}"); if (!$ret) { throw new lxException("could_not_save_file"); } diff --git a/kloxo/sbin/kloxo-wrapper.sh b/kloxo/sbin/kloxo-wrapper.sh index 074fb49d91..83a5fce6f8 100755 --- a/kloxo/sbin/kloxo-wrapper.sh +++ b/kloxo/sbin/kloxo-wrapper.sh @@ -1,11 +1,26 @@ #!/bin/sh - +# Kloxo, Hosting Control Panel +# +# Copyright (C) 2000-2009 LxLabs +# Copyright (C) 2009-2014 LxCenter +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# progname=kloxo source ../bin/common/function.sh - kill_and_save_pid wrapper; -wrapper_main; - +wrapper_main; diff --git a/kloxo/sbin/kloxo.php b/kloxo/sbin/kloxo.php index e1f8d8d5b4..385ac5775e 100755 --- a/kloxo/sbin/kloxo.php +++ b/kloxo/sbin/kloxo.php @@ -1,29 +1,50 @@ . +// +// This file is executed when lowmem flag is disabled. +// When lowmem flaq is enabled kloxo.exe is executed. +// include_once "htmllib/lib/include.php"; include_once "htmllib/lib/lxserverlib.php"; -kill_and_save_pid('lxserver'); -debug_for_backend(); - - -lxserver_main(); - - - function timed_execution() { global $global_dontlogshell; + $global_dontlogshell = true; - if (windowsOS()) { return; } + + // execute every minute timed_exec(2, "checkRestart"); - timed_exec(2 * 5, "execSisinfoc"); - $global_dontlogshell = false; + // execute every 10 minutes + timed_exec(2 * 5, "execSisinfoc"); + $global_dontlogshell = false; } function execSisinfoc() { - dprint("execing sisinfoc\n"); + log_log("cron_exec","Starting SISInfoC\n"); lxshell_background("__path_php_path", "../bin/sisinfoc.php"); } +kill_and_save_pid('lxserver'); + +debug_for_backend(); + +lxserver_main(); diff --git a/kloxo/src/kloxo.c b/kloxo/src/kloxo.c index f8ef3f3d43..aabc0cbcd0 100644 --- a/kloxo/src/kloxo.c +++ b/kloxo/src/kloxo.c @@ -2,7 +2,7 @@ * Kloxo, Hosting Control Panel * * Copyright (C) 2000-2009 LxLabs - * Copyright (C) 2009-2010 LxCenter + * Copyright (C) 2009-2014 LxCenter * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -16,6 +16,10 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * + * This executable is only used when lowmem flag is enabled. + * Otherwise the core is running as kloxo.php */ #include @@ -94,7 +98,7 @@ int run_php_prog_ssl(SSL *ssl, int sock) } } - printf("Input %d %s\n", strlen(data), data); + //printf("Input %d %s\n", strlen(data), data); bzero(buf, sizeof(buf)); //printf ("Received %d chars:'%s'\n", err, buf); @@ -109,24 +113,24 @@ int run_php_prog_ssl(SSL *ssl, int sock) if (pid == 0) { dup2(pipefd[1], 1); close(pipefd[0]); - execl("/usr/local/lxlabs/ext/php/php", "lxphp", "../bin/common/process_single.php", ftempname, NULL); - printf("Exec failed\n"); + execl("/usr/local/lxlabs/ext/php/php", "lxphp", "/usr/local/lxlabs/kloxo/bin/common/process_single.php", ftempname, NULL); + printf("Exec failed (process_single.php)\n"); exit(0); } else { close(pipefd[1]); - printf("Pipe %d\n", pipefd[0]); + // printf("Pipe %d\n", pipefd[0]); while (1) { n = read(pipefd[0], buf, sizeof(buf)); totaln += n; if (n > 0) { p = ssl_or_tcp_write(ssl, sock, buf, n); } else { - printf("Got %d\n\n", totaln); + // printf("Got %d\n\n", totaln); // Dummy Read... A Must while (1) { bzero(tmpname, sizeof(tmpname)); p = ssl_or_tcp_read(ssl, sock, tmpname, sizeof(tmpname)); - printf("Got %s\n\n", tmpname); + // printf("Got %s\n\n", tmpname); if (p <= 0) { break; } @@ -389,7 +393,7 @@ int check_restart() printf("Checking Restarts...\n"); - n = scandir("../etc/.restart/", &namelist, 0, alphasort); + n = scandir("/usr/local/lxlabs/kloxo/etc/.restart/", &namelist, 0, alphasort); if (n < 0) { perror("scandir"); return 1; @@ -401,16 +405,16 @@ int check_restart() neededstring = position + 10; if (!strcmp(neededstring, "lxcollectquota")) { printf("Running CollectQuota\n"); - close_and_system("/usr/local/lxlabs/ext/php/php ../bin/collectquota.php --just-db=true &"); + close_and_system("/usr/local/lxlabs/ext/php/php /usr/local/lxlabs/kloxo/bin/collectquota.php --just-db=true &"); } else if (!strcmp(neededstring, "openvz_tc")) { printf("Running Openvz\n"); - close_and_system("sh ../etc/openvz_tc.sh"); + close_and_system("sh /usr/local/lxlabs/kloxo/etc/openvz_tc.sh"); } else { printf("Restarting %s\n", neededstring); snprintf(cmd, sizeof(cmd), "/etc/init.d/%s restart &", neededstring); close_and_system(cmd); } - snprintf(cmd, sizeof(cmd), "../etc/.restart/%s", namelist[n]->d_name); + snprintf(cmd, sizeof(cmd), "/usr/local/lxlabs/kloxo/etc/.restart/%s", namelist[n]->d_name); unlink(cmd); } free(namelist[n]); @@ -430,7 +434,8 @@ int exec_sisinfoc() } printf("Executing Sisinfoc...\n"); - close_and_system("/usr/local/lxlabs/ext/php/php ../bin/sisinfoc.php >/dev/null 2>&1 &"); + close_and_system("/usr/local/lxlabs/ext/php/php /usr/local/lxlabs/kloxo/bin/sisinfoc.php >/dev/null 2>&1 &"); + printf("Executed in background.\n"); sisinfoc_timer = now; } @@ -461,8 +466,10 @@ int exec_scavenge() printf("Loading Scavenge time configuation...\n"); - if (!access("../etc/conf/scavenge_time.conf", R_OK)) { - fp = fopen("../etc/conf/scavenge_time.conf", "r"); + if (!access("/usr/local/lxlabs/kloxo/etc/conf/scavenge_time.conf", R_OK)) { + fp = fopen("/usr/local/lxlabs/kloxo/etc/conf/scavenge_time.conf", "r"); + printf("Found user Scavenge time configuation file...\n"); + if (fp) { fscanf(fp, "%d %d", &hour, &min); fclose(fp); @@ -470,11 +477,11 @@ int exec_scavenge() } localtime_r(&now, &tms); - printf(" Now Value: %02d:%02d\n", tms.tm_hour, tms.tm_min); - printf(" Read Value: %02d:%02d\n", hour, min); + printf(" The Time now: %02d:%02d\n", tms.tm_hour, tms.tm_min); + printf(" The Time when we run: %02d:%02d\n", hour, min); // check interval of 5 minutes - interval = 5; + interval = 1; time_match = 0; for(i = 0; i <= interval; i++) { if (tms.tm_hour == hour && tms.tm_min == min) { @@ -490,7 +497,9 @@ int exec_scavenge() if (time_match) { printf("Executing Scavenge...\n"); - close_and_system("/usr/local/lxlabs/ext/php/php ../bin/scavenge.php >/dev/null 2>&1 &"); + close_and_system("/usr/local/lxlabs/ext/php/php /usr/local/lxlabs/kloxo/bin/scavenge.php >/dev/null 2>&1 &"); + printf("Executed in background.\n"); + scavenge_timer = now + interval * 60; } else {