diff --git a/packages/tests-scheduler-lime/Makefile b/packages/tests-scheduler-lime/Makefile new file mode 100644 index 000000000..2573fd341 --- /dev/null +++ b/packages/tests-scheduler-lime/Makefile @@ -0,0 +1,48 @@ +# +# Copyright (C) 2019 Ilario Gelmetti +# +# This is free software, licensed under the GNU General Public License v3. +# + +include $(TOPDIR)/rules.mk + +GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . ) +GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . ) + +PKG_NAME:=tests-scheduler-lime +PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP) + +include $(INCLUDE_DIR)/package.mk + +define Package/$(PKG_NAME) + CATEGORY:=LiMe + TITLE:=LibreMesh tests for tests-scheduler + MAINTAINER:=Ilario Gelmetti + URL:=https://libremesh.org + DEPENDS:=+logrotate +lime-report +bandwidth-test +tests-scheduler + PKGARCH:=all +endef + +define Package/$(PKG_NAME)/config +endef + +define Package/$(PKG_NAME)/description + Schedules a few tests which gather useful information for debugging + a LibreMesh network. + The scheduling is done using the tests-scheduler package and can be run + at the network usage peak time or at the night time. +endef + +define Build/Compile +endef + +define Package/$(PKG_NAME)/install + $(INSTALL_DIR) $(1)/etc/logrotate.d/ + $(INSTALL_DIR) $(1)/etc/tests-scheduler/nightHooks.d/ + $(INSTALL_DIR) $(1)/etc/tests-scheduler/peakHooks.d/ + $(INSTALL_BIN) ./files/etc/logrotate.d/* $(1)/etc/logrotate.d + $(INSTALL_BIN) ./files/etc/tests-scheduler/nightHooks.d/* $(1)/etc/tests-scheduler/nightHooks.d/ + $(INSTALL_BIN) ./files/etc/tests-scheduler/peakHooks.d/* $(1)/etc/tests-scheduler/peakHooks.d/ +endef + +$(eval $(call BuildPackage,$(PKG_NAME))) diff --git a/packages/tests-scheduler-lime/files/etc/logrotate.d/tests-peak-lime-report b/packages/tests-scheduler-lime/files/etc/logrotate.d/tests-peak-lime-report new file mode 100644 index 000000000..5339cfab9 --- /dev/null +++ b/packages/tests-scheduler-lime/files/etc/logrotate.d/tests-peak-lime-report @@ -0,0 +1,6 @@ +/tmp/tests-peak-lime-report { + daily + compress + rotate 5 + size 100k +} diff --git a/packages/tests-scheduler-lime/files/etc/tests-scheduler/nightHooks.d/10-bandwidth-test b/packages/tests-scheduler-lime/files/etc/tests-scheduler/nightHooks.d/10-bandwidth-test new file mode 100644 index 000000000..431c9e74c --- /dev/null +++ b/packages/tests-scheduler-lime/files/etc/tests-scheduler/nightHooks.d/10-bandwidth-test @@ -0,0 +1,2 @@ +#!/bin/sh +(date && bandwidth-test) >> /tmp/tests-night-bandwidth-test diff --git a/packages/tests-scheduler-lime/files/etc/tests-scheduler/peakHooks.d/10-lime-report b/packages/tests-scheduler-lime/files/etc/tests-scheduler/peakHooks.d/10-lime-report new file mode 100644 index 000000000..fd26dc2f4 --- /dev/null +++ b/packages/tests-scheduler-lime/files/etc/tests-scheduler/peakHooks.d/10-lime-report @@ -0,0 +1,2 @@ +#!/bin/sh +lime-report >> /tmp/tests-peak-lime-report diff --git a/packages/tests-scheduler/Makefile b/packages/tests-scheduler/Makefile new file mode 100644 index 000000000..37a6f23a1 --- /dev/null +++ b/packages/tests-scheduler/Makefile @@ -0,0 +1,51 @@ +# +# Copyright (C) 2019 Ilario Gelmetti +# +# This is free software, licensed under the GNU General Public License v3. +# + +include $(TOPDIR)/rules.mk + +GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . ) +GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . ) + +PKG_NAME:=tests-scheduler +PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP) + +include $(INCLUDE_DIR)/package.mk + +define Package/$(PKG_NAME) + CATEGORY:=Utilities + TITLE:=Tests scheduler at peak and night time + MAINTAINER:=Ilario Gelmetti + URL:=https://libremesh.org + DEPENDS:=+at + PKGARCH:=all +endef + +define Package/$(PKG_NAME)/config +endef + +define Package/$(PKG_NAME)/description + Monitors the number of clients of the network for each hour, + uses this information for scheduling tests at the peak time and/or at the night time. +endef + +define Build/Compile +endef + +define Package/$(PKG_NAME)/install + $(INSTALL_DIR) $(1)/bin/ + $(INSTALL_DIR) $(1)/etc/uci-defaults/ + $(INSTALL_DIR) $(1)/etc/init.d/ + $(INSTALL_DIR) $(1)/etc/config/ + $(INSTALL_DIR) $(1)/etc/tests-scheduler/peakHooks.d/ + $(INSTALL_DIR) $(1)/etc/tests-scheduler/nightHooks.d/ + $(INSTALL_BIN) ./files/bin/tests-scheduler-at.lua $(1)/bin/tests-scheduler-at + $(INSTALL_BIN) ./files/bin/tests-scheduler-probe.lua $(1)/bin/tests-scheduler-probe + $(INSTALL_BIN) ./files/etc/uci-defaults/* $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/etc/init.d/* $(1)/etc/init.d + $(INSTALL_CONF) ./files/etc/config/tests-scheduler $(1)/etc/config/ +endef + +$(eval $(call BuildPackage,$(PKG_NAME))) diff --git a/packages/tests-scheduler/files/bin/tests-scheduler-at.lua b/packages/tests-scheduler/files/bin/tests-scheduler-at.lua new file mode 100644 index 000000000..b291decd7 --- /dev/null +++ b/packages/tests-scheduler/files/bin/tests-scheduler-at.lua @@ -0,0 +1,179 @@ +#!/usr/bin/lua + +local JSON = require("luci.jsonc") +local utils = require("lime.utils") +local fs = require("nixio.fs") +local libuci = require("uci") +local SharedState = require("shared-state") +require("nixio.util") + +local function config_uci_get(sectionname, option) + local result + result = libuci:cursor():get("tests-scheduler",sectionname,option) + return result +end + +local dataFile = arg[1] or config_uci_get("probe","data_file") +local peakHooksDir = arg[2] or config_uci_get("at","peak_hooks_dir") +local nightHooksDir = arg[3] or config_uci_get("at","night_hooks_dir") + +-- check if a file exists +function file_exists(file) + local f = io.open(file, "rb") + if f then f:close() end + return f ~= nil +end + +-- get all lines from a file, returns an empty +-- list/table if the file does not exist +function lines_from_tonumber(file) + if not file_exists(file) then return {} end + local lines = {} + for line in io.lines(file) do + lines[#lines + 1] = tonumber(line) or "missing" + end + return lines +end + +local function do_split(str,pat) + local tbl = {} + str:gsub(pat, function(x) tbl[#tbl+1]=x end) + return tbl +end + +local function max(t) + if #t == 0 then return nil, nil end + local key, value = 1, tonumber(t[1]) + for i = 2, #t do + temp = tonumber(t[i]) + if value < temp then + key, value = i, temp + end + end + return key, value +end + +local function min(t) + if #t == 0 then return nil, nil end + local key, value = 1, tonumber(t[1]) + for i = 2, #t do + temp = tonumber(t[i]) + if value > temp then + key, value = i, temp + end + end + return key, value +end + +local data = lines_from_tonumber(dataFile) + +for _,val in pairs(data) do + if val == "missing" then + io.stderr:write("Data from at least one whole day is needed in ".. + dataFile..", stopping.\n") + os.exit(1) + end +end + +io.stderr:write("Found enough data in "..dataFile..", continuing.\n") + +local peakHour1,_ = max(data) +local peakHour = peakHour1 - 1 + +local peakDirFun = fs.dir(peakHooksDir) or {} +local peakIndex = 0 +for hook in fs.dir(peakHooksDir) do + local peakMinute = (peakIndex * 5) % 60 + local peakMinutePad = string.format("%02d", peakMinute) + local peakAt = "echo '"..peakHooksDir.."/"..hook.."' | at -Mv "..peakHour..":".. + peakMinutePad.." 2>&1" + local handlePeakAt = io.popen(peakAt, 'r') + local peakAtRaw = handlePeakAt:read("*a") + handlePeakAt:close() + local peakAtTime = do_split(peakAtRaw,"%C+")[1] + utils.log("Scheduled time:\t"..peakAtTime.."\tCommand:\t"..peakHooksDir.."/"..hook) + peakIndex = peakIndex + 1 +end + +local nightDirFun = fs.dir(nightHooksDir) or os.exit(0) +if not nightDirFun(1) then + io.stderr:write("No tests configured to be run at the night time.\n") + os.exit(0) +end +local nightDirFun = fs.dir(nightHooksDir) + +local nightHours = {} + +for i = 1,6 do + nightHour1,_ = min(data) + nightHours[i] = nightHour1 - 1 + data[nightHour1] = math.huge +end + +local getCommand = "shared-state get tests-scheduler-night" +local handleAllTimesJson = io.popen(getCommand, "r") +local allTimesJson = handleAllTimesJson:read("*a") +handleAllTimesJson:close() +local allTimes = {} +for _,value in pairs(JSON.parse(allTimesJson)) do + allTimes[#allTimes + 1] = tonumber(value.data) +end + +local hitsHours = {0,0,0,0,0,0} +for _,val in pairs(allTimes) do + for i = 1,6 do + local valHourFract = val / 60 + local valHour = valHourFract - (valHourFract % 1) + if valHour == nightHours[i] then + hitsHours[i] = hitsHours[i] + 1 + break + end + end +end + +local minHourIndex,_ = min(hitsHours) +local nightHour = nightHours[minHourIndex] + +local hits5minute = {0,0,0,0,0,0,0,0,0,0,0,0} +for _,val in pairs(allTimes) do + local valHourFract = val / 60 + local valHour = valHourFract - (valHourFract % 1) + if valHour == nightHour then + local val5minuteFract = (val % 60) / 5 + local val5minute = val5minuteFract - (val5minuteFract % 1) + hits5minute[val5minute + 1] = hits5minute[val5minute + 1] + 1 + end +end + +local min5minute1,_ = min(hits5minute) +local min5minute = min5minute1 - 1 +local nightMinute = min5minute * 5 +local myTime = nightHour * 60 + nightMinute +local hostname = io.input("/proc/sys/kernel/hostname"):read("*line") +local timeToLive = 24*60 + +nixio.openlog("shared-state") +local sharedState = SharedState("tests-scheduler-night", nixio.syslog) +sharedState.lock() +sharedState.load() +sharedState.insert(hostname, myTime, timeToLive) +sharedState.save() +sharedState.unlock() +nixio.closelog() +--local handle = io.popen("shared-state insert tests-scheduler-night", "w") +--handle:write(JSON.stringify(myTimeTable)) +--handle:close() + +local nightIndex = 0 +for hook in nightDirFun do + local nightMinuteIdx = (nightMinute + nightIndex) % 60 + local nightMinuteIdxPad = string.format("%02d", nightMinuteIdx) + local nightAt = "echo '"..nightHooksDir.."/"..hook.."' | at -Mv "..nightHour..":".. + nightMinuteIdxPad.." 2>&1" + local handleNightAt = io.popen(nightAt, 'r') + local nightAtRaw = handleNightAt:read("*a") + handleNightAt:close() + local nightAtTime = do_split(nightAtRaw,"%C+")[1] + utils.log("Scheduled time:\t"..nightAtTime.."\tCommand:\t"..nightHooksDir.."/"..hook) + nightIndex = nightIndex + 1 +end diff --git a/packages/tests-scheduler/files/bin/tests-scheduler-probe.lua b/packages/tests-scheduler/files/bin/tests-scheduler-probe.lua new file mode 100644 index 000000000..026043632 --- /dev/null +++ b/packages/tests-scheduler/files/bin/tests-scheduler-probe.lua @@ -0,0 +1,110 @@ +#!/usr/bin/lua + +local libuci_loaded, libuci = pcall(require, "uci") + +local function config_uci_get(option) + local result + if libuci_loaded then + result = libuci:cursor():get("tests-scheduler","probe",option) + else + result = nil + end + return result +end + +local defaultTestsList = { + "ip neigh show nud reachable", + "ping6 -n -c 2 ff02::1%br-lan | grep '(DUP!)'", + "batctl tl | grep '(0x'" + } + +local temp = config_uci_get("test") +local testsList = type(temp) == "table" and temp or defaultTestsList + +-- stability indicates how much of the previous measurement data +-- (which could be from the previous day) is going to be considered +-- 0 means that just the very last is used, avoid using 1 +-- consider that more than one measurement is performed each hour +local stability = tonumber(config_uci_get("stability")) or 0.9 + +local dataFile = config_uci_get("data_file") or "/tmp/tests-scheduler-probe-data" + +local function do_test(test) + io.stderr:write("Command: "..test.."\n") + local handle = io.popen(test, 'r') + local output = handle:read("*a") + handle:close() + local _,count = output:gsub('\n', '\n') + return count +end + +local function do_sum(arr, length) + return length == 1 and arr[1] or arr[length] + do_sum(arr, length -1) +end + +local function do_tests_serie() + local results = {} + local i = 1 + while testsList[i] do + local test = tostring(testsList[i]) + local testResult = tonumber(do_test(test)) + if testResult > 0 then + io.stderr:write("Result: "..tostring(testResult).."\n") + results[#results + 1] = testResult + else + io.stderr:write("Failed or no output\n") + end + i = i + 1 + end + local sum = do_sum(results, #results) + return sum +end + +-- see if the file exists +function file_exists(file) + local f = io.open(file, "rb") + if f then f:close() end + return f ~= nil +end + +-- get all lines from a file, returns an empty +-- list/table if the file does not exist +function lines_from(file) + if not file_exists(file) then return {} end + local lines = {} + for line in io.lines(file) do + lines[#lines + 1] = line + end + return lines +end + +local function calculate_all_data() + data = lines_from(dataFile) + local hour = os.date("*t")["hour"] + local hour1 = hour + 1 + newResult = do_tests_serie() + for i = 1,24 do + if not data[i] then + data[i] = "" + end + end + data[hour1] = tonumber(data[hour1]) and + data[hour1] * stability + newResult * (1 - stability) or + newResult + return data,hour,newResult,data[hour1] +end + +local function save_data(data, file) + -- empty the file + io.open(file,"w"):close() + local handle = io.open(file,"a") + io.output(handle) + for _,val in pairs(data) do + io.write(val.."\n") + end + io.close(handle) +end + +local data,hour,result,cumulativeResult = calculate_all_data() +save_data(data, dataFile) +io.stdout:write(hour.."\t"..result.."\t"..cumulativeResult.."\n") diff --git a/packages/tests-scheduler/files/etc/config/tests-scheduler b/packages/tests-scheduler/files/etc/config/tests-scheduler new file mode 100644 index 000000000..4fd747b30 --- /dev/null +++ b/packages/tests-scheduler/files/etc/config/tests-scheduler @@ -0,0 +1,13 @@ +config probe 'probe' + list test 'ip neigh show nud reachable' + list test 'ping6 -n -c 2 ff02::1%br-lan | grep "(DUP!)"' + list test 'batctl tl | grep "(0x"' + option stability 0.95 + option data_file '/tmp/tests-scheduler-probe-data' + option backup_dir '/etc/tests-scheduler' + option backup_filename 'tests-scheduler-probe-data-backup' + option persist_to_flash true +config at 'at' + option peak_hooks_dir '/etc/tests-scheduler/peakHooks.d' + option night_hooks_dir '/etc/tests-scheduler/nightHooks.d' + diff --git a/packages/tests-scheduler/files/etc/init.d/tests-scheduler-probe-data-persist b/packages/tests-scheduler/files/etc/init.d/tests-scheduler-probe-data-persist new file mode 100755 index 000000000..533ffa371 --- /dev/null +++ b/packages/tests-scheduler/files/etc/init.d/tests-scheduler-probe-data-persist @@ -0,0 +1,40 @@ +#!/bin/sh /etc/rc.common +# Copyright (C) 2019 Ilario Gelmetti +USE_PROCD=1 +START=45 +STOP=01 +CONFIGURATION=tests-scheduler + +start_service() { + config_load "${CONFIGURATION}" + local persist_to_flash + local data_file + local backup_dir + local backup_filename + config_get persist_to_flash probe persist_to_flash + config_get data_file probe data_file + config_get backup_dir probe backup_dir + config_get backup_filename probe backup_filename + + procd_open_instance + procd_set_param respawn + procd_set_param command $persist_to_flash && [[ -e $backup_dir/$backup_filename ]] && cp $backup_dir/$backup_filename $data_file + procd_set_param file /etc/config/tests-scheduler + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_close_instance +} + +stop_service() { + config_load "${CONFIGURATION}" + local persist_to_flash + local data_file + local backup_dir + local backup_filename + config_get persist_to_flash probe persist_to_flash + config_get data_file probe data_file + config_get backup_dir probe backup_dir + config_get backup_filename probe backup_filename + + $persist_to_flash && [[ -e $data_file ]] && [[ -d $backup_dir ]] && cp $data_file $backup_dir/$backup_filename +} diff --git a/packages/tests-scheduler/files/etc/uci-defaults/50_tests-scheduler-probe b/packages/tests-scheduler/files/etc/uci-defaults/50_tests-scheduler-probe new file mode 100644 index 000000000..3a195043b --- /dev/null +++ b/packages/tests-scheduler/files/etc/uci-defaults/50_tests-scheduler-probe @@ -0,0 +1,12 @@ +#!/bin/sh + +unique_append() +{ + grep -qF "$1" "$2" || echo "$1" >> "$2" +} + +unique_append \ + '*/15 * * * * ((/bin/tests-scheduler-probe)&)'\ + /etc/crontabs/root + +exit 0 diff --git a/packages/tests-scheduler/files/etc/uci-defaults/51_tests-scheduler-at b/packages/tests-scheduler/files/etc/uci-defaults/51_tests-scheduler-at new file mode 100644 index 000000000..d269304a0 --- /dev/null +++ b/packages/tests-scheduler/files/etc/uci-defaults/51_tests-scheduler-at @@ -0,0 +1,15 @@ +#!/bin/sh + +hour=$(awk -v seed=$RANDOM 'BEGIN{srand(seed);print int(rand()*24)}') +minute=$(awk -v seed=$RANDOM 'BEGIN{srand(seed);print int(rand()*60)}') + +unique_append() +{ + grep -qF "$1" "$2" || echo "$minute $hour $1" >> "$2" +} + +unique_append \ + '* * * ((shared-state sync tests-scheduler-night &> /dev/null; /bin/tests-scheduler-at; shared-state sync tests-scheduler-night &> /dev/null)&)'\ + /etc/crontabs/root + +exit 0