Adds tests-scheduler for planning tests at network usage peak and minimum

packages/tests-scheduler-lime/Makefile

@@ -0,0 +1,48 @@
+# 
+# Copyright (C) 2019 Ilario Gelmetti
+#
+# This is free software, licensed under the GNU General Public License v3.
+#
+
+include $(TOPDIR)/rules.mk
+
+GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . )
+GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . )
+
+PKG_NAME:=tests-scheduler-lime
+PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP)
+
+include $(INCLUDE_DIR)/package.mk
+
+define Package/$(PKG_NAME)
+  CATEGORY:=LiMe
+  TITLE:=LibreMesh tests for tests-scheduler
+  MAINTAINER:=Ilario Gelmetti <[email protected]>
+  URL:=https://libremesh.org
+  DEPENDS:=+logrotate +lime-report +bandwidth-test +tests-scheduler
+  PKGARCH:=all
+endef
+
+define Package/$(PKG_NAME)/config
+endef
+
+define Package/$(PKG_NAME)/description
+	Schedules a few tests which gather useful information for debugging
+	a LibreMesh network.
+	The scheduling is done using the tests-scheduler package and can be run
+	at the network usage peak time or at the night time.
+endef
+
+define Build/Compile
+endef
+
+define Package/$(PKG_NAME)/install
+	$(INSTALL_DIR) $(1)/etc/logrotate.d/
+	$(INSTALL_DIR) $(1)/etc/tests-scheduler/nightHooks.d/
+	$(INSTALL_DIR) $(1)/etc/tests-scheduler/peakHooks.d/
+	$(INSTALL_BIN) ./files/etc/logrotate.d/* $(1)/etc/logrotate.d
+	$(INSTALL_BIN) ./files/etc/tests-scheduler/nightHooks.d/* $(1)/etc/tests-scheduler/nightHooks.d/
+	$(INSTALL_BIN) ./files/etc/tests-scheduler/peakHooks.d/* $(1)/etc/tests-scheduler/peakHooks.d/
+endef
+
+$(eval $(call BuildPackage,$(PKG_NAME)))

packages/tests-scheduler-lime/files/etc/logrotate.d/tests-peak-lime-report

@@ -0,0 +1,6 @@
+/tmp/tests-peak-lime-report {
+    daily
+    compress
+    rotate 5
+    size 100k
+}

packages/tests-scheduler-lime/files/etc/tests-scheduler/nightHooks.d/10-bandwidth-test

@@ -0,0 +1,2 @@
+#!/bin/sh
+(date && bandwidth-test) >> /tmp/tests-night-bandwidth-test

packages/tests-scheduler-lime/files/etc/tests-scheduler/peakHooks.d/10-lime-report

@@ -0,0 +1,2 @@
+#!/bin/sh
+lime-report >> /tmp/tests-peak-lime-report

packages/tests-scheduler/Makefile

@@ -0,0 +1,51 @@
+# 
+# Copyright (C) 2019 Ilario Gelmetti
+#
+# This is free software, licensed under the GNU General Public License v3.
+#
+
+include $(TOPDIR)/rules.mk
+
+GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . )
+GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . )
+
+PKG_NAME:=tests-scheduler
+PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP)
+
+include $(INCLUDE_DIR)/package.mk
+
+define Package/$(PKG_NAME)
+  CATEGORY:=Utilities
+  TITLE:=Tests scheduler at peak and night time
+  MAINTAINER:=Ilario Gelmetti <[email protected]>
+  URL:=https://libremesh.org
+  DEPENDS:=+at
+  PKGARCH:=all
+endef
+
+define Package/$(PKG_NAME)/config
+endef
+
+define Package/$(PKG_NAME)/description
+ 	Monitors the number of clients of the network for each hour,
+	uses this information for scheduling tests at the peak time and/or at the night time.
+endef
+
+define Build/Compile
+endef
+
+define Package/$(PKG_NAME)/install
+	$(INSTALL_DIR) $(1)/bin/
+	$(INSTALL_DIR) $(1)/etc/uci-defaults/
+	$(INSTALL_DIR) $(1)/etc/init.d/
+	$(INSTALL_DIR) $(1)/etc/config/
+	$(INSTALL_DIR) $(1)/etc/tests-scheduler/peakHooks.d/
+	$(INSTALL_DIR) $(1)/etc/tests-scheduler/nightHooks.d/
+	$(INSTALL_BIN) ./files/bin/tests-scheduler-at.lua $(1)/bin/tests-scheduler-at
+	$(INSTALL_BIN) ./files/bin/tests-scheduler-probe.lua $(1)/bin/tests-scheduler-probe
+	$(INSTALL_BIN) ./files/etc/uci-defaults/* $(1)/etc/uci-defaults
+	$(INSTALL_BIN) ./files/etc/init.d/* $(1)/etc/init.d
+	$(INSTALL_CONF) ./files/etc/config/tests-scheduler $(1)/etc/config/
+endef
+
+$(eval $(call BuildPackage,$(PKG_NAME)))

packages/tests-scheduler/files/bin/tests-scheduler-at.lua

@@ -0,0 +1,179 @@
+#!/usr/bin/lua
+
+local JSON = require("luci.jsonc")
+local utils = require("lime.utils")
+local fs = require("nixio.fs")
+local libuci = require("uci")
+local SharedState = require("shared-state")
+require("nixio.util")
+
+local function config_uci_get(sectionname, option)
+	local result
+	result = libuci:cursor():get("tests-scheduler",sectionname,option)
+	return result
+end
+
+local dataFile = arg[1] or config_uci_get("probe","data_file")
+local peakHooksDir = arg[2] or config_uci_get("at","peak_hooks_dir")
+local nightHooksDir = arg[3] or config_uci_get("at","night_hooks_dir")
+
+-- check if a file exists
+function file_exists(file)
+	local f = io.open(file, "rb")
+	if f then f:close() end
+	return f ~= nil
+end
+
+-- get all lines from a file, returns an empty 
+-- list/table if the file does not exist
+function lines_from_tonumber(file)
+	if not file_exists(file) then return {} end
+	local lines = {}
+	for line in io.lines(file) do 
+		lines[#lines + 1] = tonumber(line) or "missing"
+	end
+	return lines
+end
+
+local function do_split(str,pat)
+	local tbl = {}
+	str:gsub(pat, function(x) tbl[#tbl+1]=x end)
+	return tbl
+end
+
+local function max(t)
+	if #t == 0 then return nil, nil end
+	local key, value = 1, tonumber(t[1])
+	for i = 2, #t do
+		temp = tonumber(t[i])
+		if value < temp then
+			key, value = i, temp
+		end
+	end
+    return key, value
+end
+
+local function min(t)
+	if #t == 0 then return nil, nil end
+	local key, value = 1, tonumber(t[1])
+	for i = 2, #t do
+		temp = tonumber(t[i])
+		if value > temp then
+			key, value = i, temp
+		end
+	end
+    return key, value
+end
+
+local data = lines_from_tonumber(dataFile)
+
+for _,val in pairs(data) do
+	if val == "missing" then
+		io.stderr:write("Data from at least one whole day is needed in "..
+		  dataFile..", stopping.\n")
+		os.exit(1)
+	end
+end
+
+io.stderr:write("Found enough data in "..dataFile..", continuing.\n")
+
+local peakHour1,_ = max(data)
+local peakHour = peakHour1 - 1
+
+local peakDirFun = fs.dir(peakHooksDir) or {}
+local peakIndex = 0
+for hook in fs.dir(peakHooksDir) do
+	local peakMinute = (peakIndex * 5) % 60
+	local peakMinutePad = string.format("%02d", peakMinute)
+	local peakAt = "echo '"..peakHooksDir.."/"..hook.."' | at -Mv "..peakHour..":"..
+	  peakMinutePad.." 2>&1"
+	local handlePeakAt = io.popen(peakAt, 'r')
+	local peakAtRaw = handlePeakAt:read("*a")
+	handlePeakAt:close()
+	local peakAtTime = do_split(peakAtRaw,"%C+")[1]
+	utils.log("Scheduled time:\t"..peakAtTime.."\tCommand:\t"..peakHooksDir.."/"..hook)
+	peakIndex = peakIndex + 1
+end
+
+local nightDirFun = fs.dir(nightHooksDir) or os.exit(0)
+if not nightDirFun(1) then
+	io.stderr:write("No tests configured to be run at the night time.\n")
+	os.exit(0)
+end
+local nightDirFun = fs.dir(nightHooksDir)
+
+local nightHours = {}
+
+for i = 1,6 do
+	nightHour1,_ = min(data)
+	nightHours[i] = nightHour1 - 1
+	data[nightHour1] = math.huge
+end
+
+local getCommand = "shared-state get tests-scheduler-night"
+local handleAllTimesJson = io.popen(getCommand, "r")
+local allTimesJson = handleAllTimesJson:read("*a")
+handleAllTimesJson:close()
+local allTimes = {}
+for _,value in pairs(JSON.parse(allTimesJson)) do
+	allTimes[#allTimes + 1] = tonumber(value.data)
+end
+
+local hitsHours = {0,0,0,0,0,0}
+for _,val in pairs(allTimes) do
+	for i = 1,6 do
+		local valHourFract = val / 60
+		local valHour = valHourFract - (valHourFract % 1)
+		if valHour == nightHours[i] then
+			hitsHours[i] = hitsHours[i] + 1
+			break
+		end
+	end
+end
+
+local minHourIndex,_ = min(hitsHours)
+local nightHour = nightHours[minHourIndex]
+
+local hits5minute = {0,0,0,0,0,0,0,0,0,0,0,0}
+for _,val in pairs(allTimes) do
+	local valHourFract = val / 60
+	local valHour = valHourFract - (valHourFract % 1)
+	if valHour == nightHour then
+		local val5minuteFract = (val % 60) / 5
+		local val5minute = val5minuteFract - (val5minuteFract % 1)
+		hits5minute[val5minute + 1] = hits5minute[val5minute + 1] + 1
+	end
+end
+
+local min5minute1,_ = min(hits5minute)
+local min5minute = min5minute1 - 1
+local nightMinute = min5minute * 5
+local myTime = nightHour * 60 + nightMinute 
+local hostname = io.input("/proc/sys/kernel/hostname"):read("*line")
+local timeToLive = 24*60
+
+nixio.openlog("shared-state")
+local sharedState = SharedState("tests-scheduler-night", nixio.syslog)
+sharedState.lock()
+sharedState.load()
+sharedState.insert(hostname, myTime, timeToLive)
+sharedState.save()
+sharedState.unlock()
+nixio.closelog()
+--local handle = io.popen("shared-state insert tests-scheduler-night", "w")
+--handle:write(JSON.stringify(myTimeTable))
+--handle:close()
+
+local nightIndex = 0
+for hook in nightDirFun do
+	local nightMinuteIdx = (nightMinute + nightIndex) % 60
+	local nightMinuteIdxPad = string.format("%02d", nightMinuteIdx)
+	local nightAt = "echo '"..nightHooksDir.."/"..hook.."' | at -Mv "..nightHour..":"..
+	  nightMinuteIdxPad.." 2>&1"
+	local handleNightAt = io.popen(nightAt, 'r')
+	local nightAtRaw = handleNightAt:read("*a")
+	handleNightAt:close()
+	local nightAtTime = do_split(nightAtRaw,"%C+")[1]
+	utils.log("Scheduled time:\t"..nightAtTime.."\tCommand:\t"..nightHooksDir.."/"..hook)
+	nightIndex = nightIndex + 1
+end