CA-401651: stunnel_cache: run the cache expiry code periodically

Previously it'd only run when we added or removed entries, but on an
idle system we'd keep a large number of connections open that we don't
need, and this then exceeded the connection limits on the coordinator.

Signed-off-by: Edwin Török <[email protected]>

ocaml/libs/stunnel/stunnel_cache.ml

@@ -42,9 +42,9 @@ type endpoint = {host: string; port: int}
 (* Need to limit the absolute number of stunnels as well as the maximum age *)
 let max_stunnel = 70
 
-let max_age = 180. *. 60. (* seconds *)
+let max_age = ref (180. *. 60.) (* seconds *)
 
-let max_idle = 5. *. 60. (* seconds *)
+let max_idle = ref (5. *. 60.) (* seconds *)
 
 (* The add function adds the new stunnel before doing gc, so the cache *)
 (* can briefly contain one more than maximum. *)
@@ -104,6 +104,7 @@ let unlocked_gc () =
   let to_gc = ref [] in
   (* Find the ones which are too old *)
   let now = Unix.gettimeofday () in
+  let max_age = !max_age and max_idle = !max_idle in
   Tbl.iter !stunnels (fun idx stunnel ->
       match Hashtbl.find_opt !times idx with
       | Some time ->

ocaml/libs/stunnel/stunnel_cache.mli

@@ -46,3 +46,9 @@ val flush : unit -> unit
 
 val gc : unit -> unit
 (** GCs old stunnels *)
+
+val max_age : float ref
+(** maximum time a connection is kept in the stunnel cache, counted from the time it got initially added to the cache *)
+
+val max_idle : float ref
+(** maximum time a connection is kept in the stunnel cache, counted from the most recent time it got (re)added to the cache. *)

ocaml/xapi/xapi_globs.ml

@@ -1017,6 +1017,10 @@ let conn_limit_unix = ref 1024
 
 let conn_limit_clientcert = ref 800
 
+let stunnel_cache_max_age = ref (180. *. 60.)
+
+let stunnel_cache_max_idle = ref (5. *. 60.)
+
 let trace_log_dir = ref "/var/log/dt/zipkinv2/json"
 
 let export_interval = ref 30.
@@ -1138,6 +1142,8 @@ let xapi_globs_spec =
   ; ("conn_limit_tcp", Int conn_limit_tcp)
   ; ("conn_limit_unix", Int conn_limit_unix)
   ; ("conn_limit_clientcert", Int conn_limit_clientcert)
+  ; ("stunnel_cache_max_age", Float stunnel_cache_max_age)
+  ; ("stunnel_cache_max_idle", Float stunnel_cache_max_idle)
   ; ("export_interval", Float export_interval)
   ; ("max_spans", Int max_spans)
   ; ("max_traces", Int max_traces)

ocaml/xapi/xapi_periodic_scheduler_init.ml

@@ -114,6 +114,10 @@ let register ~__context =
           Xapi_host.alert_if_tls_verification_was_emergency_disabled ~__context
       )
   ) ;
+  let stunnel_period = !Stunnel_cache.max_idle /. 2. in
+  Xapi_periodic_scheduler.add_to_queue "Check stunnel cache expiry"
+    (Xapi_periodic_scheduler.Periodic stunnel_period) stunnel_period
+    Stunnel_cache.gc ;
   if
     master
     && Db.Pool.get_update_sync_enabled ~__context