Use a single data volume for each experiment (#231)

* Changes no-index experiments (name the host exp.) to use a single data volume and write multiple subdirs rather than having a separate volume/mount for every single datatype. * Changes an unknown variable into a static string, since we already know what the value is in this file. * Adds a missing single quote to properly terminate a string. * Adds a missing VolumeMount for the host experiment.
m-lab · Jul 19, 2019 · 64af9fb · 64af9fb
1 parent c13a726
commit 64af9fb
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 21 deletions.
diff --git a/k8s/daemonsets/core/host.jsonnet b/k8s/daemonsets/core/host.jsonnet
@@ -12,7 +12,7 @@ exp.ExperimentNoIndex('host', nodeinfo_datatypes, true) + {
             name: 'nodeinfo',
             image: 'measurementlab/nodeinfo:v1.2',
             args: [
-              '-datadir=/var/spool/nodeinfo',
+              '-datadir=/var/spool/host',
               '-wait=1h',
               '-prometheusx.listen-address=127.0.0.1:9990',
               '-config=/etc/nodeinfo/config.json',
@@ -23,7 +23,8 @@ exp.ExperimentNoIndex('host', nodeinfo_datatypes, true) + {
                 name: 'nodeinfo-config',
                 readOnly: true,
               },
-            ] + [exp.VolumeMount('nodeinfo', d) for d in nodeinfo_datatypes],
+              exp.VolumeMount('host'),
+            ],
           },
           exp.RBACProxy('nodeinfo', 9990),
         ],

diff --git a/k8s/daemonsets/experiments/ndt.jsonnet b/k8s/daemonsets/experiments/ndt.jsonnet
@@ -32,8 +32,7 @@ exp.Experiment('ndt', 2, ['ndt5', 'ndt7']) + {
                 readOnly: true,
               },
               exp.uuid.volumemount,
-              exp.VolumeMount('ndt', 'ndt5'),
-              exp.VolumeMount('ndt', 'ndt7'),
+              exp.VolumeMount('ndt'),
             ],
             ports: [
               {

diff --git a/k8s/daemonsets/templates.jsonnet b/k8s/daemonsets/templates.jsonnet
@@ -26,17 +26,17 @@ local uuid = {
   },
 };
 
-local volume(name, datatype) = {
+local volume(name) = {
   hostPath: {
-    path: '/cache/data/' + name + '/' + datatype,
+    path: '/cache/data/' + name,
     type: 'DirectoryOrCreate',
   },
-  name: datatype + '-data',
+  name: name + '-data',
 };
 
-local VolumeMount(name, datatype) = {
-  mountPath: '/var/spool/' + name + '/' + datatype,
-  name: datatype + '-data',
+local VolumeMount(name) = {
+  mountPath: '/var/spool/' + name,
+  name: name + '-data',
 };
 
 local RBACProxy(name, port) = {
@@ -98,7 +98,7 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
               else
                 '-prometheusx.listen-address=$(PRIVATE_IP):9991'
               ,
-              '-output=' + VolumeMount(name, 'tcpinfo').mountPath,
+              '-output=' + VolumeMount(name).mountPath + '/tcpinfo',
               '-uuid-prefix-file=' + uuid.prefixfile,
             ],
             env: if hostNetworking then [] else [
@@ -117,7 +117,7 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
               },
             ],
             volumeMounts: [
-              VolumeMount(name, 'tcpinfo'),
+              VolumeMount(name),
               uuid.volumemount,
             ],
           },
@@ -129,7 +129,7 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
                 '-prometheusx.listen-address=127.0.0.1:9992'
               else
                 '-prometheusx.listen-address=$(PRIVATE_IP):9992',
-              '-outputPath=' + VolumeMount(name, 'traceroute').mountPath,
+              '-outputPath=' + VolumeMount(name).mountPath + '/traceroute',
               '-uuid-prefix-file=' + uuid.prefixfile,
             ],
             env: if hostNetworking then [] else [
@@ -148,7 +148,7 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
               },
             ],
             volumeMounts: [
-              VolumeMount(name, 'traceroute'),
+              VolumeMount(name),
               uuid.volumemount,
             ],
           },
@@ -204,14 +204,13 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
               },
             ],
             volumeMounts: [
-              VolumeMount(name, 'tcpinfo'),
-              VolumeMount(name, 'traceroute'),
+              VolumeMount(name),
               {
                 mountPath: '/etc/credentials',
                 name: 'pusher-credentials',
                 readOnly: true,
               },
-            ] + [VolumeMount(name, d) for d in datatypes],
+            ],
           },
         ] + if hostNetworking then [
           RBACProxy('tcpinfo', 9991),
@@ -233,9 +232,8 @@ local ExperimentNoIndex(name, datatypes, hostNetworking) = {
             },
           },
           uuid.volume,
-          volume(name, 'traceroute'),
-          volume(name, 'tcpinfo'),
-        ] + [volume(name, d) for d in datatypes],
+          volume(name),
+        ],
       },
     },
     updateStrategy: {
@@ -292,7 +290,7 @@ local Experiment(name, index, datatypes=[]) = ExperimentNoIndex(name, datatypes,
 
   // Returns a volumemount for a given datatype. All produced volume mounts
   // in /var/spool/name/
-  VolumeMount(name, datatype):: VolumeMount(name, datatype),
+  VolumeMount(name):: VolumeMount(name),
 
   // Helper object containing uuid-related filenames, volumes, and volumemounts.
   uuid: uuid,