first working draft of .etch / preloading lib

w/ forked gunzip-maybe and @types/gunzip-maybe so we don't need esinterop

change-type: minor

lib/dotetch-preloading/appsJson.ts

@@ -0,0 +1,79 @@
+/**
+ * `Apps.json` is the file that will inform the supervisor of what's has been preloaded, which services should be started and with which config.
+ *
+ * `Apps.json` content is a subset of the `target state` for a device in a fleet running a given release.
+ * Once we have that target fleeet, we need to go down one level to `apps` and keep only that element.
+ *
+ * In Apps.json we have the list of all the images that makes up a release.
+ */
+
+import axios from "axios"
+
+/**
+ * Derives Apps.json from target state obtained from the api
+ *
+ * This requires merge of https://github.com/balena-io/open-balena-api/pull/1081 in open-balena-api
+ *
+ * @param {string} app_id - app_id
+ * @param {string} release_id - release_id
+ * @param {string} app_id - app_id === fleet_id /!\ fleet_id != fleet_uuid
+ * @param {string} api - api server url
+ * @param {string} apiToken - token to access api
+ * @returns {json} - apps.json object
+ */
+const getAppsJson = async ({ app_id, api, token }: any) => {
+  const headers = {
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `${token}`,
+    },
+  }
+
+  try {
+    const appRes = await axios({
+      url: `${api}/v6/application(${app_id})`,
+      ...headers,
+    })
+    const uuid = appRes?.data?.d?.[0]?.uuid
+    console.log("GOT UUID ", uuid)
+    // if (!uuid) throw Error(`Error: can't find fleet (${app_id}`);
+
+    const { data } = await axios({
+      url: `${api}/device/v3/fleet/${uuid}/state`,
+      ...headers,
+    })
+
+    return data[uuid] // get down one level to transform the target state into a valid Apps.json
+  } catch (error) {
+    console.error("\n\n==> getAppsJson error:", error)
+  }
+}
+
+/**
+ * Takes a apps.json and returns the list of images for an app & release.
+ * If apps_id and/or release_id is unkown it will return first.
+ * // TODO: return all instead of first when no app or release is specified.
+ */
+interface ImageIdsInput {
+  appsJson: any //TODO: get propertype for appsJson V3
+}
+
+interface Image {
+  image_name: string
+  image_hash: string
+}
+
+const getImageIds = ({ appsJson }: ImageIdsInput): Image[] => {
+  //TODO: prepare for multiapps and loop on apps instead of getting only 1st
+  const appId = Object.keys(appsJson.apps)[0]
+  const releaseId = Object.keys(appsJson.apps?.[appId]?.releases)[0]
+  console.log(`==> appId: ${appId} & releaseId: ${releaseId}`)
+  const imageKeys = Object.keys(appsJson.apps?.[appId]?.releases?.[releaseId]?.services)
+  const imageNames = imageKeys.map((key) => appsJson.apps?.[appId]?.releases?.[releaseId]?.services[key].image)
+  return imageNames.map((image) => {
+    const [image_name, image_hash] = image.split("@")
+    return { image_name, image_hash }
+  })
+}
+
+export { getAppsJson, getImageIds }

lib/dotetch-preloading/baseImage.ts

@@ -0,0 +1,31 @@
+/**
+ * Get the base image we're going to preload assets in (balenaos.img)
+ * */
+
+interface StreamBaseImageIn {
+  pipeStreamFrom: NodeJS.ReadableStream
+  pipeStreamTo: NodeJS.WritableStream
+}
+
+/**
+ * Awaitable pipe stream from input to output
+ */
+const streamBaseImage = ({ pipeStreamFrom, pipeStreamTo }: StreamBaseImageIn): Promise<boolean> =>
+  new Promise((resolve, reject) => {
+    console.log("== Start streaming base image (balenaOs) @streamBaseImage ==")
+
+    pipeStreamFrom.pipe(pipeStreamTo)
+
+    pipeStreamFrom.on("end", function () {
+      // we're good we can continue the process
+      console.log("== End of base image streaming (balenaOs) @streamBaseImage ==")
+      resolve(true)
+    })
+
+    pipeStreamFrom.on("error", function (error) {
+      // something went wrong
+      reject(error)
+    })
+  })
+
+export { streamBaseImage }

lib/dotetch-preloading/digestStream.ts

@@ -0,0 +1,30 @@
+/**
+ * minimal typescript reimplementation of https://github.com/jeffbski/digest-stream/blob/master/lib/digest-stream.js
+ *
+ * This will let a stream pass-thru then returns a sha256 hash + size of the content.
+ *
+ */
+import { Transform } from "stream";
+import { createHash } from "crypto";
+
+const digestStream = (exfiltrate: Function): Transform => {
+	const digester = createHash("sha256");
+	let length = 0;
+
+	const hashThrough = new Transform({
+		transform(chunk: Buffer, _, callback) {
+			digester.update(chunk);
+			length += chunk.length;
+			this.push(chunk);
+			callback();
+		},
+	});
+
+	hashThrough.on("end", () => {
+		exfiltrate(digester.digest("hex"), length);
+	});
+
+	return hashThrough;
+};
+
+export { digestStream };

lib/dotetch-preloading/docker-parse-image.ts

@@ -0,0 +1,47 @@
+/**
+ * Typescript version
+ * https://github.com/mafintosh/docker-parse-image/blob/master/index.js
+ */
+
+export interface DockerParsedImage {
+  registry: string | null
+  namespace?: string | null
+  repository: string | null
+  tag?: string | null
+  name: string
+  fullname: string
+}
+
+const dockerParseImage = (image: string): DockerParsedImage => {
+  const registryArray = image.split("/")
+
+  let registry = registryArray[0]
+  let namespace = registryArray[1]
+  const repository = registryArray[2].split("@")[0]
+  let tag = registryArray[2].split("@")[1]
+
+  if (!namespace && registry && !registry.includes(":") && !registry.includes(".")) {
+    namespace = registry
+    registry = ""
+  }
+
+  registry = registry ? `${registry}` : ""
+  namespace = namespace && namespace !== "library" ? `${namespace}` : ""
+  tag = tag && tag !== "latest" ? `:${tag}` : ""
+
+  const name = `${registry}${namespace}${repository}${tag}`
+  const fullname = `${registry}${namespace || "library/"}${repository}${tag || ":latest"}`
+
+  const result = {
+    registry: registry || null,
+    namespace: namespace || null,
+    repository: repository || null,
+    tag: tag || null,
+    name,
+    fullname,
+  }
+
+  return result
+}
+
+export { dockerParseImage }

lib/dotetch-preloading/expandImg.mjs

@@ -0,0 +1,170 @@
+import fs from 'fs';
+import path from 'path';
+import { spawn, spawnSync } from 'child_process';
+// import pLimit from 'p-limit';
+// import gunzip from "extract-zip";
+import gunzip from "gunzip-maybe"
+import { promisify } from "util";
+
+const IMAGES_BASE = path.resolve('images-base');
+const IMAGES_EXPANDED = path.resolve('images-expanded');
+console.log(`IMAGES_BASE: ${IMAGES_BASE}`);
+console.log(`IMAGES_EXPANDED: ${IMAGES_EXPANDED}`);
+
+// In bytes:
+const SECTOR_SIZE = 512
+const MBR_SIZE = 512
+const GPT_SIZE = SECTOR_SIZE * 34
+const MBR_BOOTSTRAP_CODE_SIZE = 446
+
+/** 
+ * createDDArgs
+ * dd helper
+ * @param {string} partitionTableDiskImage 
+ * @param {string} nameImage 
+ * @param {int} resizeMultiplier 
+ * return {Array} argsList 
+ *  // obs is the output block size and ibs is the input block size. If you specify bs without ibs or obs this is used for both.
+    // Seek will just "inflate" the output file. 
+    // Seek=7 means that at the beginning of the output file, 
+    // 7 "empty" blocks with output block size=obs=4096bytes will be inserted.
+    // This is a way to create very big files quickly.
+    // Or to skip over data at the start which you do not want to alter. 
+    // Empty blocks only result if the output file initially did not have that much data.
+ */
+function createDDArgs(inImageName, outImageName, resizeMultiplier, partitionStartBytes) {
+    const partitionTableLabel = 'GPT' || 'DOS';
+    const argsListMore = {}
+    argsListMore.sizing = [`count=${MBR_BOOTSTRAP_CODE_SIZE}`, 'seek=5'];
+    if (partitionTableLabel === 'DOS') {
+      argsListMore.sizing =  [ `skip=${MBR_SIZE}`, `seek=${MBR_SIZE}`, `count=${partitionStartBytes - MBR_SIZE}`];
+    }
+    if (partitionTableLabel === 'GPT') {
+        argsListMore.sizing = [ `skip=${GPT_SIZE}`, `seek=${GPT_SIZE}`, `count=${partitionStartBytes - GPT_SIZE}`];
+    }
+    console.log(partitionTableLabel,'partitionTableLabel', argsListMore.sizing, 'partitionStartBytes', partitionStartBytes);
+
+    const argsList = [ 
+        `if=${inImageName}`, 
+        `of=${outImageName}`,
+
+        `ibs=${1024 * resizeMultiplier}`,
+        // `bs=${resizeMultiplier}M`, // one MiB * resizeMultiplier
+        `obs=1024`,
+        'conv=notrunc',
+        'status=progress',
+        // `iflag=count_bytes, skip_bytes`, // count and skip in bytes
+        // `oflag=seek_bytes`// seek in bytes
+        ...argsListMore.sizing
+    ];
+    return argsList;
+}
+
+// fork() exec() spawn() spawnSync()
+//https://github.com/adriano-di-giovanni/node-df/blob/master/lib/index.js
+const getPartitions = async (image) => {
+    // const diskutilResults = await spawn('diskutil', ['list']);
+    // console.log('diskutil', await diskutilResults);
+    const partitions = spawn('df', ['-hkP'], { 
+        // cwd: '/',
+        // windowsHide: true,
+        stdio: [
+        /* Standard: stdin, stdout, stderr */
+        // 'inherit',
+        'ignore',
+        /* Custom: pipe:3, pipe:4, pipe:5 */
+        'pipe', process.stderr
+    ]});
+    const partitionsResults = {partitions: [], partitionsLength: 0};
+
+    partitions.stdout.on('data', data => {
+        const parsedDf = parseDf(data);
+        // const strData = splitDf(data);
+        // partitionsResults.partitionArrayLength = strData.length;
+        // // console.log('strData.length', strData.length);
+        // const columnHeaders = strData.shift();
+        // console.log('columnHeaders', columnHeaders);
+        // const formatted = formatDf(strData, columnHeaders);
+        partitionsResults.partitions = parsedDf.partitions;
+        partitionsResults.partitionsLength = parsedDf.partitionsLength;
+        return partitionsResults;
+    });
+
+    // partitions.stderr.on('data', data => {
+    //     assert(false, 'NOPE stderr');
+    // });
+
+    partitions.on('close', code => {
+        console.log('Child exited with', code, 'and stdout has been saved');
+        console.log('partitionsResults', partitionsResults);
+        return partitionsResults;
+    });
+    return partitionsResults;
+}
+
+const parseDf = (data) => {
+   const strData = splitDf(data);
+   const columnHeaders = strData.shift();
+   const formatted = formatDf(strData, columnHeaders);
+   return {partitions: formatted, partitionsLength: formatted.length};
+}
+
+const splitDf = (data) => {
+    return data.toString()
+        .replace(/ +(?= )/g,'') //replace multiple spaces between device parameters with one space
+        .split('\n') //split by newline
+        .map((line) => line.split(' ')); //split each device by one space
+}
+
+const formatDf = (strData, columnHeaders) => {
+    return strData.map((devDisk) => {
+        const partitionObj = {};
+        for ( const [index,value] of devDisk.entries()) {
+            partitionObj[columnHeaders[index]] = value
+        }
+        return partitionObj;
+    });
+}
+
+export const expandImg = async (img, partitionSizeStart = 1) => {
+    if (!img) {
+        throw new Error(`No img: "${img}"`);
+    }
+    const unzippedPath = `${IMAGES_BASE}/unzipped/`
+    if (img.includes("zip")) {
+        // await gunzip(img, {dir: `${IMAGES_BASE}/unzipped/`});
+        await gunzip(`${IMAGES_BASE}/zipped/${img}`, {dir: unzippedPath});
+    }
+    // else {
+
+    // const diskutilResults = await spawn('diskutil', ['list']);
+    // console.log('diskutil', await diskutilResults);
+    const generateRandomName = Math.random().toString(36).substring(2, 15);
+
+    const inImageName = `${unzippedPath}${img.split('.').slice(0, -1).join('.')}`;
+    const outImageName = `${IMAGES_EXPANDED}/${generateRandomName}.img`;
+    const argsList = await createDDArgs(inImageName, outImageName, 7, partitionSizeStart);
+    await spawn('dd', argsList, { 
+        cwd: '/',
+        windowsHide: true,
+        stdio: [
+        /* Standard: stdin, stdout, stderr */
+        'ignore',
+        /* Custom: pipe:3, pipe:4, pipe:5 */
+        'pipe', process.stderr
+    ]});
+    return generateRandomName;
+};
+
+// strace dd if=/dev/disk5 of=./images-expanded/tuckers.img bs=4M conv=notrunc
+// dd if=/dev/disk5 of=./images-expanded/tuckers.img bs=4M conv="notrunc"
+// bs=4M
+
+const getImages = async () => {
+    const image = 'balena-cloud-preloaded-raspberrypi4-64-2022.1.1-v12.11.0.img.zip'
+    const {partitions, partitionsLength} = await getPartitions(image);
+    console.log('partitions', await partitions, 'partitionsLength', partitionsLength);
+    const imageName = await expandImg(image)
+    console.log('imageName', await imageName);
+}
+getImages()

lib/dotetch-preloading/images.ts

@@ -0,0 +1,23 @@
+/** Prepare injectable files for all images */
+
+const getImagesConfigurationFiles = (manifests: any) => {
+  const dockerImageOverlay2Imagedb = "docker/image/overlay2/imagedb"
+  console.log("MANIFESTS => ", manifests)
+  return manifests
+    .map(({ configManifestV2, imageId }: any) => {
+      const shortImage_id = imageId.split(":")[1]
+      return [
+        {
+          header: { name: `${dockerImageOverlay2Imagedb}/content/sha256/${shortImage_id}`, mode: 644 },
+          content: JSON.stringify(configManifestV2),
+        },
+        {
+          header: { name: `${dockerImageOverlay2Imagedb}/metadata/sha256/${shortImage_id}/lastUpdated`, mode: 644 },
+          content: new Date().toISOString(),
+        },
+      ]
+    })
+    .flat()
+}
+
+export { getImagesConfigurationFiles }

lib/dotetch-preloading/index.ts

@@ -0,0 +1,3 @@
+import { streamPreloadingAssets } from "./streamPreloadingAssets";
+
+export { streamPreloadingAssets };