docker add steedosctl from appsmith

deploy/enterprise/Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:focal
 
-# The env variables are needed for Appsmith server to correctly handle non-roman scripts like Arabic.
+# The env variables are needed for Steedos server to correctly handle non-roman scripts like Arabic.
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ARG DEBIAN_FRONTEND=noninteractive
@@ -96,9 +96,21 @@ RUN rm -rf \
 COPY ./fs/etc /etc
 COPY ./fs/opt /opt
 
+
+RUN cd ./utils && npm install --only=prod && npm install --only=prod -g . && cd - \
+  && chmod 0644 /etc/cron.d/* \
+  && chmod +x entrypoint.sh renew-certificate.sh healthcheck.sh templates/nginx-app.conf.sh /watchtower-hooks/*.sh \
+  # Disable setuid/setgid bits for the files inside container.
+  && find / \( -path /proc -prune \) -o \( \( -perm -2000 -o -perm -4000 \) -print -exec chmod -s '{}' + \) || true 
+
+
+ENV PATH /opt/steedos/utils/node_modules/.bin:/opt/java/bin:$PATH
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check=/watchtower-hooks/pre-check.sh
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update=/watchtower-hooks/pre-update.sh
+
 # ENV defaults
 ENV ROOT_URL=http://127.0.0.1:3000 \
-    PORT=3000 \
     NODE_ENV=production 
 
 # ------------------------------------------------------------------------

deploy/enterprise/README.md

@@ -166,12 +166,6 @@ The container runs multiple processes, including the Steedos Platform server, Ng
 
 Supervisord comes with a web interface for managing the various processes, available at <http://localhost/supervisor/>, as well as a command line interface towards the same goal.
 
-Here's a screenshot of the web interface listing all the processes managed:
-
-<p>
-  <img src="https://raw.githubusercontent.com/steedos/steedos-platform/master/deploy/docker/images/steedos_supervisord_ui.png" width="80%">
-</p>
-
 The command line interface can also be used to perform operations like restarting the Steedos Platform server, or restarting Nginx etc. For example, the following command (run in the installation folder) can be used to get a status of all running processes:
 
 ```sh
@@ -192,4 +186,4 @@ If you encounter any errors during this process, please reach out to [**support@
 
 ## Special Thanks
 
-- [Appsmith](https://github.com/appsmithorg/appsmith)
+- [Steedos](https://github.com/appsmithorg/appsmith)

deploy/enterprise/build-multi-arch.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 docker buildx create --use
-docker buildx build --platform linux/arm64,linux/amd64 --push -t steedos/steedos-enterprise  .
+docker buildx build --platform linux/arm64,linux/amd64 -t steedos/steedos-enterprise  .

deploy/enterprise/fs/opt/steedos/templates/docker.env.sh

@@ -33,4 +33,6 @@ NPM_CACHE_ENABLED=true
 NPM_CACHE_FOLDER=/steedos-stacks/unpkg
 NPM_CACHE_PACKAGE_CONTENT=true
 
+PORT=3000
+
 EOF

deploy/enterprise/fs/opt/steedos/utils/bin/backup.js

@@ -0,0 +1,199 @@
+const fsPromises = require('fs/promises');
+const path = require('path');
+const os = require('os');
+const shell = require('shelljs');
+const utils = require('./utils');
+const Constants = require('./constants');
+const logger = require('./logger');
+const mailer = require('./mailer');
+
+const command_args = process.argv.slice(3);
+
+async function run() {
+  const timestamp = getTimeStampInISO();
+  let errorCode = 0;
+  try {
+    const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
+    shell.exec(check_supervisord_status_cmd, function (code) {
+      if (code > 0) {
+        shell.echo('application is not running, starting supervisord');
+        shell.exec('/usr/bin/supervisord');
+      }
+    });
+
+    console.log('Available free space at /steedos-stacks');
+    const availSpaceInBytes = getAvailableBackupSpaceInBytes();
+    console.log('\n');
+
+    checkAvailableBackupSpace(availSpaceInBytes);
+
+    const backupRootPath = await generateBackupRootPath();
+    const backupContentsPath = getBackupContentsPath(backupRootPath, timestamp);
+
+    await fsPromises.mkdir(backupContentsPath);
+
+    await exportDatabase(backupContentsPath);
+
+    await createGitStorageArchive(backupContentsPath);
+
+    await createManifestFile(backupContentsPath);
+    await exportDockerEnvFile(backupContentsPath);
+
+    const archivePath = await createFinalArchive(backupRootPath, timestamp);
+
+    await fsPromises.rm(backupRootPath, { recursive: true, force: true });
+
+    logger.backup_info('Finished taking a backup at' + archivePath);
+
+  } catch (err) {
+    errorCode = 1;
+    await logger.backup_error(err.stack);
+
+    if (command_args.includes('--error-mail')) {
+      const currentTS = new Date().getTime();
+      const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
+      if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) {
+        await mailer.sendBackupErrorToAdmins(err, timestamp);
+        await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
+      }
+    }
+  } finally {
+    await postBackupCleanup();
+    process.exit(errorCode);
+  }
+}
+
+async function exportDatabase(destFolder) {
+  console.log('Exporting database');
+  await executeMongoDumpCMD(destFolder, process.env.STEEDOS_MONGODB_URI)
+  console.log('Exporting database done.');
+}
+
+async function createGitStorageArchive(destFolder) {
+  console.log('Creating git-storage archive');
+
+  const gitRoot = getGitRoot(process.env.STEEDOS_GIT_ROOT);
+
+  await executeCopyCMD(gitRoot, destFolder)
+
+  console.log('Created git-storage archive');
+}
+
+async function createManifestFile(path) {
+  const version = await utils.getCurrentSteedosVersion()
+  const manifest_data = { "appsmithVersion": version }
+  await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
+}
+
+async function exportDockerEnvFile(destFolder) {
+  console.log('Exporting docker environment file');
+  const content = await fsPromises.readFile('/steedos-stacks/configuration/docker.env', { encoding: 'utf8' });
+  const cleaned_content = removeSensitiveEnvData(content)
+  await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
+  console.log('Exporting docker environment file done.');
+  console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!');
+  console.log('!!! Please ensure you have saved the STEEDOS_ENCRYPTION_SALT and STEEDOS_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.');
+  console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
+}
+
+async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
+  return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
+}
+
+async function createFinalArchive(destFolder, timestamp) {
+  console.log('Creating final archive');
+
+  const archive = `${Constants.BACKUP_PATH}/steedos-backup-${timestamp}.tar.gz`;
+  await utils.execCommand(['tar', '-cah', '-C', destFolder, '-f', archive, '.']);
+
+  console.log('Created final archive');
+
+  return archive;
+}
+
+async function postBackupCleanup() {
+  console.log('Starting the cleanup task after taking a backup.');
+  let backupArchivesLimit = getBackupArchiveLimit(process.env.STEEDOS_BACKUP_ARCHIVE_LIMIT);
+  const backupFiles = await utils.listLocalBackupFiles();
+  while (backupFiles.length > backupArchivesLimit) {
+    const fileName = backupFiles.shift();
+    await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
+  }
+  console.log('Cleanup task completed.');
+
+}
+async function executeCopyCMD(srcFolder, destFolder) {
+  return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage'])
+}
+
+function getGitRoot(gitRoot) {
+  if (gitRoot == null || gitRoot === '') {
+    gitRoot = '/steedos-stacks/git-storage';
+  }
+  return gitRoot
+}
+
+async function generateBackupRootPath() {
+  const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
+  return backupRootPath
+}
+
+function getBackupContentsPath(backupRootPath, timestamp) {
+  return backupRootPath + '/steedos-backup-' + timestamp;
+}
+
+function removeSensitiveEnvData(content) {
+  // Remove encryption and Mongodb data from docker.env
+  const output_lines = []
+  content.split(/\r?\n/).forEach(line => {
+    if (!line.startsWith("STEEDOS_ENCRYPTION") && !line.startsWith("STEEDOS_MONGODB")) {
+      output_lines.push(line);
+    }
+  });
+  return output_lines.join('\n')
+}
+
+function getBackupArchiveLimit(backupArchivesLimit) {
+  if (!backupArchivesLimit)
+    backupArchivesLimit = Constants.STEEDOS_DEFAULT_BACKUP_ARCHIVE_LIMIT;
+  return backupArchivesLimit
+}
+
+async function removeOldBackups(backupFiles, backupArchivesLimit) {
+  while (backupFiles.length > backupArchivesLimit) {
+    const fileName = backupFiles.shift();
+    await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
+  }
+  return backupFiles
+}
+
+function getTimeStampInISO() {
+  return new Date().toISOString().replace(/:/g, '-')
+}
+
+function getAvailableBackupSpaceInBytes() {
+  return parseInt(shell.exec('df --output=avail -B 1 /steedos-stacks | tail -n 1'), 10)
+}
+
+function checkAvailableBackupSpace(availSpaceInBytes) {
+  if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
+    throw new Error('Not enough space avaliable at /steedos-stacks. Please ensure availability of atleast 2GB to backup successfully.');
+  }
+}
+
+
+
+module.exports = {
+  run,
+  getTimeStampInISO,
+  getAvailableBackupSpaceInBytes,
+  checkAvailableBackupSpace,
+  generateBackupRootPath,
+  getBackupContentsPath,
+  executeMongoDumpCMD,
+  getGitRoot,
+  executeCopyCMD,
+  removeSensitiveEnvData,
+  getBackupArchiveLimit,
+  removeOldBackups
+};