Skip to content

Commit

Permalink
[fix]: fixed iob setup custom migration and iob validate (#2951)
Browse files Browse the repository at this point in the history 
* fixed 'iob validate' and 'setup custom' migration

* fix restore process for restore on migration

* add test

* make the backup name not a number on CI

* another try

* do not log backup ok if it failed

* fix backups missing postfix

* another try
  • Loading branch information
@foxriver76
foxriver76 authored Oct 25, 2024
1 parent 1ee989d commit 0320a1a
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 80 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
## __WORK IN PROGRESS__
-->

## __WORK IN PROGRESS__
* (@foxriver76) fixed `iob validate` command and `setup custom` migration

## 7.0.1 (2024-10-21) - Lucy
* (@foxriver76) fixed crash case on database migration
* (@foxriver76) fixed edge case crash cases if notifications are processed nearly simultaneously
Expand Down
6 changes: 3 additions & 3 deletions packages/cli/src/lib/setup.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1328,9 +1328,9 @@ async function processCommand(
await backup.validateBackup(name);
console.log('Backup OK');
return void exitApplicationSave(0);
} catch (err) {
console.log(`Backup check failed: ${err.message}`);
return void exitApplicationSave(1);
} catch (e) {
console.log(`Backup check failed: ${e.message}`);
return void exitApplicationSave(e instanceof IoBrokerError ? e.code : 1);
}
});
break;
Expand Down
172 changes: 97 additions & 75 deletions packages/cli/src/lib/setup/setupBackup.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ export class BackupRestore {
private readonly HOSTNAME_PLACEHOLDER_REPLACE = '$$$$__hostname__$$$$';
/** Regex to replace all occurrences of the HOSTNAME_PLACEHOLDER */
private readonly HOSTNAME_PLACEHOLDER_REGEX = /\$\$__hostname__\$\$/g;
/** Postfix for backup name */
private readonly BACKUP_POSTFIX = `_backup${tools.appNameLowerCase}`;

constructor(options: CLIBackupRestoreOptions) {
options = options || {};
Expand Down Expand Up @@ -261,7 +263,9 @@ export class BackupRestore {
const d = new Date();
name = `${d.getFullYear()}_${`0${d.getMonth() + 1}`.slice(-2)}_${`0${d.getDate()}`.slice(-2)}-${`0${d.getHours()}`.slice(
-2,
)}_${`0${d.getMinutes()}`.slice(-2)}_${`0${d.getSeconds()}`.slice(-2)}_backup${tools.appName}`;
)}_${`0${d.getMinutes()}`.slice(-2)}_${`0${d.getSeconds()}`.slice(-2)}${this.BACKUP_POSTFIX}`;
} else if (!name.endsWith(this.BACKUP_POSTFIX) && !name.endsWith(`${this.BACKUP_POSTFIX}.tar.gz`)) {
name += this.BACKUP_POSTFIX;
}

name = name.toString().replace(/\\/g, '/');
Expand Down Expand Up @@ -367,7 +371,7 @@ export class BackupRestore {

console.log(`host.${hostname} Validating backup ...`);
try {
await this._validateBackupAfterCreation(noConfig);
await this._validateTempDirectory(noConfig);
console.log(`host.${hostname} The backup is valid!`);

return await this._packBackup(name);
Expand Down Expand Up @@ -658,8 +662,14 @@ export class BackupRestore {

const backupBaseDir = path.join(this.tmpDir, 'backup');

const config: ioBroker.IoBrokerJson = await fs.readJSON(path.join(backupBaseDir, 'config.json'));
const backupHostName = config.system?.hostname || hostname;
let backupHostName = hostname;
// Note: on backups created during migration no config exists
let config: ioBroker.IoBrokerJson | undefined;

if (await fs.pathExists(path.join(backupBaseDir, 'config.json'))) {
config = (await fs.readJSON(path.join(backupBaseDir, 'config.json'))) as ioBroker.IoBrokerJson;
backupHostName = config.system?.hostname || hostname;
}

// we need to find the host obj for the compatibility check
const objFd = await open(path.join(backupBaseDir, 'objects.jsonl'));
Expand Down Expand Up @@ -692,8 +702,10 @@ export class BackupRestore {
}

// restore ioBroker.json
fs.writeFileSync(tools.getConfigFileName(), JSON.stringify(config, null, 2));
await this.connectToNewDatabase(config);
if (config) {
fs.writeFileSync(tools.getConfigFileName(), JSON.stringify(config, null, 2));
await this.connectToNewDatabase(config);
}

console.log(`host.${hostname} Clear all objects and states...`);
await this.cleanDatabase(false);
Expand Down Expand Up @@ -746,7 +758,7 @@ export class BackupRestore {
const { force, restartOnFinish, dontDeleteAdapters } = options;

const backupBaseDir = path.join(this.tmpDir, 'backup');
const isJsonl = await fs.pathExists(path.join(backupBaseDir, 'config.json'));
const isJsonl = await fs.pathExists(path.join(backupBaseDir, 'objects.jsonl'));

if (isJsonl) {
const exitCode = await this._restoreJsonlBackup(options);
Expand Down Expand Up @@ -966,27 +978,34 @@ export class BackupRestore {
}

/**
* Validates the backup.json and all json files inside the backup after (in temporary directory), here we only abort if backup.json is corrupted
* Validates a JSONL-style backup and all json files inside the backup (in temporary directory)
*
* @param noConfig if the backup does not contain a `config.json` (used by setup custom migration)
*/
private async _validateBackupAfterCreation(noConfig = false): Promise<void> {
private async _validateTempDirectory(noConfig = false): Promise<void> {
const backupBaseDir = path.join(this.tmpDir, 'backup');

if (!noConfig) {
await fs.readJSON(path.join(backupBaseDir, 'config.json'));
console.log(`host.${this.hostname} "config.json" is valid`);
}

if (!(await fs.pathExists(path.join(backupBaseDir, 'objects.jsonl')))) {
throw new Error('Backup does not contain valid objects');
}

console.log(`host.${this.hostname} "objects.jsonl" exists`);

if (!(await fs.pathExists(path.join(backupBaseDir, 'states.jsonl')))) {
throw new Error('Backup does not contain valid states');
}

console.log(`host.${this.hostname} "states.jsonl" exists`);

await this._validateDatabaseFiles();

console.log(`host.${this.hostname} JSONL lines are valid`);

// we check all other json files, we assume them as optional, because user created files may be no valid json
try {
this._checkDirectory(path.join(backupBaseDir, 'files'));
Expand Down Expand Up @@ -1034,7 +1053,7 @@ export class BackupRestore {
*
* @param _name - index or name of the backup
*/
validateBackup(_name: string | number): Promise<void> | undefined {
async validateBackup(_name: string | number): Promise<void> {
let backups;
let name = typeof _name === 'number' ? _name.toString() : _name;

Expand All @@ -1046,12 +1065,12 @@ export class BackupRestore {
console.log('Please specify one of the backup names:');

for (const t in backups) {
console.log(`${backups[t]} or ${backups[t].replace(`_backup${tools.appName}.tar.gz`, '')} or ${t}`);
console.log(`${backups[t]} or ${backups[t].replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${t}`);
}
} else {
console.warn(`No backups found. Create a backup, using "${tools.appName} backup" first`);
}
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
}
// If number
if (parseInt(name, 10).toString() === name.toString()) {
Expand All @@ -1064,95 +1083,98 @@ export class BackupRestore {
console.log('Please specify one of the backup names:');
for (const t in backups) {
console.log(
`${backups[t]} or ${backups[t].replace(`_backup${tools.appName}.tar.gz`, '')} or ${t}`,
`${backups[t]} or ${backups[t].replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${t}`,
);
}
} else {
console.log(`No existing backups. Create a backup, using "${tools.appName} backup" first`);
}
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);

throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
}
console.log(`host.${this.hostname} Using backup file ${name}`);
}

name = name.toString().replace(/\\/g, '/');
if (!name.includes('/')) {
name = BackupRestore.getBackupDir() + name;
const regEx = new RegExp(`_backup${tools.appName}`, 'i');
const regEx = new RegExp(this.BACKUP_POSTFIX, 'i');
if (!regEx.test(name)) {
name += `_backup${tools.appName}`;
name += this.BACKUP_POSTFIX;
}
if (!name.match(/\.tar\.gz$/i)) {
name += '.tar.gz';
}
}
if (!fs.existsSync(name)) {
console.error(`host.${this.hostname} Cannot find ${name}`);
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
}

if (fs.existsSync(`${this.tmpDir}/backup/backup.json`)) {
fs.unlinkSync(`${this.tmpDir}/backup/backup.json`);
}

return new Promise(resolve => {
tar.extract(
{
file: name,
cwd: this.tmpDir,
},
undefined,
err => {
if (err) {
console.error(`host.${this.hostname} Cannot extract from file "${name}": ${err.message}`);
return void this.processExit(EXIT_CODES.INVALID_ARGUMENTS);
}
if (!fs.existsSync(`${this.tmpDir}/backup/backup.json`)) {
console.error(
`host.${this.hostname} Validation failed. Cannot find extracted file from file "${this.tmpDir}/backup/backup.json"`,
);
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
}
try {
await tar.extract({
file: name,
cwd: this.tmpDir,
});
} catch (e) {
const errMessage = `Cannot extract from file "${name}": ${e.message}`;
console.error(`host.${this.hostname} ${errMessage}`);
throw new IoBrokerError({ message: 'Backup not found', code: EXIT_CODES.INVALID_ARGUMENTS });
}

console.log(`host.${this.hostname} Starting validation ...`);
let backupJSON;
try {
backupJSON = fs.readJSONSync(`${this.tmpDir}/backup/backup.json`);
} catch (err) {
console.error(
`host.${this.hostname} Backup corrupted. Backup ${name} does not contain a valid backup.json file: ${err.message}`,
);
this.removeTempBackupDir();
try {
if (fs.existsSync(path.join(this.tmpDir, 'backup', 'backup.json'))) {
this._validateLegacyTempDir();
} else {
await this._validateTempDirectory();
}
} catch (e) {
console.error(`host.${this.hostname} ${e.message}`);

return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
}
try {
this.removeTempBackupDir();
} catch (e) {
console.error(`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`);
}

if (!backupJSON || !backupJSON.objects || !backupJSON.objects.length) {
console.error(`host.${this.hostname} Backup corrupted. Backup does not contain valid objects`);
try {
this.removeTempBackupDir();
} catch (e) {
console.error(
`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`,
);
}
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
}
throw new IoBrokerError({ message: e.message, code: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP });
}

console.log(`host.${this.hostname} backup.json OK`);
try {
this.removeTempBackupDir();
} catch (e) {
console.error(`host.${this.hostname} Cannot clear temporary backup directory: ${e.message}`);
throw new IoBrokerError({ message: e.message, code: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP });
}
}

try {
this._checkDirectory(`${this.tmpDir}/backup/files`, true);
this.removeTempBackupDir();
/**
* Validate an unpacked legacy backup in the temporary directory
*/
private _validateLegacyTempDir(): void {
console.log(`host.${this.hostname} Starting validation ...`);
let backupJSON;
try {
backupJSON = fs.readJSONSync(`${this.tmpDir}/backup/backup.json`);
} catch (e) {
throw new Error(`Backup corrupted. Backup does not contain a valid backup.json file: ${e.message}`);
}

resolve();
} catch (err) {
console.error(`host.${this.hostname} Backup corrupted: ${err.message}`);
return void this.processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP);
}
},
);
});
if (!backupJSON || !backupJSON.objects || !backupJSON.objects.length) {
throw new Error(`host.${this.hostname} Backup corrupted. Backup does not contain valid objects`);
}

console.log(`host.${this.hostname} backup.json OK`);

try {
this._checkDirectory(`${this.tmpDir}/backup/files`, true);
} catch (e) {
throw new Error(`Backup corrupted: ${e.message}`);
}
}

/**
Expand Down Expand Up @@ -1204,7 +1226,7 @@ export class BackupRestore {
backups.sort((a, b) => (b > a ? 1 : b === a ? 0 : -1));
if (backups.length) {
backups.forEach((backup, i) =>
console.log(`${backup} or ${backup.replace(`_backup${tools.appName}.tar.gz`, '')} or ${i}`),
console.log(`${backup} or ${backup.replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${i}`),
);
} else {
console.warn('No backups found');
Expand All @@ -1229,7 +1251,7 @@ export class BackupRestore {
if (backups.length) {
console.log('Please specify one of the backup names:');
backups.forEach((backup, i) =>
console.log(`${backup} or ${backup.replace(`_backup${tools.appName}.tar.gz`, '')} or ${i}`),
console.log(`${backup} or ${backup.replace(`${this.BACKUP_POSTFIX}.tar.gz`, '')} or ${i}`),
);
}
} else {
Expand All @@ -1240,9 +1262,9 @@ export class BackupRestore {
name = name.toString().replace(/\\/g, '/');
if (!name.includes('/')) {
name = BackupRestore.getBackupDir() + name;
const regEx = new RegExp(`_backup${tools.appName}`, 'i');
const regEx = new RegExp(this.BACKUP_POSTFIX, 'i');
if (!regEx.test(name)) {
name += `_backup${tools.appName}`;
name += this.BACKUP_POSTFIX;
}
if (!name.match(/\.tar\.gz$/i)) {
name += '.tar.gz';
Expand Down Expand Up @@ -1272,10 +1294,10 @@ export class BackupRestore {

if (
!(await fs.pathExists(path.join(backupBasePath, 'backup.json'))) &&
!(await fs.pathExists(path.join(backupBasePath, 'config.json')))
!(await fs.pathExists(path.join(backupBasePath, 'objects.jsonl')))
) {
console.error(
`host.${this.hostname} Cannot find extracted file "${path.join(backupBasePath, 'backup.json')}" or "${path.join(backupBasePath, 'config.json')}"`,
`host.${this.hostname} Cannot find extracted file "${path.join(backupBasePath, 'backup.json')}" or "${path.join(backupBasePath, 'objects.jsonl')}"`,
);
return { exitCode: EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP, objects: this.objects, states: this.states };
}
Expand Down
10 changes: 8 additions & 2 deletions packages/controller/test/lib/testConsole.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,10 +581,16 @@ export function register(it: Mocha.TestFunction, expect: Chai.ExpectStatic, cont

// expect(found).to.be.true;

const name = Math.round(Math.random() * 10000).toString();
const name = Math.round(Math.random() * 10_000).toString();
res = await execAsync(`"${process.execPath}" "${iobExecutable}" backup ${name}`);
expect(res.stderr).to.be.not.ok;
expect(fs.existsSync(`${BackupRestore.getBackupDir() + name}.tar.gz`)).to.be.true;
expect(fs.existsSync(`${BackupRestore.getBackupDir() + name}_backupiobroker.tar.gz`)).to.be.true;
}).timeout(20_000);

it(`${testName}validates backup`, async () => {
const res = await execAsync(`"${process.execPath}" "${iobExecutable}" validate 0`);
expect(res.stderr).to.be.not.ok;
expect(res.stdout).to.include('Backup OK');
}).timeout(20_000);

// list l
Expand Down

0 comments on commit 0320a1a

Please sign in to comment.