incremental backups: completion and s3:aws integration

CyberCP/secMiddleware.py

@@ -1,6 +1,7 @@
 from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
 import json
 from django.shortcuts import HttpResponse
+import re
 
 class secMiddleware:
 
@@ -58,6 +59,14 @@ def __call__(self, request):
                     else:
                         continue
 
+                    if key == 'backupDestinations':
+                        if re.match('^[a-z|0-9]+:[a-z|0-9|\.]+\/?[A-Z|a-z|0-9|\.]*$', value) == None and value != 'local':
+                            logging.writeToFile(request.body)
+                            final_dic = {'error_message': "Data supplied is not accepted.",
+                                         "errorMessage": "Data supplied is not accepted."}
+                            final_json = json.dumps(final_dic)
+                            return HttpResponse(final_json)
+
                     if request.build_absolute_uri().find('saveSpamAssassinConfigurations') > -1 or request.build_absolute_uri().find('docker') > -1 or request.build_absolute_uri().find('cloudAPI') > -1 or request.build_absolute_uri().find('filemanager') > -1 or request.build_absolute_uri().find('verifyLogin') > -1 or request.build_absolute_uri().find('submitUserCreation') > -1:
                         continue
                     if key == 'backupDestinations' or key == 'ports' or key == 'imageByPass' or key == 'passwordByPass' or key == 'cronCommand' or key == 'emailMessage' or key == 'configData' or key == 'rewriteRules' or key == 'modSecRules' or key == 'recordContentTXT' or key == 'SecAuditLogRelevantStatus' or key == 'fileContent':

IncBackups/IncBackupsControl.py

@@ -52,6 +52,42 @@ def run(self):
         elif self.function == 'restorePoint':
             self.restorePoint()
 
+    def getAWSData(self):
+        key = self.backupDestinations.split('/')[-1]
+        path = '/home/cyberpanel/aws/%s' % (key)
+        secret = open(path, 'r').read()
+        return key, secret
+
+    def awsFunction(self, fType, backupPath = None, snapshotID = None, bType = None):
+        try:
+            if fType == 'backup':
+                key, secret = self.getAWSData()
+                command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s  && restic -r s3:s3.amazonaws.com/%s backup %s --password-file %s' % (
+                key, secret, self.website.domain, backupPath, self.passwordFile)
+                result = ProcessUtilities.outputExecutioner(command)
+                logging.statusWriter(self.statusPath, result, 1)
+                snapShotid = result.split(' ')[-2]
+                if bType == 'database':
+                    newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (bType, backupPath.split('/')[-1].strip('.sql')),
+                                               snapshotid=snapShotid,
+                                               destination=self.backupDestinations)
+                else:
+                    newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (bType, backupPath), snapshotid=snapShotid,
+                                           destination=self.backupDestinations)
+                newSnapshot.save()
+            else:
+                self.backupDestinations = self.jobid.destination
+                key, secret = self.getAWSData()
+                command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s  && restic -r s3:s3.amazonaws.com/%s restore %s --password-file %s --target /' % (
+                    key, secret, self.website, snapshotID, self.passwordFile)
+                result = ProcessUtilities.outputExecutioner(command)
+                logging.statusWriter(self.statusPath, result, 1)
+
+
+        except BaseException, msg:
+            logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
+            return 0
+
     def restoreData(self):
         try:
 
@@ -65,6 +101,8 @@ def restoreData(self):
                 command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                 result = ProcessUtilities.outputExecutioner(command)
                 logging.statusWriter(self.statusPath, result, 1)
+            else:
+                self.awsFunction('restore', '', self.jobid.snapshotid)
 
         except BaseException, msg:
             logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
@@ -83,8 +121,10 @@ def restoreDatabase(self):
                 command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                 result = ProcessUtilities.outputExecutioner(command)
                 logging.statusWriter(self.statusPath, result, 1)
+            else:
+                self.awsFunction('restore', '', self.jobid.snapshotid)
 
-            if mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(self.jobid.type.split(':')[1], '/home/cyberpanel', 'dummy', 'dummy') == 0:
+            if mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(self.jobid.type.split(':')[1].rstrip('.sql'), '/home/cyberpanel', 'dummy', 'dummy') == 0:
                 raise BaseException
 
             try:
@@ -109,6 +149,8 @@ def restoreEmail(self):
                 command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                 result = ProcessUtilities.outputExecutioner(command)
                 logging.statusWriter(self.statusPath, result, 1)
+            else:
+                self.awsFunction('restore', '', self.jobid.snapshotid)
 
         except BaseException, msg:
             logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
@@ -364,6 +406,9 @@ def backupData(self):
                 newSnapshot = JobSnapshots(job=self.jobid, type='data:%s' % (remotePath), snapshotid=snapShotid,
                                            destination=self.backupDestinations)
                 newSnapshot.save()
+            else:
+                backupPath = '/home/%s' % (self.website.domain)
+                self.awsFunction('backup', backupPath, '', 'data')
 
             logging.statusWriter(self.statusPath, 'Data for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
             return 1
@@ -402,6 +447,8 @@ def backupDatabases(self):
                     newSnapshot = JobSnapshots(job=self.jobid, type='database:%s' % (items.dbName), snapshotid=snapShotid,
                                                destination=self.backupDestinations)
                     newSnapshot.save()
+                else:
+                    self.awsFunction('backup', dbPath, '', 'database')
 
                 try:
                     os.remove('/home/cyberpanel/%s' % (items.dbName))
@@ -439,6 +486,8 @@ def emailBackup(self):
                 newSnapshot = JobSnapshots(job=self.jobid, type='email:%s' % (backupPath), snapshotid=snapShotid,
                                            destination=self.backupDestinations)
                 newSnapshot.save()
+            else:
+                self.awsFunction('backup', backupPath, '', 'email')
 
 
             logging.statusWriter(self.statusPath, 'Emails for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
@@ -462,8 +511,11 @@ def initiateRepo(self):
                 result = ProcessUtilities.outputExecutioner(command)
                 logging.statusWriter(self.statusPath, result, 1)
             else:
-                logging.statusWriter(self.statusPath, 'AWS implementation is currently pending. [5009]', 1)
-                return 0
+                key,secret = self.getAWSData()
+                command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s  && restic -r s3:s3.amazonaws.com/%s init --password-file %s' % (key, secret, self.website.domain, self.passwordFile)
+                result = ProcessUtilities.outputExecutioner(command)
+                logging.statusWriter(self.statusPath, result, 1)
+                return 1
 
             logging.statusWriter(self.statusPath, 'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
             return 1

IncBackups/IncScheduler.py

@@ -0,0 +1,99 @@
+#!/usr/local/CyberCP/bin/python2
+import os
+import os.path
+import sys
+sys.path.append('/usr/local/CyberCP')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
+
+import django
+try:
+    django.setup()
+except:
+    pass
+from IncBackupsControl import IncJobs
+from IncBackups.models import BackupJob
+from random import randint
+import argparse
+try:
+    from plogical.virtualHostUtilities import virtualHostUtilities
+    from plogical.mailUtilities import mailUtilities
+    from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
+except:
+    pass
+
+class IncScheduler():
+    logPath = '/home/cyberpanel/incbackuplogs'
+
+    @staticmethod
+    def startBackup(type):
+        try:
+            logging.statusWriter(IncScheduler.logPath, 'Starting Incremental Backup job..', 1)
+            tempPath = "/home/cyberpanel/" + str(randint(1000, 9999))
+            for job in BackupJob.objects.all():
+                logging.statusWriter(IncScheduler.logPath, 'Job Description:\n\n Destination: %s, Frequency: %s.\n ' % (job.destination, job.frequency), 1)
+                if job.frequency == type:
+                    for web in job.jobsites_set.all():
+                        logging.statusWriter(IncScheduler.logPath, 'Backing up %s.' % (web.website), 1)
+                        extraArgs = {}
+                        extraArgs['website'] = web.website
+                        extraArgs['tempPath'] = tempPath
+                        extraArgs['backupDestinations'] = job.destination
+
+                        if job.websiteData == 1:
+                            extraArgs['websiteData'] = True
+                        else:
+                            extraArgs['websiteData'] = False
+
+                        if job.websiteDatabases == 1:
+                            extraArgs['websiteDatabases'] = True
+                        else:
+                            extraArgs['websiteDatabases'] = False
+
+                        if job.websiteDataEmails == 1:
+                            extraArgs['websiteEmails'] = True
+                        else:
+                            extraArgs['websiteEmails'] = False
+
+                        extraArgs['websiteSSLs'] = False
+
+                        startJob = IncJobs('createBackup', extraArgs)
+                        startJob.start()
+
+                        ### Checking status
+
+                        while True:
+                            if os.path.exists(tempPath):
+                                result = open(tempPath, 'r').read()
+
+                                if result.find("Completed") > -1:
+
+                                    ### Removing Files
+
+                                    os.remove(tempPath)
+
+                                    logging.statusWriter(IncScheduler.logPath, 'Backed up %s.' % (web.website), 1)
+                                    break
+                                elif result.find("[5009]") > -1:
+                                    ## removing status file, so that backup can re-runn
+                                    try:
+                                        os.remove(tempPath)
+                                    except:
+                                        pass
+
+                                    logging.statusWriter(IncScheduler.logPath, 'Failed backup for %s, error: %s.' % (web.website, result), 1)
+                                    break
+        except BaseException, msg:
+            logging.writeToFile(str(msg))
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description='CyberPanel Installer')
+    parser.add_argument('function', help='Specific a function to call!')
+    args = parser.parse_args()
+
+    IncScheduler.startBackup(args.function)
+
+
+if __name__ == "__main__":
+    main()

IncBackups/models.py

@@ -12,4 +12,19 @@ class JobSnapshots(models.Model):
     job = models.ForeignKey(IncJob)
     type = models.CharField(max_length=300)
     snapshotid = models.CharField(max_length=50)
-    destination = models.CharField(max_length=200, default='')
+    destination = models.CharField(max_length=200, default='')
+
+
+class BackupJob(models.Model):
+    destination = models.CharField(max_length=300)
+    frequency = models.CharField(max_length=50)
+    websiteData = models.IntegerField()
+    websiteDatabases = models.IntegerField()
+    websiteDataEmails = models.IntegerField()
+
+
+class JobSites(models.Model):
+    job = models.ForeignKey(BackupJob)
+    website = models.CharField(max_length=300)
+
+