From 0247a172f7c21be2e8f8ebf6e49e0a673f259127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Fri, 6 Dec 2024 17:30:01 +0100 Subject: [PATCH 1/8] Added task_uuid to basic backup data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- apps/analyzer/metadata_analyzer/analyzer.py | 17 +++++++++-------- apps/analyzer/metadata_analyzer/models.py | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/apps/analyzer/metadata_analyzer/analyzer.py b/apps/analyzer/metadata_analyzer/analyzer.py index f0ca7fa..7288d33 100644 --- a/apps/analyzer/metadata_analyzer/analyzer.py +++ b/apps/analyzer/metadata_analyzer/analyzer.py @@ -22,16 +22,17 @@ def analyze(): # Convert a result from the database into the format used by the backend def _convert_result(result): backup_type = { - "F": "FULL", - "I": "INCREMENTAL", - "D": "DIFFERENTIAL", - "C": "COPY" + "F": "FULL", + "I": "INCREMENTAL", + "D": "DIFFERENTIAL", + "C": "COPY" }[result.fdi_type] return { "id": result.uuid, "sizeMB": result.data_size / 1_000_000, "creationDate": result.start_time.isoformat(), - "type": backup_type + "type": backup_type, + "taskId": result.task_uuid, } def _get_start_date(data, alert_type, backup_type): @@ -43,13 +44,13 @@ def _get_start_date(data, alert_type, backup_type): assert len(latest_alerts) == 1 return latest_alerts[0] - def update_data(): results = list(Analyzer.database.get_results()) # Batch the api calls to the backend for improved efficiency batch = [] count = 0 + for result in results: # Only send real backups if result.is_backup <= 0: @@ -78,13 +79,13 @@ def simple_rule_based_analysis(alert_limit): start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "FULL") result = Analyzer.simple_rule_based_analyzer.analyze(data, alert_limit, start_date) return result - + def simple_rule_based_analysis_diff(alert_limit): data = list(Analyzer.database.get_results()) start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "DIFFERENTIAL") result = Analyzer.simple_rule_based_analyzer.analyze_diff(data, alert_limit, start_date) return result - + def simple_rule_based_analysis_inc(alert_limit): data = list(Analyzer.database.get_results()) start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "INCREMENTAL") diff --git a/apps/analyzer/metadata_analyzer/models.py b/apps/analyzer/metadata_analyzer/models.py index 04cb40b..d07c9db 100644 --- a/apps/analyzer/metadata_analyzer/models.py +++ b/apps/analyzer/metadata_analyzer/models.py @@ -26,6 +26,7 @@ class Result(Base): saveset: Mapped[str] = mapped_column(primary_key=True) uuid: Mapped[str] task: Mapped[str] + task_uuid: Mapped[str] fdi_type: Mapped[str] is_backup: Mapped[int] state: Mapped[int] From b026b81b0970f00e3988eafb899438c80a7452e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Fri, 6 Dec 2024 17:51:51 +0100 Subject: [PATCH 2/8] Send tasks to Backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- apps/analyzer/metadata_analyzer/analyzer.py | 37 ++++++++++++++++++++- apps/analyzer/metadata_analyzer/backend.py | 5 +++ apps/analyzer/metadata_analyzer/database.py | 9 ++++- apps/analyzer/metadata_analyzer/models.py | 17 +++++++++- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/apps/analyzer/metadata_analyzer/analyzer.py b/apps/analyzer/metadata_analyzer/analyzer.py index 7288d33..1a933ef 100644 --- a/apps/analyzer/metadata_analyzer/analyzer.py +++ b/apps/analyzer/metadata_analyzer/analyzer.py @@ -1,5 +1,6 @@ import datetime + class Analyzer: def init(database, backend, simple_analyzer, simple_rule_based_analyzer): Analyzer.database = database @@ -35,6 +36,13 @@ def _convert_result(result): "taskId": result.task_uuid, } + # Convert a task from the database into the format used by the backend + def _convert_task(task): + return { + "id": task.uuid, + "displayName": task.task, + } + def _get_start_date(data, alert_type, backup_type): latest_id = Analyzer.backend.get_latest_alert_id(alert_type, backup_type) if latest_id == "": @@ -44,7 +52,7 @@ def _get_start_date(data, alert_type, backup_type): assert len(latest_alerts) == 1 return latest_alerts[0] - def update_data(): + def _send_Backups(): results = list(Analyzer.database.get_results()) # Batch the api calls to the backend for improved efficiency @@ -74,6 +82,33 @@ def update_data(): return {"count": count} + def _send_Tasks(): + tasks = list(Analyzer.database.get_tasks()) + + # Batch the api calls to the backend for improved efficiency + batch = [] + count = 0 + + for task in tasks: + + batch.append(Analyzer._convert_task(task)) + count += 1 + + # Send a full batch + if len(batch) == 100: + Analyzer.backend.send_task_data_batched(batch) + batch = [] + + # Send the remaining results + if len(batch) > 0: + Analyzer.backend.send_task_data_batched(batch) + + return {"count": count} + + def update_data(): + Analyzer._send_Tasks() + Analyzer._send_Backups() + def simple_rule_based_analysis(alert_limit): data = list(Analyzer.database.get_results()) start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "FULL") diff --git a/apps/analyzer/metadata_analyzer/backend.py b/apps/analyzer/metadata_analyzer/backend.py index 0728959..0c7f39b 100644 --- a/apps/analyzer/metadata_analyzer/backend.py +++ b/apps/analyzer/metadata_analyzer/backend.py @@ -9,6 +9,11 @@ def send_backup_data_batched(self, batch): r = requests.post(url, json=batch) r.raise_for_status() + def send_task_data_batched(self, batch): + url = self.backend_url + "tasks/batched" + r = requests.post(url, json=batch) + r.raise_for_status() + def create_size_alert(self, alert): url = self.backend_url + "alerting/size" r = requests.post(url, json=alert) diff --git a/apps/analyzer/metadata_analyzer/database.py b/apps/analyzer/metadata_analyzer/database.py index 4952497..b2e041b 100644 --- a/apps/analyzer/metadata_analyzer/database.py +++ b/apps/analyzer/metadata_analyzer/database.py @@ -1,7 +1,7 @@ import pg8000.dbapi from sqlalchemy import create_engine, select from sqlalchemy.orm import Session -from metadata_analyzer.models import BackupData, Result +from metadata_analyzer.models import BackupData, Result, Tasks import os @@ -40,3 +40,10 @@ def get_results(self): result = session.scalars(stmt) return result + + def get_tasks(self): + session = Session(self.engine) + stmt = select(Tasks) + + result = session.scalars(stmt) + return result diff --git a/apps/analyzer/metadata_analyzer/models.py b/apps/analyzer/metadata_analyzer/models.py index d07c9db..7c2e0cb 100644 --- a/apps/analyzer/metadata_analyzer/models.py +++ b/apps/analyzer/metadata_analyzer/models.py @@ -1,6 +1,7 @@ -from sqlalchemy.orm import mapped_column, Mapped, declarative_base from datetime import datetime +from sqlalchemy.orm import mapped_column, Mapped, declarative_base + Base = declarative_base() @@ -44,3 +45,17 @@ def __repr__(self): def __str__(self): return repr(self) + + +class Tasks(Base): + __tablename__ = "tasks" + + # For now I only added the most relevant columns + task: Mapped[str] = mapped_column(primary_key=True) + uuid: Mapped[str] + + def __repr__(self): + return f"""Tasks(uuid={self.uuid})""" + + def __str__(self): + return repr(self) From 36588503f84f50ed499b9113cf090978df00c42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Fri, 6 Dec 2024 17:59:05 +0100 Subject: [PATCH 3/8] Added possibility for batching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- .../app/backupData/backupData.service.spec.ts | 14 +++++++++++ .../src/app/tasks/tasks.controller.spec.ts | 24 +++++++++++++++---- .../backend/src/app/tasks/tasks.controller.ts | 7 ++++++ apps/backend/src/app/tasks/tasks.service.ts | 4 ++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/apps/backend/src/app/backupData/backupData.service.spec.ts b/apps/backend/src/app/backupData/backupData.service.spec.ts index 732c1d0..bf8a5b2 100644 --- a/apps/backend/src/app/backupData/backupData.service.spec.ts +++ b/apps/backend/src/app/backupData/backupData.service.spec.ts @@ -191,6 +191,7 @@ describe('BackupDataService', () => { expect(repository.save).toHaveBeenCalledWith(createBackupDataDtos); }); }); + describe('findOneById', () => { it('should return a backup data entity by id', async () => { const result = await service.findOneById(mockBackupDataEntity.id); @@ -200,4 +201,17 @@ describe('BackupDataService', () => { }); }); }); + + describe('createBatched', () => { + it('should create new backup data entities batched', async () => { + const createBackupDataDtos: CreateBackupDataDto[] = [ + { id: '1', sizeMB: 100, creationDate: new Date() }, + { id: '2', sizeMB: 200, creationDate: new Date() }, + ]; + + await service.createBatched(createBackupDataDtos); + + expect(repository.save).toHaveBeenCalledWith(createBackupDataDtos); + }); + }); }); diff --git a/apps/backend/src/app/tasks/tasks.controller.spec.ts b/apps/backend/src/app/tasks/tasks.controller.spec.ts index c333481..34bb6f9 100644 --- a/apps/backend/src/app/tasks/tasks.controller.spec.ts +++ b/apps/backend/src/app/tasks/tasks.controller.spec.ts @@ -32,7 +32,9 @@ describe('TasksController (e2e)', () => { app = moduleFixture.createNestApplication(); await app.init(); - repository = moduleFixture.get>(getRepositoryToken(TaskEntity)); + repository = moduleFixture.get>( + getRepositoryToken(TaskEntity) + ); }); afterAll(async () => { @@ -66,9 +68,7 @@ describe('TasksController (e2e)', () => { const id = 'ea1a2f52-5cf4-44a6-b266-175ee396a18e'; jest.spyOn(repository, 'findOneBy').mockResolvedValue(null); - await request(app.getHttpServer()) - .get(`/tasks/${id}`) - .expect(404); + await request(app.getHttpServer()).get(`/tasks/${id}`).expect(404); }); it('/tasks (POST) should create and return a task', async () => { @@ -86,4 +86,18 @@ describe('TasksController (e2e)', () => { expect(response.body).toEqual(task); }); -}); \ No newline at end of file + + it('/tasks/batched (POST) should create and return multiple tasks', async () => { + const createTaskDtos: CreateTaskDto[] = [ + { id: 'id1', displayName: 'name1' }, + { id: 'id2', displayName: 'name2' }, + ]; + + await request(app.getHttpServer()) + .post('/tasks/batched') + .send(createTaskDtos) + .expect(201); + + expect(repository.save).toHaveBeenCalledWith(createTaskDtos); + }); +}); diff --git a/apps/backend/src/app/tasks/tasks.controller.ts b/apps/backend/src/app/tasks/tasks.controller.ts index 138ef1b..79feaf0 100644 --- a/apps/backend/src/app/tasks/tasks.controller.ts +++ b/apps/backend/src/app/tasks/tasks.controller.ts @@ -45,4 +45,11 @@ export class TasksController { async create(@Body() createTaskDto: CreateTaskDto) { return this.tasksService.create(createTaskDto); } + + @Post('batched') + @ApiOperation({ summary: 'Creates multiple tasks batched.' }) + @ApiCreatedResponse() + async createBatched(@Body() createTaskDtos: CreateTaskDto[]) { + return this.tasksService.createBatched(createTaskDtos); + } } diff --git a/apps/backend/src/app/tasks/tasks.service.ts b/apps/backend/src/app/tasks/tasks.service.ts index e25fa17..221fce8 100644 --- a/apps/backend/src/app/tasks/tasks.service.ts +++ b/apps/backend/src/app/tasks/tasks.service.ts @@ -25,4 +25,8 @@ export class TasksService { async create(task: TaskEntity): Promise { return this.taskRepository.save(task); } + + async createBatched(tasks: TaskEntity[]): Promise { + await this.taskRepository.save(tasks); + } } From 353b704f0224ba94f43049a6462efc16048914f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Sat, 7 Dec 2024 09:39:29 +0100 Subject: [PATCH 4/8] Ignore unknown taskIds for batched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- .../src/app/backupData/backupData.module.ts | 25 +++++++++-------- .../src/app/backupData/backupData.service.ts | 27 +++++++++++++++++-- apps/backend/src/app/db-config.service.ts | 1 + apps/backend/src/app/tasks/tasks.service.ts | 10 +++---- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/backend/src/app/backupData/backupData.module.ts b/apps/backend/src/app/backupData/backupData.module.ts index 34cbf1c..c36762a 100644 --- a/apps/backend/src/app/backupData/backupData.module.ts +++ b/apps/backend/src/app/backupData/backupData.module.ts @@ -1,16 +1,15 @@ -import {Module} from '@nestjs/common'; -import {TypeOrmModule} from "@nestjs/typeorm"; -import {BackupDataService} from "./backupData.service"; -import {BackupDataController} from "./backupData.controller"; -import {BackupDataEntity} from "./entity/backupData.entity"; +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { BackupDataService } from './backupData.service'; +import { BackupDataController } from './backupData.controller'; +import { BackupDataEntity } from './entity/backupData.entity'; +import { TasksService } from '../tasks/tasks.service'; +import { TaskEntity } from '../tasks/entity/task.entity'; @Module({ - providers: [BackupDataService], - imports: [ - TypeOrmModule.forFeature([BackupDataEntity]), - ], - controllers: [BackupDataController], - exports: [BackupDataService], + providers: [BackupDataService, TasksService], + imports: [TypeOrmModule.forFeature([BackupDataEntity, TaskEntity])], + controllers: [BackupDataController], + exports: [BackupDataService], }) -export class BackupDataModule { -} \ No newline at end of file +export class BackupDataModule {} diff --git a/apps/backend/src/app/backupData/backupData.service.ts b/apps/backend/src/app/backupData/backupData.service.ts index 1a837f2..9a5eb2b 100644 --- a/apps/backend/src/app/backupData/backupData.service.ts +++ b/apps/backend/src/app/backupData/backupData.service.ts @@ -1,4 +1,8 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { + BadRequestException, + Injectable, + NotFoundException, +} from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Between, @@ -17,12 +21,14 @@ import { BackupDataDto } from './dto/backupData.dto'; import { BackupDataFilterDto } from './dto/backupDataFilter.dto'; import { BackupDataOrderOptionsDto } from './dto/backupDataOrderOptions.dto'; import { BackupType } from './dto/backupType'; +import { TasksService } from '../tasks/tasks.service'; @Injectable() export class BackupDataService extends PaginationService { constructor( @InjectRepository(BackupDataEntity) - private readonly backupDataRepository: Repository + private readonly backupDataRepository: Repository, + private readonly tasksService: TasksService ) { super(); } @@ -58,7 +64,14 @@ export class BackupDataService extends PaginationService { async create( createBackupDataDto: CreateBackupDataDto ): Promise { + if (!(await this.tasksService.findOne(createBackupDataDto.taskId ?? ''))) { + throw new NotFoundException( + `Task with id ${createBackupDataDto.taskId} not found` + ); + } + const entity = Object.assign(new BackupDataEntity(), createBackupDataDto); + return await this.backupDataRepository.save(entity); } @@ -69,6 +82,16 @@ export class BackupDataService extends PaginationService { async createBatched( createBackupDataDtos: CreateBackupDataDto[] ): Promise { + //ignore unknown taskIds + for (const dto of createBackupDataDtos) { + if (dto.taskId && !(await this.tasksService.findOne(dto.taskId))) { + console.warn( + `Task with id ${dto.taskId} not found - still creating the backup, but without task` + ); + dto.taskId = undefined; + } + } + const entities = createBackupDataDtos.map((dto) => Object.assign(new BackupDataEntity(), dto) ); diff --git a/apps/backend/src/app/db-config.service.ts b/apps/backend/src/app/db-config.service.ts index b8b4646..0713f86 100644 --- a/apps/backend/src/app/db-config.service.ts +++ b/apps/backend/src/app/db-config.service.ts @@ -63,6 +63,7 @@ export class DbConfigService implements TypeOrmOptionsFactory { Tasks1733397652480, ], logging: true, + logger: 'debug' }; } } diff --git a/apps/backend/src/app/tasks/tasks.service.ts b/apps/backend/src/app/tasks/tasks.service.ts index 221fce8..74c0f2c 100644 --- a/apps/backend/src/app/tasks/tasks.service.ts +++ b/apps/backend/src/app/tasks/tasks.service.ts @@ -1,4 +1,4 @@ -import { Injectable, NotFoundException } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { TaskEntity } from './entity/task.entity'; import { Repository } from 'typeorm'; @@ -14,12 +14,8 @@ export class TasksService { return this.taskRepository.find(); } - async findOne(id: string): Promise { - const task = await this.taskRepository.findOneBy({ id }); - if (!task) { - throw new NotFoundException(`Task with id ${id} not found`); - } - return task; + async findOne(id: string): Promise { + return await this.taskRepository.findOneBy({ id }); } async create(task: TaskEntity): Promise { From d0c3180065a94a4e22a7c1c56b4cbe8ffa5d4af3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Sat, 7 Dec 2024 09:53:42 +0100 Subject: [PATCH 5/8] Adjusted Backend Tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- apps/analyzer/metadata_analyzer/analyzer.py | 3 +++ .../backend/src/app/alerting/alerting.controller.spec.ts | 3 +++ .../src/app/backupData/backupData.controller.spec.ts | 5 +++++ .../src/app/backupData/backupData.service.spec.ts | 9 +++++++++ apps/backend/src/app/backupData/backupData.service.ts | 2 -- apps/backend/src/app/tasks/tasks.controller.spec.ts | 7 ------- apps/backend/src/app/tasks/tasks.service.spec.ts | 8 -------- 7 files changed, 20 insertions(+), 17 deletions(-) diff --git a/apps/analyzer/metadata_analyzer/analyzer.py b/apps/analyzer/metadata_analyzer/analyzer.py index 1a933ef..f3f81b3 100644 --- a/apps/analyzer/metadata_analyzer/analyzer.py +++ b/apps/analyzer/metadata_analyzer/analyzer.py @@ -91,6 +91,9 @@ def _send_Tasks(): for task in tasks: + if task.uuid is None or task.task is None: + continue + batch.append(Analyzer._convert_task(task)) count += 1 diff --git a/apps/backend/src/app/alerting/alerting.controller.spec.ts b/apps/backend/src/app/alerting/alerting.controller.spec.ts index d69a41d..555079c 100644 --- a/apps/backend/src/app/alerting/alerting.controller.spec.ts +++ b/apps/backend/src/app/alerting/alerting.controller.spec.ts @@ -15,6 +15,7 @@ import { CreateSizeAlertDto } from './dto/alerts/createSizeAlert.dto'; import { CREATION_DATE_ALERT, SIZE_ALERT } from '../utils/constants'; import { CreateCreationDateAlertDto } from './dto/alerts/createCreationDateAlert.dto'; import { CreationDateAlertEntity } from './entity/alerts/creationDateAlert.entity'; +import { TaskEntity } from '../tasks/entity/task.entity'; const mockedBackupDataEntity: BackupDataEntity = { id: 'backup-id', @@ -104,6 +105,8 @@ describe('AlertingController (e2e)', () => { .useValue(mockCreationDateAlertRepository) .overrideProvider(getRepositoryToken(AlertTypeEntity)) .useValue(mockAlertTypeRepository) + .overrideProvider(getRepositoryToken(TaskEntity)) + .useValue({}) .compile(); repository = module.get(getRepositoryToken(SizeAlertEntity)); diff --git a/apps/backend/src/app/backupData/backupData.controller.spec.ts b/apps/backend/src/app/backupData/backupData.controller.spec.ts index d5cc3ee..f4c117f 100644 --- a/apps/backend/src/app/backupData/backupData.controller.spec.ts +++ b/apps/backend/src/app/backupData/backupData.controller.spec.ts @@ -7,6 +7,7 @@ import { BackupDataEntity } from './entity/backupData.entity'; import { CreateBackupDataDto } from './dto/createBackupData.dto'; import { BackupDataModule } from './backupData.module'; import { BackupType } from './dto/backupType'; +import { TaskEntity } from '../tasks/entity/task.entity'; const mockBackupDataEntity: BackupDataEntity = { id: '123e4567-e89b-12d3-a456-426614174062', @@ -40,6 +41,10 @@ describe('BackupDataController (e2e)', () => { }) .overrideProvider(getRepositoryToken(BackupDataEntity)) .useValue(mockBackupDataRepository) + .overrideProvider(getRepositoryToken(TaskEntity)) + .useValue({ + findOneBy: jest.fn().mockResolvedValue(new TaskEntity()), + }) .compile(); repository = module.get(getRepositoryToken(BackupDataEntity)); diff --git a/apps/backend/src/app/backupData/backupData.service.spec.ts b/apps/backend/src/app/backupData/backupData.service.spec.ts index bf8a5b2..bfa8aea 100644 --- a/apps/backend/src/app/backupData/backupData.service.spec.ts +++ b/apps/backend/src/app/backupData/backupData.service.spec.ts @@ -9,6 +9,8 @@ import { CreateBackupDataDto } from './dto/createBackupData.dto'; import { BackupDataOrderByOptions } from './dto/backupDataOrderOptions.dto'; import { SortOrder } from '../utils/pagination/SortOrder'; import { BackupType } from './dto/backupType'; +import { TaskEntity } from '../tasks/entity/task.entity'; +import { TasksService } from '../tasks/tasks.service'; const mockBackupDataEntity: BackupDataEntity = { id: '123e4567-e89b-12d3-a456-426614174062', @@ -31,10 +33,17 @@ describe('BackupDataService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ BackupDataService, + TasksService, { provide: getRepositoryToken(BackupDataEntity), useValue: mockBackupDataRepository, }, + { + provide: getRepositoryToken(TaskEntity), + useValue: { + findOneBy: jest.fn().mockResolvedValue(new TaskEntity()), + }, + }, ], }).compile(); diff --git a/apps/backend/src/app/backupData/backupData.service.ts b/apps/backend/src/app/backupData/backupData.service.ts index 9a5eb2b..da2e3ea 100644 --- a/apps/backend/src/app/backupData/backupData.service.ts +++ b/apps/backend/src/app/backupData/backupData.service.ts @@ -125,7 +125,6 @@ export class BackupDataService extends PaginationService { from.setMinutes(0); from.setSeconds(0); from.setMilliseconds(0); - console.log(from); } if (backupDataFilterDto.toDate) { to = new Date(backupDataFilterDto.toDate); @@ -138,7 +137,6 @@ export class BackupDataService extends PaginationService { to.setSeconds(0); to.setDate(to.getDate() + 1); to.setMilliseconds(-1); - console.log(to); } //Creation date search diff --git a/apps/backend/src/app/tasks/tasks.controller.spec.ts b/apps/backend/src/app/tasks/tasks.controller.spec.ts index 34bb6f9..78162dd 100644 --- a/apps/backend/src/app/tasks/tasks.controller.spec.ts +++ b/apps/backend/src/app/tasks/tasks.controller.spec.ts @@ -64,13 +64,6 @@ describe('TasksController (e2e)', () => { expect(response.body).toEqual(task); }); - it('/tasks/:id (GET) should throw a NotFoundException if task not found', async () => { - const id = 'ea1a2f52-5cf4-44a6-b266-175ee396a18e'; - jest.spyOn(repository, 'findOneBy').mockResolvedValue(null); - - await request(app.getHttpServer()).get(`/tasks/${id}`).expect(404); - }); - it('/tasks (POST) should create and return a task', async () => { const createTaskDto: CreateTaskDto = { id: 'someId', diff --git a/apps/backend/src/app/tasks/tasks.service.spec.ts b/apps/backend/src/app/tasks/tasks.service.spec.ts index 365fb66..1d709cd 100644 --- a/apps/backend/src/app/tasks/tasks.service.spec.ts +++ b/apps/backend/src/app/tasks/tasks.service.spec.ts @@ -1,6 +1,5 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; -import { NotFoundException } from '@nestjs/common'; import { Repository } from 'typeorm'; import { TasksService } from './tasks.service'; import { TaskEntity } from './entity/task.entity'; @@ -53,13 +52,6 @@ describe('TasksService', () => { expect(await service.findOne(id)).toBe(task); }); - - it('should throw a NotFoundException if task not found', async () => { - const id = 'ea1a2f52-5cf4-44a6-b266-175ee396a18d'; - jest.spyOn(repository, 'findOneBy').mockResolvedValue(null); - - await expect(service.findOne(id)).rejects.toThrow(NotFoundException); - }); }); describe('create', () => { From 0839b75e52be1647937dd1c32d5aa27544f44eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Sat, 7 Dec 2024 10:05:47 +0100 Subject: [PATCH 6/8] Adjusted Analyzer Tests to include taskId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- apps/analyzer/tests/mock_backend.py | 4 ++++ apps/analyzer/tests/mock_database.py | 12 ++++++++---- apps/analyzer/tests/test_analyzer.py | 23 ++++++++++++++--------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/apps/analyzer/tests/mock_backend.py b/apps/analyzer/tests/mock_backend.py index a4cb95f..b253a96 100644 --- a/apps/analyzer/tests/mock_backend.py +++ b/apps/analyzer/tests/mock_backend.py @@ -1,6 +1,7 @@ class MockBackend: def __init__(self): self.backups = [] + self.tasks = [] self.size_alerts = [] self.creation_date_alerts = [] self.latest_alert_ids = {} @@ -8,6 +9,9 @@ def __init__(self): def send_backup_data_batched(self, batch): self.backups += batch + def send_task_data_batched(self, batch): + self.tasks += batch + def create_creation_date_alert(self, alert): self.creation_date_alerts.append(alert) diff --git a/apps/analyzer/tests/mock_database.py b/apps/analyzer/tests/mock_database.py index 3839999..2d20efc 100644 --- a/apps/analyzer/tests/mock_database.py +++ b/apps/analyzer/tests/mock_database.py @@ -1,6 +1,10 @@ class MockDatabase: - def __init__(self, results): - self.results = results + def __init__(self, results, tasks = []): + self.results = results + self.tasks = tasks - def get_results(self): - return iter(self.results) + def get_results(self): + return iter(self.results) + + def get_tasks(self): + return iter(self.tasks) diff --git a/apps/analyzer/tests/test_analyzer.py b/apps/analyzer/tests/test_analyzer.py index d8ee3cf..14f192f 100644 --- a/apps/analyzer/tests/test_analyzer.py +++ b/apps/analyzer/tests/test_analyzer.py @@ -4,7 +4,7 @@ from tests.mock_backend import MockBackend from tests.mock_database import MockDatabase -def _create_mock_result(task, uuid, fdi_type, data_size, start_time, is_backup=1): +def _create_mock_result(task, uuid, fdi_type, data_size, start_time, task_uuid = None, is_backup=1): mock_result = Result() mock_result.task = task mock_result.uuid = uuid @@ -12,16 +12,17 @@ def _create_mock_result(task, uuid, fdi_type, data_size, start_time, is_backup=1 mock_result.data_size = data_size mock_result.start_time = start_time mock_result.is_backup = is_backup + mock_result.task_uuid = task_uuid return mock_result def test_update_data_all_types(): mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01")) mock_result2 = _create_mock_result("foo", "2", "D", 150_000_000, datetime.fromisoformat("2000-01-02")) mock_result3 = _create_mock_result("foo", "3", "I", 200_000_000, datetime.fromisoformat("2000-01-03")) - mock_result4 = _create_mock_result("foo", "4", "C", 250_000_000, datetime.fromisoformat("2000-01-04")) + mock_result4 = _create_mock_result("foo", "4", "C", 250_000_000, datetime.fromisoformat("2000-01-04"), '123') mock_results = [mock_result1, mock_result2, mock_result3, mock_result4] - database = MockDatabase(mock_results) + database = MockDatabase(mock_results, []) backend = MockBackend() Analyzer.init(database, backend, None, None) Analyzer.update_data() @@ -30,28 +31,32 @@ def test_update_data_all_types(): "id": mock_result1.uuid, "sizeMB": mock_result1.data_size / 1_000_000, "creationDate": mock_result1.start_time.isoformat(), - "type": "FULL" + "type": "FULL", + "taskId": None }, { "id": mock_result2.uuid, "sizeMB": mock_result2.data_size / 1_000_000, "creationDate": mock_result2.start_time.isoformat(), - "type": "DIFFERENTIAL" + "type": "DIFFERENTIAL", + "taskId": None }, { "id": mock_result3.uuid, "sizeMB": mock_result3.data_size / 1_000_000, "creationDate": mock_result3.start_time.isoformat(), - "type": "INCREMENTAL" + "type": "INCREMENTAL", + "taskId": None }, { "id": mock_result4.uuid, "sizeMB": mock_result4.data_size / 1_000_000, "creationDate": mock_result4.start_time.isoformat(), - "type": "COPY" + "type": "COPY", + "taskId": '123' }] def test_update_data_not_a_backup(): - mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01"), 0) + mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01"), None, 0) - database = MockDatabase([mock_result1]) + database = MockDatabase([mock_result1], []) backend = MockBackend() Analyzer.init(database, backend, None, None) Analyzer.update_data() From da5edc383165f49c1f3aa5d9b3b8afe781574260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Obernd=C3=B6rfer?= Date: Sat, 7 Dec 2024 10:18:10 +0100 Subject: [PATCH 7/8] Added Analyzer Test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Florian Oberndörfer --- apps/analyzer/tests/test_analyzer.py | 128 ++++++++++++++++----------- 1 file changed, 77 insertions(+), 51 deletions(-) diff --git a/apps/analyzer/tests/test_analyzer.py b/apps/analyzer/tests/test_analyzer.py index 14f192f..2ce19ca 100644 --- a/apps/analyzer/tests/test_analyzer.py +++ b/apps/analyzer/tests/test_analyzer.py @@ -1,64 +1,90 @@ from datetime import datetime + from metadata_analyzer.analyzer import Analyzer -from metadata_analyzer.models import Result +from metadata_analyzer.models import Result, Tasks from tests.mock_backend import MockBackend from tests.mock_database import MockDatabase -def _create_mock_result(task, uuid, fdi_type, data_size, start_time, task_uuid = None, is_backup=1): - mock_result = Result() - mock_result.task = task - mock_result.uuid = uuid - mock_result.fdi_type = fdi_type - mock_result.data_size = data_size - mock_result.start_time = start_time - mock_result.is_backup = is_backup - mock_result.task_uuid = task_uuid - return mock_result + +def _create_mock_result(task, uuid, fdi_type, data_size, start_time, task_uuid=None, is_backup=1): + mock_result = Result() + mock_result.task = task + mock_result.uuid = uuid + mock_result.fdi_type = fdi_type + mock_result.data_size = data_size + mock_result.start_time = start_time + mock_result.is_backup = is_backup + mock_result.task_uuid = task_uuid + return mock_result + + +def _create_mock_task(uuid, task): + mock_task = Tasks() + mock_task.uuid = uuid + mock_task.task = task + return mock_task + def test_update_data_all_types(): - mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01")) - mock_result2 = _create_mock_result("foo", "2", "D", 150_000_000, datetime.fromisoformat("2000-01-02")) - mock_result3 = _create_mock_result("foo", "3", "I", 200_000_000, datetime.fromisoformat("2000-01-03")) - mock_result4 = _create_mock_result("foo", "4", "C", 250_000_000, datetime.fromisoformat("2000-01-04"), '123') - mock_results = [mock_result1, mock_result2, mock_result3, mock_result4] + mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01")) + mock_result2 = _create_mock_result("foo", "2", "D", 150_000_000, datetime.fromisoformat("2000-01-02")) + mock_result3 = _create_mock_result("foo", "3", "I", 200_000_000, datetime.fromisoformat("2000-01-03")) + mock_result4 = _create_mock_result("foo", "4", "C", 250_000_000, datetime.fromisoformat("2000-01-04"), '123') + mock_results = [mock_result1, mock_result2, mock_result3, mock_result4] + + mock_task1 = _create_mock_task("1", "task1") + mock_task2 = _create_mock_task("123", "task123") + mock_tasks = [mock_task1, mock_task2] + + database = MockDatabase(mock_results, mock_tasks) + backend = MockBackend() + Analyzer.init(database, backend, None, None) + Analyzer.update_data() + + assert backend.backups == [{ + "id": mock_result1.uuid, + "sizeMB": mock_result1.data_size / 1_000_000, + "creationDate": mock_result1.start_time.isoformat(), + "type": "FULL", + "taskId": None + }, { + "id": mock_result2.uuid, + "sizeMB": mock_result2.data_size / 1_000_000, + "creationDate": mock_result2.start_time.isoformat(), + "type": "DIFFERENTIAL", + "taskId": None + }, { + "id": mock_result3.uuid, + "sizeMB": mock_result3.data_size / 1_000_000, + "creationDate": mock_result3.start_time.isoformat(), + "type": "INCREMENTAL", + "taskId": None + }, { + "id": mock_result4.uuid, + "sizeMB": mock_result4.data_size / 1_000_000, + "creationDate": mock_result4.start_time.isoformat(), + "type": "COPY", + "taskId": '123' + }] - database = MockDatabase(mock_results, []) - backend = MockBackend() - Analyzer.init(database, backend, None, None) - Analyzer.update_data() + assert backend.tasks == [ + { + "id": "1", + "displayName": "task1" + }, + { + "id": "123", + "displayName": "task123" + } + ] - assert backend.backups == [{ - "id": mock_result1.uuid, - "sizeMB": mock_result1.data_size / 1_000_000, - "creationDate": mock_result1.start_time.isoformat(), - "type": "FULL", - "taskId": None - }, { - "id": mock_result2.uuid, - "sizeMB": mock_result2.data_size / 1_000_000, - "creationDate": mock_result2.start_time.isoformat(), - "type": "DIFFERENTIAL", - "taskId": None - }, { - "id": mock_result3.uuid, - "sizeMB": mock_result3.data_size / 1_000_000, - "creationDate": mock_result3.start_time.isoformat(), - "type": "INCREMENTAL", - "taskId": None - }, { - "id": mock_result4.uuid, - "sizeMB": mock_result4.data_size / 1_000_000, - "creationDate": mock_result4.start_time.isoformat(), - "type": "COPY", - "taskId": '123' - }] def test_update_data_not_a_backup(): - mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01"), None, 0) + mock_result1 = _create_mock_result("foo", "1", "F", 100_000_000, datetime.fromisoformat("2000-01-01"), None, 0) - database = MockDatabase([mock_result1], []) - backend = MockBackend() - Analyzer.init(database, backend, None, None) - Analyzer.update_data() + database = MockDatabase([mock_result1], []) + backend = MockBackend() + Analyzer.init(database, backend, None, None) + Analyzer.update_data() - assert backend.backups == [] + assert backend.backups == [] From aaa69c5b20dd1c7bcd98b1dd6b6fe88ec6436436 Mon Sep 17 00:00:00 2001 From: Christoph Klingenberg Date: Sun, 8 Dec 2024 16:25:14 +0100 Subject: [PATCH 8/8] test error fixed Signed-off-by: Christoph Klingenberg --- apps/backend/src/app/alerting/alerting.controller.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/backend/src/app/alerting/alerting.controller.spec.ts b/apps/backend/src/app/alerting/alerting.controller.spec.ts index f2f46ae..e75a754 100644 --- a/apps/backend/src/app/alerting/alerting.controller.spec.ts +++ b/apps/backend/src/app/alerting/alerting.controller.spec.ts @@ -108,6 +108,7 @@ describe('AlertingController (e2e)', () => { .overrideProvider(getRepositoryToken(AlertTypeEntity)) .useValue(mockAlertTypeRepository) .overrideProvider(getRepositoryToken(TaskEntity)) + .useValue({}) .overrideProvider(getRepositoryToken(MailReceiverEntity)) .useValue({}) .compile();