From 97aaef012a3e6ca6b57126bc7cc4998070580e2c Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko <30902180+vpavlenko-cv@users.noreply.github.com> Date: Mon, 15 Apr 2024 02:21:02 +0300 Subject: [PATCH] [CYB-139][CYB-151][CYB-164] Transferring the configuration in a zip archive between clusters. New rest api with UI. (#61) --- .idea/codeStyles/codeStyleConfig.xml | 5 - .idea/cybersec.iml | 9 - .idea/modules.xml | 8 - .idea/vcs.xml | 6 - .idea/workspace.xml | 53 - flink-cyber/.vscode/settings.json | 3 - .../cyber-service-common/pom.xml | 46 + .../com/cloudera/service/common/Utils.java | 36 + .../config/kafka/ClouderaKafkaProperties.java | 14 + .../ClouderaManagementKafkaProperties.java | 4 + .../kafka/ClouderaWorkerKafkaProperties.java | 4 + .../service/common/request/RequestBody.java | 18 + .../service/common/request/RequestType.java | 5 + .../service/common/response/ClusterMeta.java | 18 + .../cloudera/service/common/response/Job.java | 61 + .../service/common/response/ResponseBody.java | 20 + .../service/common/response/ResponseType.java | 5 + .../service/common/utils/ArchiveUtil.java | 146 + .../service/common/utils/FileUtil.java | 44 + .../cloudera/service/common/UtilsTest.java | 67 + .../cyber-worker-service/pom.xml | 93 + .../cyber/restcli/CliApplication.java | 14 + .../restcli/configuration/KafkaConfig.java | 53 + .../configuration/WebConfiguration.java | 15 + .../controller/KafkaListenerController.java | 124 + .../cyber/restcli/service/JobService.java | 166 + .../cloudera/cyber/restcli/utils/Utils.java | 8 + .../src/main/resources/application.properties | 1 + .../cyber/restcli/CliApplicationTests.java | 13 + .../src/test/resources/application.properties | 15 + flink-cyber/cyber-services/pom.xml | 25 + .../parser-chains-api/package-lock.json | 2579 +++-- .../routes/parserconfig/index.js | 7 + .../routes/parserconfig/mock-data/chains.json | 2 +- .../parserconfig/mock-data/clusters.json | 16 + .../parser-chains-client/angular.json | 47 +- .../parser-chains-client/package-lock.json | 9385 ++++++++--------- .../parser-chains-client/package.json | 5 + .../src/app/app-routing.module.ts | 21 +- .../src/app/app.component.html | 11 +- .../src/app/app.component.ts | 19 + .../src/app/app.module.ts | 39 +- .../chain-add-parser-page.effects.spec.ts | 2 +- .../src/app/chain-page/chain-page.effects.ts | 2 +- .../custom-form/custom-form.component.ts | 2 +- .../cluster-list-page.component.html | 57 + .../cluster-list-page.component.scss | 66 + .../cluster-list-page.component.ts | 47 + .../cluster-list-page.model.ts | 35 + .../cluster-list-page.module.ts | 31 + .../cluster-page/cluster-page.component.html | 104 + .../cluster-page/cluster-page.component.scss | 54 + .../cluster-page/cluster-page.component.ts | 147 + .../cluster-page/cluster-page.module.ts | 112 + .../dialog/upload-dialog.component.html | 10 + .../dialog/upload-dialog.component.ts | 66 + .../file-upload/file-upload.component.html | 31 + .../file-upload/file-upload.component.scss | 131 + .../file-upload/file-upload.component.ts | 172 + .../file-upload/file-upload.module.ts | 31 + .../progress/progress.component.html | 4 + .../progress/progress.component.scss | 19 + .../progress/progress.component.ts | 13 + .../status-icon/status-icon.component.html | 5 + .../status-icon/status-icon.component.scss | 31 + .../status-icon/status-icon.component.ts | 63 + .../status-icon/status-icon.module.ts | 23 + .../src/app/misc/drag-drop.directives.ts | 41 + .../src/app/services/cluster.service.ts | 29 + .../src/app/services/pipeline.service.ts | 16 +- .../src/app/services/snack-bar.service.ts | 75 + .../src/environments/environment.mock.dev.ts | 27 + .../src/environments/environment.prod.ts | 2 +- .../parser-chains-client/src/index.html | 1 + .../src/mockServiceWorker.js | 303 + .../parser-chains-client/src/mocks/browser.ts | 11 + .../src/mocks/handlers/clusterHandlers.ts | 47 + .../handlers/parserconfig/chainHandlers.ts | 78 + .../parserconfig/chainParsersHandlers.ts | 48 + .../parserconfig/customFormHandlers.ts | 48 + .../handlers/parserconfig/parserTypes.ts | 23 + .../parser-chains-client/src/styles.scss | 63 +- .../test/resources/parserconfig/chains.json | 303 + .../resources/parserconfig/cluster-1.json | 54 + .../test/resources/parserconfig/clusters.json | 56 + .../resources/parserconfig/form-configs.json | 628 ++ .../resources/parserconfig/parser-types.json | 66 + .../parser-chains-client/tsconfig.json | 3 +- .../parser-chains-config-service/pom.xml | 20 +- .../common/ApplicationConstants.java | 3 + .../FailedAllClusterReponseException.java | 16 + .../FailedClusterReponseException.java | 14 + .../common/exception/FailedJobAction.java | 23 + .../exception/JobValidationException.java | 23 + .../exception/KafkaClusterNotFound.java | 23 + .../common/exception/KafkaException.java | 23 + .../config/MainConfiguration.java | 11 +- .../config/kafka/KafkaConfig.java | 54 +- .../controller/ChainController.java | 6 +- .../controller/ClusterController.java | 55 + .../GlobalExceptionAdviceController.java | 25 + .../controller/JobController.java | 84 + .../controller/ParserController.java | 1 + .../controller/ParserSampleController.java | 13 +- .../queryservice/model/enums/JobActions.java | 32 + .../queryservice/service/ClusterService.java | 46 + .../queryservice/service/JobService.java | 64 + .../queryservice/service/KafkaService.java | 126 +- .../service/ClusterServiceTest.java | 76 + .../service/KafkaServiceTest.java | 272 +- flink-cyber/pom.xml | 5 +- 111 files changed, 11446 insertions(+), 5979 deletions(-) delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/cybersec.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml delete mode 100644 flink-cyber/.vscode/settings.json create mode 100644 flink-cyber/cyber-services/cyber-service-common/pom.xml create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/Utils.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaKafkaProperties.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaManagementKafkaProperties.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaWorkerKafkaProperties.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ClusterMeta.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseBody.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/ArchiveUtil.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/FileUtil.java create mode 100644 flink-cyber/cyber-services/cyber-service-common/src/test/java/com/cloudera/service/common/UtilsTest.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/pom.xml create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/CliApplication.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/KafkaConfig.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/WebConfiguration.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/utils/Utils.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/main/resources/application.properties create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/test/java/com/cloudera/cyber/restcli/CliApplicationTests.java create mode 100644 flink-cyber/cyber-services/cyber-worker-service/src/test/resources/application.properties create mode 100644 flink-cyber/cyber-services/pom.xml create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-api/routes/parserconfig/mock-data/clusters.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-list-page/cluster-list-page.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-list-page/cluster-list-page.component.scss create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-list-page/cluster-list-page.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-list-page/cluster-list-page.model.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-list-page/cluster-list-page.module.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/cluster-page.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/cluster-page.component.scss create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/cluster-page.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/cluster-page.module.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/dialog/upload-dialog.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/cluster-page/dialog/upload-dialog.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/file-upload.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/file-upload.component.scss create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/file-upload.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/file-upload.module.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/progress/progress.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/progress/progress.component.scss create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/file-upload/progress/progress.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/status-icon/status-icon.component.html create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/status-icon/status-icon.component.scss create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/status-icon/status-icon.component.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/cluster/component/status-icon/status-icon.module.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/misc/drag-drop.directives.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/services/cluster.service.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/services/snack-bar.service.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/environments/environment.mock.dev.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mockServiceWorker.js create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/browser.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/handlers/clusterHandlers.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/handlers/parserconfig/chainHandlers.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/handlers/parserconfig/chainParsersHandlers.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/handlers/parserconfig/customFormHandlers.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/mocks/handlers/parserconfig/parserTypes.ts create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/test/resources/parserconfig/chains.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/test/resources/parserconfig/cluster-1.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/test/resources/parserconfig/clusters.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/test/resources/parserconfig/form-configs.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/test/resources/parserconfig/parser-types.json create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedAllClusterReponseException.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedClusterReponseException.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedJobAction.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/JobValidationException.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaClusterNotFound.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaException.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/GlobalExceptionAdviceController.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/JobController.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/JobActions.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/JobService.java create mode 100644 flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/ClusterServiceTest.java diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a179..000000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/cybersec.iml b/.idea/cybersec.iml deleted file mode 100644 index d6ebd4805..000000000 --- a/.idea/cybersec.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 2816ac0a0..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddfb..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index ba57f15ff..000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - -