Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: P4ADEV-1545 fdr ingestion actity implementation retrieval via query file validation #22

Open
wants to merge 36 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
615f97d
P4ADEV-1545 FdRIngestionActity added interface implementation and its…
macacia Nov 26, 2024
9e0ada7
P4ADEV-1545 FdRIngestionActity naming refactoring
macacia Nov 26, 2024
9f58e1f
P4ADEV-1545 FdRIngestionActity added service and test
macacia Nov 27, 2024
93450f8
P4ADEV-1545 FdRIngestionActity added utils and its test
macacia Nov 27, 2024
9780f00
P4ADEV-1545 FdRIngestionActity fix
macacia Nov 27, 2024
a5d451d
P4ADEV-1545 FdRIngestionActity fix
macacia Nov 27, 2024
45edff1
P4ADEV-1545 FdRIngestionActity refactored FileUtils.java and its tests
macacia Nov 27, 2024
88e6cea
P4ADEV-1545 FdRIngestionActity added missing test
macacia Nov 27, 2024
476d008
P4ADEV-1545 FdRIngestionActity added missing test
macacia Nov 27, 2024
474ea2f
P4ADEV-1545 FdRIngestionActity resolved sonar issues
macacia Nov 27, 2024
b898521
P4ADEV-1545 FdRIngestionActity fixed with sonar issues
macacia Nov 27, 2024
0789a33
P4ADEV-1545 FdRIngestionActity resolved sonar issues
macacia Nov 27, 2024
cfbdb39
P4ADEV-1545 FdRIngestionActity rename Activity
macacia Nov 28, 2024
d86cf20
P4ADEV-1545 FdRIngestionActity rename Activity
macacia Nov 28, 2024
141cc4b
P4ADEV-1545 ReportingflowIngestionActity added documentation
macacia Nov 28, 2024
7ef1f1f
Merge branch 'main' of https://github.com/pagopa/p4pa-payhub-activiti…
macacia Nov 28, 2024
71c5830
P4ADEV-1545 ReportingFlowIngestionActivity rename Activity
macacia Nov 28, 2024
c1f2cf6
P4ADEV-1545 moved packages
macacia Nov 28, 2024
af747db
feat:P4ADEV-1545-FdRIngestionActity-implementation-retrieval-via-quer…
marcomatteuccieng Nov 28, 2024
81fdcce
P4ADEV-1545 some refactor
macacia Nov 28, 2024
b7bfaef
P4ADEV-1545 fix test
macacia Nov 28, 2024
3ada04e
P4ADEV-1545 added java docs
macacia Nov 28, 2024
bd2bf69
P4ADEV-1545 resolved sonar security issues
macacia Nov 28, 2024
00c89a5
P4ADEV-1545 resolved sonar security issues
macacia Nov 28, 2024
e1ab12c
P4ADEV-1545 fixed
macacia Nov 28, 2024
cad3093
P4ADEV-1545 fixed sonar issues
macacia Nov 28, 2024
c2cbea5
P4ADEV-1545 fixed sonar issues
macacia Nov 28, 2024
cf1d135
P4ADEV-1545 fix to prevent zip bomb
macacia Nov 29, 2024
29ad416
P4ADEV-1545 fix to prevent zip bomb
macacia Nov 29, 2024
ed282f7
P4ADEV-1545 fix
macacia Nov 29, 2024
9ed8d19
Merge remote-tracking branch 'origin/main' into P4ADEV-1545-FdRIngest…
macacia Nov 29, 2024
3628a52
P4ADEV-1545 updated version
macacia Nov 29, 2024
c45eeb5
P4ADEV-1545 fixed sonar issues
macacia Nov 29, 2024
e65f935
P4ADEV-1545 fix sonar issues
macacia Nov 29, 2024
0c319ef
P4ADEV-1545 removed unused declaration
macacia Nov 29, 2024
f373eae
P4ADEV-1545 fixed javadocs
macacia Nov 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
package it.gov.pagopa.payhub.activities.activity.fdr;
import it.gov.pagopa.payhub.activities.dto.fdr.FdRIngestionActivityResult;
package it.gov.pagopa.payhub.activities.activity.paymentsreporting;
import it.gov.pagopa.payhub.activities.dto.reportingflow.FdRIngestionActivityResult;

/**
* Interface for the FdRIngestionActivity.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package it.gov.pagopa.payhub.activities.activity.paymentsreporting;

import it.gov.pagopa.payhub.activities.activity.paymentsreporting.service.IngestionFileHandlerService;
import it.gov.pagopa.payhub.activities.activity.paymentsreporting.service.IngestionFileValidatorService;
import it.gov.pagopa.payhub.activities.activity.paymentsreporting.service.IngestionFlowRetrieverService;
import it.gov.pagopa.payhub.activities.dto.reportingflow.FdRIngestionActivityResult;
import it.gov.pagopa.payhub.activities.dto.reportingflow.IngestionFlowDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;

@Slf4j
@Component
public class FdRIngestionActivityImpl implements FdRIngestionActivity {
private final IngestionFlowRetrieverService ingestionFlowRetrieverService;
private final IngestionFileValidatorService ingestionFileValidatorService;
private final IngestionFileHandlerService ingestionFileHandlerService;

public FdRIngestionActivityImpl(IngestionFlowRetrieverService ingestionFlowRetrieverService,
IngestionFileValidatorService ingestionFileValidatorService,
IngestionFileHandlerService ingestionFileHandlerService) {
this.ingestionFlowRetrieverService = ingestionFlowRetrieverService;
this.ingestionFileValidatorService = ingestionFileValidatorService;
this.ingestionFileHandlerService = ingestionFileHandlerService;
}

@Override
public FdRIngestionActivityResult processFile(String ingestionFlowId) {
List<String> iufList = new ArrayList<>();
boolean success = true;

try {
IngestionFlowDTO ingestionFlowDTO = ingestionFlowRetrieverService.getIngestionFlow(Long.valueOf(ingestionFlowId));

ingestionFileValidatorService.validate(ingestionFlowDTO.getFilePathName(), ingestionFlowDTO.getFileName(), ingestionFlowDTO.getRequestTokenCode());

ingestionFileHandlerService.setUpProcess(ingestionFlowDTO.getFilePathName(), ingestionFlowDTO.getFileName());
} catch (Exception e) {
log.error("Error during IngestionActivity flowId {} due to: {}", ingestionFlowId, e.getMessage());
success = false;
}
return new FdRIngestionActivityResult(iufList, success);
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package it.gov.pagopa.payhub.activities.activity.fdr;
package it.gov.pagopa.payhub.activities.activity.paymentsreporting;

/**
* Interface for SendEmailIngestionFlowActivity.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package it.gov.pagopa.payhub.activities.activity.fdr;
package it.gov.pagopa.payhub.activities.activity.paymentsreporting;

/**
* Interface for the UpdateIngestionFlowStatusActivity.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package it.gov.pagopa.payhub.activities.activity.paymentsreporting.service;

import it.gov.pagopa.payhub.activities.util.AESUtils;
import it.gov.pagopa.payhub.activities.util.FileUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

@Slf4j
@Service
public class IngestionFileHandlerService {
private static final String TEMPORARY_PATH = "TEMP";

private final String dataCipherPsw;

public IngestionFileHandlerService(@Value("${data-cipher.encrypt-psw}") String dataCipherPsw) {
this.dataCipherPsw = dataCipherPsw;
}

public Path setUpProcess(String relativePath, String filename) throws IOException {
Path relativePathDir = Paths.get(relativePath);
Path encryptedFilePath = relativePathDir.resolve(filename);

Path temporaryPath = relativePathDir.resolve(TEMPORARY_PATH);
Files.createDirectories(temporaryPath);

String filenameNoCipher = filename.replace(AESUtils.CIPHER_EXTENSION, "");
Path temporaryZipFilePath = temporaryPath.resolve(filenameNoCipher);

log.debug("Decrypting file: {}", encryptedFilePath);
AESUtils.decrypt(dataCipherPsw, encryptedFilePath.toFile(), temporaryZipFilePath.toFile());

log.debug("Validating ZIP file: {}", temporaryZipFilePath);
FileUtils.isArchive(temporaryZipFilePath);

String unzippedFilename = filenameNoCipher.replace(".zip", ".xml");
Path outputUnzippedPath = temporaryPath.resolve(unzippedFilename);

log.debug("Unzipping file: {} to {}", temporaryZipFilePath, outputUnzippedPath);
FileUtils.unzip(temporaryZipFilePath, outputUnzippedPath);

log.debug("File setup process completed successfully for: {}", filename);
return outputUnzippedPath;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package it.gov.pagopa.payhub.activities.activity.paymentsreporting.service;

import it.gov.pagopa.payhub.activities.exception.InvalidIngestionFileException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

@Slf4j
@Service
public class IngestionFileValidatorService {

public void validate(String relativePath, String filename, String requestTokenCode) {
Path fileLocation = Paths.get(relativePath, filename);

validateFile(fileLocation);
validateAUTH(fileLocation.toString(), requestTokenCode);
validateMD5(fileLocation.toString());
}

private void validateFile(Path filePath) {
if (!Files.exists(filePath) || !Files.isRegularFile(filePath)) {
throw new InvalidIngestionFileException("File non trovato: " + filePath);
}
}

private void validateMD5(String filename) {
String md5 = filename.replace(".zip", ".md5");
String valueMD5 = readContentFile(md5);
String calculatedMD5 = calculateMd5(filename);
if(!valueMD5.equalsIgnoreCase(calculatedMD5)) {
throw new InvalidIngestionFileException("Error while calculating MD5 file value");
}
}

private void validateAUTH(String filename, String requestToken) {
String auth = filename.replace(".zip", ".auth");
String valueAUTH = readContentFile(auth);
if(!valueAUTH.equalsIgnoreCase(requestToken)) {
throw new InvalidIngestionFileException("Error while calculating MD5 file value");
}
}

private String readContentFile(String filename) {
try {
return Files.readString(Paths.get(filename));
} catch (IOException e) {
throw new InvalidIngestionFileException("Error while reading file: " + filename);
}
}

private String calculateMd5(String filename) {
try (InputStream is = Files.newInputStream(Paths.get(filename))) {
MessageDigest md = MessageDigest.getInstance("MD5");
Fixed Show fixed Hide fixed
DigestInputStream dis = new DigestInputStream(is, md);
IOUtils.toByteArray(dis);
return Hex.encodeHexString(md.digest());
} catch (NoSuchAlgorithmException | IOException e) {
throw new InvalidIngestionFileException("Error while calculating MD5");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package it.gov.pagopa.payhub.activities.activity.paymentsreporting.service;

import it.gov.pagopa.payhub.activities.dao.IngestionFlowDao;
import it.gov.pagopa.payhub.activities.dto.reportingflow.IngestionFlowDTO;
import it.gov.pagopa.payhub.activities.exception.IngestionFlowNotFoundException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

import java.util.Optional;

@Service
@Slf4j
public class IngestionFlowRetrieverService {

private final IngestionFlowDao ingestionFlowDao;

public IngestionFlowRetrieverService(IngestionFlowDao ingestionFlowDao) {
this.ingestionFlowDao = ingestionFlowDao;
}

public IngestionFlowDTO getIngestionFlow(Long ingestionFlowId) {
Optional<IngestionFlowDTO> ingestionFlow = ingestionFlowDao.getIngestionFlow(ingestionFlowId);

return ingestionFlow
.orElseThrow(() -> new IngestionFlowNotFoundException("Cannot found ingestionFlow having id: "+ ingestionFlowId));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package it.gov.pagopa.payhub.activities.dao;

import it.gov.pagopa.payhub.activities.dto.reportingflow.IngestionFlowDTO;

import java.util.Optional;

public interface IngestionFlowDao {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
public interface IngestionFlowDao {
public interface IngestionFlowFileDao {


/**
* * It will return the requested IngestionFlowDTO entity from its id
* */
Optional<IngestionFlowDTO> getIngestionFlow(Long ingestionFlowId);
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package it.gov.pagopa.payhub.activities.dto.fdr;
package it.gov.pagopa.payhub.activities.dto.reportingflow;

import lombok.AllArgsConstructor;
import lombok.Data;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package it.gov.pagopa.payhub.activities.dto.reportingflow;

import it.gov.pagopa.payhub.activities.dto.OrganizationDTO;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.sql.Timestamp;

@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class IngestionFlowDTO {
private Long ingestionFlowId;
private int version;
private OrganizationDTO orgId;
private String PspIdentifier;
private String flowIdentifierCode;
private Timestamp flowDateTime;
private String flowType;
private String flowTypeCode;
private String operatorName;
private String status;
private String filePathName;
private String fileName;
private Long downloadedFileSize;
private String requestTokenCode;
private Timestamp creationDate;
private Timestamp lastChangeDate;
private String fileSourceCode;
private String descriptionFileNameScraps;
private String errorCode;
private Long TotalRowsNumber;
private Long numberLinesImportedCorrectly;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package it.gov.pagopa.payhub.activities.exception;

public class IngestionFlowNotFoundException extends ActivitiesException {

public IngestionFlowNotFoundException(String message) { super(message); }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package it.gov.pagopa.payhub.activities.exception;

public class InvalidIngestionFileException extends ActivitiesException {
public InvalidIngestionFileException(String message) {
super(message);
}
}
Loading