Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix handle fichier #87

Merged
merged 4 commits into from
Apr 11, 2024
Merged

Fix handle fichier #87

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a2cacf
refactor rename isforced
Apr 11, 2024
6167a80
refactor camelCase
Apr 11, 2024
6b4acdb
refactor camelCase
Apr 11, 2024
e1901ef
Fix handle fichier pour envoyé un mail dans tout les cas
Apr 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 22 additions & 24 deletions src/main/java/fr/abes/bestppn/kafka/TopicConsumer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,44 +82,44 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
String origineNbCurrentLine = new String(ligneKbart.headers().lastHeader("nbCurrentLines").value());
ThreadContext.put("package", (filename + ";" + origineNbCurrentLine)); //Ajoute le nom de fichier dans le contexte du thread pour log4j
service.processConsumerRecord(ligneKbartDto, providerName, workInProgress.get(filename).isForced(), workInProgress.get(filename).isBypassed(), filename);
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
if (ligneKbartDto.getBestPpn() != null && !ligneKbartDto.getBestPpn().isEmpty())
workInProgress.get(filename).addNbBestPpnFindedInExecutionReport();
workInProgress.get(filename).addLineKbartToMailAttachment(ligneKbartDto);
Header lastHeader = ligneKbart.headers().lastHeader("nbLinesTotal");
if (lastHeader != null) {
int nbLignesTotal = Integer.parseInt(new String(lastHeader.value()));
if (nbLignesTotal == workInProgress.get(filename).incrementNbLignesTraiteesAndGet()) {

if (workInProgress.get(filename).getSemaphore().tryAcquire()) {
workInProgress.get(filename).setNbtotalLinesInExecutionReport(nbLignesTotal);
handleFichier(filename);
}
}
}
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
//on ne décrémente pas le nb de thread si l'objet de suivi a été supprimé après la production des messages dans le second topic
if (workInProgress.get(filename) != null)
workInProgress.get(filename).decrementThreads();
}
});
} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}

} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}
}


Expand All @@ -135,9 +135,7 @@ private void handleFichier(String filename) {
}
} while (workInProgress.get(filename).getNbActiveThreads() > 1);
try {
if (workInProgress.get(filename).isOnError()) {
log.error("Fichier " + filename + " : Une erreur s'est produite dans le traitement du fichier");
} else {
if (!workInProgress.get(filename).isOnError()) {
String providerName = Utils.extractProvider(filename);
service.commitDatas(providerName, filename);
//quel que soit le résultat du traitement, on envoie le rapport par mail
Expand All @@ -151,7 +149,7 @@ private void handleFichier(String filename) {
log.error("Le nom du fichier " + filename + " n'est pas correct. " + e);
emailService.sendProductionErrorEmail(filename, e.getMessage());
} finally {
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).incrementNbLignesTraiteesAndGet());
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).getNbLignesTraitees());
workInProgress.remove(filename);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/fr/abes/bestppn/service/BestPpnService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public BestPpnService(WsService service, NoticeService noticeService, TopicProdu
this.checkUrlService = checkUrlService;
}

public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKafka, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean isForced, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
List<String> messages = new ArrayList<>();
Map<String, Integer> ppnElecScoredList = new HashMap<>();
Set<String> ppnPrintResultList = new HashSet<>();
Expand All @@ -76,7 +76,7 @@ public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKa
feedPpnListFromDat(kbart, ppnElecScoredList, ppnPrintResultList, provider, isSendLogs, messages);
}

return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, injectKafka, isSendLogs, messages);
return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, isForced, isSendLogs, messages);
}

private void feedPpnListFromOnline(LigneKbartDto kbart, String provider, Map<String, Integer> ppnElecScoredList, Set<String> ppnPrintResultList, boolean isSendLogs, List<String> messages) throws IOException, URISyntaxException, IllegalArgumentException, BestPpnException {
Expand Down
Loading