Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Correction écrasement fichier .bad #88

Merged
merged 13 commits into from
Apr 15, 2024
46 changes: 22 additions & 24 deletions src/main/java/fr/abes/bestppn/kafka/TopicConsumer.java
Original file line number Diff line number Diff line change
Expand Up @@ -82,44 +82,44 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
String origineNbCurrentLine = new String(ligneKbart.headers().lastHeader("nbCurrentLines").value());
ThreadContext.put("package", (filename + ";" + origineNbCurrentLine)); //Ajoute le nom de fichier dans le contexte du thread pour log4j
service.processConsumerRecord(ligneKbartDto, providerName, workInProgress.get(filename).isForced(), workInProgress.get(filename).isBypassed(), filename);
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
if (ligneKbartDto.getBestPpn() != null && !ligneKbartDto.getBestPpn().isEmpty())
workInProgress.get(filename).addNbBestPpnFindedInExecutionReport();
workInProgress.get(filename).addLineKbartToMailAttachment(ligneKbartDto);
Header lastHeader = ligneKbart.headers().lastHeader("nbLinesTotal");
if (lastHeader != null) {
int nbLignesTotal = Integer.parseInt(new String(lastHeader.value()));
if (nbLignesTotal == workInProgress.get(filename).incrementNbLignesTraiteesAndGet()) {

if (workInProgress.get(filename).getSemaphore().tryAcquire()) {
workInProgress.get(filename).setNbtotalLinesInExecutionReport(nbLignesTotal);
handleFichier(filename);
}
}
}
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
//on ne décrémente pas le nb de thread si l'objet de suivi a été supprimé après la production des messages dans le second topic
if (workInProgress.get(filename) != null)
workInProgress.get(filename).decrementThreads();
}
});
} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}

} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}
}


Expand All @@ -135,9 +135,7 @@ private void handleFichier(String filename) {
}
} while (workInProgress.get(filename).getNbActiveThreads() > 1);
try {
if (workInProgress.get(filename).isOnError()) {
log.error("Fichier " + filename + " : Une erreur s'est produite dans le traitement du fichier");
} else {
if (!workInProgress.get(filename).isOnError()) {
String providerName = Utils.extractProvider(filename);
service.commitDatas(providerName, filename);
//quel que soit le résultat du traitement, on envoie le rapport par mail
Expand All @@ -151,7 +149,7 @@ private void handleFichier(String filename) {
log.error("Le nom du fichier " + filename + " n'est pas correct. " + e);
emailService.sendProductionErrorEmail(filename, e.getMessage());
} finally {
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).incrementNbLignesTraiteesAndGet());
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).getNbLignesTraitees());
workInProgress.remove(filename);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/fr/abes/bestppn/service/BestPpnService.java
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public BestPpnService(WsService service, NoticeService noticeService, TopicProdu
this.checkUrlService = checkUrlService;
}

public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKafka, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean isForced, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
List<String> messages = new ArrayList<>();
Map<String, Integer> ppnElecScoredList = new HashMap<>();
Set<String> ppnPrintResultList = new HashSet<>();
Expand All @@ -76,7 +76,7 @@ public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKa
feedPpnListFromDat(kbart, ppnElecScoredList, ppnPrintResultList, provider, isSendLogs, messages);
}

return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, injectKafka, isSendLogs, messages);
return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, isForced, isSendLogs, messages);
}

private void feedPpnListFromOnline(LigneKbartDto kbart, String provider, Map<String, Integer> ppnElecScoredList, Set<String> ppnPrintResultList, boolean isSendLogs, List<String> messages) throws IOException, URISyntaxException, IllegalArgumentException, BestPpnException {
Expand Down
13 changes: 8 additions & 5 deletions src/main/java/fr/abes/bestppn/service/WsService.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.lang.Nullable;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

Expand Down Expand Up @@ -95,15 +97,16 @@ private ResultWsSudocDto getResultWsSudocDto(String type, String id, @Nullable S
ResultWsSudocDto result = new ResultWsSudocDto();
try {
result = mapper.readValue((provider != null && !provider.isEmpty()) ? getRestCall(url, type, id, provider) : getRestCall(url, type, id), ResultWsSudocDto.class);
} catch (RestClientException ex) {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
} catch (IllegalArgumentException ex) {
if (ex.getMessage().equals("argument \"content\" is null")) {
} catch (HttpClientErrorException ex) {
if (ex.getStatusCode() == HttpStatus.BAD_REQUEST && ex.getMessage().contains("Aucune notice ne correspond à la recherche")) {
log.info("Aucuns ppn correspondant à l'identifiant " + id);
} else {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
}
} catch (RestClientException ex) {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
} catch (JsonProcessingException ex) {
throw new RestClientException(ex.getMessage());
} finally {
Expand Down
Loading