Skip to content

Commit

Permalink
Merge pull request #88 from abes-esr/develop
Browse files Browse the repository at this point in the history 
Correction écrasement fichier .bad
  • Loading branch information
@SamuelQuetin
SamuelQuetin authored Apr 15, 2024
2 parents 439a211 + 3c35176 commit faf24e8
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 31 deletions.
46 changes: 22 additions & 24 deletions src/main/java/fr/abes/bestppn/kafka/TopicConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,44 +82,44 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
String origineNbCurrentLine = new String(ligneKbart.headers().lastHeader("nbCurrentLines").value());
ThreadContext.put("package", (filename + ";" + origineNbCurrentLine)); //Ajoute le nom de fichier dans le contexte du thread pour log4j
service.processConsumerRecord(ligneKbartDto, providerName, workInProgress.get(filename).isForced(), workInProgress.get(filename).isBypassed(), filename);
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
if (ligneKbartDto.getBestPpn() != null && !ligneKbartDto.getBestPpn().isEmpty())
workInProgress.get(filename).addNbBestPpnFindedInExecutionReport();
workInProgress.get(filename).addLineKbartToMailAttachment(ligneKbartDto);
Header lastHeader = ligneKbart.headers().lastHeader("nbLinesTotal");
if (lastHeader != null) {
int nbLignesTotal = Integer.parseInt(new String(lastHeader.value()));
if (nbLignesTotal == workInProgress.get(filename).incrementNbLignesTraiteesAndGet()) {

if (workInProgress.get(filename).getSemaphore().tryAcquire()) {
workInProgress.get(filename).setNbtotalLinesInExecutionReport(nbLignesTotal);
handleFichier(filename);
}
}
}
} catch (IOException | URISyntaxException | RestClientException | IllegalDoiException e) {
//erreurs non bloquantes, on n'arrête pas le programme
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
}
log.error(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(ligneKbartDto, e.getMessage());
workInProgress.get(filename).addNbLinesWithErrorsInExecutionReport();
} finally {
//on ne décrémente pas le nb de thread si l'objet de suivi a été supprimé après la production des messages dans le second topic
if (workInProgress.get(filename) != null)
workInProgress.get(filename).decrementThreads();
}
});
} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}

} catch(IllegalProviderException | JsonProcessingException e){
workInProgress.get(filename).setIsOnError(true);
log.warn(e.getMessage());
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}
}


Expand All @@ -135,9 +135,7 @@ private void handleFichier(String filename) {
}
} while (workInProgress.get(filename).getNbActiveThreads() > 1);
try {
if (workInProgress.get(filename).isOnError()) {
log.error("Fichier " + filename + " : Une erreur s'est produite dans le traitement du fichier");
} else {
if (!workInProgress.get(filename).isOnError()) {
String providerName = Utils.extractProvider(filename);
service.commitDatas(providerName, filename);
//quel que soit le résultat du traitement, on envoie le rapport par mail
Expand All @@ -151,7 +149,7 @@ private void handleFichier(String filename) {
log.error("Le nom du fichier " + filename + " n'est pas correct. " + e);
emailService.sendProductionErrorEmail(filename, e.getMessage());
} finally {
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).incrementNbLignesTraiteesAndGet());
log.info("Traitement terminé pour fichier " + filename + " / nb lignes " + workInProgress.get(filename).getNbLignesTraitees());
workInProgress.remove(filename);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/fr/abes/bestppn/service/BestPpnService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public BestPpnService(WsService service, NoticeService noticeService, TopicProdu
this.checkUrlService = checkUrlService;
}

public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKafka, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean isForced, boolean isSendLogs) throws IOException, BestPpnException, URISyntaxException, RestClientException, IllegalArgumentException, IllegalDoiException {
List<String> messages = new ArrayList<>();
Map<String, Integer> ppnElecScoredList = new HashMap<>();
Set<String> ppnPrintResultList = new HashSet<>();
Expand All @@ -76,7 +76,7 @@ public BestPpn getBestPpn(LigneKbartDto kbart, String provider, boolean injectKa
feedPpnListFromDat(kbart, ppnElecScoredList, ppnPrintResultList, provider, isSendLogs, messages);
}

return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, injectKafka, isSendLogs, messages);
return getBestPpnByScore(kbart, ppnElecScoredList, ppnPrintResultList, isForced, isSendLogs, messages);
}

private void feedPpnListFromOnline(LigneKbartDto kbart, String provider, Map<String, Integer> ppnElecScoredList, Set<String> ppnPrintResultList, boolean isSendLogs, List<String> messages) throws IOException, URISyntaxException, IllegalArgumentException, BestPpnException {
Expand Down
13 changes: 8 additions & 5 deletions src/main/java/fr/abes/bestppn/service/WsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.lang.Nullable;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

Expand Down Expand Up @@ -95,15 +97,16 @@ private ResultWsSudocDto getResultWsSudocDto(String type, String id, @Nullable S
ResultWsSudocDto result = new ResultWsSudocDto();
try {
result = mapper.readValue((provider != null && !provider.isEmpty()) ? getRestCall(url, type, id, provider) : getRestCall(url, type, id), ResultWsSudocDto.class);
} catch (RestClientException ex) {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
} catch (IllegalArgumentException ex) {
if (ex.getMessage().equals("argument \"content\" is null")) {
} catch (HttpClientErrorException ex) {
if (ex.getStatusCode() == HttpStatus.BAD_REQUEST && ex.getMessage().contains("Aucune notice ne correspond à la recherche")) {
log.info("Aucuns ppn correspondant à l'identifiant " + id);
} else {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
}
} catch (RestClientException ex) {
log.info("URL : {} / id : {} / provider : {} : Erreur dans l'acces au webservice.", url, id, provider);
throw ex;
} catch (JsonProcessingException ex) {
throw new RestClientException(ex.getMessage());
} finally {
Expand Down

0 comments on commit faf24e8

Please sign in to comment.