Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

merge dev to test #93

Merged
merged 6 commits into from
Apr 25, 2024
Merged

merge dev to test #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
85a9020
CDE-409 : ajustement suite à modification des dates dans kbart2kafka …
pierre-maraval Apr 25, 2024
e1f31d0
CDE-411 ajout du cas, bestppnexeption en FORCE
Apr 25, 2024
e8ff8b6
Merge pull request #91 from abes-esr/CDE-409-format-de-dates-de-publi…
SamuelQuetin Apr 25, 2024
7741d23
Merge pull request #92 from abes-esr/CDE-411_en_FORCE_lignes_en_erreu…
pierre-maraval Apr 25, 2024
af699ed
Suppression timeout sur lecture des messages suite à fausse manip
pierre-maraval Apr 25, 2024
368e97e
Merge pull request #94 from abes-esr/fix_supp_timeout
SamuelQuetin Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions src/main/java/fr/abes/bestppn/kafka/TopicConsumer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,13 @@ public void initExecutor() {
*/
@KafkaListener(topics = {"${topic.name.source.kbart}"}, groupId = "${topic.groupid.source.kbart}", containerFactory = "kafkaKbartListenerContainerFactory", concurrency = "${spring.kafka.concurrency.nbThread}")
public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {

String filename = ligneKbart.key();
if (!this.workInProgress.containsKey(filename)) {
//nouveau fichier trouvé dans le topic, on initialise les variables partagées
workInProgress.put(filename, new KafkaWorkInProgress(ligneKbart.key().contains("_FORCE"), ligneKbart.key().contains("_BYPASS")));
}
try {
//traitement de chaque ligne kbart
if (!this.workInProgress.containsKey(ligneKbart.key())) {
//nouveau fichier trouvé dans le topic, on initialise les variables partagées
workInProgress.put(filename, new KafkaWorkInProgress(ligneKbart.key().contains("_FORCE"), ligneKbart.key().contains("_BYPASS")));
}

LigneKbartDto ligneKbartDto = mapper.readValue(ligneKbart.value(), LigneKbartDto.class);
String providerName = Utils.extractProvider(ligneKbart.key());
executorService.execute(() -> {
Expand All @@ -90,6 +88,8 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
} catch (BestPpnException e) {
if (!workInProgress.get(filename).isForced()) {
workInProgress.get(filename).setIsOnError(true);
} else {
workInProgress.get(filename).addKbartToSend(ligneKbartDto);
}
log.error(e.getMessage());
ligneKbartDto.setErrorType(e.getMessage());
Expand All @@ -102,7 +102,6 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
if (lastHeader != null) {
int nbLignesTotal = Integer.parseInt(new String(lastHeader.value()));
if (nbLignesTotal == workInProgress.get(filename).incrementNbLignesTraiteesAndGet()) {

if (workInProgress.get(filename).getSemaphore().tryAcquire()) {
workInProgress.get(filename).setNbtotalLinesInExecutionReport(nbLignesTotal);
handleFichier(filename);
Expand All @@ -120,8 +119,8 @@ public void kbartFromkafkaListener(ConsumerRecord<String, String> ligneKbart) {
workInProgress.get(filename).addLineKbartToMailAttachementWithErrorMessage(new LigneKbartDto(), e.getMessage());
workInProgress.get(filename).addNbLinesWithInputDataErrorsInExecutionReport();
}
}

}

private void handleFichier(String filename) {
//on attend que l'ensemble des threads aient terminé de travailler avant de lancer le commit
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/fr/abes/bestppn/model/dto/kafka/LigneKbartDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,11 +159,11 @@ public String getAuthor() {

@JsonIgnore
public String getAnneeFromDate_monograph_published_print() {
return this.dateMonographPublishedPrint;
return this.dateMonographPublishedPrint.substring(0, 4);
}

@JsonIgnore
public String getAnneeFromDate_monograph_published_online() {
return this.dateMonographPublishedOnline;
return this.dateMonographPublishedOnline.substring(0, 4);
}
}
4 changes: 2 additions & 2 deletions src/main/java/fr/abes/bestppn/service/BestPpnService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,10 @@ private void feedPpnListFromDat(LigneKbartDto kbart, Map<String, Integer> ppnEle
if (isSendLogs) messages.add(message);
ResultWsSudocDto resultCallWs = null;
if (!kbart.getAnneeFromDate_monograph_published_online().isEmpty()) {
log.debug("Appel dat2ppn : date_monograph_published_online : " + kbart.getAnneeFromDate_monograph_published_online() + " / publication_title : " + kbart.getPublicationTitle() + " auteur : " + kbart.getAuthor());
log.debug("Appel dat2ppn : date_monograph_published_online : " + kbart.getDateMonographPublishedOnline() + " / publication_title : " + kbart.getPublicationTitle() + " auteur : " + kbart.getAuthor());
resultCallWs = service.callDat2Ppn(kbart.getAnneeFromDate_monograph_published_online(), kbart.getAuthor(), kbart.getPublicationTitle(), providerName);
} else if (ppnElecScoredList.isEmpty() && !kbart.getAnneeFromDate_monograph_published_print().isEmpty()) {
log.debug("Appel dat2ppn : date_monograph_published_print : " + kbart.getAnneeFromDate_monograph_published_print() + " / publication_title : " + kbart.getPublicationTitle() + " auteur : " + kbart.getAuthor());
log.debug("Appel dat2ppn : date_monograph_published_print : " + kbart.getDateMonographPublishedPrint() + " / publication_title : " + kbart.getPublicationTitle() + " auteur : " + kbart.getAuthor());
resultCallWs = service.callDat2Ppn(kbart.getAnneeFromDate_monograph_published_print(), kbart.getAuthor(), kbart.getPublicationTitle(), providerName);
}
if(resultCallWs != null && !resultCallWs.getPpns().isEmpty()) {
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/fr/abes/bestppn/service/WsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,15 @@ private ResultWsSudocDto getResultWsSudocDto(String type, String id, @Nullable S
return result;
}

/**
* Appel ws dat2ppn
* @param date date au format YYYY à rechercher
* @param author auteur à rechercher
* @param title titre à rechercher
* @param providerName nom du provider
* @return résultat de l'appel à dat2ppn
* @throws JsonProcessingException erreur construction objet résultat
*/
public ResultWsSudocDto callDat2Ppn(String date, String author, String title, String providerName) throws JsonProcessingException {
SearchDatWebDto searchDatWebDto = new SearchDatWebDto(title, providerName);
if (author != null && !author.isEmpty()) {
Expand Down
Loading