added committed while harvesting

teamdigitale · Nov 21, 2024 · 5ca8aec · 5ca8aec
1 parent ee21450
commit 5ca8aec
Show file tree

Hide file tree

Showing 8 changed files with 95 additions and 9 deletions.
diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterUpdateCommitDateEvent.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterUpdateCommitDateEvent.java
@@ -0,0 +1,13 @@
+package it.gov.innovazione.ndc.eventhandler.event;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.time.Instant;
+
+@Builder
+@Data
+public class HarvesterUpdateCommitDateEvent {
+    private final String runId;
+    private final Instant commitDate;
+}
diff --git a/...a/it/gov/innovazione/ndc/eventhandler/handler/HarvesterCommitDateUpdateEventListener.java b/...a/it/gov/innovazione/ndc/eventhandler/handler/HarvesterCommitDateUpdateEventListener.java
@@ -0,0 +1,42 @@
+package it.gov.innovazione.ndc.eventhandler.handler;
+
+import it.gov.innovazione.ndc.eventhandler.NdcEventHandler;
+import it.gov.innovazione.ndc.eventhandler.NdcEventWrapper;
+import it.gov.innovazione.ndc.eventhandler.event.HarvesterUpdateCommitDateEvent;
+import it.gov.innovazione.ndc.harvester.service.HarvesterRunService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class HarvesterCommitDateUpdateEventListener implements NdcEventHandler {
+
+    private static final Collection<Class<?>> SUPPORTED_EVENTS = List.of(HarvesterUpdateCommitDateEvent.class);
+
+    private final HarvesterRunService harvesterRunService;
+
+    @Override
+    public boolean canHandle(NdcEventWrapper<?> event) {
+        return SUPPORTED_EVENTS.contains(event.getPayload().getClass());
+    }
+
+    @Override
+    public void handle(NdcEventWrapper<?> event) {
+        HarvesterUpdateCommitDateEvent payload = (HarvesterUpdateCommitDateEvent) event.getPayload();
+        if (Objects.isNull(payload) || Objects.isNull(payload.getCommitDate())) {
+            log.warn("Received invalid HarvesterUpdateCommitDateEvent: {}", payload);
+            return;
+        }
+        harvesterRunService.getAllRuns().stream()
+                .filter(harvesterRun -> payload.getRunId().equals(harvesterRun.getId()))
+                .findAny()
+                .map(harvesterRun -> harvesterRun.withRevisionCommittedAt(payload.getCommitDate()))
+                .ifPresent(harvesterRunService::updateHarvesterRunCommittedAt);
+    }
+}
diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryService.java b/src/main/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryService.java
@@ -1,5 +1,7 @@
 package it.gov.innovazione.ndc.harvester;
 
+import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher;
+import it.gov.innovazione.ndc.eventhandler.event.HarvesterUpdateCommitDateEvent;
 import it.gov.innovazione.ndc.harvester.exception.InvalidAssetFolderException;
 import it.gov.innovazione.ndc.harvester.model.CvPath;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
@@ -11,6 +13,7 @@
 import it.gov.innovazione.ndc.harvester.util.GitUtils;
 import it.gov.innovazione.ndc.harvester.util.PropertiesUtils;
 import it.gov.innovazione.ndc.harvester.util.Version;
+import it.gov.innovazione.ndc.model.harvester.HarvesterRun;
 import it.gov.innovazione.ndc.service.logging.HarvesterStage;
 import it.gov.innovazione.ndc.service.logging.LoggingContext;
 import lombok.SneakyThrows;
@@ -19,6 +22,7 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -40,19 +44,22 @@ public class AgencyRepositoryService {
     private final ControlledVocabularyFolderScanner controlledVocabularyFolderScanner;
     private final SchemaFolderScanner schemaFolderScanner;
     private final List<String> lowerSkipWords;
+    private final NdcEventPublisher eventPublisher;
 
     public AgencyRepositoryService(FileUtils fileUtils,
                                    GitUtils gitUtils,
                                    OntologyFolderScanner ontologyFolderScanner,
                                    ControlledVocabularyFolderScanner controlledVocabularyFolderScanner,
                                    SchemaFolderScanner schemaFolderScanner,
-                                   AgencyRepositoryServiceProperties agencyRepositoryServiceProperties) {
+                                   AgencyRepositoryServiceProperties agencyRepositoryServiceProperties,
+                                   NdcEventPublisher eventPublisher) {
         this.fileUtils = fileUtils;
         this.gitUtils = gitUtils;
         this.ontologyFolderScanner = ontologyFolderScanner;
         this.controlledVocabularyFolderScanner = controlledVocabularyFolderScanner;
         this.schemaFolderScanner = schemaFolderScanner;
         this.lowerSkipWords = PropertiesUtils.lowerSkipWords(agencyRepositoryServiceProperties.getSkipWords(), MIN_SKIP_WORD_LENGTH);
+        this.eventPublisher = eventPublisher;
     }
 
     public Path cloneRepo(String repoUrl) throws IOException {
@@ -62,7 +69,12 @@ public Path cloneRepo(String repoUrl) throws IOException {
     public Path cloneRepo(String repoUrl, String revision) throws IOException {
         Path cloneDir = fileUtils.createTempDirectory(TEMP_DIR_PREFIX);
         log.info("Cloning repo {} @ revision {}, at location {}", repoUrl, revision, cloneDir);
-        gitUtils.cloneRepo(repoUrl, cloneDir.toFile(), revision);
+        Instant instant = gitUtils.cloneRepoAndGetLastCommitDate(repoUrl, cloneDir.toFile(), revision);
+        eventPublisher.publishEvent("harvester", "harvester.get.commit.date", null, "harvester",
+                HarvesterUpdateCommitDateEvent.builder()
+                        .runId(HarvesterRun.getCurrentRunId())
+                        .commitDate(instant)
+                        .build());
         return cloneDir;
     }
 

diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/util/GitUtils.java b/src/main/java/it/gov/innovazione/ndc/harvester/util/GitUtils.java
@@ -28,6 +28,10 @@ public class GitUtils {
     private final FileUtils fileUtils;
 
     public void cloneRepo(String repoUrl, File destination, String revision) {
+        cloneRepoAndGetLastCommitDate(repoUrl, destination, revision);
+    }
+
+    public Instant cloneRepoAndGetLastCommitDate(String repoUrl, File destination, String revision) {
         try {
             Git call = Git.cloneRepository()
                     .setURI(repoUrl)
@@ -37,6 +41,7 @@ public void cloneRepo(String repoUrl, File destination, String revision) {
             if (StringUtils.isNotBlank(revision)) {
                 call.checkout().setName(revision).call();
             }
+            return safelyGetLastCommitDate(call);
         } catch (GitAPIException e) {
             throw new GitRepoCloneException(String.format("Cannot clone repo '%s'", repoUrl), e);
         }
@@ -74,8 +79,7 @@ public Optional<Instant> getCommitDate(String repositoryUrl, String revision) {
             return Optional.empty();
         }
 
-
-        Optional<Instant> instant = gitOpt.map(this::safelyGetRevCommits);
+        Optional<Instant> instant = gitOpt.map(this::safelyGetLastCommitDate);
         tryRemoveDirectory(tempDirectory.get());
         return instant;
     }
@@ -105,7 +109,7 @@ private Optional<Path> safelyGetTempDirectory(String repositoryUrl, String revis
         }
     }
 
-    private Instant safelyGetRevCommits(Git git) {
+    private Instant safelyGetLastCommitDate(Git git) {
         try {
             return StreamSupport.stream(git.log().call().spliterator(), false)
                     .findFirst()

diff --git a/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java b/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java
@@ -1,5 +1,7 @@
 package it.gov.innovazione.ndc.model.harvester;
 
+import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContext;
+import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils;
 import lombok.Builder;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
@@ -25,6 +27,14 @@ public class HarvesterRun {
     private final Status status;
     private final String reason;
 
+    public static String getCurrentRunId() {
+        HarvestExecutionContext context = HarvestExecutionContextUtils.getContext();
+        if (context == null) {
+            return null;
+        }
+        return context.getRunId();
+    }
+
     public enum Status {
         SUCCESS, UNCHANGED, ALREADY_RUNNING, RUNNING, NDC_ISSUES_PRESENT, FAILURE
     }

diff --git a/src/main/java/it/gov/innovazione/ndc/service/logging/NDCHarvesterLoggerUtils.java b/src/main/java/it/gov/innovazione/ndc/service/logging/NDCHarvesterLoggerUtils.java
@@ -34,7 +34,7 @@ public static void overrideContext(LoggingContext context) {
     public static void clearContext() {
         SEEN_MESSAGES.get().clear();
         CONTEXT_HOLDER.remove();
-        log.info("Context cleared");
+        log.info("Contexts cleared");
     }
 
     public static boolean notSeen(String message) {

diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties
@@ -1,4 +1,4 @@
-harvester.repositories=https://github.com/FrankMaverick/Leo-OpenData
+harvester.repositories=https://github.com/FrankMaverick/Leo-OpenData,https://github.com/italia/daf-ontologie-vocabolari-controllati,https://github.com/InailUfficio5/inail-ndc
 
 virtuoso.sparql=http://localhost:8890/sparql-auth
 virtuoso.sparql-graph-store=http://localhost:8890/sparql-graph-crud-auth

diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryServiceTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryServiceTest.java
@@ -1,5 +1,6 @@
 package it.gov.innovazione.ndc.harvester;
 
+import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher;
 import it.gov.innovazione.ndc.harvester.model.CvPath;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
 import it.gov.innovazione.ndc.harvester.scanners.ControlledVocabularyFolderScanner;
@@ -23,6 +24,7 @@
 import static it.gov.innovazione.ndc.harvester.SemanticAssetType.SCHEMA;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -31,17 +33,20 @@ public class AgencyRepositoryServiceTest {
     FileUtils fileUtils;
     GitUtils gitUtils;
     AgencyRepositoryService agencyRepoService;
+    NdcEventPublisher eventPublisher;
 
     @BeforeEach
     public void setup() {
         fileUtils = mock(FileUtils.class);
+        eventPublisher = mock(NdcEventPublisher.class);
         when(fileUtils.getLowerCaseFileName(any())).thenCallRealMethod();
+        doNothing().when(eventPublisher).publishEvent(any(), any(), any(), any(), any());
         gitUtils = mock(GitUtils.class);
         OntologyFolderScanner ontologyScanner = new OntologyFolderScanner(fileUtils, OntologyFolderScannerProperties.forWords("aligns"));
         ControlledVocabularyFolderScanner cvScanner = new ControlledVocabularyFolderScanner(fileUtils, ControlledVocabularyFolderScannerProperties.forWords());
         SchemaFolderScanner schemaScanner = new SchemaFolderScanner(fileUtils);
         agencyRepoService = new AgencyRepositoryService(fileUtils, gitUtils, ontologyScanner,
-                cvScanner, schemaScanner, AgencyRepositoryServiceProperties.forWords("scriptR2RML"));
+                cvScanner, schemaScanner, AgencyRepositoryServiceProperties.forWords("scriptR2RML"), eventPublisher);
     }
 
     @Test
@@ -52,7 +57,7 @@ void shouldCloneTheRepoInTempDir() throws IOException {
 
         assertThat(clonedTempDir).isEqualTo(Path.of("temp"));
         verify(fileUtils).createTempDirectory(TEMP_DIR_PREFIX);
-        verify(gitUtils).cloneRepo("someURI", new File("temp"), null);
+        verify(gitUtils).cloneRepoAndGetLastCommitDate("someURI", new File("temp"), null);
     }
 
     /**