Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Data xfer3 #180

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Data xfer3 #180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f76262e
Get unique IDs before counting in MetadataMatchCountIndexValueMaker.
ChemMitch Jun 10, 2023
20becea
commented out delombok
ChemMitch Jun 10, 2023
e5d008b
Merge branch 'master' into dataXfer3
ChemMitch Jun 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ...-autoconfigure/src/main/java/gsrs/imports/indexers/MetadataMatchCountIndexValueMaker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,12 @@ public void createIndexableValues(ImportMetadata importMetadata, Consumer<Indexa
.map(MatchedKeyValue::getMatchingRecords)
.flatMap(Collection::stream)
.filter(m->m. getSourceName().equals(USED_SOURCE))
.map(m->m.getRecordId().getIdString())
.distinct()
.count();
consumer.accept(IndexableValue.simpleFacetStringValue(IMPORT_METADATA_MATCH_COUNT_FACET, Long.toString(matchCount)));
log.trace("created string facet {} with value {}", IMPORT_METADATA_MATCH_COUNT_FACET, matchCount);
log.trace("created string facet {} with value {} for record {}", IMPORT_METADATA_MATCH_COUNT_FACET, matchCount,
importMetadata.getRecordId());
matchedRecordSummary.getMatches().stream()
.filter(m->m.getMatchingRecords().stream().anyMatch(r->r.getSourceName().equals(USED_SOURCE)))
.forEach(r->r.getMatchingRecords()
Expand Down
19 changes: 1 addition & 18 deletions ...-boot-autoconfigure/src/main/java/gsrs/stagingarea/service/DefaultStagingAreaService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,24 +226,8 @@ public String createRecord(ImportRecordParameters parameters) {
if(performMatching){
log.trace("going to match");
List<MatchableKeyValueTuple> definitionalValueTuples = getMatchables(domainObject);
definitionalValueTuples.forEach(t -> log.trace("key: {}, value: {}", t.getKey(), t.getValue()));
//definitionalValueTuples.forEach(t -> log.trace("key: {}, value: {}", t.getKey(), t.getValue()));
persistDefinitionalValues(definitionalValueTuples, instanceId, recordId, parameters.getEntityClassName());

//event driven: each step in process sends an event (pub/sub) look in ... indexing
// validation, when done would trigger the next event via
// event manager type of thing
// passively: daemon running in background looks for records with a given status and then performs
// the next step
// will
//try {
// MatchedRecordSummary summary = findMatches(domainObject.getClass().getName(), definitionalValueTuples, recordId.toString());
//log.trace("Matches: ");
//summary.getMatches().forEach(m -> {
//log.trace("Matching key: {} = {}", m.getTupleUsedInMatching().getKey(), m.getTupleUsedInMatching().getValue());
//});
//} catch (ClassNotFoundException e) {
// log.error("Error looking for matches", e);
//}
}
if( performIndexing ) {
log.trace("going to index");
Expand Down Expand Up @@ -567,7 +551,6 @@ public MatchedRecordSummary findMatchesForJson(String qualifiedEntityType, Strin
Object domainObject;
try {
log.trace("going deserialize object of class {}", qualifiedEntityType);
log.trace(entityJson);
domainObject = deserializeObject(qualifiedEntityType, entityJson);
} catch (JsonProcessingException e) {
log.error("Error deserializing imported object.", e);
Expand Down
4 changes: 2 additions & 2 deletions gsrs-spring-legacy-indexer/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
</dependency>
</dependencies>
<build>
<plugins>
<!--plugins>
<plugin>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
Expand All @@ -143,6 +143,6 @@
<sourcepath>${delombok.output}</sourcepath>
</configuration>
</plugin>
</plugins>
</plugins-->
</build>
</project>
74 changes: 73 additions & 1 deletion ...c/test/java/gsrs/dataexchange/imports/indexers/MetadataMatchCountIndexValueMakerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,80 @@ public void createIndexableValuesTest() throws NoSuchFieldException, IllegalAcce
serviceField.set(indexValueMaker1, stagingAreaService);;
indexValueMaker1.createIndexableValues(metadata, indexedValues::add);
Assertions.assertTrue(indexedValues.stream().anyMatch(i->i.name().equals(MetadataMatchCountIndexValueMaker.IMPORT_METADATA_MATCH_COUNT_FACET)
&& (i.value().equals("2"))));
&& (i.value().equals("1"))));
Assertions.assertTrue(indexedValues.stream().anyMatch(i->i.name().equals(MetadataMatchCountIndexValueMaker.IMPORT_METADATA_MATCH_KEY_FACET)
&& i.value().equals("Factor 1")));
}

@Test
public void createIndexableValuesTest2() throws NoSuchFieldException, IllegalAccessException {
ImportMetadata metadata = new ImportMetadata();
metadata.setAccess( Collections.singleton(new Group("protected")));
metadata.setReason("test");
String sourceName = "Unique Data Source";
metadata.setSourceName(sourceName);
metadata.setEntityClassName("ix.ginas.models.v1.Substance");
UUID record1IdValue = UUID.randomUUID();
String record1Id = record1IdValue.toString();
metadata.setRecordId(record1IdValue);
StagingAreaService stagingAreaService = mock(StagingAreaService.class);
ImportDataRepository importDataRepository = mock(ImportDataRepository.class);
MatchedRecordSummary matchedRecordSummary = new MatchedRecordSummary();
List<MatchedKeyValue> matchedKeyValues = new ArrayList<>();
List<MatchedKeyValue.MatchingRecordReference> matchingRecords = new ArrayList<>();
MatchedKeyValue.MatchingRecordReference match1= new MatchedKeyValue.MatchingRecordReference();
match1.setMatchedKey("Factor 1");
match1.setSourceName("GSRS");
EntityUtils.Key recordId=EntityUtils.Key.of(ImportMetadata.class, record1Id);
match1.setRecordId( recordId);
matchingRecords.add(match1);

String record2Id = UUID.randomUUID().toString();
EntityUtils.Key recordId2=EntityUtils.Key.of(ImportMetadata.class, record2Id);
MatchedKeyValue.MatchingRecordReference match2= new MatchedKeyValue.MatchingRecordReference();
match2.setRecordId( recordId2);
match2.setMatchedKey("Factor 1");
match2.setSourceName("GSRS");
matchingRecords.add(match2);

MatchedKeyValue mkv1= new MatchedKeyValue();
mkv1.setMatchingRecords(matchingRecords);
MatchableKeyValueTuple tuple1 =MatchableKeyValueTuple.builder()
.key("Factor 1")
.value("Value 1")
.build();
mkv1.setTupleUsedInMatching(tuple1);
matchedKeyValues.add(mkv1);
MatchedKeyValue mkv2= new MatchedKeyValue();
mkv2.setMatchingRecords(matchingRecords);
MatchableKeyValueTuple tuple2 =MatchableKeyValueTuple.builder()
.key("Factor 2")
.value("Value 2")
.build();
mkv2.setTupleUsedInMatching(tuple2);
matchedKeyValues.add(mkv2);
matchedRecordSummary.setMatches(matchedKeyValues);
List<MatchableKeyValueTuple> query = new ArrayList<>();
query.add(tuple1);
query.add(tuple2);
matchedRecordSummary.setQuery(query);
String recordJson = "";
when(stagingAreaService.findMatchesForJson(metadata.getEntityClassName(), recordJson, null)).thenReturn(matchedRecordSummary);
when(stagingAreaService.findMatchesForJson(metadata.getEntityClassName(), recordJson, record1Id)).thenReturn(matchedRecordSummary);
when(stagingAreaService.findMatchesForJson(metadata.getEntityClassName(), recordJson, record2Id)).thenReturn(matchedRecordSummary);
when(importDataRepository.retrieveByInstanceID(metadata.getInstanceId())).thenReturn(recordJson);

List<IndexableValue> indexedValues = new ArrayList<>();
MetadataMatchCountIndexValueMaker indexValueMaker1 = new MetadataMatchCountIndexValueMaker();
Field repoField = indexValueMaker1.getClass().getDeclaredField("importDataRepository");
repoField.setAccessible(true);
Field serviceField = indexValueMaker1.getClass().getDeclaredField("stagingAreaService");
serviceField.setAccessible(true);
repoField.set(indexValueMaker1, importDataRepository);
serviceField.set(indexValueMaker1, stagingAreaService);
String expectedMatchCount = Integer.toString(2);
indexValueMaker1.createIndexableValues(metadata, indexedValues::add);
Assertions.assertTrue(indexedValues.stream().anyMatch(i->i.name().equals(MetadataMatchCountIndexValueMaker.IMPORT_METADATA_MATCH_COUNT_FACET)
&& (i.value().equals(expectedMatchCount))));
}
}