diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7fbdc9a2..22d8190ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added * LearnLib now supports JPMS modules. All artifacts now provide a `module-info` descriptor except of the distribution artifacts (for Maven-less environments) which only provide an `Automatic-Module-Name` due to non-modular dependencies. Note that while this is a Java 9+ feature, LearnLib still supports Java 8 byte code for the remaining class files. +* Added an `InterningMembershipOracle` (including refinements) to the `learnlib-cache` artifact that interns query responses to reduce memory consumption of large data structures. This exports the internal concepts of the DHC learner (which no longer interns query responses automatically). ### Changed diff --git a/build-parent/pom.xml b/build-parent/pom.xml index 6adae30f93..6d5ad6a634 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -63,6 +63,7 @@ limitations under the License. de/learnlib/filter/reuse/**/Reuse*Builder.class + de/learnlib/filter/cache/**/*Interning*.class de/learnlib/filter/statistic/**/DFA*.class de/learnlib/filter/statistic/**/Mealy*.class de/learnlib/filter/statistic/**/Moore*.class diff --git a/filters/cache/pom.xml b/filters/cache/pom.xml index 1ca54ea773..9df06a8d31 100644 --- a/filters/cache/pom.xml +++ b/filters/cache/pom.xml @@ -69,6 +69,12 @@ limitations under the License. slf4j-api + + + de.learnlib.tooling + annotations + + de.learnlib diff --git a/filters/cache/src/main/java/de/learnlib/filter/cache/InterningMembershipOracle.java b/filters/cache/src/main/java/de/learnlib/filter/cache/InterningMembershipOracle.java new file mode 100644 index 0000000000..ac711068d7 --- /dev/null +++ b/filters/cache/src/main/java/de/learnlib/filter/cache/InterningMembershipOracle.java @@ -0,0 +1,101 @@ +/* Copyright (C) 2013-2024 TU Dortmund University + * This file is part of LearnLib, http://www.learnlib.de/. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.learnlib.filter.cache; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + +import de.learnlib.oracle.MembershipOracle; +import de.learnlib.oracle.MembershipOracle.MealyMembershipOracle; +import de.learnlib.oracle.MembershipOracle.MooreMembershipOracle; +import de.learnlib.query.DefaultQuery; +import de.learnlib.query.Query; +import de.learnlib.tooling.annotation.refinement.GenerateRefinement; +import de.learnlib.tooling.annotation.refinement.Generic; +import de.learnlib.tooling.annotation.refinement.Interface; +import de.learnlib.tooling.annotation.refinement.Mapping; +import net.automatalib.word.Word; + +/** + * A {@link MembershipOracle} that interns query outputs. May be used to reduce memory consumption of data structures + * that store a lot of query responses. Typically, this oracle only makes sense for output types that are not already + * interned by the JVM (such as {@link Boolean}s in case of {@link DFAMembershipOracle}s). + */ +@GenerateRefinement(name = "InterningMealyMembershipOracle", + packageName = "de.learnlib.filter.cache.mealy", + generics = {@Generic(value = "I", desc = "input symbol type"), + @Generic(value = "O", desc = "output symbol type")}, + parentGenerics = {@Generic("I"), @Generic(clazz = Word.class, generics = "O")}, + interfaces = @Interface(clazz = MealyMembershipOracle.class, + generics = {@Generic("I"), @Generic("O")}), + typeMappings = @Mapping(from = MembershipOracle.class, + to = MealyMembershipOracle.class, + generics = {@Generic("I"), @Generic("O")})) +@GenerateRefinement(name = "InterningMooreMembershipOracle", + packageName = "de.learnlib.filter.cache.moore", + generics = {@Generic(value = "I", desc = "input symbol type"), + @Generic(value = "O", desc = "output symbol type")}, + parentGenerics = {@Generic("I"), @Generic(clazz = Word.class, generics = "O")}, + interfaces = @Interface(clazz = MooreMembershipOracle.class, + generics = {@Generic("I"), @Generic("O")}), + typeMappings = @Mapping(from = MembershipOracle.class, + to = MooreMembershipOracle.class, + generics = {@Generic("I"), @Generic("O")})) +public class InterningMembershipOracle implements MembershipOracle { + + private final MembershipOracle delegate; + private final Map> cache; + + public InterningMembershipOracle(MembershipOracle delegate) { + this.delegate = delegate; + this.cache = new WeakHashMap<>(); + } + + @Override + public void processQueries(Collection> queries) { + final List> delegates = new ArrayList<>(queries.size()); + + for (Query q : queries) { + delegates.add(new DefaultQuery<>(q)); + } + + this.delegate.processQueries(delegates); + + final Iterator> origIter = queries.iterator(); + final Iterator> delegateIter = delegates.iterator(); + + while (origIter.hasNext() && delegateIter.hasNext()) { + final Query origNext = origIter.next(); + final DefaultQuery delegateNext = delegateIter.next(); + final D delegateOutput = delegateNext.getOutput(); + + // Since the GC may free our references during the lookup, repeat until we have a (non-null) cache hit. + D origOutput; + do { + origOutput = cache.computeIfAbsent(delegateOutput, k -> new WeakReference<>(delegateOutput)).get(); + } while (origOutput == null); + + origNext.answer(origOutput); + } + + assert !origIter.hasNext() && !delegateIter.hasNext(); + } +} diff --git a/filters/cache/src/main/java/module-info.java b/filters/cache/src/main/java/module-info.java index 4df43d4094..41e2b4c508 100644 --- a/filters/cache/src/main/java/module-info.java +++ b/filters/cache/src/main/java/module-info.java @@ -39,6 +39,7 @@ // only required by documentation requires static de.learnlib.oracle.parallelism; + requires static de.learnlib.tooling.annotation; exports de.learnlib.filter.cache; exports de.learnlib.filter.cache.dfa; diff --git a/filters/cache/src/test/java/de/learnlib/filter/cache/InterningMembershipOracleTest.java b/filters/cache/src/test/java/de/learnlib/filter/cache/InterningMembershipOracleTest.java new file mode 100644 index 0000000000..a92f360b0e --- /dev/null +++ b/filters/cache/src/test/java/de/learnlib/filter/cache/InterningMembershipOracleTest.java @@ -0,0 +1,80 @@ +/* Copyright (C) 2013-2024 TU Dortmund University + * This file is part of LearnLib, http://www.learnlib.de/. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.learnlib.filter.cache; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import de.learnlib.oracle.SingleQueryOracle.SingleQueryOracleMealy; +import de.learnlib.query.DefaultQuery; +import net.automatalib.common.util.Pair; +import net.automatalib.word.Word; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class InterningMembershipOracleTest { + + @Test + public void testInterning() { + final DefaultQuery> q1 = new DefaultQuery<>(Word.epsilon(), Word.fromString("qwe")); + final DefaultQuery> q2 = new DefaultQuery<>(Word.fromString("asd"), Word.epsilon()); + + final Oracle oracle = new Oracle<>(); + final InterningMembershipOracle> interning = new InterningMembershipOracle<>(oracle); + + interning.processQueries(Arrays.asList(q1, q2)); + + final Word o1 = q1.getOutput(); + + Assert.assertSame(q1.getOutput(), o1); + Assert.assertSame(q2.getOutput(), o1); + + // repeated queries + interning.processQueries(Arrays.asList(q1, q2)); + + Assert.assertSame(q1.getOutput(), o1); + Assert.assertSame(q2.getOutput(), o1); + + // check executed queries + Assert.assertEquals(oracle.count, 4); + Assert.assertEquals(oracle.queries, + Set.of(Pair.of(Word.epsilon(), Word.fromString("qwe")), + Pair.of(Word.fromString("asd"), Word.epsilon()))); + // check that oracle actually returns different objects + Assert.assertNotSame(oracle.answerQuery(q1.getInput()), o1); + Assert.assertEquals(oracle.answerQuery(q1.getInput()), o1); + } + + private static class Oracle implements SingleQueryOracleMealy { + + private int count; + private final Set, Word>> queries; + + Oracle() { + this.count = 0; + this.queries = new HashSet<>(); + } + + @Override + public Word answerQuery(Word prefix, Word suffix) { + count++; + queries.add(Pair.of(prefix, suffix)); + return Word.fromString("abcdef"); + } + } + +}