Fixed multiple bugs in the expert mode:

- crash when adjusting conditions with nominal attributes (issue 126). - `mincov` parameter not set to the number of uncovered examples in the expert classification rules - complementary conditions not supported in the expert knowledge
adaa-polsl · Nov 22, 2024 · 24243cd · 24243cd
1 parent b4472d9
commit 24243cd
Show file tree

Hide file tree

Showing 16 changed files with 166 additions and 82 deletions.
diff --git a/adaa.analytics.rules/build.gradle b/adaa.analytics.rules/build.gradle
@@ -5,7 +5,7 @@ plugins {
     id 'java'
 }
 
-version = '2.1.22'
+version = '2.1.23'
 java {
     sourceCompatibility = JavaVersion.VERSION_1_8
 }

diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/AbstractFinder.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/AbstractFinder.java
@@ -30,6 +30,7 @@
 import adaa.analytics.rules.logic.representation.condition.ElementaryCondition;
 import adaa.analytics.rules.logic.representation.rule.Rule;
 import adaa.analytics.rules.utils.Logger;
+import adaa.analytics.rules.utils.Pair;
 
 /**
  * Abstract base class for growing and pruning procedures for all types of rules (classification, regression, survival).
@@ -350,7 +351,7 @@ public void postprocess(
 
 	/**
 	 * Abstract method representing all procedures which induce an elementary condition.
-	 * 
+	 *
 	 * @param rule Current rule.
 	 * @param trainSet Training set.
 	 * @param uncoveredByRuleset Set of examples uncovered by the model.
@@ -360,12 +361,12 @@ public void postprocess(
 	 * @return Induced elementary condition.
 	 */
 	protected abstract ElementaryCondition induceCondition(
-		final Rule rule,
-		final IExampleSet trainSet,
-		final Set<Integer> uncoveredByRuleset,
-		final Set<Integer> coveredByRule, 
-		final Set<IAttribute> allowedAttributes,
-		Object... extraParams);
+			final Rule rule,
+			final IExampleSet trainSet,
+			final Set<Integer> uncoveredByRuleset,
+			final Set<Integer> coveredByRule,
+			final Set<IAttribute> allowedAttributes,
+			Pair<String, Object>... extraParams);
 
 	/**
 	 * Maps a set of attribute names to a set of attributes.

diff --git a/...s/src/main/java/adaa/analytics/rules/logic/induction/ApproximateClassificationFinder.java b/...s/src/main/java/adaa/analytics/rules/logic/induction/ApproximateClassificationFinder.java
@@ -11,6 +11,7 @@
 import adaa.analytics.rules.logic.representation.valueset.SingletonSet;
 import adaa.analytics.rules.logic.representation.valueset.SingletonSetComplement;
 import adaa.analytics.rules.utils.Logger;
+import adaa.analytics.rules.utils.Pair;
 import tech.tablesaw.api.DoubleColumn;
 
 import java.util.*;
@@ -289,7 +290,7 @@ protected ElementaryCondition induceCondition(
             Set<Integer> uncoveredPositives,
             Set<Integer> coveredByRule,
             Set<IAttribute> allowedAttributes,
-            Object... extraParams) {
+            Pair<String,Object>... extraParams) {
 
         if (allowedAttributes.size() == 0) {
             return null;

diff --git a/....rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationExpertFinder.java b/....rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationExpertFinder.java
@@ -27,6 +27,7 @@
 import adaa.analytics.rules.logic.representation.valueset.SingletonSet;
 import adaa.analytics.rules.logic.representation.valueset.Universum;
 import adaa.analytics.rules.utils.Logger;
+import adaa.analytics.rules.utils.Pair;
 import org.apache.commons.lang3.SerializationUtils;
 import tech.tablesaw.api.DoubleColumn;
 
@@ -85,17 +86,20 @@ public void adjust(
 		CompoundCondition expertPremise = rule.getPremise();
 		rule.setPremise(new CompoundCondition());
 
-		HashSet<Integer> covered = new HashSet<Integer>();
+		//HashSet<Integer> covered = new HashSet<Integer>();
+		Set<Integer> covered = new IntegerBitSet(dataset.size());
 
 		// bit vectors for faster operations on coverings
 
 		covered.addAll(rule.getCoveredPositives());
 		covered.addAll(rule.getCoveredNegatives());
-		
+
 		for (ConditionBase cnd : expertPremise.getSubconditions()) {
 			ElementaryCondition ec = (ElementaryCondition)cnd;
 			ElementaryCondition newCondition;
-
+
+			boolean alreadyAdded = false;
+
 			if (ec.isAdjustable()) {
 				// determine attribute
 				Set<IAttribute> attr = new TreeSet<IAttribute>(new AttributeComparator());
@@ -113,13 +117,24 @@ public void adjust(
 					ec.evaluate(dataset, mustBeCovered);
 					mustBeCovered.retainAll(rule.getCoveredPositives());
 				}
-
-				newCondition = induceCondition(
-						rule, dataset, mustBeCovered, covered, attr);
-				newCondition.setType(Type.FORCED);
-				tryAddCondition(rule, null, newCondition, dataset, covered, uncoveredPositives);
-
-			} else {
+
+				if (mustBeCovered.size() > 0) {
+					newCondition = induceCondition(
+							rule, dataset, mustBeCovered, covered, attr, new Pair<String, Object>("disable_precision_control", (Object) true));
+
+					// use original condition if adjustment fails
+					if (newCondition != null) {
+						newCondition.setType(Type.FORCED);
+						tryAddCondition(rule, null, newCondition, dataset, covered, mustBeCovered);
+						alreadyAdded = true;
+					} else {
+						Logger.log("Unable to adjust condition: " + ec, Level.FINE);
+					}
+				} else {
+					Logger.log("Adjustable condition does not cover any positives: " + ec, Level.FINE);
+				}
+			}
+			if (!alreadyAdded) {
 				// add condition as it is without verification
 				IntegerBitSet conditionCovered = new IntegerBitSet(dataset.size());
 				newCondition = SerializationUtils.clone(ec);
@@ -305,7 +320,9 @@ public int grow(
 			int preferredCounter = knowledge.getPreferredAttributesPerRule();
 
 			do {
-				ElementaryCondition condition = induceCondition(rule, dataset, uncoveredPositives, covered, localAllowed, rule.getCoveredPositives());
+				ElementaryCondition condition = induceCondition(rule, dataset, uncoveredPositives, covered, localAllowed,
+						new Pair<String,Object>("covered_positives", rule.getCoveredPositives()));
+
 				carryOn = tryAddCondition(rule, null, condition, dataset, covered,uncoveredPositives);
 				// fixme: we are not sure if condition was added
 				if (carryOn) {
@@ -340,7 +357,8 @@ public int grow(
 
 			do {
 				ElementaryCondition condition = induceCondition(
-						rule, dataset, uncoveredPositives, covered, allowedAttributes, rule.getCoveredPositives());
+						rule, dataset, uncoveredPositives, covered, allowedAttributes,
+							new Pair<String,Object>("covered_positives", rule.getCoveredPositives()));
 
 				if (params.getSelectBestCandidate()) {
 					carryOn = tryAddCondition(currentRule, rule, condition, dataset, covered, uncoveredPositives);
@@ -367,9 +385,8 @@ public int grow(
 	 */
 	@Override
 	protected boolean checkCandidate(ElementaryCondition cnd, double classId, double p, double n, double new_p, double P,double uncoveredSize,  int ruleOrderNum) {
-		return new_p >= params.getAbsoluteMinimumCovered(P)
-				&& p >= params.getAbsoluteMinimumCoveredAll(P)
-				&& !knowledge.isForbidden(cnd.getAttribute(), cnd.getValueSet(), (int)classId);
+		boolean ok = super.checkCandidate(cnd, classId, p, n, new_p, P, uncoveredSize, ruleOrderNum);
+		return ok && !knowledge.isForbidden(cnd.getAttribute(), cnd.getValueSet(), (int)classId);
 
 	}
 

diff --git a/...ics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationExpertSnC.java b/...ics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationExpertSnC.java
@@ -229,9 +229,18 @@ public ClassificationRuleSet run(IExampleSet dataset)
 					uncoveredPositives.removeAll(rule.getCoveredPositives());
 					uncovered.removeAll(rule.getCoveredPositives());
 					uncovered.removeAll(rule.getCoveredNegatives());
-
+
+					double uncovered_p = 0;
+					if (dataset.getAttributes().getWeight() == null) {
+						uncovered_p = uncoveredPositives.size();
+					} else {
+						for (int id : uncoveredPositives) {
+							uncovered_p += weightDataColumnDoubleAdapter.getDouble(id);
+						}
+					}
+
 					// stop if no positive examples remaining
-					if (uncoveredPositives.size() == 0) {
+					if (uncovered_p <= params.getMaximumUncoveredFraction() * weighted_P) {
 						carryOn = false;
 					}
 

diff --git a/...lytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationFinder.java b/...lytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationFinder.java
@@ -32,6 +32,7 @@
 import adaa.analytics.rules.logic.representation.valueset.SingletonSet;
 import adaa.analytics.rules.logic.representation.valueset.SingletonSetComplement;
 import adaa.analytics.rules.utils.Logger;
+import adaa.analytics.rules.utils.Pair;
 import tech.tablesaw.api.DoubleColumn;
 
 
@@ -465,7 +466,7 @@ public void postprocess(
 
 	/**
 	 * Induces an elementary condition.
-	 * 
+	 *
 	 * @param rule Current rule.
 	 * @param trainSet Training set.
 	 * @param uncoveredPositives Set of positive examples uncovered by the model.
@@ -476,12 +477,12 @@ public void postprocess(
 	 */
 	@Override
 	protected ElementaryCondition induceCondition(
-		Rule rule,
-		IExampleSet trainSet,
-		Set<Integer> uncoveredPositives,
-		Set<Integer> coveredByRule, 
-		Set<IAttribute> allowedAttributes,
-		Object... extraParams) {
+			Rule rule,
+			IExampleSet trainSet,
+			Set<Integer> uncoveredPositives,
+			Set<Integer> coveredByRule,
+			Set<IAttribute> allowedAttributes,
+			Pair<String, Object>... extraParams) {
 
 		if (allowedAttributes.size() == 0) {
 			return null;
@@ -495,7 +496,17 @@ protected ElementaryCondition induceCondition(
 		double P = rule.getWeighted_P();
 		double N = rule.getWeighted_N();
 
-		double apriori_prec = params.isControlAprioriPrecision()
+
+		boolean controlPrecision = params.isControlAprioriPrecision();
+		// override precision verification
+		for (Pair<String, Object> param : extraParams) {
+			if (param.getFirst().equals("disable_precision_control") && (Boolean)param.getSecond()) {
+				controlPrecision = false;
+				break;
+			}
+		}
+
+		double apriori_prec = controlPrecision
 				? P / (P + N)
 				: Double.MIN_VALUE;
 

diff --git a/...analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationSnC.java b/...analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ClassificationSnC.java
@@ -165,9 +165,12 @@ public RuleSetBase run(IExampleSet dataset) {
 					uncovered.removeAll(rule.getCoveredNegatives());
 
 					uncovered_p = 0;
-
-					for (int id : uncoveredPositives) {
-						uncovered_p += dataset.getAttributes().getWeight() == null ? 1.0 : weightDataColumnDoubleAdapter.getDouble(id);
+					if (dataset.getAttributes().getWeight() == null) {
+						uncovered_p = uncoveredPositives.size();
+					} else {
+						for (int id : uncoveredPositives) {
+							uncovered_p += weightDataColumnDoubleAdapter.getDouble(id);
+						}
 					}
 
 					Logger.log("Uncovered positives" + uncovered_p + "\n", Level.FINER);

diff --git a/...tics.rules/src/main/java/adaa/analytics/rules/logic/induction/RegressionExpertFinder.java b/...tics.rules/src/main/java/adaa/analytics/rules/logic/induction/RegressionExpertFinder.java
@@ -62,6 +62,8 @@ public void adjust(
 
 			for (ConditionBase cnd : expertPremise.getSubconditions()) {
 				ElementaryCondition ec = (ElementaryCondition)cnd;
+				boolean alreadyAdded = false;
+
 				if (ec.isAdjustable()) {
 
 					// update covering information - needed for automatic induction
@@ -84,23 +86,26 @@ public void adjust(
 
 					} else {
 						// condition in other form - find best condition using this attribute with non-empty intersection with specified condition
-						mustBeCovered = new HashSet<Integer>();
-						for (int i : covered) {
-							if (ec.evaluate(dataset.getExample(i))) {
-								mustBeCovered.add(i);
-							}
-						}	
+						mustBeCovered = new IntegerBitSet(dataset.size());
+						ec.evaluate(dataset, mustBeCovered);
+						mustBeCovered.retainAll(rule.getCoveredPositives());
 					}
-
-					ElementaryCondition newCondition = induceCondition(
-							rule, dataset, mustBeCovered, covered, attr);
-
-					if (newCondition != null) {
-						newCondition.setType(Type.FORCED);
-						rule.getPremise().addSubcondition(newCondition);
+
+					if (!mustBeCovered.isEmpty()) {
+						ElementaryCondition newCondition = induceCondition(
+								rule, dataset, mustBeCovered, covered, attr);
+
+						if (newCondition != null) {
+							newCondition.setType(Type.FORCED);
+							rule.getPremise().addSubcondition(newCondition);
+							alreadyAdded = true;
+						}
+					} else {
+						Logger.log("Adjustable condition does not cover any positives: " + ec, Level.FINE);
 					}
-
-				} else {
+				}
+
+				if (!alreadyAdded) {
 					rule.getPremise().addSubcondition((ElementaryCondition)SerializationUtils.clone(ec));
 				}
 			}

diff --git a/....analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/RegressionFinder.java b/....analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/RegressionFinder.java
@@ -14,7 +14,6 @@
  ******************************************************************************/
 package adaa.analytics.rules.logic.induction;
 
-import adaa.analytics.rules.data.metadata.EColumnSortDirections;
 import adaa.analytics.rules.logic.representation.*;
 
 import adaa.analytics.rules.data.IAttribute;
@@ -29,6 +28,7 @@
 import adaa.analytics.rules.logic.representation.valueset.SingletonSet;
 import adaa.analytics.rules.logic.representation.valueset.SingletonSetComplement;
 import adaa.analytics.rules.utils.Logger;
+import adaa.analytics.rules.utils.Pair;
 import tech.tablesaw.api.DoubleColumn;
 
 import java.security.InvalidParameterException;
@@ -55,7 +55,7 @@ protected ElementaryCondition induceCondition_mean(
 			final Set<Integer> uncovered,
 			final Set<Integer> covered,
 			final Set<IAttribute> allowedAttributes,
-			Object... extraParams) {
+			Pair<String,Object>... extraParams) {
 
 		RegressionExampleSet set = (dataset instanceof RegressionExampleSet) ? (RegressionExampleSet)dataset : null;
 		if (set == null) {
@@ -300,12 +300,12 @@ class Stats{
 
 	@Override
 	protected ElementaryCondition induceCondition(
-		final Rule rule,
-		final IExampleSet dataset,
-		final Set<Integer> uncovered, 
-		final Set<Integer> covered, 
-		final Set<IAttribute> allowedAttributes,
-		Object... extraParams) {
+			final Rule rule,
+			final IExampleSet dataset,
+			final Set<Integer> uncovered,
+			final Set<Integer> covered,
+			final Set<IAttribute> allowedAttributes,
+			Pair<String, Object>... extraParams) {
 
 		if (allowedAttributes.size() == 0) {
 			return null;

diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/RuleParser.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/RuleParser.java
@@ -173,9 +173,10 @@ public static ElementaryCondition parseElementaryCondition(String s, IAttributes
 	    	if (Pattern.compile("Any").matcher(valueString).find()) {
 	    		valueSet = new Universum();
 	    	} else if (attributeMeta.isNominal()) { 
-	    		regex = Pattern.compile("\\{(?<discrete>.+)\\}");
+	    		regex = Pattern.compile("(?<negation>(!?))\\{(?<discrete>.+)\\}");
 		    	matcher = regex.matcher(valueString);
 		    	if (matcher.find()) {
+					String negation = matcher.group("negation");
 		    		String value = matcher.group("discrete");
 
 					if (value.equals("NaN") && isSurvival) {
@@ -188,7 +189,11 @@ public static ElementaryCondition parseElementaryCondition(String s, IAttributes
 							Logger.log("Invalid value <" + value + "> of the nominal attribute <" + attribute + ">" + "\n", Level.WARNING);
 							return null;
 						}
-						valueSet = new SingletonSet(v, mapping);
+						if (negation.isEmpty()) {
+							valueSet = new SingletonSet(v, mapping);
+						} else {
+							valueSet = new SingletonSetComplement(v, mapping);
+						}
 					}
 
 		    	}