Merge branch 'master' into scala2.13

CHANGES

@@ -1,3 +1,9 @@
+3.0.1
+
+  Bugfix for self-union or self-intersection of indexes (caused 
+a cartesian product).
+  Additional convenience methods for Index, e.g. unionLeft, unionRight, etc.
+
 3.0.0
 
   Support for indexes (k-mer databases) written as parquet files. 

README.md

@@ -280,7 +280,7 @@ For both notebook and REPL use, it is recommended to consult the API docs
 You can add Discount as a dependency using the following syntax (SBT):
 
 `
- libraryDependencies += "com.jnpersson" %% "discount" % "3.0.0"
+ libraryDependencies += "com.jnpersson" %% "discount" % "3.0.1"
 `
 
 Please note that Discount is still under heavy development and the API may change slightly even between minor versions.

build.sbt

@@ -1,6 +1,6 @@
 name := "Discount"
 
-version := "3.0.0"
+version := "3.0.1"
 
 scalaVersion := "2.13.10"
 

discount-aws.sh

@@ -12,7 +12,7 @@ BUCKET=s3://my-bucket/discount
 
 DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")"
 
-aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.0.jar" $BUCKET/
+aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" $BUCKET/
 
 #aws s3 sync "$DISCOUNT_HOME/resources/PASHA" $BUCKET/PASHA/
 
@@ -23,7 +23,7 @@ aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.0.jar" $BUCKET
 #To set SPLIT or other variables, uncomment below.
 COMMAND=( \
 #  --conf $SPLIT \
-  --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.0.0.jar $*)
+  --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.0.1.jar $*)
 
 #Turn off paging for output
 export AWS_PAGER=""

discount-gcloud.sh

@@ -33,6 +33,6 @@ PROPERTIES=$MAXRES
 DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")"
 
 exec gcloud --verbosity=info  dataproc jobs submit spark --region $REGION --cluster $CLUSTER \
-  --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.0.jar" \
+  --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" \
   --properties $PROPERTIES -- "$@"
 

discount-shell.sh

@@ -25,4 +25,4 @@ exec $SPARK/bin/spark-shell \
   --master $MASTER \
   --conf $MEMORY \
   --conf $LOCAL_DIR \
-  --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.0.jar"
+  --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar"

discount.sh

@@ -32,4 +32,4 @@ exec $SPARK/bin/spark-submit \
   --conf $MEMORY \
   --conf $LOCAL_DIR \
   --master $MASTER \
-  --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.0.jar" $*
+  --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" $*

docs/com/jnpersson/discount/bucket/BucketStats$.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="CountReducer.html" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>

docs/com/jnpersson/discount/bucket/BucketStats.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="CountReducer.html" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>

docs/com/jnpersson/discount/bucket/CountReducer.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>

docs/com/jnpersson/discount/bucket/CountersSubtractReducer.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="CountReducer.html" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>
@@ -210,7 +210,9 @@ <h4 id="signature" class="signature">
       </h4>
 
 
-          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>Implements the <a href="../spark/package$$Rule$$CountersSubtract$.html" class="extype" name="com.jnpersson.discount.spark.Rule.CountersSubtract">com.jnpersson.discount.spark.Rule.CountersSubtract</a> reduction rule</p></div><div class="toggleContainer block">
+          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>Implements the <a href="../spark/package$$Rule$$CountersSubtract$.html" class="extype" name="com.jnpersson.discount.spark.Rule.CountersSubtract">com.jnpersson.discount.spark.Rule.CountersSubtract</a> reduction rule.
+For each k-mer we calculate count_1 - count_2 and set the result to this value.
+Only positive counts are preserved in the output.</p></div><div class="toggleContainer block">
           <span class="toggle">
             Linear Supertypes
           </span>

docs/com/jnpersson/discount/bucket/KmerBucket.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="CountReducer.html" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="KmerSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>

docs/com/jnpersson/discount/bucket/KmerSubtractReducer.html

@@ -144,16 +144,16 @@ <h1>Packages</h1>
                         <a href="CountReducer.html" title="A reducer that handles k-mer count values stored in the longsForK(k) + 1 tag position.">CountReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule"></a>
-                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule">CountersSubtractReducer</a>
+                        <a class="class" href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule."></a>
+                        <a href="CountersSubtractReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.CountersSubtract reduction rule.">CountersSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g."></a>
                         <a href="KmerBucket.html" title="A bucket that maintains some number of super-mers and associated tags (e.g.">KmerBucket</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
-                        <a class="class" href="" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule"></a>
-                        <a href="" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule">KmerSubtractReducer</a>
+                        <a class="class" href="" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule."></a>
+                        <a href="" title="Implements the com.jnpersson.discount.spark.Rule.KmersSubtract reduction rule.">KmerSubtractReducer</a>
                       </li><li class="current-entities indented4">
                         <span class="separator"></span>
                         <a class="class" href="LeftReducer.html" title="Implements the com.jnpersson.discount.spark.Rule.Left reduction rule"></a>
@@ -210,7 +210,8 @@ <h4 id="signature" class="signature">
       </h4>
 
 
-          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>Implements the <a href="../spark/package$$Rule$$KmersSubtract$.html" class="extype" name="com.jnpersson.discount.spark.Rule.KmersSubtract">com.jnpersson.discount.spark.Rule.KmersSubtract</a> reduction rule</p></div><div class="toggleContainer block">
+          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>Implements the <a href="../spark/package$$Rule$$KmersSubtract$.html" class="extype" name="com.jnpersson.discount.spark.Rule.KmersSubtract">com.jnpersson.discount.spark.Rule.KmersSubtract</a> reduction rule.
+k-mers are kept if they existed in bucket A, but not in bucket B.</p></div><div class="toggleContainer block">
           <span class="toggle">
             Linear Supertypes
           </span>