diff --git a/.github/workflows/build-and-test-2.12.yml b/.github/workflows/build-and-test-2.12.yml index 5e1d8dbabd94..3ecadc7e1d9f 100644 --- a/.github/workflows/build-and-test-2.12.yml +++ b/.github/workflows/build-and-test-2.12.yml @@ -28,12 +28,12 @@ jobs: run: ./build/mvn clean install $MAVEN_CLI_OPTS -DskipTests -T4 - name: Unit tests id: test - run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS -o + run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS continue-on-error: true - name: Unit tests (retry) id: test-retry if: steps.test.outcome=='failure' - run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS -o + run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS - name: Integration Tests run: mvn failsafe:integration-test failsafe:verify $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS - name: Remove geomesa artifacts diff --git a/.github/workflows/build-and-test-2.13.yml b/.github/workflows/build-and-test-2.13.yml index 01e6e0026d2f..25fab2f11d0e 100644 --- a/.github/workflows/build-and-test-2.13.yml +++ b/.github/workflows/build-and-test-2.13.yml @@ -30,12 +30,12 @@ jobs: run: ./build/mvn clean install $MAVEN_CLI_OPTS -DskipTests -T4 - name: Unit tests id: test - run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS -o + run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS continue-on-error: true - name: Unit tests (retry) id: test-retry if: steps.test.outcome=='failure' - run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS -o + run: mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS - name: Integration Tests run: mvn failsafe:integration-test failsafe:verify $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS - name: Remove geomesa artifacts diff --git a/docs/common.py b/docs/common.py index cc369ee6d11b..d34593c4b15b 100644 --- a/docs/common.py +++ b/docs/common.py @@ -83,9 +83,9 @@ .. |geotools_version| replace:: %(gt_version)s -.. |accumulo_supported_versions| replace:: versions %(accumulo_version)s and %(accumulo_version_recommended)s +.. |accumulo_supported_versions| replace:: versions 2.0.1 and %(accumulo_version_recommended)s -.. |accumulo_required_version| replace:: %(accumulo_version)s or %(accumulo_version_recommended)s +.. |accumulo_required_version| replace:: 2.0.1 or %(accumulo_version_recommended)s .. |hbase_required_version| replace:: %(hbase_version)s diff --git a/docs/user/upgrade.rst b/docs/user/upgrade.rst index 7831669194d3..9266b6be6935 100644 --- a/docs/user/upgrade.rst +++ b/docs/user/upgrade.rst @@ -136,6 +136,7 @@ Dependency Version Upgrades The following dependencies have been upgraded: +* accumulo ``2.0.1`` -> ``2.1.2`` * aircompressor ``0.21`` -> ``0.25`` * antlr ``4.7.1`` -> ``4.7.2`` * aws-java-sdk ``1.11.179`` -> ``1.12.625`` @@ -143,7 +144,7 @@ The following dependencies have been upgraded: * cassandra-driver ``3.11.3`` -> ``3.11.5`` * com.clearspring.analytics ``2.9.2`` -> ``2.9.8`` * com.fasterxml.jackson ``2.14.1`` -> ``2.16.1`` -* com.jayway.jsonpath ``2.7.0`` -> ``2.8.0`` +* com.jayway.jsonpath ``2.7.0`` -> ``2.9.0`` * com.typesafe:config ``1.4.2`` -> ``1.4.3`` * commons-cli ``1.2`` -> ``1.6.0`` * commons-codec ``1.15`` -> ``1.16.0`` diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/pom.xml b/geomesa-accumulo/geomesa-accumulo-datastore/pom.xml index f6c7d88e5aee..d09c85fb557f 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-datastore/pom.xml @@ -20,6 +20,14 @@ org.locationtech.geomesa geomesa-index-api_${scala.binary.version} + + org.locationtech.geomesa + geomesa-accumulo-indices_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-accumulo-iterators_${scala.binary.version} + org.locationtech.geomesa geomesa-utils_${scala.binary.version} @@ -117,8 +125,17 @@ specs2-mock_${scala.binary.version} - org.apache.accumulo - accumulo-minicluster + org.testcontainers + testcontainers + + + org.geomesa.testcontainers + testcontainers-accumulo + + + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test io.dropwizard.metrics diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStore.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStore.scala index e5f6c9c34cf0..d5abaf54ed90 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStore.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStore.scala @@ -33,7 +33,7 @@ import org.locationtech.geomesa.index.index.z3.{XZ3Index, Z3Index} import org.locationtech.geomesa.index.metadata.{GeoMesaMetadata, MetadataStringSerializer} import org.locationtech.geomesa.index.utils.Explainer import org.locationtech.geomesa.utils.conf.FeatureExpiration.{FeatureTimeExpiration, IngestTimeExpiration} -import org.locationtech.geomesa.utils.conf.IndexId +import org.locationtech.geomesa.utils.conf.{FeatureExpiration, IndexId} import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.AttributeOptions @@ -207,9 +207,24 @@ class AccumuloDataStore(val connector: AccumuloClient, override val config: Accu stats.configureStatCombiner(connector, sft) } sft.getFeatureExpiration.foreach { - case IngestTimeExpiration(ttl) => AgeOffIterator.set(this, sft, ttl) - case FeatureTimeExpiration(dtg, _, ttl) => DtgAgeOffIterator.set(this, sft, ttl, dtg) - case e => throw new IllegalArgumentException(s"Unexpected feature expiration: $e") + case IngestTimeExpiration(ttl) => + val tableOps = connector.tableOperations() + getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach { table => + AgeOffIterator.set(tableOps, table, sft, ttl) + } + + case FeatureTimeExpiration(dtg, _, ttl) => + val tableOps = connector.tableOperations() + manager.indices(sft).foreach { index => + val indexSft = index match { + case joinIndex: AttributeJoinIndex => joinIndex.indexSft + case _ => sft + } + DtgAgeOffIterator.set(tableOps, indexSft, index, ttl, dtg) + } + + case e => + throw new IllegalArgumentException(s"Unexpected feature expiration: $e") } } @@ -236,7 +251,16 @@ class AccumuloDataStore(val connector: AccumuloClient, override val config: Accu // check any previous age-off - previously age-off wasn't tied to the sft metadata if (!sft.isFeatureExpirationEnabled && !previous.isFeatureExpirationEnabled) { // explicitly set age-off in the feature type if found - val configured = AgeOffIterator.expiry(this, previous).orElse(DtgAgeOffIterator.expiry(this, previous)) + val tableOps = connector.tableOperations() + val tables = getAllIndexTableNames(previous.getTypeName).filter(tableOps.exists) + val ageOff = tables.foldLeft[Option[FeatureExpiration]](None) { (res, table) => + res.orElse(AgeOffIterator.expiry(tableOps, table)) + } + val configured = ageOff.orElse { + tables.foldLeft[Option[FeatureExpiration]](None) { (res, table) => + res.orElse(DtgAgeOffIterator.expiry(tableOps, previous, table)) + } + } configured.foreach(sft.setFeatureExpiration) } @@ -253,19 +277,35 @@ class AccumuloDataStore(val connector: AccumuloClient, override val config: Accu stats.configureStatCombiner(connector, sft) } + val tableOps = connector.tableOperations() + val previousTables = getAllIndexTableNames(previous.getTypeName).filter(tableOps.exists) + val tables = getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists) + if (previous.isVisibilityRequired != sft.isVisibilityRequired) { - VisibilityIterator.clear(this, previous) + previousTables.foreach(VisibilityIterator.clear(tableOps, _)) if (sft.isVisibilityRequired) { - VisibilityIterator.set(this, sft) + tables.foreach(VisibilityIterator.set(tableOps, _)) } } - AgeOffIterator.clear(this, previous) - DtgAgeOffIterator.clear(this, previous) + previousTables.foreach { table => + AgeOffIterator.clear(tableOps, table) + DtgAgeOffIterator.clear(tableOps, table) + } sft.getFeatureExpiration.foreach { - case IngestTimeExpiration(ttl) => AgeOffIterator.set(this, sft, ttl) - case FeatureTimeExpiration(dtg, _, ttl) => DtgAgeOffIterator.set(this, sft, ttl, dtg) + case IngestTimeExpiration(ttl) => + tables.foreach(AgeOffIterator.set(tableOps, _, sft, ttl)) + + case FeatureTimeExpiration(dtg, _, ttl) => + manager.indices(sft).foreach { index => + val indexSft = index match { + case joinIndex: AttributeJoinIndex => joinIndex.indexSft + case _ => sft + } + DtgAgeOffIterator.set(tableOps, indexSft, index, ttl, dtg) + } + case e => throw new IllegalArgumentException(s"Unexpected feature expiration: $e") } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloIndexAdapter.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloIndexAdapter.scala index b60864a55489..440168856eb2 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloIndexAdapter.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloIndexAdapter.scala @@ -16,7 +16,7 @@ import org.apache.hadoop.io.Text import org.geotools.api.feature.simple.{SimpleFeature, SimpleFeatureType} import org.locationtech.geomesa.accumulo.data.AccumuloIndexAdapter.{AccumuloIndexWriter, AccumuloResultsToFeatures, ZIterPriority} import org.locationtech.geomesa.accumulo.data.AccumuloQueryPlan.{BatchScanPlan, EmptyPlan} -import org.locationtech.geomesa.accumulo.index.{AccumuloJoinIndex, JoinIndex} +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, JoinIndex} import org.locationtech.geomesa.accumulo.iterators.ArrowIterator.AccumuloArrowResultsToFeatures import org.locationtech.geomesa.accumulo.iterators.BinAggregatingIterator.AccumuloBinResultsToFeatures import org.locationtech.geomesa.accumulo.iterators.DensityIterator.AccumuloDensityResultsToFeatures @@ -178,8 +178,8 @@ class AccumuloIndexAdapter(ds: AccumuloDataStore) extends IndexAdapter[AccumuloD } index match { - case i: AccumuloJoinIndex => - i.createQueryPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + case i: AttributeJoinIndex => + AccumuloJoinIndexAdapter.createQueryPlan(ds, i, filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) case _ => val (iter, eToF, reduce) = if (strategy.hints.isBinQuery) { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloJoinIndex.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloJoinIndexAdapter.scala similarity index 58% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloJoinIndex.scala rename to geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloJoinIndexAdapter.scala index f3fe12be3cfc..d6b6e3208d0a 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloJoinIndex.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloJoinIndexAdapter.scala @@ -6,7 +6,7 @@ * http://www.opensource.org/licenses/apache2.0.php. ***********************************************************************/ -package org.locationtech.geomesa.accumulo.index +package org.locationtech.geomesa.accumulo.data import org.apache.accumulo.core.client.IteratorSetting import org.apache.accumulo.core.data.{Key, Range, Value} @@ -17,17 +17,18 @@ import org.geotools.feature.simple.SimpleFeatureTypeBuilder import org.geotools.util.factory.Hints import org.locationtech.geomesa.accumulo.data.AccumuloIndexAdapter.AccumuloResultsToFeatures import org.locationtech.geomesa.accumulo.data.AccumuloQueryPlan._ +import org.locationtech.geomesa.accumulo.index.AttributeJoinIndex import org.locationtech.geomesa.accumulo.data.{AccumuloDataStore, AccumuloIndexAdapter, AccumuloQueryPlan} import org.locationtech.geomesa.accumulo.iterators.ArrowIterator.AccumuloArrowResultsToFeatures import org.locationtech.geomesa.accumulo.iterators.BinAggregatingIterator.AccumuloBinResultsToFeatures import org.locationtech.geomesa.accumulo.iterators.DensityIterator.AccumuloDensityResultsToFeatures import org.locationtech.geomesa.accumulo.iterators.StatsIterator.AccumuloStatsResultsToFeatures import org.locationtech.geomesa.accumulo.iterators._ -import org.locationtech.geomesa.filter.{FilterHelper, andOption, partitionPrimarySpatials, partitionPrimaryTemporals} +import org.locationtech.geomesa.filter.{andOption, partitionPrimarySpatials, partitionPrimaryTemporals} import org.locationtech.geomesa.index.api.QueryPlan.{FeatureReducer, ResultsToFeatures} import org.locationtech.geomesa.index.api._ import org.locationtech.geomesa.index.conf.QueryHints -import org.locationtech.geomesa.index.index.attribute.{AttributeIndex, AttributeIndexKey, AttributeIndexValues} +import org.locationtech.geomesa.index.index.attribute.AttributeIndex import org.locationtech.geomesa.index.index.id.IdIndex import org.locationtech.geomesa.index.iterators.StatsScan import org.locationtech.geomesa.index.planning.LocalQueryRunner.{ArrowDictionaryHook, LocalTransformReducer} @@ -40,42 +41,9 @@ import scala.util.Try /** * Mixin trait to add join support to the normal attribute index class */ -trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], AttributeIndexKey] { - - this: AttributeIndex => +object AccumuloJoinIndexAdapter { import org.locationtech.geomesa.index.conf.QueryHints.RichHints - import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType - - import scala.collection.JavaConverters._ - - private val attribute = attributes.head - private val attributeIndex = sft.indexOf(attribute) - private val descriptor = sft.getDescriptor(attributeIndex) - private val binding = descriptor.getType.getBinding - val indexSft = IndexValueEncoder.getIndexSft(sft) - - override val name: String = JoinIndex.name - override val identifier: String = GeoMesaFeatureIndex.identifier(name, version, attributes) - - abstract override def getFilterStrategy( - filter: Filter, - transform: Option[SimpleFeatureType]): Option[FilterStrategy] = { - super.getFilterStrategy(filter, transform).flatMap { strategy => - // verify that it's ok to return join plans, and filter them out if not - if (!requiresJoin(strategy.secondary, transform)) { - Some(strategy) - } else if (!JoinIndex.AllowJoinPlans.get) { - None - } else { - val primary = strategy.primary.getOrElse(Filter.INCLUDE) - val bounds = FilterHelper.extractAttributeBounds(primary, attribute, binding) - val joinMultiplier = 9f + bounds.values.length // 10 plus 1 per additional range being scanned - val multiplier = strategy.costMultiplier * joinMultiplier - Some(FilterStrategy(strategy.index, strategy.primary, strategy.secondary, strategy.temporal, multiplier)) - } - } - } /** * Create a query plan against a join index - if possible, will use the reduced index-values to scan @@ -91,15 +59,20 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A * @param numThreads query threads * @return */ - def createQueryPlan(filter: FilterStrategy, - tables: Seq[String], - ranges: Seq[org.apache.accumulo.core.data.Range], - colFamily: Option[Text], - schema: SimpleFeatureType, - ecql: Option[Filter], - hints: Hints, - numThreads: Int): AccumuloQueryPlan = { - + def createQueryPlan( + ds: AccumuloDataStore, + index: AttributeJoinIndex, + filter: FilterStrategy, + tables: Seq[String], + ranges: Seq[org.apache.accumulo.core.data.Range], + colFamily: Option[Text], + schema: SimpleFeatureType, + ecql: Option[Filter], + hints: Hints, + numThreads: Int): AccumuloQueryPlan = { + + // TODO seems like this should be using 'schema' here, which may be a reduced version of the indexSft due to col groups + val indexSft = index.indexSft lazy val sort = hints.getSortFields lazy val max = hints.getMaxFeatures lazy val project = hints.getProjection @@ -108,15 +81,15 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A def plan( iters: Seq[IteratorSetting], kvsToFeatures: ResultsToFeatures[Entry[Key, Value]], - reduce: Option[FeatureReducer]) = + reduce: Option[FeatureReducer]): BatchScanPlan = BatchScanPlan(filter, tables, ranges, iters, colFamily, kvsToFeatures, reduce, sort, max, project, numThreads) val transform = hints.getTransformSchema // used when remote processing is disabled lazy val returnSchema = hints.getTransformSchema.getOrElse(indexSft) - lazy val fti = visibilityIter(indexSft) ++ FilterTransformIterator.configure(indexSft, this, ecql, hints).toSeq - lazy val resultsToFeatures = AccumuloResultsToFeatures(this, returnSchema) + lazy val fti = visibilityIter(index) ++ FilterTransformIterator.configure(indexSft, index, ecql, hints).toSeq + lazy val resultsToFeatures = AccumuloResultsToFeatures(index, returnSchema) lazy val localReducer = { val arrowHook = Some(ArrowDictionaryHook(ds.stats, filter.filter)) Some(new LocalTransformReducer(returnSchema, None, None, None, hints, arrowHook)) @@ -127,10 +100,10 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A if (indexSft.indexOf(hints.getBinTrackIdField) != -1 && hints.getBinGeomField.forall(indexSft.indexOf(_) != -1) && hints.getBinLabelField.forall(indexSft.indexOf(_) != -1) && - supportsFilter(ecql)) { - if (ds.asInstanceOf[AccumuloDataStore].config.remote.bin) { - val iter = BinAggregatingIterator.configure(indexSft, this, ecql, hints) - val iters = visibilityIter(indexSft) :+ iter + index.supportsFilter(ecql)) { + if (ds.config.remote.bin) { + val iter = BinAggregatingIterator.configure(indexSft, index, ecql, hints) + val iters = visibilityIter(index) :+ iter plan(iters, new AccumuloBinResultsToFeatures(), None) } else { if (hints.isSkipReduce) { @@ -142,14 +115,14 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A } } else { // have to do a join against the record table - createJoinPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + createJoinPlan(ds, index, filter, tables, ranges, colFamily, ecql, hints) } } else if (hints.isArrowQuery) { // check to see if we can execute against the index values - if (canUseIndexSchema(ecql, transform)) { - if (ds.asInstanceOf[AccumuloDataStore].config.remote.bin) { - val (iter, reduce) = ArrowIterator.configure(indexSft, this, ds.stats, filter.filter, ecql, hints) - val iters = visibilityIter(indexSft) :+ iter + if (index.canUseIndexSchema(ecql, transform)) { + if (ds.config.remote.bin) { + val (iter, reduce) = ArrowIterator.configure(indexSft, index, ds.stats, filter.filter, ecql, hints) + val iters = visibilityIter(index) :+ iter plan(iters, new AccumuloArrowResultsToFeatures(), Some(reduce)) } else { if (hints.isSkipReduce) { @@ -159,20 +132,20 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A } plan(fti, resultsToFeatures, localReducer) } - } else if (canUseIndexSchemaPlusKey(ecql, transform)) { + } else if (index.canUseIndexSchemaPlusKey(ecql, transform)) { val transformSft = transform.getOrElse { throw new IllegalStateException("Must have a transform for attribute key plus value scan") } // first filter and apply the transform - val filterTransformIter = FilterTransformIterator.configure(indexSft, this, ecql, hints, 23).get + val filterTransformIter = FilterTransformIterator.configure(indexSft, index, ecql, hints, 23).get // clear the transforms as we've already accounted for them hints.clearTransforms() // next add the attribute value from the row key - val rowValueIter = AttributeKeyValueIterator.configure(this, transformSft, 24) - if (ds.asInstanceOf[AccumuloDataStore].config.remote.bin) { + val rowValueIter = AttributeKeyValueIterator.configure(index.asInstanceOf[AttributeIndex], transformSft, 24) + if (ds.config.remote.bin) { // finally apply the arrow iterator on the resulting features - val (iter, reduce) = ArrowIterator.configure(transformSft, this, ds.stats, None, None, hints) - val iters = visibilityIter(indexSft) ++ Seq(filterTransformIter, rowValueIter, iter) + val (iter, reduce) = ArrowIterator.configure(transformSft, index, ds.stats, None, None, hints) + val iters = visibilityIter(index) ++ Seq(filterTransformIter, rowValueIter, iter) plan(iters, new AccumuloArrowResultsToFeatures(), Some(reduce)) } else { if (hints.isSkipReduce) { @@ -184,14 +157,14 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A } } else { // have to do a join against the record table - createJoinPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + createJoinPlan(ds, index, filter, tables, ranges, colFamily, ecql, hints) } } else if (hints.isDensityQuery) { // check to see if we can execute against the index values - val weightIsAttribute = hints.getDensityWeight.contains(attribute) - if (supportsFilter(ecql) && (weightIsAttribute || hints.getDensityWeight.forall(indexSft.indexOf(_) != -1))) { - if (ds.asInstanceOf[AccumuloDataStore].config.remote.bin) { - val visIter = visibilityIter(indexSft) + val weightIsAttribute = hints.getDensityWeight.contains(index.attributes.head) + if (index.supportsFilter(ecql) && (weightIsAttribute || hints.getDensityWeight.forall(indexSft.indexOf(_) != -1))) { + if (ds.config.remote.bin) { + val visIter = visibilityIter(index) val iters = if (weightIsAttribute) { // create a transform sft with the attribute added val transform = { @@ -199,7 +172,7 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A builder.setNamespaceURI(null: String) builder.setName(indexSft.getTypeName + "--attr") builder.setAttributes(indexSft.getAttributeDescriptors) - builder.add(sft.getDescriptor(attribute)) + builder.add(index.sft.getDescriptor(index.attributes.head)) if (indexSft.getGeometryDescriptor != null) { builder.setDefaultGeometry(indexSft.getGeometryDescriptor.getLocalName) } @@ -209,11 +182,11 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A tmp } // priority needs to be between vis iter (21) and density iter (25) - val keyValueIter = AttributeKeyValueIterator.configure(this, transform, 23) - val densityIter = DensityIterator.configure(transform, this, ecql, hints) + val keyValueIter = AttributeKeyValueIterator.configure(index.asInstanceOf[AttributeIndex], transform, 23) + val densityIter = DensityIterator.configure(transform, index, ecql, hints) visIter :+ keyValueIter :+ densityIter } else { - visIter :+ DensityIterator.configure(indexSft, this, ecql, hints) + visIter :+ DensityIterator.configure(indexSft, index, ecql, hints) } plan(iters, new AccumuloDensityResultsToFeatures(), None) } else { @@ -226,14 +199,14 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A } } else { // have to do a join against the record table - createJoinPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + createJoinPlan(ds, index, filter, tables, ranges, colFamily, ecql, hints) } } else if (hints.isStatsQuery) { // check to see if we can execute against the index values - if (Try(Stat(indexSft, hints.getStatsQuery)).isSuccess && supportsFilter(ecql)) { - if (ds.asInstanceOf[AccumuloDataStore].config.remote.bin) { - val iter = StatsIterator.configure(indexSft, this, ecql, hints) - val iters = visibilityIter(indexSft) :+ iter + if (Try(Stat(indexSft, hints.getStatsQuery)).isSuccess && index.supportsFilter(ecql)) { + if (ds.config.remote.bin) { + val iter = StatsIterator.configure(indexSft, index, ecql, hints) + val iters = visibilityIter(index) :+ iter val reduce = Some(StatsScan.StatsReducer(indexSft, hints)) plan(iters, new AccumuloStatsResultsToFeatures(), reduce) } else { @@ -246,35 +219,37 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A } } else { // have to do a join against the record table - createJoinPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + createJoinPlan(ds, index, filter, tables, ranges, colFamily, ecql, hints) } - } else if (canUseIndexSchema(ecql, transform)) { + } else if (index.canUseIndexSchema(ecql, transform)) { // we can use the index value // transform has to be non-empty to get here and can only include items // in the index value (not the index keys aka the attribute indexed) val transformSft = transform.getOrElse { throw new IllegalStateException("Must have a transform for attribute value scan") } - val iter = FilterTransformIterator.configure(indexSft, this, ecql, hints.getTransform, hints.getSampling) + val iter = FilterTransformIterator.configure(indexSft, index, ecql, hints.getTransform, hints.getSampling) // add the attribute-level vis iterator if necessary - val iters = visibilityIter(schema) ++ iter.toSeq + val iters = visibilityIter(index) ++ iter.toSeq // need to use transform to convert key/values - val toFeatures = AccumuloResultsToFeatures(this, transformSft) + val toFeatures = AccumuloResultsToFeatures(index, transformSft) plan(iters, toFeatures, None) - } else if (canUseIndexSchemaPlusKey(ecql, transform)) { + } else if (index.canUseIndexSchemaPlusKey(ecql, transform)) { // we can use the index PLUS the value val transformSft = transform.getOrElse { throw new IllegalStateException("Must have a transform for attribute key plus value scan") } - val iter = FilterTransformIterator.configure(indexSft, this, ecql, hints.getTransform, hints.getSampling) + val iter = FilterTransformIterator.configure(indexSft, index, ecql, hints.getTransform, hints.getSampling) // add the attribute-level vis iterator if necessary - val iters = visibilityIter(schema) ++ iter.toSeq :+ AttributeKeyValueIterator.configure(this, transformSft) + val iters = + visibilityIter(index) ++ iter.toSeq :+ + AttributeKeyValueIterator.configure(index.asInstanceOf[AttributeIndex], transformSft) // need to use transform to convert key/values - val toFeatures = AccumuloResultsToFeatures(this, transformSft) + val toFeatures = AccumuloResultsToFeatures(index, transformSft) plan(iters, toFeatures, None) } else { // have to do a join against the record table - createJoinPlan(filter, tables, ranges, colFamily, schema, ecql, hints, numThreads) + createJoinPlan(ds, index, filter, tables, ranges, colFamily, ecql, hints) } if (ranges.nonEmpty) { qp } else { EmptyPlan(qp.filter, qp.reducer) } @@ -284,42 +259,43 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A * Gets a query plan comprised of a join against the record table. This is the slowest way to * execute a query, so we avoid it if possible. */ - private def createJoinPlan(filter: FilterStrategy, - tables: Seq[String], - ranges: Seq[org.apache.accumulo.core.data.Range], - colFamily: Option[Text], - schema: SimpleFeatureType, - ecql: Option[Filter], - hints: Hints, - numThreads: Int): AccumuloQueryPlan = { + private def createJoinPlan( + ds: AccumuloDataStore, + index: AttributeJoinIndex, + filter: FilterStrategy, + tables: Seq[String], + ranges: Seq[org.apache.accumulo.core.data.Range], + colFamily: Option[Text], + ecql: Option[Filter], + hints: Hints): AccumuloQueryPlan = { import org.locationtech.geomesa.filter.ff import org.locationtech.geomesa.index.conf.QueryHints.RichHints import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType // apply any secondary filters or transforms against the record table - val recordIndex = ds.manager.indices(sft, IndexMode.Read).find(_.name == IdIndex.name).getOrElse { + val recordIndex = ds.manager.indices(index.sft, IndexMode.Read).find(_.name == IdIndex.name).getOrElse { throw new RuntimeException("Id index does not exist for join query: " + - ds.manager.indices(sft, IndexMode.Read).map(_.identifier).mkString(", ")) + ds.manager.indices(index.sft, IndexMode.Read).map(_.identifier).mkString(", ")) } // break out the st filter to evaluate against the attribute table val (stFilter, ecqlFilter) = ecql.map { f => - val (geomFilters, otherFilters) = partitionPrimarySpatials(f, sft) - val (temporalFilters, nonSTFilters) = partitionPrimaryTemporals(otherFilters, sft) + val (geomFilters, otherFilters) = partitionPrimarySpatials(f, index.sft) + val (temporalFilters, nonSTFilters) = partitionPrimaryTemporals(otherFilters, index.sft) (andOption(geomFilters ++ temporalFilters), andOption(nonSTFilters)) }.getOrElse((None, None)) val (recordColFamily, recordSchema) = { - val (cf, s) = ds.adapter.groups.group(sft, hints.getTransformDefinition, ecqlFilter) + val (cf, s) = ds.adapter.groups.group(index.sft, hints.getTransformDefinition, ecqlFilter) (Some(new Text(AccumuloIndexAdapter.mapColumnFamily(recordIndex)(cf))), s) } // since each range is a single row, it wouldn't be very efficient to do any aggregating scans // instead, handle them with the local query runner - val resultSft = hints.getTransformSchema.getOrElse(sft) + val resultSft = hints.getTransformSchema.getOrElse(index.sft) val recordIterators = { val recordIter = FilterTransformIterator.configure(recordSchema, recordIndex, ecqlFilter, hints).toSeq - if (sft.getVisibilityLevel != VisibilityLevel.Attribute) { recordIter } else { + if (index.sft.getVisibilityLevel != VisibilityLevel.Attribute) { recordIter } else { Seq(KryoVisibilityRowEncoder.configure(recordSchema)) ++ recordIter } } @@ -328,67 +304,33 @@ trait AccumuloJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], A val reducer = new LocalTransformReducer(resultSft, None, None, None, hints, hook) val recordTables = recordIndex.getTablesForQuery(filter.filter) - val recordThreads = ds.asInstanceOf[AccumuloDataStore].config.queries.recordThreads + val recordThreads = ds.config.queries.recordThreads // function to join the attribute index scan results to the record table // have to pull the feature id from the row val joinFunction: JoinFunction = { - val prefix = sft.getTableSharingBytes - val idToBytes = GeoMesaFeatureIndex.idToBytes(sft) + val prefix = index.sft.getTableSharingBytes + val idToBytes = GeoMesaFeatureIndex.idToBytes(index.sft) kv => { val row = kv.getKey.getRow - new Range(new Text(ByteArrays.concat(prefix, idToBytes(getIdFromRow(row.getBytes, 0, row.getLength, null))))) + new Range(new Text(ByteArrays.concat(prefix, idToBytes(index.getIdFromRow(row.getBytes, 0, row.getLength, null))))) } } val joinQuery = BatchScanPlan(filter, recordTables, Seq.empty, recordIterators, recordColFamily, toFeatures, Some(reducer), hints.getSortFields, hints.getMaxFeatures, hints.getProjection, recordThreads) - val attributeIters = visibilityIter(indexSft) ++ - FilterTransformIterator.configure(indexSft, this, stFilter, None, hints.getSampling).toSeq + val attributeIters = visibilityIter(index) ++ + FilterTransformIterator.configure(index.indexSft, index, stFilter, None, hints.getSampling).toSeq JoinPlan(filter, tables, ranges, attributeIters, colFamily, recordThreads, joinFunction, joinQuery) } - private def visibilityIter(schema: SimpleFeatureType): Seq[IteratorSetting] = sft.getVisibilityLevel match { - case VisibilityLevel.Feature => Seq.empty - case VisibilityLevel.Attribute => Seq(KryoVisibilityRowEncoder.configure(schema)) - } - - /** - * Does the query require a join against the record table, or can it be satisfied - * in a single scan - * - * @param filter non-attribute filter being evaluated, if any - * @param transform transform being applied, if any - * @return - */ - private def requiresJoin(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = - !canUseIndexSchema(filter, transform) && !canUseIndexSchemaPlusKey(filter, transform) - - /** - * Determines if the given filter and transform can operate on index encoded values. - */ - private def canUseIndexSchema(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = { - // verify that transform *does* exist and only contains fields in the index sft, - // and that filter *does not* exist or can be fulfilled by the index sft - supportsTransform(transform) && supportsFilter(filter) - } - - /** - * Determines if the given filter and transform can operate on index encoded values - * in addition to the values actually encoded in the attribute index keys - */ - private def canUseIndexSchemaPlusKey(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = { - transform.exists { t => - val attributes = t.getAttributeDescriptors.asScala.map(_.getLocalName) - attributes.forall(a => a == attribute || indexSft.indexOf(a) != -1) && supportsFilter(filter) + private def visibilityIter(index: AttributeJoinIndex): Seq[IteratorSetting] = { + import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType + index.sft.getVisibilityLevel match { + case VisibilityLevel.Feature => Seq.empty + case VisibilityLevel.Attribute => Seq(KryoVisibilityRowEncoder.configure(index.indexSft)) } } - - private def supportsTransform(transform: Option[SimpleFeatureType]): Boolean = - transform.exists(_.getAttributeDescriptors.asScala.map(_.getLocalName).forall(indexSft.indexOf(_) != -1)) - - private def supportsFilter(filter: Option[Filter]): Boolean = - filter.forall(FilterHelper.propertyNames(_, sft).forall(indexSft.indexOf(_) != -1)) } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloWritableFeature.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloWritableFeature.scala index abe55f84e48b..43dd8f29ae27 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloWritableFeature.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/AccumuloWritableFeature.scala @@ -9,7 +9,7 @@ package org.locationtech.geomesa.accumulo.data import org.geotools.api.feature.simple.{SimpleFeature, SimpleFeatureType} -import org.locationtech.geomesa.accumulo.index.{AccumuloJoinIndex, IndexValueEncoder} +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, IndexValueEncoder, ReducedIndexValues} import org.locationtech.geomesa.features.{ScalaSimpleFeature, SimpleFeatureSerializer} import org.locationtech.geomesa.index.api.WritableFeature.{AttributeLevelWritableFeature, FeatureWrapper} import org.locationtech.geomesa.index.api.{GeoMesaFeatureIndex, KeyValue, WritableFeature} @@ -21,13 +21,10 @@ import org.locationtech.geomesa.utils.index.VisibilityLevel * * @param delegate base writable feature */ -abstract class AccumuloWritableFeature(delegate: WritableFeature) extends WritableFeature { - +abstract class AccumuloWritableFeature(delegate: WritableFeature) extends WritableFeature with ReducedIndexValues { override val feature: SimpleFeature = delegate.feature override val values: Seq[KeyValue] = delegate.values override val id: Array[Byte] = delegate.id - - def indexValues: Seq[KeyValue] } object AccumuloWritableFeature { @@ -40,7 +37,7 @@ object AccumuloWritableFeature { indices: Seq[GeoMesaFeatureIndex[_, _]]): FeatureWrapper[WritableFeature] = { // make sure to provide our index values for attribute join indices if we need them val base = WritableFeature.wrapper(sft, groups) - if (indices.forall(i => !i.isInstanceOf[AccumuloJoinIndex])) { base } else { + if (indices.forall(i => !i.isInstanceOf[AttributeJoinIndex])) { base } else { val serializer = IndexValueEncoder(sft) if (sft.getVisibilityLevel == VisibilityLevel.Attribute) { new AccumuloAttributeLevelFeatureWrapper(base, serializer) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/AccumuloGeoMesaStats.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/AccumuloGeoMesaStats.scala index 9db311e4a03a..7fe6a3f4ae3e 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/AccumuloGeoMesaStats.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/AccumuloGeoMesaStats.scala @@ -11,7 +11,9 @@ package org.locationtech.geomesa.accumulo.data.stats import org.apache.accumulo.core.client.AccumuloClient import org.apache.hadoop.io.Text import org.geotools.api.feature.simple.SimpleFeatureType +import org.locationtech.geomesa.accumulo.combiners.StatsCombiner import org.locationtech.geomesa.accumulo.data._ +import org.locationtech.geomesa.accumulo.util.TableUtils import org.locationtech.geomesa.index.stats.GeoMesaStats.{GeoMesaStatWriter, StatUpdater} import org.locationtech.geomesa.index.stats.MetadataBackedStats.{StatsMetadataSerializer, WritableStat} import org.locationtech.geomesa.index.stats._ @@ -75,6 +77,7 @@ class AccumuloGeoMesaStats(ds: AccumuloDataStore, val metadata: AccumuloBackedMe def configureStatCombiner(connector: AccumuloClient, sft: SimpleFeatureType): Unit = { import MetadataBackedStats._ + TableUtils.createTableIfNeeded(connector, metadata.table) StatsCombiner.configure(sft, connector, metadata.table, metadata.typeNameSeparator.toString) val keys = Seq(CountKey, BoundsKeyPrefix, TopKKeyPrefix, FrequencyKeyPrefix, HistogramKeyPrefix) @@ -149,12 +152,10 @@ class AccumuloGeoMesaStats(ds: AccumuloDataStore, val metadata: AccumuloBackedMe object AccumuloGeoMesaStats { - val CombinerName = "stats-combiner" - def apply(ds: AccumuloDataStore): AccumuloGeoMesaStats = { val table = s"${ds.config.catalog}_stats" new AccumuloGeoMesaStats(ds, new AccumuloBackedMetadata(ds.connector, table, new StatsMetadataSerializer(ds))) } - private [stats] val executor = ExitingExecutor(new ScheduledThreadPoolExecutor(3), force = true) + private[stats] val executor = ExitingExecutor(new ScheduledThreadPoolExecutor(3), force = true) } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/AccumuloContainer.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/AccumuloContainer.scala new file mode 100644 index 000000000000..7c308cb25b3e --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/AccumuloContainer.scala @@ -0,0 +1,90 @@ +/*********************************************************************** + * Copyright (c) 2013-2024 Commonwealth Computer Research, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License, Version 2.0 + * which accompanies this distribution and is available at + * http://www.opensource.org/licenses/apache2.0.php. + ***********************************************************************/ + +package org.locationtech.geomesa.accumulo + +import com.typesafe.scalalogging.StrictLogging +import org.apache.accumulo.core.client.security.tokens.PasswordToken +import org.apache.accumulo.core.security.{Authorizations, NamespacePermission, SystemPermission} +import org.geomesa.testcontainers.AccumuloContainer +import org.locationtech.geomesa.utils.io.WithClose +import org.testcontainers.utility.DockerImageName + +import java.util.concurrent.atomic.AtomicBoolean +import scala.util.Try + +case object AccumuloContainer extends StrictLogging { + + val ImageName = + DockerImageName.parse("ghcr.io/geomesa/accumulo-uno") + .withTag(sys.props.getOrElse("accumulo.docker.tag", "2.1.2")) + + val Namespace = "gm" + + lazy val instanceName = Container.getInstanceName + lazy val zookeepers = Container.getZookeepers + lazy val user = Container.getUsername + lazy val password = Container.getPassword + + lazy val Container: AccumuloContainer = { + val container = tryContainer.get + WithClose(container.client()) { client => + client.namespaceOperations().create(Namespace) + val secOps = client.securityOperations() + secOps.changeUserAuthorizations(Users.root.name, Users.root.auths) + Seq(Users.admin, Users.user).foreach { case UserWithAuths(name, password, auths) => + secOps.createLocalUser(name, new PasswordToken(password)) + SystemPermissions.foreach(p => secOps.grantSystemPermission(name, p)) + NamespacePermissions.foreach(p => secOps.grantNamespacePermission(name, Namespace, p)) + client.securityOperations().changeUserAuthorizations(name, auths) + } + } + container + } + + private lazy val tryContainer: Try[AccumuloContainer] = Try { + logger.info("Starting Accumulo container") + val container = new AccumuloContainer(ImageName).withGeoMesaDistributedRuntime() + initialized.getAndSet(true) + container.start() + logger.info("Started Accumulo container") + container + } + + private val initialized = new AtomicBoolean(false) + + sys.addShutdownHook({ + if (initialized.get) { + logger.info("Stopping Accumulo container") + tryContainer.foreach(_.stop()) + logger.info("Stopped Accumulo container") + } + }) + + case class UserWithAuths(name: String, password: String, auths: Authorizations) + + object Users { + val root = UserWithAuths("root", "secret", new Authorizations("admin", "user", "system")) + val admin = UserWithAuths("admin", "secret", new Authorizations("admin", "user")) + val user = UserWithAuths("user", "secret", new Authorizations("user")) + } + + private val SystemPermissions = Seq( + SystemPermission.CREATE_NAMESPACE, + SystemPermission.ALTER_NAMESPACE, + SystemPermission.DROP_NAMESPACE + ) + + private val NamespacePermissions = Seq( + NamespacePermission.READ, + NamespacePermission.WRITE, + NamespacePermission.CREATE_TABLE, + NamespacePermission.ALTER_TABLE, + NamespacePermission.DROP_TABLE + ) +} diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/MiniCluster.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/MiniCluster.scala deleted file mode 100644 index 2cdc053ce2b5..000000000000 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/MiniCluster.scala +++ /dev/null @@ -1,103 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2013-2024 Commonwealth Computer Research, Inc. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License, Version 2.0 - * which accompanies this distribution and is available at - * http://www.opensource.org/licenses/apache2.0.php. - ***********************************************************************/ - -package org.locationtech.geomesa.accumulo - -import com.typesafe.scalalogging.LazyLogging -import org.apache.accumulo.core.client.security.tokens.PasswordToken -import org.apache.accumulo.core.security.{Authorizations, NamespacePermission, SystemPermission} -import org.apache.accumulo.minicluster.{MiniAccumuloCluster, MiniAccumuloConfig} -import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl -import org.locationtech.geomesa.utils.io.{PathUtils, WithClose} - -import java.io.{File, FileWriter} -import java.nio.file.Files -import scala.collection.JavaConverters._ -import scala.util.Try - -case object MiniCluster extends LazyLogging { - - private val miniClusterTempDir = Files.createTempDirectory("gm-mini-acc-") - - private val systemPermissions = Seq( - SystemPermission.CREATE_NAMESPACE, - SystemPermission.ALTER_NAMESPACE, - SystemPermission.DROP_NAMESPACE - ) - - private val namespacePermissions = Seq( - NamespacePermission.READ, - NamespacePermission.WRITE, - NamespacePermission.CREATE_TABLE, - NamespacePermission.ALTER_TABLE, - NamespacePermission.DROP_TABLE - ) - - val namespace = "gm" - - lazy val cluster: MiniAccumuloCluster = tryCluster.get - - lazy private val tryCluster: Try[MiniAccumuloCluster] = Try { - logger.info(s"Starting Accumulo minicluster at $miniClusterTempDir") - - val config = new MiniAccumuloConfig(miniClusterTempDir.toFile, Users.root.password) - sys.props.get("geomesa.accumulo.test.tablet.servers").map(_.toInt).foreach(config.setNumTservers) - config.setDefaultMemory(256, org.apache.accumulo.minicluster.MemoryUnit.MEGABYTE) // default is 128MB - - // Use reflection to access a package-private method to set system properties before starting - // the minicluster. It is possible that this could break with future versions of Accumulo. - val configGetImpl = config.getClass.getDeclaredMethod("getImpl") - val systemProps = Map.empty[String, String] ++ - Option("zookeeper.jmx.log4j.disable").flatMap(key => sys.props.get(key).map(key -> _)) - configGetImpl.setAccessible(true) - configGetImpl.invoke(config).asInstanceOf[MiniAccumuloConfigImpl].setSystemProperties(systemProps.asJava) - - val cluster = new MiniAccumuloCluster(config) - // required for zookeeper 3.5 - WithClose(new FileWriter(new File(miniClusterTempDir.toFile, "conf/zoo.cfg"), true)) { writer => - writer.write("admin.enableServer=false\n") // disable the admin server, which tries to bind to 8080 - writer.write("4lw.commands.whitelist=*\n") // enable 'ruok', which the minicluster uses to check zk status - writer.write("snapshot.compression.method=gz\n") - } - cluster.start() - - // set up users and authorizations - val connector = cluster.createAccumuloClient(Users.root.name, new PasswordToken(Users.root.password)) - connector.namespaceOperations().create(namespace) - Seq(Users.root, Users.admin, Users.user).foreach { case UserWithAuths(name, password, auths) => - if (name != Users.root.name) { - connector.securityOperations().createLocalUser(name, new PasswordToken(password)) - systemPermissions.foreach(p => connector.securityOperations().grantSystemPermission(name, p)) - namespacePermissions.foreach(p => connector.securityOperations().grantNamespacePermission(name, namespace, p)) - } - connector.securityOperations().changeUserAuthorizations(name, auths) - } - - connector.close() - - logger.info("Started Accumulo minicluster") - - cluster - } - - sys.addShutdownHook({ - logger.info("Stopping Accumulo minicluster") - try { cluster.stop() } finally { - PathUtils.deleteRecursively(miniClusterTempDir) - } - logger.info("Stopped Accumulo minicluster") - }) - - case class UserWithAuths(name: String, password: String, auths: Authorizations) - - object Users { - val root = UserWithAuths("root", "secret", new Authorizations("admin", "user", "system")) - val admin = UserWithAuths("admin", "secret", new Authorizations("admin", "user")) - val user = UserWithAuths("user", "secret", new Authorizations("user")) - } -} diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/TestWithDataStore.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/TestWithDataStore.scala index a4df7119ddbf..998c041a8add 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/TestWithDataStore.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/TestWithDataStore.scala @@ -27,22 +27,22 @@ import scala.collection.JavaConverters._ trait TestWithDataStore extends Specification { // we use class name to prevent spillage between unit tests - lazy val catalog = s"${MiniCluster.namespace}.${getClass.getSimpleName}" + lazy val catalog = s"${AccumuloContainer.Namespace}.${getClass.getSimpleName}" // note the table needs to be different to prevent tests from conflicting with each other lazy val dsParams: Map[String, String] = Map( - AccumuloDataStoreParams.InstanceNameParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, AccumuloDataStoreParams.CatalogParam.key -> catalog ) lazy val ds = DataStoreFinder.getDataStore(dsParams.asJava).asInstanceOf[AccumuloDataStore] - lazy val root = MiniCluster.Users.root - lazy val admin = MiniCluster.Users.admin - lazy val user = MiniCluster.Users.user + lazy val root = AccumuloContainer.Users.root + lazy val admin = AccumuloContainer.Users.admin + lazy val user = AccumuloContainer.Users.user override def map(fragments: => Fragments): Fragments = fragments ^ fragmentFactory.step { ds.delete() diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/audit/AccumuloQueryEventTransformTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/audit/AccumuloQueryEventTransformTest.scala index 0ee970cde114..a9168b1e456b 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/audit/AccumuloQueryEventTransformTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/audit/AccumuloQueryEventTransformTest.scala @@ -9,11 +9,9 @@ package org.locationtech.geomesa.accumulo.audit import org.apache.accumulo.core.client.BatchWriterConfig -import org.apache.accumulo.core.client.security.tokens.PasswordToken import org.apache.accumulo.core.security.Authorizations import org.junit.runner.RunWith -import org.locationtech.geomesa.accumulo.MiniCluster -import org.locationtech.geomesa.accumulo.MiniCluster.Users +import org.locationtech.geomesa.accumulo.AccumuloContainer import org.locationtech.geomesa.index.audit.QueryEvent import org.locationtech.geomesa.utils.io.WithClose import org.specs2.mutable.Specification @@ -24,7 +22,7 @@ class AccumuloQueryEventTransformTest extends Specification { import scala.collection.JavaConverters._ - lazy val connector = MiniCluster.cluster.createAccumuloClient(Users.root.name, new PasswordToken(Users.root.password)) + lazy val connector = AccumuloContainer.Container.client() "AccumuloQueryEventTransform" should { "Convert from and to mutations" in { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreAuthTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreAuthTest.scala index aa38e6791bcc..4bb0b77cb17b 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreAuthTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreAuthTest.scala @@ -16,7 +16,7 @@ import org.geotools.filter.text.ecql.ECQL import org.junit.runner.RunWith import org.locationtech.geomesa.accumulo.TestWithFeatureType import org.locationtech.geomesa.features.ScalaSimpleFeature -import org.locationtech.geomesa.security.{AuthorizationsProvider, DefaultAuthorizationsProvider, FilteringAuthorizationsProvider, SecurityUtils} +import org.locationtech.geomesa.security.{AuthorizationsProvider, DefaultAuthorizationsProvider, FilteringAuthorizationsProvider, GEOMESA_AUTH_PROVIDER_IMPL, SecurityUtils} import org.locationtech.geomesa.utils.collection.SelfClosingIterator import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes import org.specs2.runner.JUnitRunner @@ -32,8 +32,6 @@ class AccumuloDataStoreAuthTest extends TestWithFeatureType { import scala.collection.JavaConverters._ - sequential - override val spec = "name:String:index=join,dtg:Date,*geom:Point:srid=4326" step { @@ -79,12 +77,12 @@ class AccumuloDataStoreAuthTest extends TestWithFeatureType { } "fail when auth provider system property does not match an actual class" >> { - System.setProperty(AuthorizationsProvider.AUTH_PROVIDER_SYS_PROPERTY, "my.fake.Clas") + GEOMESA_AUTH_PROVIDER_IMPL.threadLocalValue.set("my.fake.Clas") try { val params = dsParams ++ Map(AuthsParam.key -> "user,admin,system") DataStoreFinder.getDataStore(params.asJava).asInstanceOf[AccumuloDataStore] must throwAn[IllegalArgumentException] } finally { - System.clearProperty(AuthorizationsProvider.AUTH_PROVIDER_SYS_PROPERTY) + GEOMESA_AUTH_PROVIDER_IMPL.threadLocalValue.remove() } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreDeleteTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreDeleteTest.scala index cfdc72b327aa..bc6d900eabf6 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreDeleteTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreDeleteTest.scala @@ -30,8 +30,6 @@ class AccumuloDataStoreDeleteTest extends Specification with TestWithMultipleSft import scala.collection.JavaConverters._ - sequential - lazy val tableOps = ds.connector.tableOperations() def createFeature(schema: String = "name:String:index=join,*geom:Point:srid=4326,dtg:Date") = { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFactoryTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFactoryTest.scala index 1aa2eeca2efc..73b0b51bde4b 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFactoryTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFactoryTest.scala @@ -11,7 +11,7 @@ package org.locationtech.geomesa.accumulo.data import org.geotools.api.data.DataStoreFinder import org.junit.runner.RunWith -import org.locationtech.geomesa.accumulo.MiniCluster +import org.locationtech.geomesa.accumulo.AccumuloContainer import org.specs2.mutable.Specification import org.specs2.runner.JUnitRunner @@ -23,17 +23,17 @@ class AccumuloDataStoreFactoryTest extends Specification { import scala.collection.JavaConverters._ // we use class name to prevent spillage between unit tests - lazy val catalog = s"${MiniCluster.namespace}.${getClass.getSimpleName}" + lazy val catalog = s"${AccumuloContainer.Namespace}.${getClass.getSimpleName}" "AccumuloDataStoreFactory" should { "create a password authenticated store" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue val ds = DataStoreFinder.getDataStore(params) @@ -48,11 +48,11 @@ class AccumuloDataStoreFactoryTest extends Specification { "create a keytab authenticated store" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.KeytabPathParam.key -> "/path/to/keytab", - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.KeytabPathParam.key -> "/path/to/keytab", + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue // TODO GEOMESA-2797 test kerberos @@ -60,12 +60,12 @@ class AccumuloDataStoreFactoryTest extends Specification { "not accept password and keytab" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, - AccumuloDataStoreParams.KeytabPathParam.key -> "/path/to/keytab", - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, + AccumuloDataStoreParams.KeytabPathParam.key -> "/path/to/keytab", + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue DataStoreFinder.getDataStore(params) must throwAn[IllegalArgumentException] @@ -73,9 +73,9 @@ class AccumuloDataStoreFactoryTest extends Specification { "not accept a missing instanceId" in { val params = Map( - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue @@ -84,10 +84,10 @@ class AccumuloDataStoreFactoryTest extends Specification { "not accept a missing zookeepers" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue DataStoreFinder.getDataStore(params) must throwAn[IOException] @@ -95,10 +95,10 @@ class AccumuloDataStoreFactoryTest extends Specification { "not accept a missing user" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.PasswordParam.key -> MiniCluster.Users.root.password, - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.PasswordParam.key -> AccumuloContainer.password, + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue DataStoreFinder.getDataStore(params) must throwAn[IOException] @@ -106,10 +106,10 @@ class AccumuloDataStoreFactoryTest extends Specification { "not accept a missing password and keytab" in { val params = Map( - AccumuloDataStoreParams.InstanceIdParam.key -> MiniCluster.cluster.getInstanceName, - AccumuloDataStoreParams.ZookeepersParam.key -> MiniCluster.cluster.getZooKeepers, - AccumuloDataStoreParams.UserParam.key -> MiniCluster.Users.root.name, - AccumuloDataStoreParams.CatalogParam.key -> catalog + AccumuloDataStoreParams.InstanceNameParam.key -> AccumuloContainer.instanceName, + AccumuloDataStoreParams.ZookeepersParam.key -> AccumuloContainer.zookeepers, + AccumuloDataStoreParams.UserParam.key -> AccumuloContainer.user, + AccumuloDataStoreParams.CatalogParam.key -> catalog ).asJava AccumuloDataStoreFactory.canProcess(params) must beTrue DataStoreFinder.getDataStore(params) must throwAn[IOException] diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFilterTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFilterTest.scala index 97d3e02d1730..78898fe1da36 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFilterTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreFilterTest.scala @@ -20,8 +20,6 @@ import org.specs2.runner.JUnitRunner @RunWith(classOf[JUnitRunner]) class AccumuloDataStoreFilterTest extends Specification with TestWithFeatureType { - sequential - // note: index=full on the geometry tests a regression bug in stats (GEOMESA-1292) override val spec = "name:String,dtg:Date,*geom:Geometry:srid=4326:index=full;geomesa.mixed.geometries='true'" @@ -39,7 +37,10 @@ class AccumuloDataStoreFilterTest extends Specification with TestWithFeatureType sf.setAttribute(2, "POLYGON((-120 45, -120 50, -125 50, -125 45, -120 45))") sf } - addFeatures(Seq(point, polygon)) + + step { + addFeatures(Seq(point, polygon)) + } "AccumuloDataStore" should { "query by point type" in { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreIdlTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreIdlTest.scala index 85b26beed9ef..a65ea37ed874 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreIdlTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreIdlTest.scala @@ -22,19 +22,19 @@ class AccumuloDataStoreIdlTest extends Specification with TestWithFeatureType { import org.locationtech.geomesa.filter.ff - sequential - override val spec = "dtg:Date,*geom:Point:srid=4326" - addFeatures((-180 to 180).map { lon => - val sf = new ScalaSimpleFeature(sft, lon.toString) - sf.setAttribute(0, "2015-01-01T00:00:00.000Z") - sf.setAttribute(1, s"POINT($lon ${lon / 10})") - sf - }) - val srs = CRS.toSRS(org.locationtech.geomesa.utils.geotools.CRS_EPSG_4326) + step { + addFeatures((-180 to 180).map { lon => + val sf = new ScalaSimpleFeature(sft, lon.toString) + sf.setAttribute(0, "2015-01-01T00:00:00.000Z") + sf.setAttribute(1, s"POINT($lon ${lon / 10})") + sf + }) + } + "AccumuloDataStore" should { "handle IDL correctly" in { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreJsonTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreJsonTest.scala index 78f31fca66f9..df54dbd9955d 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreJsonTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreJsonTest.scala @@ -21,8 +21,6 @@ import org.specs2.runner.JUnitRunner @RunWith(classOf[JUnitRunner]) class AccumuloDataStoreJsonTest extends Specification with TestWithFeatureType { - sequential - override val spec = "json:String:json=true,*geom:Point:srid=4326" def getJson(x: Double, y: Double, props: String = "{}"): String = { @@ -49,7 +47,9 @@ class AccumuloDataStoreJsonTest extends Specification with TestWithFeatureType { sf4.setAttribute(0, """["a1","a2","a3"]""") sf4.setAttribute(1, "POINT(45 63)") - addFeatures(Seq(sf0, sf1, sf2, sf3, sf4)) + step { + addFeatures(Seq(sf0, sf1, sf2, sf3, sf4)) + } "AccumuloDataStore" should { "support json attributes" in { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreNullAttributeVisibilityTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreNullAttributeVisibilityTest.scala index a4c49485c1f0..98bc0ed7b6a9 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreNullAttributeVisibilityTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreNullAttributeVisibilityTest.scala @@ -28,8 +28,6 @@ class AccumuloDataStoreNullAttributeVisibilityTest extends TestWithFeatureType { import scala.collection.JavaConverters._ - sequential - override val spec = "some_id:String,dtg:Date,*geo_location:Point:srid=4326,number:Integer,text:String;geomesa.visibility.level='attribute'" val visibility = "admin,user,user,user,user" @@ -100,6 +98,5 @@ class AccumuloDataStoreNullAttributeVisibilityTest extends TestWithFeatureType { val features = queryByAuths("user,admin", "INCLUDE") features must containTheSameElementsAs(Seq(complete_feature, null_string_feature)) } - } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreQueryTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreQueryTest.scala index 5bf6bd45b0f5..3d187b08f14b 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreQueryTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreQueryTest.scala @@ -50,10 +50,11 @@ class AccumuloDataStoreQueryTest extends Specification with TestWithMultipleSfts import scala.collection.JavaConverters._ - sequential - val defaultSft = createNewSchema("name:String:index=join,geom:Point:srid=4326,dtg:Date") - addFeature(ScalaSimpleFeature.create(defaultSft, "fid-1", "name1", "POINT(45 49)", "2010-05-07T12:30:00.000Z")) + + step { + addFeature(ScalaSimpleFeature.create(defaultSft, "fid-1", "name1", "POINT(45 49)", "2010-05-07T12:30:00.000Z")) + } "AccumuloDataStore" should { "return an empty iterator correctly" in { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreSortTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreSortTest.scala index 7d2a4fa7c514..075b3eb929ed 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreSortTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreSortTest.scala @@ -23,8 +23,6 @@ class AccumuloDataStoreSortTest extends Specification with TestWithFeatureType { import org.locationtech.geomesa.filter.ff - sequential - override val spec = "name:String:index=join,age:Int:index=full,weight:Double,dtg:Date,*geom:Point:srid=4326" lazy val features = Seq.tabulate(5) { i => diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreStatsTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreStatsTest.scala index e2bde9f38a73..4d4d0feffe0e 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreStatsTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreStatsTest.scala @@ -18,7 +18,7 @@ import org.geotools.geometry.jts.ReferencedEnvelope import org.geotools.util.factory.Hints import org.junit.runner.RunWith import org.locationtech.geomesa.accumulo.TestWithMultipleSfts -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.AttributeJoinIndex import org.locationtech.geomesa.features.ScalaSimpleFeature import org.locationtech.geomesa.index.conf.QueryHints.{EXACT_COUNT, QUERY_INDEX} import org.locationtech.geomesa.index.index.z2.Z2Index @@ -409,7 +409,7 @@ class AccumuloDataStoreStatsTest extends Specification with TestWithMultipleSfts // deleting the "name" index table to show that the QUERY_INDEX hint is being passed through ds.manager.indices(sft).collectFirst { - case i: AccumuloJoinIndex if i.attributes.head == "name" => i.getTableNames().head + case i: AttributeJoinIndex if i.attributes.head == "name" => i.getTableNames().head }.foreach { ds.connector.tableOperations().delete(_) } val filters = Seq("bbox(geom,0,0,10,5)", "name < '7'") diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreTest.scala index 61e5a2ec421f..51c51e8c5879 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloDataStoreTest.scala @@ -47,18 +47,17 @@ class AccumuloDataStoreTest extends Specification with TestWithMultipleSfts { import scala.collection.JavaConverters._ - sequential - val defaultSpec = "name:String,geom:Point:srid=4326,dtg:Date" - val defaultSft = createNewSchema(defaultSpec) - val defaultTypeName = defaultSft.getTypeName - - addFeature(defaultPoint(defaultSft)) - addFeature(defaultPoint(defaultSft, id = "f2")) - + lazy val defaultSft = createNewSchema(defaultSpec) + lazy val defaultTypeName = defaultSft.getTypeName val defaultGeom = WKTUtils.read("POINT(45.0 49.0)") + step { + addFeature(defaultPoint(defaultSft)) + addFeature(defaultPoint(defaultSft, id = "f2")) + } + def defaultPoint(sft: SimpleFeatureType, id: String = "f1", name: String = "testType", @@ -176,9 +175,9 @@ class AccumuloDataStoreTest extends Specification with TestWithMultipleSfts { val results = ds.getFeatureSource(defaultTypeName).getFeatures(query) val features = SelfClosingIterator(results.features).toList - "results schema should match" >> { results.getSchema mustEqual defaultSft } - "geometry should be set" >> { forall(features)(_.getDefaultGeometry mustEqual defaultGeom) } - "result length should be 1" >> { features must haveLength(2) } + results.getSchema mustEqual defaultSft + features must haveLength(2) + forall(features)(_.getDefaultGeometry mustEqual defaultGeom) } "create a schema with custom record splitting options with table sharing off" in { @@ -348,37 +347,26 @@ class AccumuloDataStoreTest extends Specification with TestWithMultipleSfts { val originalSchema = "name:String,dtg:Date,*geom:Point:srid=4326" val sftName = createNewSchema(originalSchema).getTypeName - "prevent changing default geometry" >> { - val modified = - SimpleFeatureTypes.createType(sftName, "name:String,dtg:Date,geom:Point:srid=4326,*geom2:Point:srid=4326") - modified.getUserData.putAll(ds.getSchema(sftName).getUserData) - ds.updateSchema(sftName, modified) should throwAn[UnsupportedOperationException] - val retrievedSchema = SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) - retrievedSchema mustEqual originalSchema - } - "prevent changing attribute order" >> { - val modified = SimpleFeatureTypes.createType(sftName, "dtg:Date,name:String,*geom:Point:srid=4326") - modified.getUserData.putAll(ds.getSchema(sftName).getUserData) - ds.updateSchema(sftName, modified) should throwA[UnsupportedOperationException] - val retrievedSchema = SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) - retrievedSchema mustEqual originalSchema - } - "prevent removing attributes" >> { - val modified = SimpleFeatureTypes.createType(sftName, "dtg:Date,*geom:Point:srid=4326") - modified.getUserData.putAll(ds.getSchema(sftName).getUserData) - ds.updateSchema(sftName, modified) should throwA[UnsupportedOperationException] - val retrievedSchema = SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) - retrievedSchema mustEqual originalSchema - } - "allow adding attributes" >> { - // note: we actually modify the schema here so this check is last - val newSchema = "name:String,dtg:Date,*geom:Point:srid=4326,newField:String" - val modified = SimpleFeatureTypes.createType(sftName, newSchema) + def modify(spec: String): Unit = { + val modified = SimpleFeatureTypes.createType(sftName, spec) modified.getUserData.putAll(ds.getSchema(sftName).getUserData) ds.updateSchema(sftName, modified) - val retrievedSchema = SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) - retrievedSchema mustEqual newSchema } + + // "prevent changing default geometry" >> { + modify("name:String,dtg:Date,geom:Point:srid=4326,*geom2:Point:srid=4326") must throwAn[UnsupportedOperationException] + SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) mustEqual originalSchema + // "prevent changing attribute order" >> { + modify("dtg:Date,name:String,*geom:Point:srid=4326") must throwA[UnsupportedOperationException] + SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) mustEqual originalSchema + // "prevent removing attributes" >> { + modify("dtg:Date,*geom:Point:srid=4326") must throwA[UnsupportedOperationException] + SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) mustEqual originalSchema + // "allow adding attributes" >> { + // note: we actually modify the schema here so this check is last + val newSchema = "name:String,dtg:Date,*geom:Point:srid=4326,newField:String" + modify(newSchema) + SimpleFeatureTypes.encodeType(ds.getSchema(sftName)) mustEqual newSchema } "Provide a feature update implementation" in { @@ -491,51 +479,43 @@ class AccumuloDataStoreTest extends Specification with TestWithMultipleSfts { sf }) - "with out of order attributes" >> { - val query = new Query(sftName, ECQL.toFilter("bbox(geom,49,49,60,60)"), "geom", "dtg", "label") - val features = - SelfClosingIterator(ds.getFeatureSource(sftName).getFeatures(query).features).toList.sortBy(_.getID) - features must haveSize(5) - (0 until 5).foreach { i => - features(i).getID mustEqual s"f$i" - features(i).getAttributeCount mustEqual 3 - features(i).getAttribute("label") mustEqual s"label$i" - features(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) - features(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") - } - success + def query(filter: Filter, transforms: String*): List[SimpleFeature] = { + var query = new Query(sftName, filter, transforms: _*) + SelfClosingIterator(ds.getFeatureSource(sftName).getFeatures(query).features).toList.sortBy(_.getID) } - "with only date and geom" >> { - val query = new Query(sftName, ECQL.toFilter("bbox(geom,49,49,60,60)"), "geom", "dtg") - val features = - SelfClosingIterator(ds.getFeatureSource(sftName).getFeatures(query).features).toList.sortBy(_.getID) - features must haveSize(5) - (0 until 5).foreach { i => - features(i).getID mustEqual s"f$i" - features(i).getAttributeCount mustEqual 2 - features(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) - features(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") - } - success + // "with out of order attributes" >> { + val features0 = query(ECQL.toFilter("bbox(geom,49,49,60,60)"), "geom", "dtg", "label") + features0 must haveSize(5) + foreach(0 until 5) { i => + features0(i).getID mustEqual s"f$i" + features0(i).getAttributeCount mustEqual 3 + features0(i).getAttribute("label") mustEqual s"label$i" + features0(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) + features0(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") } - "with all attributes" >> { - val query = new Query(sftName, ECQL.toFilter("bbox(geom,49,49,60,60)"), - "geom", "dtg", "label", "score", "trackId") - val features = - SelfClosingIterator(ds.getFeatureSource(sftName).getFeatures(query).features).toList.sortBy(_.getID) - features must haveSize(5) - (0 until 5).foreach { i => - features(i).getID mustEqual s"f$i" - features(i).getAttributeCount mustEqual 5 - features(i).getAttribute("label") mustEqual s"label$i" - features(i).getAttribute("trackId") mustEqual s"trk$i" - features(i).getAttribute("score") mustEqual i.toDouble - features(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) - features(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") - } - success + // "with only date and geom" >> { + val features1 = query(ECQL.toFilter("bbox(geom,49,49,60,60)"), "geom", "dtg") + features1 must haveSize(5) + foreach(0 until 5) { i => + features1(i).getID mustEqual s"f$i" + features1(i).getAttributeCount mustEqual 2 + features1(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) + features1(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") + } + + // "with all attributes" >> { + val features2 = query(ECQL.toFilter("bbox(geom,49,49,60,60)"), "geom", "dtg", "label", "score", "trackId") + features2 must haveSize(5) + foreach(0 until 5) { i => + features2(i).getID mustEqual s"f$i" + features2(i).getAttributeCount mustEqual 5 + features2(i).getAttribute("label") mustEqual s"label$i" + features2(i).getAttribute("trackId") mustEqual s"trk$i" + features2(i).getAttribute("score") mustEqual i.toDouble + features2(i).getAttribute("dtg") mustEqual java.util.Date.from(java.time.LocalDateTime.parse(s"2014-01-01T0$i:00:00.000Z", GeoToolsDateFormat).toInstant(java.time.ZoneOffset.UTC)) + features2(i).getAttribute("geom") mustEqual WKTUtils.read(s"POINT(5$i 50)") } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureReaderTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureReaderTest.scala index 988cdab36bc2..45b2fe0931b6 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureReaderTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureReaderTest.scala @@ -28,8 +28,6 @@ import scala.reflect.ClassTag @RunWith(classOf[JUnitRunner]) class AccumuloFeatureReaderTest extends Specification with TestWithFeatureType { - sequential - override def spec = s"name:String,dtg:Date,*geom:Point" val features = (0 until 100).map { i => @@ -40,7 +38,9 @@ class AccumuloFeatureReaderTest extends Specification with TestWithFeatureType { sf } - addFeatures(features) + step { + addFeatures(features) + } val filter = ECQL.toFilter("bbox(geom, -10, -10, 10, 10) and dtg during 2010-05-07T00:00:00.000Z/2010-05-08T00:00:00.000Z") diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureWriterTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureWriterTest.scala index 2e3ce8f1f705..8af00325397d 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureWriterTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloFeatureWriterTest.scala @@ -43,7 +43,7 @@ class AccumuloFeatureWriterTest extends Specification with TestWithMultipleSfts lazy val sfts = Seq(/*logical, */millis) override def before: Any = { - sfts.foreach{ sft => + sfts.foreach { sft => ds.manager.indices(sft).flatMap(_.getTableNames()).foreach { name => val deleter = ds.connector.createBatchDeleter(name, new Authorizations(), 5, new BatchWriterConfig()) deleter.setRanges(Collections.singletonList(new org.apache.accumulo.core.data.Range())) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloPartitioningTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloPartitioningTest.scala index dbfba10588f9..43bbc13d25d8 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloPartitioningTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/AccumuloPartitioningTest.scala @@ -34,8 +34,6 @@ class AccumuloPartitioningTest extends Specification with TestWithFeatureType { // note: using `Seq.foreach; ok` instead of `foreach(Seq)` shaves several seconds off the time to run this test - sequential - override val spec: String = s"name:String:index=true,attr:String,dtg:Date,*geom:Point:srid=4326;${Configs.TablePartitioning}=${TimePartition.Name}" diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZIntervalTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZIntervalTest.scala index 6e9a9587a21e..e1b77c9db5ad 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZIntervalTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZIntervalTest.scala @@ -22,8 +22,6 @@ import org.specs2.runner.JUnitRunner @RunWith(classOf[JUnitRunner]) class ZIntervalTest extends Specification with TestWithMultipleSfts { - sequential - val key = SimpleFeatureTypes.Configs.IndexZ3Interval val spec = "name:String,dtg:Date,*geom:Point:srid=4326" diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZLineTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZLineTest.scala index 79f4800096bf..b047e373f14f 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZLineTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/data/ZLineTest.scala @@ -26,17 +26,17 @@ class ZLineTest extends Specification with TestWithFeatureType { import scala.collection.JavaConverters._ - sequential - override val spec = "name:String,dtg:Date,*geom:LineString:srid=4326" - addFeatures({ - val sf = new ScalaSimpleFeature(sft, "fid1") - sf.setAttribute("name", "fred") - sf.setAttribute("dtg", "2015-01-01T12:00:00.000Z") - sf.setAttribute("geom", "LINESTRING(47.28515625 25.576171875, 48 26, 49 27)") - Seq(sf) - }) + step { + addFeatures({ + val sf = new ScalaSimpleFeature(sft, "fid1") + sf.setAttribute("name", "fred") + sf.setAttribute("dtg", "2015-01-01T12:00:00.000Z") + sf.setAttribute("geom", "LINESTRING(47.28515625 25.576171875, 48 26, 49 27)") + Seq(sf) + }) + } def printR(e: java.util.Map.Entry[Key, Value]): Unit = { val row = Key.toPrintableString(e.getKey.getRow.getBytes, 0, e.getKey.getRow.getLength, e.getKey.getRow.getLength) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/AttributeIndexStrategyTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/AttributeIndexStrategyTest.scala index 5812bd9f8753..1855f08acd93 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/AttributeIndexStrategyTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/AttributeIndexStrategyTest.scala @@ -43,8 +43,6 @@ import scala.collection.JavaConverters._ @RunWith(classOf[JUnitRunner]) class AttributeIndexStrategyTest extends Specification with TestWithFeatureType { - sequential - override val spec = "name:String:index=full,age:Integer:index=join,count:Long:index=join," + "weight:Double:index=join,height:Float:index=join,admin:Boolean:index=join," + "*geom:Point:srid=4326,dtg:Date,indexedDtg:Date:index=join,fingers:List[String]:index=join," + diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/BinLineStringTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/BinLineStringTest.scala index 4c344ffeda18..bfd5075d4d22 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/BinLineStringTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/BinLineStringTest.scala @@ -29,8 +29,6 @@ class BinLineStringTest extends Specification with TestWithFeatureType { import org.locationtech.geomesa.utils.geotools.GeoToolsDateFormat - sequential - override val spec = "name:String,track:String,dtgList:List[Date],dtg:Date,*geom:LineString:srid=4326" val features = @@ -50,7 +48,9 @@ class BinLineStringTest extends Specification with TestWithFeatureType { sf } - addFeatures(features) + step { + addFeatures(features) + } def getQuery(filter: String, dtg: Option[String] = None, label: Option[String] = None): Query = { val query = new Query(sftName, ECQL.toFilter(filter)) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigurableIndexesTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigurableIndexesTest.scala index 804b66646838..d6b37dabb71b 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigurableIndexesTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigurableIndexesTest.scala @@ -38,7 +38,10 @@ class ConfigurableIndexesTest extends Specification with TestWithFeatureType { sf.setAttribute(2, s"POINT(4$i 5$i)") sf } - addFeatures(features) + + step { + addFeatures(features) + } "AccumuloDataStore" should { "only create the z3 index" >> { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigureShardsTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigureShardsTest.scala index a4774887ff98..69be786e7e17 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigureShardsTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigureShardsTest.scala @@ -24,8 +24,6 @@ class ConfigureShardsTest extends Specification with TestWithFeatureType { import scala.collection.JavaConverters._ - sequential - val spec = "name:String,dtg:Date,*geom:Point:srid=4326;geomesa.z3.splits='8'" val features: Seq[ScalaSimpleFeature] = { @@ -47,9 +45,12 @@ class ConfigureShardsTest extends Specification with TestWithFeatureType { } } + step { + addFeatures(features) + } + "Indexes" should { "configure from spec" >> { - addFeatures(features) var shardSet: Set[Long] = Set[Long]() val index = ds.manager.indices(sft).find(_.name == Z3Index.name) index must beSome diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/CoveringAttributeIndexTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/CoveringAttributeIndexTest.scala index a853325a2e9d..548c5b5b73c7 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/CoveringAttributeIndexTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/CoveringAttributeIndexTest.scala @@ -21,20 +21,20 @@ import org.specs2.runner.JUnitRunner @RunWith(classOf[JUnitRunner]) class CoveringAttributeIndexTest extends Specification with TestWithFeatureType { - sequential - override val spec = "name:String:index=full,age:Integer:index=join,weight:Double:index=join," + "height:Double,dtg:Date,*geom:Point:srid=4326" val geom = WKTUtils.read("POINT(45.0 49.0)") - addFeatures({ - (0 until 10).map { i => - val dtg = s"2014-01-1${i}T12:00:00.000Z" - val attrs = Array(s"${i}name$i", s"$i", s"${i * 2.0}", s"${i * 3.0}", dtg, geom) - ScalaSimpleFeatureFactory.buildFeature(sft, attrs, i.toString) - } - }) + step { + addFeatures({ + (0 until 10).map { i => + val dtg = s"2014-01-1${i}T12:00:00.000Z" + val attrs = Array(s"${i}name$i", s"$i", s"${i * 2.0}", s"${i * 3.0}", dtg, geom) + ScalaSimpleFeatureFactory.buildFeature(sft, attrs, i.toString) + } + }) + } val joinIndicator = "Join Plan:" diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/Z3IdxStrategyTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/Z3IdxStrategyTest.scala index 711305c0d814..f67268e8ae85 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/Z3IdxStrategyTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/Z3IdxStrategyTest.scala @@ -34,8 +34,6 @@ class Z3IdxStrategyTest extends Specification with TestWithFeatureType { import scala.collection.JavaConverters._ - sequential // note: test doesn't need to be sequential but it actually runs faster this way - val spec = "name:String,track:String,dtg:Date,*geom:Point:srid=4326;geomesa.indexes.enabled='z3'" val features = @@ -64,7 +62,10 @@ class Z3IdxStrategyTest extends Specification with TestWithFeatureType { sf.setAttributes(Array[AnyRef](name, track, dtg, geom)) sf } - addFeatures(features) + + step { + addFeatures(features) + } def runQuery(filter: String, transforms: Array[String] = null): Iterator[SimpleFeature] = runQuery(new Query(sftName, ECQL.toFilter(filter), transforms: _*)) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/ArrowBatchIteratorTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/ArrowBatchIteratorTest.scala index 18deac73f14d..4fa530b15436 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/ArrowBatchIteratorTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/ArrowBatchIteratorTest.scala @@ -41,8 +41,6 @@ class ArrowBatchIteratorTest extends TestWithMultipleSfts with Mockito { import scala.collection.JavaConverters._ - sequential - lazy val pointSft = createNewSchema("name:String:index=join,team:String:index-value=true,age:Int,weight:Int,dtg:Date,*geom:Point:srid=4326") lazy val lineSft = createNewSchema("name:String:index=join,team:String:index-value=true,age:Int,weight:Int,dtg:Date,*geom:LineString:srid=4326") lazy val listSft = createNewSchema("names:List[String],team:String,dtg:Date,*geom:Point:srid=4326") @@ -80,9 +78,11 @@ class ArrowBatchIteratorTest extends TestWithMultipleSfts with Mockito { "name IN('name0', 'name1')" ).map(ECQL.toFilter) - addFeatures(pointFeatures) - addFeatures(lineFeatures) - addFeatures(listFeatures) + step { + addFeatures(pointFeatures) + addFeatures(lineFeatures) + addFeatures(listFeatures) + } val sfts = Seq((pointSft, pointFeatures), (lineSft, lineFeatures)) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexFilteringIteratorTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexFilteringIteratorTest.scala index 6e7f6cdc2e1d..38a6f7d91539 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexFilteringIteratorTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexFilteringIteratorTest.scala @@ -31,8 +31,6 @@ class AttributeIndexFilteringIteratorTest extends Specification with TestWithFea import org.locationtech.geomesa.filter.ff - sequential - override val spec = s"name:String:index=join,age:Integer:index=join,dtg:Date,*geom:Point:srid=4326" val features = List("a", "b", "c", "d").flatMap { name => @@ -46,7 +44,9 @@ class AttributeIndexFilteringIteratorTest extends Specification with TestWithFea } } - addFeatures(features) + step { + addFeatures(features) + } def checkStrategies[T](query: Query, strategy: NamedIndex, explain: Explainer = ExplainNull): MatchResult[Any] = { val plan = ds.getQueryPlan(query, explainer = explain) diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexIteratorTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexIteratorTest.scala index f95cfafbd67a..b4d5ca1979d2 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexIteratorTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/AttributeIndexIteratorTest.scala @@ -29,8 +29,6 @@ import java.util.{Collections, Date, TimeZone} @RunWith(classOf[JUnitRunner]) class AttributeIndexIteratorTest extends Specification with TestWithFeatureType { - sequential - val spec = "name:String:index=join,age:Integer:index=join,scars:List[String]:index=join,dtg:Date:index=join," + "*geom:Point:srid=4326;override.index.dtg.join=true" @@ -67,149 +65,146 @@ class AttributeIndexIteratorTest extends Specification with TestWithFeatureType "AttributeIndexIterator" should { - "return correct results" >> { - - "for string equals" >> { - val filter = "name = 'b'" - val results = query(filter, Array("geom", "dtg", "name")) + "work for string equals" >> { + val filter = "name = 'b'" + val results = query(filter, Array("geom", "dtg", "name")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(3).foreach - foreach(results.map(_.getAttribute("name").asInstanceOf[String]))(_ must contain("b")) - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(3).foreach + foreach(results.map(_.getAttribute("name").asInstanceOf[String]))(_ must contain("b")) + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "for string less than" >> { - val filter = "name < 'b'" - val results = query(filter, Array("geom", "dtg", "name")) + "work for string less than" >> { + val filter = "name < 'b'" + val results = query(filter, Array("geom", "dtg", "name")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(3).foreach - foreach(results.map(_.getAttribute("name").asInstanceOf[String]))(_ must contain("a")) - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(3).foreach + foreach(results.map(_.getAttribute("name").asInstanceOf[String]))(_ must contain("a")) + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "for string greater than" >> { - val filter = "name > 'b'" - val results = query(filter, Array("geom", "dtg", "name")) + "work for string greater than" >> { + val filter = "name > 'b'" + val results = query(filter, Array("geom", "dtg", "name")) - results must haveSize(8) - results.map(_.getAttributeCount) must contain(3).foreach - results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("c")).exactly(4) - results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("d")).exactly(4) - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + results must haveSize(8) + results.map(_.getAttributeCount) must contain(3).foreach + results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("c")).exactly(4) + results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("d")).exactly(4) + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "for string greater than or equals" >> { - val filter = "name >= 'b'" - val results = query(filter, Array("geom", "dtg", "name")) - - results must haveSize(12) - results.map(_.getAttributeCount) must contain(3).foreach - results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("b")).exactly(4) - results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("c")).exactly(4) - results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("d")).exactly(4) - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + "work for string greater than or equals" >> { + val filter = "name >= 'b'" + val results = query(filter, Array("geom", "dtg", "name")) + + results must haveSize(12) + results.map(_.getAttributeCount) must contain(3).foreach + results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("b")).exactly(4) + results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("c")).exactly(4) + results.map(_.getAttribute("name").asInstanceOf[String]) must contain(beEqualTo("d")).exactly(4) + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "for date tequals" >> { - val filter = "dtg TEQUALS 2014-01-02T00:00:00.000Z" - val results = query(filter, Array("geom", "dtg")) + "work for date tequals" >> { + val filter = "dtg TEQUALS 2014-01-02T00:00:00.000Z" + val results = query(filter, Array("geom", "dtg")) - results must haveSize(20) - results.map(_.getAttributeCount) must contain(2).foreach - } + results must haveSize(20) + results.map(_.getAttributeCount) must contain(2).foreach + } - "for date equals" >> { - val filter = "dtg = '2014-01-02T00:00:00.000Z'" - val results = query(filter, Array("geom", "dtg")) + "work for date equals" >> { + val filter = "dtg = '2014-01-02T00:00:00.000Z'" + val results = query(filter, Array("geom", "dtg")) - results must haveSize(20) - results.map(_.getAttributeCount) must contain(2).foreach - } + results must haveSize(20) + results.map(_.getAttributeCount) must contain(2).foreach + } - "for date between" >> { - val filter = "dtg BETWEEN '2014-01-01T00:00:00.000Z' AND '2014-01-03T00:00:00.000Z'" - val results = query(filter, Array("geom", "dtg")) + "work for date between" >> { + val filter = "dtg BETWEEN '2014-01-01T00:00:00.000Z' AND '2014-01-03T00:00:00.000Z'" + val results = query(filter, Array("geom", "dtg")) - results must haveSize(20) - results.map(_.getAttributeCount) must contain(2).foreach - } + results must haveSize(20) + results.map(_.getAttributeCount) must contain(2).foreach + } - "for int less than" >> { - val filter = "age < 2" - val results = query(filter, Array("geom", "dtg", "age")) + "work for int less than" >> { + val filter = "age < 2" + val results = query(filter, Array("geom", "dtg", "age")) - results must haveSize(5) - results.map(_.getAttributeCount) must contain(3).foreach - results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(1).foreach - foreach(results.map(_.getAttribute("geom").toString))(_ must contain("POINT (45 45)")) - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + results must haveSize(5) + results.map(_.getAttributeCount) must contain(3).foreach + results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(1).foreach + foreach(results.map(_.getAttribute("geom").toString))(_ must contain("POINT (45 45)")) + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "for int greater than or equals" >> { - val filter = "age >= 3" - val results = query(filter, Array("geom", "dtg", "age")) - - results must haveSize(10) - results.map(_.getAttributeCount) must contain(3).foreach - results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(3).exactly(5) - results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(4).exactly(5) - results.map(_.getAttribute("geom").toString) must contain(beEqualTo("POINT (47 47)")).exactly(5) - results.map(_.getAttribute("geom").toString) must contain(beEqualTo("POINT (48 48)")).exactly(5) - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + "work for int greater than or equals" >> { + val filter = "age >= 3" + val results = query(filter, Array("geom", "dtg", "age")) + + results must haveSize(10) + results.map(_.getAttributeCount) must contain(3).foreach + results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(3).exactly(5) + results.map(_.getAttribute("age").asInstanceOf[Int]) must contain(4).exactly(5) + results.map(_.getAttribute("geom").toString) must contain(beEqualTo("POINT (47 47)")).exactly(5) + results.map(_.getAttribute("geom").toString) must contain(beEqualTo("POINT (48 48)")).exactly(5) + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "not including attribute queried on" >> { - val filter = "name = 'b'" - val results = query(filter, Array("geom", "dtg")) + "work not including attribute queried on" >> { + val filter = "name = 'b'" + val results = query(filter, Array("geom", "dtg")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(2).foreach - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - forall(results.map(_.getAttribute("dtg").asInstanceOf[Date]))(_ mustEqual dateToIndex) - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(2).foreach + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + forall(results.map(_.getAttribute("dtg").asInstanceOf[Date]))(_ mustEqual dateToIndex) + } - "not including geom" >> { - val filter = "name = 'b'" - val results = query(filter, Array("dtg")) + "work not including geom" >> { + val filter = "name = 'b'" + val results = query(filter, Array("dtg")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(1).foreach - results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(1).foreach + results.map(_.getAttribute("dtg").asInstanceOf[Date]) must contain(dateToIndex).foreach + } - "not including dtg" >> { - val filter = "name = 'b'" - val results = query(filter, Array("geom")) + "work not including dtg" >> { + val filter = "name = 'b'" + val results = query(filter, Array("geom")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(1).foreach - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(1).foreach + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)", "POINT (46 46)", "POINT (47 47)", "POINT (48 48)") + } - "not including geom or dtg" >> { - val filter = "name = 'b'" - val results = query(filter, Array("name")) + "work not including geom or dtg" >> { + val filter = "name = 'b'" + val results = query(filter, Array("name")) - results must haveSize(4) - results.map(_.getAttributeCount) must contain(1).foreach - foreach(results.map(_.getAttribute("name").toString))(_ must contain("b")) - } + results must haveSize(4) + results.map(_.getAttributeCount) must contain(1).foreach + foreach(results.map(_.getAttribute("name").toString))(_ must contain("b")) + } - "with additional filter applied" >> { - val filter = "name = 'b' AND BBOX(geom, 44.5, 44.5, 45.5, 45.5)" - val results = query(filter, Array("geom", "dtg", "name")) + "work with additional filter applied" >> { + val filter = "name = 'b' AND BBOX(geom, 44.5, 44.5, 45.5, 45.5)" + val results = query(filter, Array("geom", "dtg", "name")) - results must haveSize(1) - results.map(_.getAttributeCount) must contain(3).foreach // geom gets added back in - results.map(_.getAttribute("name").toString) must contain("b") - results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)") - } + results must haveSize(1) + results.map(_.getAttributeCount) must contain(3).foreach // geom gets added back in + results.map(_.getAttribute("name").toString) must contain("b") + results.map(_.getAttribute("geom").toString) must contain("POINT (45 45)") } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DensityIteratorTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DensityIteratorTest.scala index 61ae1e7bf797..ed8371eec170 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DensityIteratorTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DensityIteratorTest.scala @@ -34,8 +34,6 @@ class DensityIteratorTest extends Specification with TestWithFeatureType { import scala.collection.JavaConverters._ - sequential - override val spec: String = "an_id:Int,attr:Double,dtg:Date," + "*geom:Point:srid=4326," + diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffTest.scala index 16dbe9ac527c..5f7a1c026053 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffTest.scala @@ -8,18 +8,18 @@ package org.locationtech.geomesa.accumulo.iterators -import org.apache.accumulo.core.client.security.tokens.PasswordToken import org.geotools.api.data.{DataStore, DataStoreFinder} import org.geotools.api.feature.simple.SimpleFeature import org.geotools.api.filter.Filter import org.junit.runner.RunWith import org.locationtech.geomesa.accumulo.data.AccumuloDataStoreParams -import org.locationtech.geomesa.accumulo.{MiniCluster, TestWithFeatureType} +import org.locationtech.geomesa.accumulo.{AccumuloContainer, TestWithFeatureType} import org.locationtech.geomesa.features.ScalaSimpleFeature import org.locationtech.geomesa.security.SecurityUtils import org.locationtech.geomesa.utils.collection.SelfClosingIterator import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.Configs +import org.locationtech.geomesa.utils.io.WithClose import org.specs2.mutable.Specification import org.specs2.runner.JUnitRunner @@ -101,14 +101,16 @@ class DtgAgeOffTest extends Specification with TestWithFeatureType { } // Scans all GeoMesa Accumulo tables directly and verifies the number of records that the `root` user can see. - private def scanDirect(expected: Int) = { - val conn = MiniCluster.cluster.createAccumuloClient(MiniCluster.Users.root.name, new PasswordToken(MiniCluster.Users.root.password)) - conn.tableOperations().list().asScala.filter(t => t.contains("DtgAgeOffTest_DtgAgeOffTest")).forall { tableName => - val scanner = conn.createScanner(tableName, MiniCluster.Users.root.auths) - val count = scanner.asScala.size - scanner.close() - count mustEqual expected + private def scanDirect(expected: Int): Unit = { + WithClose(AccumuloContainer.Container.client()) { conn => + val tables = conn.tableOperations().list().asScala.filter(_.contains("DtgAgeOffTest_DtgAgeOffTest")) + tables must not(beEmpty) + forall(tables) { tableName => + val scanner = conn.createScanner(tableName, AccumuloContainer.Users.root.auths) + val count = scanner.asScala.size + scanner.close() + count mustEqual expected + } } - conn.close() } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/KryoLazyStatsIteratorTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/KryoLazyStatsIteratorTest.scala index dc7a470ac939..a4b14c4201aa 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/KryoLazyStatsIteratorTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/iterators/KryoLazyStatsIteratorTest.scala @@ -24,16 +24,16 @@ class KryoLazyStatsIteratorTest extends Specification with TestWithFeatureType { import org.locationtech.geomesa.index.iterators.StatsScan.decodeStat - sequential - override val spec = "idt:java.lang.Integer:index=full,attr:java.lang.Long:index=join,dtg:Date,*geom:Point:srid=4326" - addFeatures((0 until 150).toArray.map { i => - val attrs = Array(i.asInstanceOf[AnyRef], (i * 2).asInstanceOf[AnyRef], "2012-01-01T19:00:00Z", "POINT(-77 38)") - val sf = new ScalaSimpleFeature(sft, i.toString) - sf.setAttributes(attrs) - sf - }) + step { + addFeatures((0 until 150).toArray.map { i => + val attrs = Array(i.asInstanceOf[AnyRef], (i * 2).asInstanceOf[AnyRef], "2012-01-01T19:00:00Z", "POINT(-77 38)") + val sf = new ScalaSimpleFeature(sft, i.toString) + sf.setAttributes(attrs) + sf + }) + } def getQuery(statString: String, ecql: Option[String] = None): Query = { val query = new Query(sftName, ECQL.toFilter("dtg DURING 2012-01-01T18:30:00.000Z/2012-01-01T19:30:00.000Z " + diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/util/GeoMesaBatchWriterConfigTest.scala b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/util/GeoMesaBatchWriterConfigTest.scala index 4e3db0c95332..ffcfff26a377 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/util/GeoMesaBatchWriterConfigTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/util/GeoMesaBatchWriterConfigTest.scala @@ -23,8 +23,6 @@ class GeoMesaBatchWriterConfigTest extends Specification { import AccumuloProperties.BatchWriterProperties - sequential - "GeoMesaBatchWriterConfig" should { "have defaults set" in { bwc.getMaxMemory must be equalTo BatchWriterProperties.WRITER_MEMORY_BYTES.toBytes.get diff --git a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/pom.xml b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/pom.xml index 6f433bab6878..ee77bf99ff35 100644 --- a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/pom.xml @@ -18,12 +18,17 @@ org.locationtech.geomesa - geomesa-accumulo-datastore_${scala.binary.version} + geomesa-accumulo-iterators_${scala.binary.version} org.geotools gt-api + + org.apache.accumulo + accumulo-core + provided + commons-codec commons-codec @@ -64,6 +69,12 @@ shade + + + geomesa-accumulo-distributed-runtime.properties + src/main/resources/geomesa-accumulo-distributed-runtime.properties + + com.fasterxml.jackson diff --git a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/resources/geomesa-accumulo-distributed-runtime.properties b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/resources/geomesa-accumulo-distributed-runtime.properties new file mode 100644 index 000000000000..fab677966886 --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/resources/geomesa-accumulo-distributed-runtime.properties @@ -0,0 +1 @@ +# marker file that lets us find the distributed-runtime jar for loading into unit tests diff --git a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala new file mode 100644 index 000000000000..d9aad07e11ed --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala @@ -0,0 +1,17 @@ +/*********************************************************************** + * Copyright (c) 2013-2024 Commonwealth Computer Research, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License, Version 2.0 + * which accompanies this distribution and is available at + * http://www.opensource.org/licenses/apache2.0.php. + ***********************************************************************/ + +package org.locationtech.geomesa.accumulo.data.stats + +/** + * The class is kept here for back-compatibility on already configured tables. Note that it violates + * split-packaging with accumulo-datastore, but this module only builds a shaded jar so packages get + * flattened out. + */ +@deprecated("Moved to org.locationtech.geomesa.accumulo.combiners.StatsCombiner") +class StatsCombiner extends org.locationtech.geomesa.accumulo.combiners.StatsCombiner diff --git a/geomesa-accumulo/geomesa-accumulo-indices/pom.xml b/geomesa-accumulo/geomesa-accumulo-indices/pom.xml new file mode 100644 index 000000000000..181606d8abdd --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-indices/pom.xml @@ -0,0 +1,115 @@ + + + + 4.0.0 + + org.locationtech.geomesa + geomesa-accumulo_2.12 + 5.0.0-SNAPSHOT + + + geomesa-accumulo-indices_2.12 + GeoMesa Accumulo Indices + + + UTF-8 + + + + + org.locationtech.geomesa + geomesa-index-api_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-utils_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-hadoop-utils_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-filter_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-feature-all_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-arrow-gt_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-z3_${scala.binary.version} + + + org.geotools + gt-main + + + org.geotools + gt-cql + + + org.geotools + gt-metadata + + + org.geotools + gt-render + + + org.locationtech.jts + jts-core + + + commons-io + commons-io + + + com.github.ben-manes.caffeine + caffeine + + + + + org.apache.accumulo + accumulo-core + + + org.apache.accumulo + accumulo-start + + + org.apache.hadoop + hadoop-client + + + + + org.specs2 + specs2-core_${scala.binary.version} + + + org.specs2 + specs2-junit_${scala.binary.version} + + + org.specs2 + specs2-mock_${scala.binary.version} + + + + javax.media + jai_core + test + + + org.slf4j + jul-to-slf4j + + + + diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/resources/META-INF/services/org.locationtech.geomesa.index.api.GeoMesaFeatureIndexFactory b/geomesa-accumulo/geomesa-accumulo-indices/src/main/resources/META-INF/services/org.locationtech.geomesa.index.api.GeoMesaFeatureIndexFactory similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/resources/META-INF/services/org.locationtech.geomesa.index.api.GeoMesaFeatureIndexFactory rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/resources/META-INF/services/org.locationtech.geomesa.index.api.GeoMesaFeatureIndexFactory diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloFeatureIndexFactory.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloFeatureIndexFactory.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloFeatureIndexFactory.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/AccumuloFeatureIndexFactory.scala diff --git a/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/AttributeJoinIndex.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/AttributeJoinIndex.scala new file mode 100644 index 000000000000..1ea2c53f7b89 --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/AttributeJoinIndex.scala @@ -0,0 +1,90 @@ +/*********************************************************************** + * Copyright (c) 2013-2024 Commonwealth Computer Research, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License, Version 2.0 + * which accompanies this distribution and is available at + * http://www.opensource.org/licenses/apache2.0.php. + ***********************************************************************/ + +package org.locationtech.geomesa.accumulo.index + +import org.locationtech.geomesa.filter.FilterHelper +import org.locationtech.geomesa.index.api._ +import org.locationtech.geomesa.index.index.attribute.{AttributeIndex, AttributeIndexKey, AttributeIndexValues} +import org.geotools.api.feature.simple.SimpleFeatureType +import org.geotools.api.filter.Filter + +/** + * Mixin trait to add join support to the normal attribute index class + */ +trait AttributeJoinIndex extends GeoMesaFeatureIndex[AttributeIndexValues[Any], AttributeIndexKey] { + + this: AttributeIndex => + + import scala.collection.JavaConverters._ + + private val attribute = attributes.head + private val attributeIndex = sft.indexOf(attribute) + private val descriptor = sft.getDescriptor(attributeIndex) + private val binding = descriptor.getType.getBinding + val indexSft: SimpleFeatureType = IndexValueEncoder.getIndexSft(sft) + + override val name: String = JoinIndex.name + override val identifier: String = GeoMesaFeatureIndex.identifier(name, version, attributes) + + abstract override def getFilterStrategy( + filter: Filter, + transform: Option[SimpleFeatureType]): Option[FilterStrategy] = { + super.getFilterStrategy(filter, transform).flatMap { strategy => + // verify that it's ok to return join plans, and filter them out if not + if (!requiresJoin(strategy.secondary, transform)) { + Some(strategy) + } else if (!JoinIndex.AllowJoinPlans.get) { + None + } else { + val primary = strategy.primary.getOrElse(Filter.INCLUDE) + val bounds = FilterHelper.extractAttributeBounds(primary, attribute, binding) + val joinMultiplier = 9f + bounds.values.length // 10 plus 1 per additional range being scanned + val multiplier = strategy.costMultiplier * joinMultiplier + Some(FilterStrategy(strategy.index, strategy.primary, strategy.secondary, strategy.temporal, multiplier)) + } + } + } + + /** + * Does the query require a join against the record table, or can it be satisfied + * in a single scan + * + * @param filter non-attribute filter being evaluated, if any + * @param transform transform being applied, if any + * @return + */ + private def requiresJoin(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = + !canUseIndexSchema(filter, transform) && !canUseIndexSchemaPlusKey(filter, transform) + + /** + * Determines if the given filter and transform can operate on index encoded values. + */ + def canUseIndexSchema(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = { + // verify that transform *does* exist and only contains fields in the index sft, + // and that filter *does not* exist or can be fulfilled by the index sft + supportsTransform(transform) && supportsFilter(filter) + } + + /** + * Determines if the given filter and transform can operate on index encoded values + * in addition to the values actually encoded in the attribute index keys + */ + def canUseIndexSchemaPlusKey(filter: Option[Filter], transform: Option[SimpleFeatureType]): Boolean = { + transform.exists { t => + val attributes = t.getAttributeDescriptors.asScala.map(_.getLocalName) + attributes.forall(a => a == attribute || indexSft.indexOf(a) != -1) && supportsFilter(filter) + } + } + + def supportsTransform(transform: Option[SimpleFeatureType]): Boolean = + transform.exists(_.getAttributeDescriptors.asScala.map(_.getLocalName).forall(indexSft.indexOf(_) != -1)) + + def supportsFilter(filter: Option[Filter]): Boolean = + filter.forall(FilterHelper.propertyNames(_, sft).forall(indexSft.indexOf(_) != -1)) +} diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/IndexValueEncoder.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/IndexValueEncoder.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/IndexValueEncoder.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/IndexValueEncoder.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala similarity index 94% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala index 6994104c7591..1ff10b10fc04 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/JoinIndex.scala @@ -9,7 +9,6 @@ package org.locationtech.geomesa.accumulo.index import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature import org.locationtech.geomesa.index.api.ShardStrategy.AttributeShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -24,7 +23,7 @@ class JoinIndex(ds: GeoMesaDataStore[_], attribute: String, secondaries: Seq[String], mode: IndexMode) - extends AttributeIndex(ds, sft, attribute, secondaries, mode) with AccumuloJoinIndex { + extends AttributeIndex(ds, sft, attribute, secondaries, mode) with AttributeJoinIndex { override val keySpace: AttributeIndexKeySpace = new AttributeIndexKeySpace(sft, AttributeShardStrategy(sft), attribute) { @@ -33,7 +32,7 @@ class JoinIndex(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accmulo/dr/ScalaDocs.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/ReducedIndexValues.scala similarity index 68% rename from geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accmulo/dr/ScalaDocs.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/ReducedIndexValues.scala index 17da00128593..f9abcd8ca78f 100644 --- a/geomesa-accumulo/geomesa-accumulo-distributed-runtime/src/main/scala/org/locationtech/geomesa/accmulo/dr/ScalaDocs.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/ReducedIndexValues.scala @@ -6,11 +6,13 @@ * http://www.opensource.org/licenses/apache2.0.php. ***********************************************************************/ -package org.locationtech.geomesa.accmulo.dr +package org.locationtech.geomesa.accumulo.index -/** - * Placeholder class to force generation of -javadocs and -sources jars for maven publishing - */ -object ScalaDocs { +import org.locationtech.geomesa.index.api.KeyValue +/** + * Marker trait for reduced values used in a join index + */ +trait ReducedIndexValues { + def indexValues: Seq[KeyValue] } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala similarity index 95% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala index 4e1a8e411d37..743aad69c186 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV2.scala @@ -8,7 +8,7 @@ package org.locationtech.geomesa.accumulo.index.legacy -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.AttributeJoinIndex import org.locationtech.geomesa.utils.index.IndexMode.IndexMode import org.geotools.api.feature.simple.SimpleFeatureType // noinspection ScalaDeprecation @@ -24,7 +24,7 @@ class JoinIndexV2(ds: GeoMesaDataStore[_], attribute: String, dtg: Option[String], mode: IndexMode) - extends AttributeIndexV2(ds, sft, attribute, dtg, mode) with AccumuloJoinIndex { + extends AttributeIndexV2(ds, sft, attribute, dtg, mode) with AttributeJoinIndex { override val keySpace: AttributeIndexKeySpace = new AttributeIndexKeySpaceV2(sft, attribute) { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala similarity index 88% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala index 2ba4c3f43640..a4f1877d1145 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV3.scala @@ -9,8 +9,7 @@ package org.locationtech.geomesa.accumulo.index.legacy import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, ReducedIndexValues} import org.locationtech.geomesa.index.api.ShardStrategy.NoShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -24,7 +23,7 @@ class JoinIndexV3(ds: GeoMesaDataStore[_], attribute: String, dtg: Option[String], mode: IndexMode) - extends AttributeIndexV3(ds, sft, attribute, dtg, mode) with AccumuloJoinIndex { + extends AttributeIndexV3(ds, sft, attribute, dtg, mode) with AttributeJoinIndex { import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType @@ -35,7 +34,7 @@ class JoinIndexV3(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala similarity index 88% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala index 2f30a7428c6a..466f1514aaa9 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV4.scala @@ -9,8 +9,7 @@ package org.locationtech.geomesa.accumulo.index.legacy import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, ReducedIndexValues} import org.locationtech.geomesa.index.api.ShardStrategy.NoShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -24,7 +23,7 @@ class JoinIndexV4(ds: GeoMesaDataStore[_], attribute: String, secondaries: Seq[String], mode: IndexMode) - extends AttributeIndexV4(ds, sft, attribute, secondaries, mode) with AccumuloJoinIndex { + extends AttributeIndexV4(ds, sft, attribute, secondaries, mode) with AttributeJoinIndex { import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType @@ -35,7 +34,7 @@ class JoinIndexV4(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala similarity index 88% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala index cc923ff75a40..2709a7cf109c 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV5.scala @@ -9,8 +9,7 @@ package org.locationtech.geomesa.accumulo.index.legacy import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, ReducedIndexValues} import org.locationtech.geomesa.index.api.ShardStrategy.AttributeShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -25,7 +24,7 @@ class JoinIndexV5(ds: GeoMesaDataStore[_], attribute: String, secondaries: Seq[String], mode: IndexMode) - extends AttributeIndexV5(ds, sft, attribute, secondaries, mode) with AccumuloJoinIndex { + extends AttributeIndexV5(ds, sft, attribute, secondaries, mode) with AttributeJoinIndex { import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType @@ -39,7 +38,7 @@ class JoinIndexV5(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } else { @@ -50,7 +49,7 @@ class JoinIndexV5(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala similarity index 88% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala index 056a6070a2e6..799ecf244b23 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV6.scala @@ -9,8 +9,7 @@ package org.locationtech.geomesa.accumulo.index.legacy import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, ReducedIndexValues} import org.locationtech.geomesa.index.api.ShardStrategy.AttributeShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -25,7 +24,7 @@ class JoinIndexV6(ds: GeoMesaDataStore[_], attribute: String, secondaries: Seq[String], mode: IndexMode) - extends AttributeIndexV6(ds, sft, attribute, secondaries, mode) with AccumuloJoinIndex { + extends AttributeIndexV6(ds, sft, attribute, secondaries, mode) with AttributeJoinIndex { import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType @@ -39,7 +38,7 @@ class JoinIndexV6(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } else { @@ -50,7 +49,7 @@ class JoinIndexV6(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala similarity index 88% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala rename to geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala index eff8bdc4fad3..c3ec1895082e 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala +++ b/geomesa-accumulo/geomesa-accumulo-indices/src/main/scala/org/locationtech/geomesa/accumulo/index/legacy/JoinIndexV7.scala @@ -9,8 +9,7 @@ package org.locationtech.geomesa.accumulo.index.legacy import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloWritableFeature -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex +import org.locationtech.geomesa.accumulo.index.{AttributeJoinIndex, ReducedIndexValues} import org.locationtech.geomesa.index.api.ShardStrategy.AttributeShardStrategy import org.locationtech.geomesa.index.api.{RowKeyValue, WritableFeature} import org.locationtech.geomesa.index.geotools.GeoMesaDataStore @@ -24,7 +23,7 @@ class JoinIndexV7(ds: GeoMesaDataStore[_], attribute: String, secondaries: Seq[String], mode: IndexMode) - extends AttributeIndexV7(ds, sft, attribute, secondaries, mode) with AccumuloJoinIndex { + extends AttributeIndexV7(ds, sft, attribute, secondaries, mode) with AttributeJoinIndex { import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType @@ -35,7 +34,7 @@ class JoinIndexV7(ds: GeoMesaDataStore[_], id: Array[Byte], lenient: Boolean): RowKeyValue[AttributeIndexKey] = { val kv = super.toIndexKey(writable, tier, id, lenient) - kv.copy(values = writable.asInstanceOf[AccumuloWritableFeature].indexValues) + kv.copy(values = writable.asInstanceOf[ReducedIndexValues].indexValues) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-iterators/pom.xml b/geomesa-accumulo/geomesa-accumulo-iterators/pom.xml new file mode 100644 index 000000000000..50d917d3470f --- /dev/null +++ b/geomesa-accumulo/geomesa-accumulo-iterators/pom.xml @@ -0,0 +1,134 @@ + + + + 4.0.0 + + org.locationtech.geomesa + geomesa-accumulo_2.12 + 5.0.0-SNAPSHOT + + + geomesa-accumulo-iterators_2.12 + GeoMesa Accumulo Iterators + + + UTF-8 + + + + + org.locationtech.geomesa + geomesa-index-api_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-accumulo-indices_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-utils_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-hadoop-utils_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-filter_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-feature-all_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-arrow-gt_${scala.binary.version} + + + org.locationtech.geomesa + geomesa-z3_${scala.binary.version} + + + org.geotools + gt-main + + + org.geotools + gt-cql + + + org.geotools + gt-metadata + + + org.geotools + gt-render + + + org.locationtech.jts + jts-core + + + commons-io + commons-io + + + com.github.ben-manes.caffeine + caffeine + + + + + org.apache.accumulo + accumulo-core + + + org.apache.accumulo + accumulo-start + + + org.apache.hadoop + hadoop-client + + + + + org.specs2 + specs2-core_${scala.binary.version} + + + org.specs2 + specs2-junit_${scala.binary.version} + + + org.specs2 + specs2-mock_${scala.binary.version} + + + org.apache.arrow + arrow-vector + tests + + + org.slf4j + log4j-over-slf4j + + + joda-time + joda-time + + + + + + javax.media + jai_core + test + + + org.slf4j + jul-to-slf4j + + + + diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/combiners/StatsCombiner.scala similarity index 82% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/combiners/StatsCombiner.scala index 40ebb1c8f75e..184b18ca91b4 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/data/stats/StatsCombiner.scala +++ b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/combiners/StatsCombiner.scala @@ -6,38 +6,37 @@ * http://www.opensource.org/licenses/apache2.0.php. ***********************************************************************/ -package org.locationtech.geomesa.accumulo.data.stats +package org.locationtech.geomesa.accumulo.combiners import com.typesafe.scalalogging.LazyLogging import org.apache.accumulo.core.client.{AccumuloClient, IteratorSetting} import org.apache.accumulo.core.data.{Key, Value} import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope import org.apache.accumulo.core.iterators.{Combiner, IteratorEnvironment, SortedKeyValueIterator} +import org.apache.hadoop.io.Text import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloBackedMetadata.SingleRowAccumuloMetadata -import org.locationtech.geomesa.accumulo.data.stats.AccumuloGeoMesaStats.CombinerName -import org.locationtech.geomesa.accumulo.util.TableUtils import org.locationtech.geomesa.index.metadata.KeyValueStoreMetadata import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes import org.locationtech.geomesa.utils.stats.{Stat, StatSerializer} -import scala.collection.JavaConverters._ import scala.util.control.NonFatal /** - * Combiner for serialized stats. Should be one instance configured per catalog table. Simple feature - * types and columns with stats should be set in the configuration. - */ + * Combiner for serialized stats. Should be one instance configured per catalog table. Simple feature + * types and columns with stats should be set in the configuration. + */ class StatsCombiner extends Combiner with LazyLogging { import StatsCombiner.{SeparatorOption, SftOption} + import scala.collection.JavaConverters._ + private var serializers: Map[String, StatSerializer] = _ private var separator: Char = '~' override def init(source: SortedKeyValueIterator[Key, Value], - options: java.util.Map[String, String], - env: IteratorEnvironment): Unit = { + options: java.util.Map[String, String], + env: IteratorEnvironment): Unit = { super.init(source, options, env) serializers = options.asScala.toMap.collect { case (k, v) if k.startsWith(SftOption) => @@ -54,7 +53,7 @@ class StatsCombiner extends Combiner with LazyLogging { KeyValueStoreMetadata.decodeRow(key.getRow.getBytes, separator)._1 } catch { // back compatible check - case NonFatal(_) => SingleRowAccumuloMetadata.getTypeName(key.getRow) + case NonFatal(_) => StatsCombiner.SingleRowMetadata.getTypeName(key.getRow) } val serializer = serializers(sftName) @@ -87,19 +86,21 @@ class StatsCombiner extends Combiner with LazyLogging { object StatsCombiner { + import scala.collection.JavaConverters._ + + val CombinerName = "stats-combiner" + val SftOption = "sft-" val SeparatorOption = "sep" def configure(sft: SimpleFeatureType, connector: AccumuloClient, table: String, separator: String): Unit = { - TableUtils.createTableIfNeeded(connector, table) - val sftKey = getSftKey(sft) val sftOpt = SimpleFeatureTypes.encodeType(sft) getExisting(connector, table) match { case None => attach(connector, table, options(separator) + (sftKey -> sftOpt)) case Some(existing) => - val existingSfts = existing.getOptions.asScala.filter(_._1.startsWith(StatsCombiner.SftOption)) + val existingSfts = existing.getOptions.asScala.filter(_._1.startsWith(SftOption)) if (!existingSfts.get(sftKey).contains(sftOpt)) { connector.tableOperations().removeIterator(table, CombinerName, java.util.EnumSet.allOf(classOf[IteratorScope])) attach(connector, table, existingSfts.toMap ++ options(separator) + (sftKey -> sftOpt)) @@ -110,7 +111,7 @@ object StatsCombiner { def remove(sft: SimpleFeatureType, connector: AccumuloClient, table: String, separator: String): Unit = { getExisting(connector, table).foreach { existing => val sftKey = getSftKey(sft) - val existingSfts = existing.getOptions.asScala.filter(_._1.startsWith(StatsCombiner.SftOption)) + val existingSfts = existing.getOptions.asScala.filter(_._1.startsWith(SftOption)) if (existingSfts.asJava.containsKey(sftKey)) { connector.tableOperations().removeIterator(table, CombinerName, java.util.EnumSet.allOf(classOf[IteratorScope])) if (existingSfts.size > 1) { @@ -137,7 +138,7 @@ object StatsCombiner { } private def options(separator: String): Map[String, String] = - Map(StatsCombiner.SeparatorOption -> separator, "all" -> "true") + Map(SeparatorOption -> separator, "all" -> "true") private def getSftKey(sft: SimpleFeatureType): String = s"$SftOption${sft.getTypeName}" @@ -147,4 +148,19 @@ object StatsCombiner { options.foreach { case (k, v) => is.addOption(k, v) } connector.tableOperations().attachIterator(table, is) } + + /** + * Code copied from org.locationtech.geomesa.accumulo.data.AccumuloBackedMetadata.SingleRowAccumuloMetadata, + * just kept around for back compatibility + */ + private object SingleRowMetadata { + + private val MetadataTag = "~METADATA" + private val MetadataRowKeyRegex = (MetadataTag + """_(.*)""").r + + def getTypeName(row: Text): String = { + val MetadataRowKeyRegex(typeName) = row.toString + typeName + } + } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala similarity index 71% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala index bafbdc20e85e..5dedcd49cad9 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala +++ b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AgeOffIterator.scala @@ -10,11 +10,11 @@ package org.locationtech.geomesa.accumulo.iterators import com.typesafe.scalalogging.LazyLogging import org.apache.accumulo.core.client.IteratorSetting +import org.apache.accumulo.core.client.admin.TableOperations import org.apache.accumulo.core.data.{Key, Value} import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope import org.apache.accumulo.core.iterators.{Filter, IteratorEnvironment, SortedKeyValueIterator} import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloDataStore import org.locationtech.geomesa.index.filters.AgeOffFilter import org.locationtech.geomesa.utils.conf.FeatureExpiration import org.locationtech.geomesa.utils.conf.FeatureExpiration.IngestTimeExpiration @@ -68,9 +68,9 @@ object AgeOffIterator extends LazyLogging { is } - def expiry(ds: AccumuloDataStore, sft: SimpleFeatureType): Option[FeatureExpiration] = { + def expiry(tableOps: TableOperations, table: String): Option[FeatureExpiration] = { try { - list(ds, sft).map { is => + list(tableOps, table).map { is => val expiry = java.time.Duration.parse(is.getOptions.get(AgeOffFilter.Configuration.ExpiryOpt)).toMillis IngestTimeExpiration(Duration(expiry, TimeUnit.MILLISECONDS)) } @@ -79,29 +79,17 @@ object AgeOffIterator extends LazyLogging { } } - def list(ds: AccumuloDataStore, sft: SimpleFeatureType): Option[IteratorSetting] = { + def list(tableOps: TableOperations, table: String): Option[IteratorSetting] = { import org.locationtech.geomesa.utils.conversions.ScalaImplicits.RichIterator - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).iterator.filter(tableOps.exists).flatMap { table => - IteratorScope.values.iterator.flatMap(scope => Option(tableOps.getIteratorSetting(table, Name, scope))).headOption - }.headOption + IteratorScope.values.iterator.flatMap(scope => Option(tableOps.getIteratorSetting(table, Name, scope))).headOption } - def set(ds: AccumuloDataStore, sft: SimpleFeatureType, expiry: Duration): Unit = { - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).foreach { table => - if (tableOps.exists(table)) { - tableOps.attachIterator(table, configure(sft, expiry)) // all scopes - } - } - } + def set(tableOps: TableOperations, table: String, sft: SimpleFeatureType, expiry: Duration): Unit = + tableOps.attachIterator(table, configure(sft, expiry)) // all scopes - def clear(ds: AccumuloDataStore, sft: SimpleFeatureType): Unit = { - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach { table => - if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { - tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) - } + def clear(tableOps: TableOperations, table: String): Unit = { + if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { + tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ArrowIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ArrowIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ArrowIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ArrowIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AttributeKeyValueIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AttributeKeyValueIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AttributeKeyValueIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/AttributeKeyValueIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BaseAggregatingIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BaseAggregatingIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BaseAggregatingIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BaseAggregatingIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BinAggregatingIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BinAggregatingIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BinAggregatingIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/BinAggregatingIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DensityIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DensityIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DensityIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DensityIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala similarity index 68% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala index a1cf714855e6..08e77236e6e1 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala +++ b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/DtgAgeOffIterator.scala @@ -10,12 +10,11 @@ package org.locationtech.geomesa.accumulo.iterators import com.typesafe.scalalogging.LazyLogging import org.apache.accumulo.core.client.IteratorSetting +import org.apache.accumulo.core.client.admin.TableOperations import org.apache.accumulo.core.data.{Key, Value} import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope import org.apache.accumulo.core.iterators.{IteratorEnvironment, SortedKeyValueIterator} import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloDataStore -import org.locationtech.geomesa.accumulo.index.AccumuloJoinIndex import org.locationtech.geomesa.index.api.GeoMesaFeatureIndex import org.locationtech.geomesa.index.filters.{AgeOffFilter, DtgAgeOffFilter} import org.locationtech.geomesa.utils.conf.FeatureExpiration @@ -74,9 +73,9 @@ object DtgAgeOffIterator extends LazyLogging { is } - def expiry(ds: AccumuloDataStore, sft: SimpleFeatureType): Option[FeatureExpiration] = { + def expiry(tableOps: TableOperations, sft: SimpleFeatureType, table: String): Option[FeatureExpiration] = { try { - list(ds, sft).map { is => + list(tableOps, table).map { is => val attribute = sft.getDescriptor(is.getOptions.get(DtgAgeOffFilter.Configuration.DtgOpt).toInt).getLocalName val expiry = java.time.Duration.parse(is.getOptions.get(AgeOffFilter.Configuration.ExpiryOpt)).toMillis FeatureTimeExpiration(attribute, sft.indexOf(attribute), Duration(expiry, TimeUnit.MILLISECONDS)) @@ -86,35 +85,25 @@ object DtgAgeOffIterator extends LazyLogging { } } - def list(ds: AccumuloDataStore, sft: SimpleFeatureType): Option[IteratorSetting] = { + def list(tableOps: TableOperations, table: String): Option[IteratorSetting] = { import org.locationtech.geomesa.utils.conversions.ScalaImplicits.RichIterator - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).iterator.filter(tableOps.exists).flatMap { table => - IteratorScope.values.iterator.flatMap(scope => Option(tableOps.getIteratorSetting(table, Name, scope))).headOption - }.headOption + IteratorScope.values.iterator.flatMap(scope => Option(tableOps.getIteratorSetting(table, Name, scope))).headOption } - def set(ds: AccumuloDataStore, sft: SimpleFeatureType, expiry: Duration, dtg: String): Unit = { - val tableOps = ds.connector.tableOperations() - ds.manager.indices(sft).foreach { index => - val indexSft = index match { - case joinIndex: AccumuloJoinIndex => joinIndex.indexSft - case _ => sft - } - index.getTableNames(None).foreach { table => - if (tableOps.exists(table)) { - tableOps.attachIterator(table, configure(indexSft, index, expiry, Option(dtg))) // all scopes - } - } + def set( + tableOps: TableOperations, + sft: SimpleFeatureType, + index: GeoMesaFeatureIndex[_, _], + expiry: Duration, + dtg: String): Unit = { + index.getTableNames(None).foreach { table => + tableOps.attachIterator(table, configure(sft, index, expiry, Option(dtg))) // all scopes } } - def clear(ds: AccumuloDataStore, sft: SimpleFeatureType): Unit = { - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach { table => - if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { - tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) - } + def clear(tableOps: TableOperations, table: String): Unit = { + if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { + tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/FilterTransformIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/FilterTransformIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/FilterTransformIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/FilterTransformIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/KryoVisibilityRowEncoder.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/KryoVisibilityRowEncoder.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/KryoVisibilityRowEncoder.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/KryoVisibilityRowEncoder.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala similarity index 94% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala index f96662e4dc93..01687330d389 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala +++ b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/ProjectVersionIterator.scala @@ -39,7 +39,7 @@ class ProjectVersionIterator extends SortedKeyValueIterator[Key, Value] { columnFamilies: util.Collection[ByteSequence], inclusive: Boolean): Unit = {} - override def deepCopy(env: IteratorEnvironment) = throw new NotImplementedError() + override def deepCopy(env: IteratorEnvironment): ProjectVersionIterator = throw new NotImplementedError() } object ProjectVersionIterator { diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/RowFilterIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/RowFilterIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/RowFilterIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/RowFilterIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S2Iterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S2Iterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S2Iterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S2Iterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S3Iterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S3Iterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S3Iterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/S3Iterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/StatsIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/StatsIterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/StatsIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/StatsIterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala similarity index 59% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala index 1b4a59138304..87804683d14f 100644 --- a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala +++ b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/VisibilityIterator.scala @@ -12,8 +12,6 @@ import org.apache.accumulo.core.client.IteratorSetting import org.apache.accumulo.core.client.admin.TableOperations import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope import org.apache.accumulo.core.iterators.user.ReqVisFilter -import org.geotools.api.feature.simple.SimpleFeatureType -import org.locationtech.geomesa.accumulo.data.AccumuloDataStore object VisibilityIterator { @@ -23,17 +21,9 @@ object VisibilityIterator { def set(tableOps: TableOperations, table: String): Unit = tableOps.attachIterator(table, new IteratorSetting(Priority, Name, classOf[ReqVisFilter])) - def set(ds: AccumuloDataStore, sft: SimpleFeatureType): Unit = { - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach(set(tableOps, _)) - } - - def clear(ds: AccumuloDataStore, sft: SimpleFeatureType): Unit = { - val tableOps = ds.connector.tableOperations() - ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach { table => - if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { - tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) - } + def clear(tableOps: TableOperations, table: String): Unit = { + if (IteratorScope.values.exists(scope => tableOps.getIteratorSetting(table, Name, scope) != null)) { + tableOps.removeIterator(table, Name, java.util.EnumSet.allOf(classOf[IteratorScope])) } } } diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z2Iterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z2Iterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z2Iterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z2Iterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z3Iterator.scala b/geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z3Iterator.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-datastore/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z3Iterator.scala rename to geomesa-accumulo/geomesa-accumulo-iterators/src/main/scala/org/locationtech/geomesa/accumulo/iterators/Z3Iterator.scala diff --git a/geomesa-accumulo/geomesa-accumulo-jobs/pom.xml b/geomesa-accumulo/geomesa-accumulo-jobs/pom.xml index da7113c9c905..095cd26050b5 100644 --- a/geomesa-accumulo/geomesa-accumulo-jobs/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-jobs/pom.xml @@ -70,8 +70,17 @@ tests - org.apache.accumulo - accumulo-minicluster + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test + + + org.testcontainers + testcontainers + + + org.geomesa.testcontainers + testcontainers-accumulo diff --git a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/pom.xml b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/pom.xml index f089112b440f..191a2448b6e0 100644 --- a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/pom.xml @@ -49,11 +49,19 @@ org.apache.accumulo accumulo-core + ${accumulo-20.version} compile org.apache.accumulo accumulo-hadoop-mapreduce + ${accumulo-20.version} + compile + + + org.apache.thrift + libthrift + ${thrift-accumulo-20.version} compile @@ -118,8 +126,13 @@ tests - org.apache.accumulo - accumulo-minicluster + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test + + + org.testcontainers + testcontainers diff --git a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/pom.xml b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/pom.xml index 2ba6df3efc43..9bd2d1ff07f1 100644 --- a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/pom.xml @@ -49,31 +49,16 @@ org.apache.accumulo accumulo-core - ${accumulo-21.version} compile - - - org.apache.logging.log4j - * - - org.apache.accumulo accumulo-hadoop-mapreduce - ${accumulo-21.version} compile - - - org.apache.logging.log4j - * - - org.apache.thrift libthrift - ${thrift-accumulo-21.version} compile @@ -138,8 +123,17 @@ tests - org.apache.accumulo - accumulo-minicluster + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test + + + org.testcontainers + testcontainers + + + org.geomesa.testcontainers + testcontainers-accumulo diff --git a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/src/test/scala/org/locationtech/geomesa/accumulo/spark/AccumuloSparkProviderTest.scala b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/src/test/scala/org/locationtech/geomesa/accumulo/spark/AccumuloSparkProviderTest.scala similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/src/test/scala/org/locationtech/geomesa/accumulo/spark/AccumuloSparkProviderTest.scala rename to geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/src/test/scala/org/locationtech/geomesa/accumulo/spark/AccumuloSparkProviderTest.scala diff --git a/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/src/test/shared-resources b/geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/src/test/shared-resources similarity index 100% rename from geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo20/src/test/shared-resources rename to geomesa-accumulo/geomesa-accumulo-spark-runtime-accumulo21/src/test/shared-resources diff --git a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh index 6fa412eaa0b5..3d44aad34c22 100755 --- a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh +++ b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh @@ -67,14 +67,14 @@ function dependencies() { # add accumulo 2.1 jars if needed if version_ge "${accumulo_version}" 2.1.0; then gavs+=( - "org.apache.thrift:libthrift:%%thrift-accumulo-21.version%%:jar" + "org.apache.thrift:libthrift:%%thrift.version%%:jar" "io.opentelemetry:opentelemetry-api:1.19.0:jar" "io.opentelemetry:opentelemetry-context:1.19.0:jar" "io.micrometer:micrometer-core:1.9.6:jar" ) else gavs+=( - "org.apache.thrift:libthrift:%%thrift.version%%:jar" + "org.apache.thrift:libthrift:0.12.0:jar" ) fi diff --git a/geomesa-accumulo/geomesa-accumulo-tools/pom.xml b/geomesa-accumulo/geomesa-accumulo-tools/pom.xml index 2ceaef54b562..7c5fb1454380 100644 --- a/geomesa-accumulo/geomesa-accumulo-tools/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-tools/pom.xml @@ -129,8 +129,17 @@ tests - org.apache.accumulo - accumulo-minicluster + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test + + + org.testcontainers + testcontainers + + + org.geomesa.testcontainers + testcontainers-accumulo diff --git a/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/data/AccumuloAgeOffCommand.scala b/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/data/AccumuloAgeOffCommand.scala index a1a38bd9ef87..7fc10bbf46e2 100644 --- a/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/data/AccumuloAgeOffCommand.scala +++ b/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/data/AccumuloAgeOffCommand.scala @@ -9,6 +9,7 @@ package org.locationtech.geomesa.accumulo.tools.data import com.beust.jcommander.{Parameter, ParameterException, Parameters} +import org.apache.accumulo.core.client.IteratorSetting import org.locationtech.geomesa.accumulo.data.AccumuloDataStore import org.locationtech.geomesa.accumulo.iterators.{AgeOffIterator, DtgAgeOffIterator} import org.locationtech.geomesa.accumulo.tools.data.AccumuloAgeOffCommand.AccumuloAgeOffParams @@ -45,8 +46,16 @@ class AccumuloAgeOffCommand extends AccumuloDataStoreCommand { } lazy val mutable = SimpleFeatureTypes.mutable(sft) if (params.list) { - Command.user.info(s"Attribute age-off: ${DtgAgeOffIterator.list(ds, sft).getOrElse("None")}") - Command.user.info(s"Timestamp age-off: ${AgeOffIterator.list(ds, sft).getOrElse("None")}") + val tableOps = ds.connector.tableOperations() + val tables = ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists) + val dtgAgeOff = tables.foldLeft[Option[IteratorSetting]](None) { (res, table) => + res.orElse(DtgAgeOffIterator.list(tableOps, table)) + } + val ageOff = tables.foldLeft[Option[IteratorSetting]](None) { (res, table) => + res.orElse(AgeOffIterator.list(tableOps, table)) + } + Command.user.info(s"Attribute age-off: ${dtgAgeOff.getOrElse("None")}") + Command.user.info(s"Timestamp age-off: ${ageOff.getOrElse("None")}") } else if (params.set && params.dtgField == null) { if (Prompt.confirm(s"Configuring ingest-time-based age-off for schema '${params.featureName}' " + s"with expiry ${params.expiry}. Continue (y/n)? ")) { @@ -64,8 +73,11 @@ class AccumuloAgeOffCommand extends AccumuloDataStoreCommand { // clear the iterator configs if expiration wasn't configured in the schema, // as we can't detect that in updateSchema if (mutable.getUserData.remove(Configs.FeatureExpiration) == null) { - AgeOffIterator.clear(ds, sft) - DtgAgeOffIterator.clear(ds, sft) + val tableOps = ds.connector.tableOperations() + ds.getAllIndexTableNames(sft.getTypeName).filter(tableOps.exists).foreach { table => + AgeOffIterator.clear(tableOps, table) + DtgAgeOffIterator.clear(tableOps, table) + } } else { ds.updateSchema(sft.getTypeName, mutable) } diff --git a/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/stats/AccumuloStatsConfigureCommand.scala b/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/stats/AccumuloStatsConfigureCommand.scala index ec89d84b4525..bbedf4d7650c 100644 --- a/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/stats/AccumuloStatsConfigureCommand.scala +++ b/geomesa-accumulo/geomesa-accumulo-tools/src/main/scala/org/locationtech/geomesa/accumulo/tools/stats/AccumuloStatsConfigureCommand.scala @@ -9,8 +9,8 @@ package org.locationtech.geomesa.accumulo.tools.stats import com.beust.jcommander.Parameters +import org.locationtech.geomesa.accumulo.combiners.StatsCombiner import org.locationtech.geomesa.accumulo.data.AccumuloDataStore -import org.locationtech.geomesa.accumulo.data.stats.StatsCombiner import org.locationtech.geomesa.accumulo.tools.stats.AccumuloStatsConfigureCommand.AccumuloStatsConfigureParams import org.locationtech.geomesa.accumulo.tools.{AccumuloDataStoreCommand, AccumuloDataStoreParams} import org.locationtech.geomesa.tools.Command diff --git a/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/IngestCommandTest.scala b/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/IngestCommandTest.scala index 56f6b4c42d61..4a523a4f17aa 100644 --- a/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/IngestCommandTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/IngestCommandTest.scala @@ -12,7 +12,7 @@ import com.beust.jcommander.ParameterException import com.typesafe.config.{ConfigFactory, ConfigRenderOptions} import org.apache.commons.io.IOUtils import org.junit.runner.RunWith -import org.locationtech.geomesa.accumulo.MiniCluster +import org.locationtech.geomesa.accumulo.AccumuloContainer import org.locationtech.geomesa.accumulo.tools.{AccumuloDataStoreCommand, AccumuloRunner} import org.locationtech.geomesa.utils.collection.SelfClosingIterator import org.locationtech.geomesa.utils.io.WithClose @@ -30,11 +30,11 @@ class IngestCommandTest extends Specification { def baseArgs: Array[String] = Array( "ingest", - "--instance", MiniCluster.cluster.getInstanceName, - "--zookeepers", MiniCluster.cluster.getZooKeepers, - "--user", MiniCluster.Users.root.name, - "--password", MiniCluster.Users.root.password, - "--catalog", s"${MiniCluster.namespace}.${getClass.getSimpleName}${sftCounter.getAndIncrement()}", + "--instance", AccumuloContainer.instanceName, + "--zookeepers", AccumuloContainer.zookeepers, + "--user", AccumuloContainer.user, + "--password", AccumuloContainer.password, + "--catalog", s"${AccumuloContainer.Namespace}.${getClass.getSimpleName}${sftCounter.getAndIncrement()}", "--compact-stats", "false" ) diff --git a/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/ShpIngestTest.scala b/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/ShpIngestTest.scala index 5dd7373f0188..f9723d9a64bf 100644 --- a/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/ShpIngestTest.scala +++ b/geomesa-accumulo/geomesa-accumulo-tools/src/test/scala/org/locationtech/geomesa/accumulo/tools/ingest/ShpIngestTest.scala @@ -14,7 +14,7 @@ import org.geotools.data.store.ReprojectingFeatureCollection import org.geotools.referencing.CRS import org.geotools.util.factory.Hints import org.junit.runner.RunWith -import org.locationtech.geomesa.accumulo.MiniCluster +import org.locationtech.geomesa.accumulo.AccumuloContainer import org.locationtech.geomesa.convert.Modes import org.locationtech.geomesa.features.ScalaSimpleFeature import org.locationtech.geomesa.tools.Command.CommandException @@ -49,11 +49,11 @@ class ShpIngestTest extends Specification { def createCommand(file: String): AccumuloIngestCommand = { val command = new AccumuloIngestCommand() - command.params.user = MiniCluster.Users.root.name - command.params.instance = MiniCluster.cluster.getInstanceName - command.params.zookeepers = MiniCluster.cluster.getZooKeepers - command.params.password = MiniCluster.Users.root.password - command.params.catalog = s"${MiniCluster.namespace}.${getClass.getSimpleName}${sftCounter.getAndIncrement()}" + command.params.user = AccumuloContainer.user + command.params.instance = AccumuloContainer.instanceName + command.params.zookeepers = AccumuloContainer.zookeepers + command.params.password = AccumuloContainer.password + command.params.catalog = s"${AccumuloContainer.Namespace}.${getClass.getSimpleName}${sftCounter.getAndIncrement()}" command.params.force = true command.params.files = Collections.singletonList(new File(dir.toFile, s"$file.shp").getAbsolutePath) command.params.compact = false diff --git a/geomesa-accumulo/pom.xml b/geomesa-accumulo/pom.xml index 6525d6b18e33..d6339cd66eb0 100644 --- a/geomesa-accumulo/pom.xml +++ b/geomesa-accumulo/pom.xml @@ -20,6 +20,8 @@ geomesa-accumulo-dist geomesa-accumulo-distributed-runtime geomesa-accumulo-gs-plugin + geomesa-accumulo-indices + geomesa-accumulo-iterators geomesa-accumulo-jobs geomesa-accumulo-spark geomesa-accumulo-spark-runtime-accumulo20 diff --git a/geomesa-cassandra/geomesa-cassandra-datastore/pom.xml b/geomesa-cassandra/geomesa-cassandra-datastore/pom.xml index c20063d36e29..97a483e28763 100644 --- a/geomesa-cassandra/geomesa-cassandra-datastore/pom.xml +++ b/geomesa-cassandra/geomesa-cassandra-datastore/pom.xml @@ -90,6 +90,12 @@ + + org.apache.thrift + libthrift + ${thrift-cassandra.version} + test + org.cassandraunit diff --git a/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/guard/GraduatedQueryGuard.scala b/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/guard/GraduatedQueryGuard.scala index db105ef320cb..7a983b90b0d4 100644 --- a/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/guard/GraduatedQueryGuard.scala +++ b/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/guard/GraduatedQueryGuard.scala @@ -103,6 +103,10 @@ object GraduatedQueryGuard extends LazyLogging { }, "Graduated query guard percentages must be in range (0,1]") val percentageLimit: Option[Double] = samplePercent + + override def toString: String = + s"SizeAndLimits(sizeLimit=$sizeLimit, durationLimit=$durationLimit, " + + s"percentageLimit=$percentageLimit, sampleAttribute=$sampleAttribute)" } def disabled(typeName: String): Boolean = diff --git a/geomesa-lambda/geomesa-lambda-datastore/pom.xml b/geomesa-lambda/geomesa-lambda-datastore/pom.xml index cad729485b26..a1844c01c812 100644 --- a/geomesa-lambda/geomesa-lambda-datastore/pom.xml +++ b/geomesa-lambda/geomesa-lambda-datastore/pom.xml @@ -117,8 +117,9 @@ tests - org.apache.accumulo - accumulo-minicluster + org.locationtech.geomesa + geomesa-accumulo-distributed-runtime_${scala.binary.version} + test org.apache.kafka @@ -147,6 +148,10 @@ org.testcontainers kafka + + org.geomesa.testcontainers + testcontainers-accumulo + diff --git a/geomesa-lambda/geomesa-lambda-datastore/src/test/scala/org/locationtech/geomesa/lambda/LambdaContainerTest.scala b/geomesa-lambda/geomesa-lambda-datastore/src/test/scala/org/locationtech/geomesa/lambda/LambdaContainerTest.scala index a3c343e0087e..162b76c0dbaa 100644 --- a/geomesa-lambda/geomesa-lambda-datastore/src/test/scala/org/locationtech/geomesa/lambda/LambdaContainerTest.scala +++ b/geomesa-lambda/geomesa-lambda-datastore/src/test/scala/org/locationtech/geomesa/lambda/LambdaContainerTest.scala @@ -8,7 +8,7 @@ package org.locationtech.geomesa.lambda -import org.locationtech.geomesa.accumulo.MiniCluster +import org.locationtech.geomesa.accumulo.AccumuloContainer import org.locationtech.geomesa.kafka.KafkaContainerTest import org.locationtech.geomesa.lambda.LambdaContainerTest.TestClock @@ -22,10 +22,10 @@ class LambdaContainerTest extends KafkaContainerTest { val offsetManager = new InMemoryOffsetManager lazy val dsParams = Map( - "lambda.accumulo.instance.id" -> MiniCluster.cluster.getInstanceName, - "lambda.accumulo.zookeepers" -> MiniCluster.cluster.getZooKeepers, - "lambda.accumulo.user" -> MiniCluster.Users.root.name, - "lambda.accumulo.password" -> MiniCluster.Users.root.password, + "lambda.accumulo.instance.id" -> AccumuloContainer.instanceName, + "lambda.accumulo.zookeepers" -> AccumuloContainer.zookeepers, + "lambda.accumulo.user" -> AccumuloContainer.user, + "lambda.accumulo.password" -> AccumuloContainer.password, // note the table needs to be different to prevent testing errors "lambda.accumulo.catalog" -> sftName, "lambda.kafka.brokers" -> brokers, diff --git a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh index f9f9dd8aeafa..8f32f835bc70 100755 --- a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh +++ b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh @@ -79,14 +79,14 @@ function dependencies() { # add accumulo 2.1 jars if needed if version_ge "${accumulo_version}" 2.1.0; then gavs+=( - "org.apache.thrift:libthrift:%%thrift-accumulo-21.version%%:jar" + "org.apache.thrift:libthrift:%%thrift.version%%:jar" "io.opentelemetry:opentelemetry-api:1.19.0:jar" "io.opentelemetry:opentelemetry-context:1.19.0:jar" "io.micrometer:micrometer-core:1.9.6:jar" ) else gavs+=( - "org.apache.thrift:libthrift:%%thrift.version%%:jar" + "org.apache.thrift:libthrift:0.12.0:jar" ) fi diff --git a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/package.scala b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/package.scala index f33f914b0ea5..e8f3a2d87d72 100644 --- a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/package.scala +++ b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/package.scala @@ -16,7 +16,7 @@ import org.locationtech.geomesa.utils.geotools.GeoMesaParam.ReadWriteFlag package object security { val GEOMESA_AUDIT_PROVIDER_IMPL: SystemProperty = SystemProperty("geomesa.audit.provider.impl") - val GEOMESA_AUTH_PROVIDER_IMPL : SystemProperty = SystemProperty("geomesa.auth.provider.impl") + val GEOMESA_AUTH_PROVIDER_IMPL : SystemProperty = SystemProperty(AuthorizationsProvider.AUTH_PROVIDER_SYS_PROPERTY) val AuthsParam = new GeoMesaParam[String]( diff --git a/geomesa-spark/geomesa-spark-jts/src/test/scala/org/locationtech/geomesa/spark/jts/docs/JTSDocsTest.scala b/geomesa-spark/geomesa-spark-jts/src/test/scala/org/locationtech/geomesa/spark/jts/docs/JTSDocsTest.scala index d3978d4dbf2a..4fea4c532818 100644 --- a/geomesa-spark/geomesa-spark-jts/src/test/scala/org/locationtech/geomesa/spark/jts/docs/JTSDocsTest.scala +++ b/geomesa-spark/geomesa-spark-jts/src/test/scala/org/locationtech/geomesa/spark/jts/docs/JTSDocsTest.scala @@ -20,8 +20,16 @@ import org.specs2.runner.JUnitRunner */ @RunWith(classOf[JUnitRunner]) class JTSDocsTest extends Specification with TestEnvironment { + + sequential + + // before + step { + // Trigger initialization of spark session + val _ = spark + } + "jts documentation example" should { - sequential import org.apache.spark.sql.types._ import org.locationtech.geomesa.spark.jts._ @@ -68,14 +76,19 @@ class JTSDocsTest extends Specification with TestEnvironment { df.as[Point].first shouldEqual point } - val chicagoDF = dfBlank.withColumn("geom", st_makePoint(10.0, 10.0)) - "should search chicago" >> { import org.locationtech.geomesa.spark.jts._ import spark.implicits._ + + val chicagoDF = dfBlank.withColumn("geom", st_makePoint(10.0, 10.0)) chicagoDF.where(st_contains(st_makeBBOX(0.0, 0.0, 90.0, 90.0), $"geom")) chicagoDF.count() shouldEqual 1 } } + + // after + step { + spark.stop() + } } diff --git a/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/DataFrameFunctionsTest.scala b/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/DataFrameFunctionsTest.scala index 53066310bbbc..e4456e8f90ec 100644 --- a/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/DataFrameFunctionsTest.scala +++ b/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/DataFrameFunctionsTest.scala @@ -23,19 +23,19 @@ import java.{lang => jl} @RunWith(classOf[JUnitRunner]) class DataFrameFunctionsTest extends Specification with LazyLogging with SpatialRelations { - type DFRelation = (Column, Column) => TypedColumn[Any, jl.Boolean] - sequential - "DataFrame functions" should { + sequential - val spark = SparkSQLTestUtils.createSparkSession() - val sc = spark.sqlContext - sc.withJTS + var spark: SparkSession = _ + var dfBlank: DataFrame = _ - var dfBlank: DataFrame = null + "DataFrame functions" should { // before step { + spark = SparkSQLTestUtils.createSparkSession() + val sc = spark.sqlContext + sc.withJTS dfBlank = spark.createDataFrame(spark.sparkContext.makeRDD(Seq(Row())), StructType(Seq.empty)) } diff --git a/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/SparkSQLGeometricDistanceFunctionsTest.scala b/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/SparkSQLGeometricDistanceFunctionsTest.scala index 8b725e3adb4c..e38a06096571 100644 --- a/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/SparkSQLGeometricDistanceFunctionsTest.scala +++ b/geomesa-spark/geomesa-spark-sql/src/test/scala/org/locationtech/geomesa/spark/SparkSQLGeometricDistanceFunctionsTest.scala @@ -9,7 +9,8 @@ package org.locationtech.geomesa.spark import com.typesafe.scalalogging.LazyLogging -import org.geotools.api.data.DataStoreFinder +import org.apache.spark.sql.{SQLContext, SparkSession} +import org.geotools.api.data.{DataStore, DataStoreFinder} import org.junit.runner.RunWith import org.locationtech.geomesa.utils.text.WKTUtils import org.locationtech.jts.geom.Point @@ -23,24 +24,31 @@ class SparkSQLGeometricDistanceFunctionsTest extends Specification with LazyLogg import scala.collection.JavaConverters._ - "sql geometric distance functions" should { + sequential + + val dsParams: JMap[String, String] = Map("cqengine" -> "true", "geotools" -> "true").asJava - sequential - val dsParams: JMap[String, String] = Map("cqengine" -> "true", "geotools" -> "true").asJava + var ds: DataStore = _ + var spark: SparkSession = _ + var sc: SQLContext = _ - val ds = DataStoreFinder.getDataStore(dsParams) - val spark = SparkSQLTestUtils.createSparkSession() - val sc = spark.sqlContext + step { + ds = DataStoreFinder.getDataStore(dsParams) + spark = SparkSQLTestUtils.createSparkSession() + sc = spark.sqlContext SparkSQLTestUtils.ingestChicago(ds) val df = spark.read - .format("geomesa") - .options(dsParams) - .option("geomesa.feature", "chicago") - .load() + .format("geomesa") + .options(dsParams) + .option("geomesa.feature", "chicago") + .load() logger.debug(df.schema.treeString) df.createOrReplaceTempView("chicago") + } + + "sql geometric distance functions" should { "st_aggregateDistanceSpheroid" >> { "should work with window functions" >> { @@ -104,4 +112,8 @@ class SparkSQLGeometricDistanceFunctionsTest extends Specification with LazyLogg } } + step { + ds.dispose() + spark.close() + } } diff --git a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/conf/GeoMesaSystemProperties.scala b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/conf/GeoMesaSystemProperties.scala index 1b22ec2b5089..e3b53f9cc53c 100644 --- a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/conf/GeoMesaSystemProperties.scala +++ b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/conf/GeoMesaSystemProperties.scala @@ -45,7 +45,7 @@ object GeoMesaSystemProperties extends LazyLogging { Suffixes.Memory.bytes(value) match { case Success(b) => Some(b) case Failure(e) => - logger.warn(s"Invalid bytes for property $property=$value: $e") + logger.warn(s"Invalid bytes for property $property=$value: ${e.toString}") Option(default).flatMap(Suffixes.Memory.bytes(_).toOption) } } diff --git a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/converters/FastConverter.scala b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/converters/FastConverter.scala index 754f5313c804..67bf5a482e6c 100644 --- a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/converters/FastConverter.scala +++ b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/converters/FastConverter.scala @@ -69,7 +69,9 @@ object FastConverter extends StrictLogging { i += 1 } - logger.warn(s"Could not convert '$value' (of type ${value.getClass.getName}) to ${binding.getName}", new Exception()) + val msg = s"Could not convert '$value' (of type ${value.getClass.getName}) to ${binding.getName}" + logger.warn(msg) + logger.debug(msg, new Exception()) null.asInstanceOf[T] } diff --git a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/text/Suffixes.scala b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/text/Suffixes.scala index 68d4c7e82654..084c86d47052 100644 --- a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/text/Suffixes.scala +++ b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/text/Suffixes.scala @@ -68,7 +68,7 @@ object Suffixes { Failure(new ArithmeticException(s"Arithmetic overflow parsing '$s'")) } } recoverWith { - case e => Failure(new NumberFormatException(s"Error parsing memory property from input '$s': $e")) + case e => Failure(new NumberFormatException(s"Error parsing memory property from input '$s': ${e.toString}")) } } else { Failure(new NumberFormatException(s"Unable to match memory pattern from input '$s'")) diff --git a/pom.xml b/pom.xml index d3ddf8d902d2..2fd000112de7 100644 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ 2.10.1 5.1.0 3.6.12 - 2.8.0 + 2.9.0 4.0.3 3.0.0 3.2.6 @@ -112,18 +112,20 @@ 2.6 - 4.20.4 + 4.20.5 4.13.2 1.19.3 + 1.0.0 2.28.2 - 2.0.1 - 2.1.1 + 2.1.2 + 2.0.1 3.9.1 - 0.12.0 - 0.17.0 + 0.17.0 + 0.12.0 + 0.12.0 3.3.6 3.5.0 2.5.7-hadoop3 @@ -169,7 +171,7 @@ 3.5.1 3.4.4 - 3.2.3 + 3.2.5 3.2.3 3.1.1 3.3.0 @@ -188,9 +190,9 @@ --add-opens=java.base/java.nio=ALL-UNNAMED --illegal-access=warn -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=y ${maven.test.memory} ${maven.test.opens} - 4 - true + 2 false + 2.1.2 7-alpine 15.1 15-3.3 @@ -204,7 +206,7 @@ 2024 - 2.1.1 + 2.1.2 2.5.7 4.1.5 3.11.14 @@ -399,6 +401,16 @@ geomesa-accumulo-datastore_${scala.binary.version} ${project.version} + + org.locationtech.geomesa + geomesa-accumulo-iterators_${scala.binary.version} + ${project.version} + + + org.locationtech.geomesa + geomesa-accumulo-indices_${scala.binary.version} + ${project.version} + org.locationtech.geomesa geomesa-accumulo-distributed-runtime_${scala.binary.version} @@ -1838,7 +1850,7 @@ ${accumulo.version} provided - + org.apache.hadoop hadoop-client-api @@ -1851,6 +1863,10 @@ log4j * + + org.apache.logging.log4j + * + @@ -1859,7 +1875,7 @@ ${accumulo.version} provided - + org.apache.hadoop hadoop-client-api @@ -1868,6 +1884,10 @@ org.apache.hadoop hadoop-client-runtime + + org.apache.logging.log4j + * + @@ -1876,7 +1896,7 @@ ${accumulo.version} provided - + org.apache.hadoop hadoop-client-api @@ -1885,6 +1905,10 @@ org.apache.hadoop hadoop-client-runtime + + org.apache.logging.log4j + * + @@ -1899,7 +1923,7 @@ ${accumulo.version} provided - + org.apache.hadoop hadoop-client-api @@ -1908,6 +1932,10 @@ org.apache.hadoop hadoop-client-runtime + + org.apache.logging.log4j + * + @@ -2569,40 +2597,6 @@ - - org.apache.accumulo - accumulo-minicluster - ${accumulo.version} - test - - - log4j - * - - - org.slf4j - slf4j-log4j12 - - - - org.apache.hadoop - hadoop-client-api - - - org.apache.hadoop - hadoop-client-minicluster - - - org.apache.hadoop - hadoop-client-runtime - - - - org.openjfx - * - - - org.apache.hbase hbase-common @@ -2764,6 +2758,12 @@ ${testcontainers.version} test + + org.geomesa.testcontainers + testcontainers-accumulo + ${testcontainers.accumulo.version} + test + @@ -2879,7 +2879,9 @@ *:* + module-info.class META-INF/MANIFEST.MF + META-INF/DEPENDENCIES META-INF/*.SF META-INF/*.DSA META-INF/*.RSA @@ -2896,6 +2898,8 @@ ${module.name} + + @@ -2933,13 +2937,13 @@ org.apache.maven.surefire - surefire-junit4 + surefire-junit47 ${maven.surefire.plugin.version} - ${test.fork.count} - ${test.fork.reuse} + ${test.fork.count} + true 0 1 ${test.enable.assertions} @@ -2961,6 +2965,7 @@ true logging.properties slf4j + ${test.accumulo.docker.tag} ${test.postgis.docker.tag} ${test.postgres.docker.tag} ${test.redis.docker.tag}