From b563fb528d17ef68ed51fbd2b01670750f59da7b Mon Sep 17 00:00:00 2001 From: Emilio Date: Mon, 4 Mar 2024 21:00:49 +0000 Subject: [PATCH] GEOMESA-3335 Use accumulo-access for visibility expressions (#3049) --- build/cqs.tsv | 50 ++++++++++--------- .../common/s3/S3VisibilityObserver.scala | 16 +++--- .../index/planning/LocalQueryRunner.scala | 37 ++++++++------ geomesa-security/pom.xml | 4 ++ .../security/VisibilityEvaluator.scala | 3 ++ .../filter/VisibilityFilterFunction.scala | 13 ++--- pom.xml | 6 +++ 7 files changed, 73 insertions(+), 56 deletions(-) diff --git a/build/cqs.tsv b/build/cqs.tsv index 1d42f3c64eec..45fc0e454cfe 100644 --- a/build/cqs.tsv +++ b/build/cqs.tsv @@ -34,7 +34,7 @@ com.google.re2j:re2j 1.6 compile com.googlecode.concurrent-trees:concurrent-trees 2.6.1 compile com.googlecode.cqengine:cqengine 3.6.0 compile com.googlecode.json-simple:json-simple 1.1.1 compile -com.jayway.jsonpath:json-path 2.8.0 compile +com.jayway.jsonpath:json-path 2.9.0 compile com.thoughtworks.paranamer:paranamer 2.8 compile com.typesafe.scala-logging:scala-logging_2.12 3.9.5 compile com.typesafe:config 1.4.3 compile @@ -50,9 +50,9 @@ io.dropwizard.metrics:metrics-core 3.2.6 compile io.dropwizard.metrics:metrics-graphite 3.2.6 compile io.dropwizard.metrics:metrics-jvm 3.2.6 compile io.github.azagniotov:dropwizard-metrics-cloudwatch 1.0.13 compile -io.micrometer:micrometer-commons 1.10.6 compile -io.micrometer:micrometer-core 1.10.6 compile -io.micrometer:micrometer-observation 1.10.6 compile +io.micrometer:micrometer-commons 1.11.1 compile +io.micrometer:micrometer-core 1.11.1 compile +io.micrometer:micrometer-observation 1.11.1 compile io.netty:netty-buffer 4.1.106.Final compile io.netty:netty-codec 4.1.106.Final compile io.netty:netty-common 4.1.106.Final compile @@ -63,11 +63,11 @@ io.netty:netty-transport-classes-epoll 4.1.106.Final compile io.netty:netty-transport-native-epoll linux-x86_64:4.1.106.Final compile io.netty:netty-transport-native-unix-common 4.1.106.Final compile io.opentelemetry:opentelemetry-api 1.15.0 compile -io.opentelemetry:opentelemetry-api 1.25.0 compile +io.opentelemetry:opentelemetry-api 1.27.0 compile io.opentelemetry:opentelemetry-context 1.15.0 compile -io.opentelemetry:opentelemetry-context 1.25.0 compile +io.opentelemetry:opentelemetry-context 1.27.0 compile io.opentelemetry:opentelemetry-semconv 1.15.0-alpha compile -io.opentelemetry:opentelemetry-semconv 1.25.0-alpha compile +io.opentelemetry:opentelemetry-semconv 1.27.0-alpha compile io.prometheus:simpleclient 0.16.0 compile io.prometheus:simpleclient_common 0.16.0 compile io.prometheus:simpleclient_dropwizard 0.16.0 compile @@ -111,6 +111,7 @@ it.geosolutions.jaiext.warp:jt-warp 1.1.24 compile it.geosolutions.jaiext.zonal:jt-zonal 1.1.24 compile it.geosolutions.jgridshift:jgridshift-core 1.3 compile jakarta.annotation:jakarta.annotation-api 1.3.5 compile +jakarta.inject:jakarta.inject-api 2.0.0 compile jakarta.inject:jakarta.inject-api 2.0.1 compile jakarta.transaction:jakarta.transaction-api 1.3.3 compile javax.annotation:javax.annotation-api 1.2 compile @@ -122,17 +123,18 @@ net.jodah:typetools 0.6.1 compile net.sf.geographiclib:GeographicLib-Java 1.49 compile net.sf.saxon:Saxon-HE 12.4 compile org.antlr:antlr4-runtime 4.7.2 compile +org.apache.accumulo:accumulo-access 1.0.0-beta compile org.apache.accumulo:accumulo-core 2.0.1 compile -org.apache.accumulo:accumulo-core 2.1.1 compile +org.apache.accumulo:accumulo-core 2.1.2 compile org.apache.accumulo:accumulo-hadoop-mapreduce 2.0.1 compile -org.apache.accumulo:accumulo-hadoop-mapreduce 2.1.1 compile +org.apache.accumulo:accumulo-hadoop-mapreduce 2.1.2 compile org.apache.arrow:arrow-format 15.0.0 compile org.apache.arrow:arrow-memory-core 15.0.0 compile org.apache.arrow:arrow-memory-netty 15.0.0 compile org.apache.arrow:arrow-vector 15.0.0 compile org.apache.avro:avro 1.11.3 compile org.apache.commons:commons-collections4 4.4 compile -org.apache.commons:commons-compress 1.25.0 compile +org.apache.commons:commons-compress 1.26.0 compile org.apache.commons:commons-configuration2 2.9.0 compile org.apache.commons:commons-crypto 1.1.0 compile org.apache.commons:commons-csv 1.10.0 compile @@ -143,8 +145,8 @@ org.apache.commons:commons-text 1.11.0 compile org.apache.curator:curator-client 5.6.0 compile org.apache.curator:curator-framework 5.6.0 compile org.apache.curator:curator-recipes 5.6.0 compile -org.apache.datasketches:datasketches-java 3.3.0 compile -org.apache.datasketches:datasketches-memory 2.1.0 compile +org.apache.datasketches:datasketches-java 4.1.0 compile +org.apache.datasketches:datasketches-memory 2.2.0 compile org.apache.hbase.thirdparty:hbase-shaded-gson 4.1.5 compile org.apache.hbase.thirdparty:hbase-shaded-miscellaneous 4.1.5 compile org.apache.hbase.thirdparty:hbase-shaded-netty 4.1.5 compile @@ -183,7 +185,7 @@ org.apiguardian:apiguardian-api 1.1.0 compile org.apiguardian:apiguardian-api 1.1.2 compile org.calrissian.mango:mango-core 3.0.0 compile org.checkerframework:checker-qual 3.37.0 compile -org.checkerframework:checker-qual 3.41.0 compile +org.checkerframework:checker-qual 3.42.0 compile org.eclipse.collections:eclipse-collections 11.1.0 compile org.eclipse.collections:eclipse-collections-api 11.1.0 compile org.eclipse.emf:org.eclipse.emf.common 2.15.0 compile @@ -239,16 +241,16 @@ org.objenesis:objenesis 2.6 compile org.objenesis:objenesis 3.3 compile org.parboiled:parboiled-core 1.4.1 compile org.parboiled:parboiled-scala_2.12 1.3.1 compile -org.postgresql:postgresql 42.7.1 compile +org.postgresql:postgresql 42.7.2 compile org.scala-lang.modules:scala-parser-combinators_2.12 2.3.0 compile org.scala-lang.modules:scala-xml_2.12 2.2.0 compile -org.scala-lang:scala-library 2.12.18 compile -org.scala-lang:scala-reflect 2.12.18 compile +org.scala-lang:scala-library 2.12.19 compile +org.scala-lang:scala-reflect 2.12.19 compile org.slf4j:slf4j-api 1.7.36 compile org.threeten:threeten-extra 1.7.1 compile org.xmlresolver:xmlresolver 5.2.2 compile org.xmlresolver:xmlresolver data:5.2.2 compile -org.yaml:snakeyaml 1.33 compile +org.yaml:snakeyaml 2.0 compile picocontainer:picocontainer 1.2 compile redis.clients:jedis 5.1.0 compile si.uom:si-quantity 2.1 compile @@ -266,7 +268,7 @@ com.datastax.cassandra:cassandra-driver-mapping 3.11.5 provided io.confluent:kafka-avro-serializer 6.2.7 provided io.confluent:kafka-schema-registry-client 6.2.7 provided javax.media:jai_core 1.1.3 provided -org.apache.accumulo:accumulo-start 2.0.1 provided +org.apache.accumulo:accumulo-start 2.1.2 provided org.apache.commons:commons-text 1.10.0 provided org.apache.hadoop:hadoop-annotations 3.3.6 provided org.apache.hadoop:hadoop-client 3.3.6 provided @@ -288,7 +290,7 @@ org.apache.sedona:sedona-spark-common-3.4_2.12 1.5.0 provided org.apache.spark:spark-catalyst_2.12 3.5.0 provided org.apache.spark:spark-core_2.12 3.5.0 provided org.apache.spark:spark-sql_2.12 3.5.0 provided -org.scala-lang:scala-compiler 2.12.18 provided +org.scala-lang:scala-compiler 2.12.19 provided org.slf4j:slf4j-reload4j 1.7.36 provided org.springframework.security:spring-security-core 5.8.9 provided @@ -299,8 +301,7 @@ commons-lang:commons-lang 2.6 test io.confluent:kafka-schema-registry 6.2.7 test io.confluent:kafka-schema-registry tests:6.2.7 test junit:junit 4.13.2 test -org.apache.accumulo:accumulo-minicluster 2.0.1 test -org.apache.accumulo:accumulo-test 2.0.1 test +org.apache.accumulo:accumulo-test 2.1.2 test org.apache.arrow:arrow-vector tests:15.0.0 test org.apache.cassandra:cassandra-all 3.11.14 test org.apache.cassandra:cassandra-thrift 3.11.14 test @@ -315,14 +316,15 @@ org.apache.logging.log4j:log4j-core 2.22.1 test org.apache.sedona:sedona-common 1.5.0 test org.cassandraunit:cassandra-unit 3.7.1.0 test org.codehaus.groovy:groovy-jsr223 3.0.20 test +org.geomesa.testcontainers:testcontainers-accumulo 1.0.0 test org.geotools:gt-epsg-hsql 30.2 test org.jruby:jruby 9.4.5.0 test org.mockito:mockito-core 2.28.2 test org.mortbay.jetty:jetty 6.1.26 test org.slf4j:jul-to-slf4j 1.7.36 test -org.specs2:specs2-core_2.12 4.20.4 test -org.specs2:specs2-junit_2.12 4.20.4 test -org.specs2:specs2-mock_2.12 4.20.4 test +org.specs2:specs2-core_2.12 4.20.5 test +org.specs2:specs2-junit_2.12 4.20.5 test +org.specs2:specs2-mock_2.12 4.20.5 test org.testcontainers:kafka 1.19.3 test org.testcontainers:postgresql 1.19.3 test org.testcontainers:testcontainers 1.19.3 test diff --git a/geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common/src/main/scala/org/locationtech/geomesa/fs/storage/common/s3/S3VisibilityObserver.scala b/geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common/src/main/scala/org/locationtech/geomesa/fs/storage/common/s3/S3VisibilityObserver.scala index b742a8ad5994..5c0f399679db 100644 --- a/geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common/src/main/scala/org/locationtech/geomesa/fs/storage/common/s3/S3VisibilityObserver.scala +++ b/geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common/src/main/scala/org/locationtech/geomesa/fs/storage/common/s3/S3VisibilityObserver.scala @@ -9,10 +9,10 @@ package org.locationtech.geomesa.fs.storage.common.s3 import com.amazonaws.services.s3.AmazonS3 +import org.apache.accumulo.access.AccessExpression import org.apache.hadoop.fs.Path import org.geotools.api.feature.simple.SimpleFeature -import org.locationtech.geomesa.security.VisibilityEvaluator.VisibilityAnd -import org.locationtech.geomesa.security.{SecurityUtils, VisibilityEvaluator} +import org.locationtech.geomesa.security.SecurityUtils import java.nio.charset.StandardCharsets import java.util.Base64 @@ -37,14 +37,10 @@ class S3VisibilityObserver(s3: AmazonS3, path: Path, tag: String) extends S3Obje override protected def tags(): Iterable[(String, String)] = { if (visibilities.isEmpty) { Seq.empty } else { - val expressions = visibilities.flatMap { v => - VisibilityEvaluator.parse(v) match { - case VisibilityAnd(clauses) => clauses - case e => Seq(e) - } - } - val vis = VisibilityAnd(expressions.toSeq).expression - Seq(tag -> Base64.getEncoder.encodeToString(vis.getBytes(StandardCharsets.UTF_8))) + val vis = visibilities.mkString("(", ")&(", ")") + // this call simplifies and de-duplicates the expression + val expression = AccessExpression.of(vis, /*normalize = */true).getExpression + Seq(tag -> Base64.getEncoder.encodeToString(expression.getBytes(StandardCharsets.UTF_8))) } } } diff --git a/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/LocalQueryRunner.scala b/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/LocalQueryRunner.scala index cc15f7d4e6ef..31b6a7d3e5a0 100644 --- a/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/LocalQueryRunner.scala +++ b/geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/planning/LocalQueryRunner.scala @@ -9,6 +9,7 @@ package org.locationtech.geomesa.index.planning import com.typesafe.scalalogging.LazyLogging +import org.apache.accumulo.access.{AccessEvaluator, Authorizations} import org.geotools.api.data.Query import org.geotools.api.feature.simple.{SimpleFeature, SimpleFeatureType} import org.geotools.api.filter.Filter @@ -25,7 +26,7 @@ import org.locationtech.geomesa.index.planning.LocalQueryRunner.ArrowDictionaryH import org.locationtech.geomesa.index.planning.QueryRunner.QueryResult import org.locationtech.geomesa.index.stats.GeoMesaStats import org.locationtech.geomesa.index.utils.{Explainer, FeatureSampler, Reprojection, SortingSimpleFeatureIterator} -import org.locationtech.geomesa.security.{AuthorizationsProvider, SecurityUtils, VisibilityEvaluator} +import org.locationtech.geomesa.security.{AuthorizationsProvider, SecurityUtils} import org.locationtech.geomesa.utils.bin.BinaryEncodeCallback.ByteStreamCallback import org.locationtech.geomesa.utils.bin.BinaryOutputEncoder import org.locationtech.geomesa.utils.bin.BinaryOutputEncoder.EncodingOptions @@ -36,7 +37,7 @@ import org.locationtech.geomesa.utils.stats.Stat import org.locationtech.jts.geom.Envelope import java.io.ByteArrayOutputStream -import java.nio.charset.StandardCharsets +import scala.util.control.NonFatal /** * Query runner that handles transforms, visibilities and analytic queries locally. Subclasses are responsible @@ -122,8 +123,6 @@ object LocalQueryRunner extends LazyLogging { import org.locationtech.geomesa.index.conf.QueryHints.RichHints import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType - import scala.collection.JavaConverters._ - case class ArrowDictionaryHook(stats: GeoMesaStats, filter: Option[Filter]) /** @@ -135,7 +134,7 @@ object LocalQueryRunner extends LazyLogging { def visible(provider: Option[AuthorizationsProvider]): SimpleFeature => Boolean = { provider match { case None => noAuthVisibilityCheck - case Some(p) => authVisibilityCheck(_, p.getAuthorizations.asScala.map(_.getBytes(StandardCharsets.UTF_8)).toSeq) + case Some(p) => new AuthVisibilityCheck(p.getAuthorizations) } } @@ -425,14 +424,24 @@ object LocalQueryRunner extends LazyLogging { } /** - * Parses any visibilities in the feature and compares with the user's authorizations - * - * @param f simple feature to check - * @param auths authorizations for the current user - * @return true if feature is visible to the current user, otherwise false - */ - private def authVisibilityCheck(f: SimpleFeature, auths: Seq[Array[Byte]]): Boolean = { - val vis = SecurityUtils.getVisibility(f) - vis == null || VisibilityEvaluator.parse(vis).evaluate(auths) + * Parses any visibilities in the feature and compares with the user's authorizations + * + * @param auths authorizations for the current user + */ + private class AuthVisibilityCheck(auths: java.util.List[String]) extends (SimpleFeature => Boolean) { + + private val access = AccessEvaluator.of(Authorizations.of(auths)) + private val cache = scala.collection.mutable.Map.empty[String, Boolean] + + /** + * Checks auths against the feature's visibility + * + * @param f feature + * @return true if feature is visible to the current user, otherwise false + */ + override def apply(f: SimpleFeature): Boolean = { + val vis = SecurityUtils.getVisibility(f) + vis == null || cache.getOrElseUpdate(vis, try { access.canAccess(vis) } catch { case NonFatal(_) => false }) + } } } diff --git a/geomesa-security/pom.xml b/geomesa-security/pom.xml index 7ec49635a0b4..c90f1d1f0b76 100644 --- a/geomesa-security/pom.xml +++ b/geomesa-security/pom.xml @@ -15,6 +15,10 @@ org.locationtech.geomesa geomesa-utils_${scala.binary.version} + + org.apache.accumulo + accumulo-access + diff --git a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/VisibilityEvaluator.scala b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/VisibilityEvaluator.scala index 7bce61c2138c..70755c8367a2 100644 --- a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/VisibilityEvaluator.scala +++ b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/VisibilityEvaluator.scala @@ -19,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap /** * Evaluates visibilities against authorizations. Abstracted from Accumulo visibility code */ +@deprecated("Replaced with org.apache.accumulo:accumulo-access") object VisibilityEvaluator { private val Parser = new VisibilityEvaluator() @@ -63,6 +64,7 @@ object VisibilityEvaluator { /** * Parsed visibility that can be evaluated */ + @deprecated("Replaced with org.apache.accumulo:accumulo-access") sealed trait VisibilityExpression { /** @@ -153,6 +155,7 @@ object VisibilityEvaluator { private def isValidAuthChar(b: Byte): Boolean = validAuthChars(0xff & b) } +@deprecated("Replaced with org.apache.accumulo:accumulo-access") class VisibilityEvaluator private extends BasicParser { import org.locationtech.geomesa.security.VisibilityEvaluator._ diff --git a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/filter/VisibilityFilterFunction.scala b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/filter/VisibilityFilterFunction.scala index 44841d851e3a..7aaa38e56c94 100644 --- a/geomesa-security/src/main/scala/org/locationtech/geomesa/security/filter/VisibilityFilterFunction.scala +++ b/geomesa-security/src/main/scala/org/locationtech/geomesa/security/filter/VisibilityFilterFunction.scala @@ -8,26 +8,24 @@ package org.locationtech.geomesa.security.filter +import org.apache.accumulo.access.{AccessEvaluator, Authorizations} import org.geotools.api.feature.simple.SimpleFeature import org.geotools.api.filter.capability.FunctionName import org.geotools.api.filter.expression.Expression import org.geotools.filter.FunctionExpressionImpl import org.geotools.filter.capability.FunctionNameImpl import org.geotools.filter.capability.FunctionNameImpl.parameter -import org.locationtech.geomesa.security.{AuthUtils, SecurityUtils, VisibilityEvaluator} +import org.locationtech.geomesa.security.{AuthUtils, SecurityUtils} -import java.nio.charset.StandardCharsets import java.util.Collections import scala.util.Try class VisibilityFilterFunction extends FunctionExpressionImpl(VisibilityFilterFunction.Name) { - import scala.collection.JavaConverters._ - private val cache = scala.collection.mutable.Map.empty[String, java.lang.Boolean] - private val auths = - VisibilityFilterFunction.provider.getAuthorizations.asScala.map(_.getBytes(StandardCharsets.UTF_8)) + private val auths = Authorizations.of(VisibilityFilterFunction.provider.getAuthorizations) + private val access = AccessEvaluator.of(auths) private var expression: Expression = _ @@ -46,8 +44,7 @@ class VisibilityFilterFunction extends FunctionExpressionImpl(VisibilityFilterFu expression.evaluate(obj).asInstanceOf[String] } if (vis == null || vis.isEmpty) { java.lang.Boolean.FALSE } else { - cache.getOrElseUpdate(vis, - Try(Boolean.box(VisibilityEvaluator.parse(vis).evaluate(auths.toSeq))).getOrElse(java.lang.Boolean.FALSE)) + cache.getOrElseUpdate(vis, Try(Boolean.box(access.canAccess(vis))).getOrElse(java.lang.Boolean.FALSE)) } case _ => java.lang.Boolean.FALSE diff --git a/pom.xml b/pom.xml index c3f9f4005437..dcbba0f99b36 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 0.41 + 1.0.0-beta 15.0.0 1.11.3 1.12.625 @@ -1305,6 +1306,11 @@ ${gt.version} + + org.apache.accumulo + accumulo-access + ${accumulo.access.version} + com.esotericsoftware kryo-shaded