From 4d6f6478127d0c08acd7ac7803f7d0546aa01c00 Mon Sep 17 00:00:00 2001 From: Emilio Lahr-Vivaz Date: Wed, 31 Jul 2024 08:56:04 -0400 Subject: [PATCH] Docs - Make data store params more discoverable --- docs/user/accumulo/usage.rst | 61 +++++++++---------- docs/user/cassandra/install.rst | 1 - docs/user/cassandra/usage.rst | 40 ++++++------ docs/user/filesystem/usage.rst | 37 ++++++----- docs/user/hbase/usage.rst | 57 +++++++++-------- docs/user/kafka/usage.rst | 41 +++++-------- docs/user/lambda/usage.rst | 55 ++++++++--------- docs/user/postgis/usage.rst | 43 +++++++------ docs/user/redis/usage.rst | 38 ++++++------ .../geomesa/utils/geotools/GeoMesaParam.scala | 8 ++- 10 files changed, 180 insertions(+), 201 deletions(-) diff --git a/docs/user/accumulo/usage.rst b/docs/user/accumulo/usage.rst index b04643561a63..e6b13bc6dd43 100644 --- a/docs/user/accumulo/usage.rst +++ b/docs/user/accumulo/usage.rst @@ -1,37 +1,9 @@ -Using the Accumulo Data Store Programmatically -============================================== - -Creating a Data Store ---------------------- - -An instance of an Accumulo data store can be obtained through the normal GeoTools discovery methods, assuming -that the GeoMesa code is on the classpath: - -.. code-block:: java - - Map parameters = new HashMap<>; - parameters.put("accumulo.instance.name", "myInstance"); - parameters.put("accumulo.zookeepers", "myZoo1,myZoo2,myZoo3"); - parameters.put("accumulo.user", "myUser"); - parameters.put("accumulo.password", "myPassword"); - parameters.put("accumulo.catalog", "myNamespace.myTable"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - -Instead of specifying the cluster connection explicitly, an appropriate ``accumulo-client.properties`` may be added -to the classpath. See the -`Accumulo documentation `_ -for information on the necessary configuration keys. Any explicit data store parameters will take precedence over -the configuration file. - -More information on using GeoTools can be found in the `GeoTools user guide `_. - .. _accumulo_parameters: Accumulo Data Store Parameters ------------------------------- +============================== -The Accumulo Data Store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for an Accumulo data store (required parameters are marked with ``*``): ====================================== ======= ========================================================================== Parameter Type Description @@ -69,4 +41,31 @@ Parameter Type Description ``geomesa.partition.scan.parallel`` Boolean For partitioned schemas, execute scans in parallel instead of sequentially ====================================== ======= ========================================================================== -Note: it is an error to specify both ``accumulo.password`` and ``accumulo.keytab.path``. +.. note:: + + It is an error to specify both ``accumulo.password`` and ``accumulo.keytab.path``. + +Instead of specifying the cluster connection explicitly, an appropriate ``accumulo-client.properties`` may be added +to the classpath. See the +`Accumulo documentation `_ +for information on the necessary configuration keys. Any explicit data store parameters will take precedence over +the configuration file. + +Programmatic Access +------------------- + +An instance of an Accumulo data store can be obtained through the normal GeoTools discovery methods, assuming +that the GeoMesa code is on the classpath: + +.. code-block:: java + + Map parameters = new HashMap<>; + parameters.put("accumulo.instance.name", "myInstance"); + parameters.put("accumulo.zookeepers", "myZoo1,myZoo2,myZoo3"); + parameters.put("accumulo.user", "myUser"); + parameters.put("accumulo.password", "myPassword"); + parameters.put("accumulo.catalog", "myNamespace.myTable"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. \ No newline at end of file diff --git a/docs/user/cassandra/install.rst b/docs/user/cassandra/install.rst index a3be602c8ada..5b28c5d39b63 100644 --- a/docs/user/cassandra/install.rst +++ b/docs/user/cassandra/install.rst @@ -71,7 +71,6 @@ Building from Source GeoMesa Cassandra may also be built from source. For more information, refer to the instructions on `GitHub `__. -`GitHub `__. The remainder of the instructions in this chapter assume the use of the binary GeoMesa Cassandra distribution. If you have built from source, the distribution is created in the ``target`` directory of ``geomesa-cassandra/geomesa-cassandra-dist``. diff --git a/docs/user/cassandra/usage.rst b/docs/user/cassandra/usage.rst index 60a5c65e18a8..0cda76eb0d6b 100644 --- a/docs/user/cassandra/usage.rst +++ b/docs/user/cassandra/usage.rst @@ -1,27 +1,9 @@ -Using the Cassandra DataStore Programmatically -============================================== - -Creating a Data Store ---------------------- - -An instance of a Cassandra data store can be obtained through the normal GeoTools discovery methods, -assuming that the GeoMesa code is on the classpath. - -.. code-block:: java - - Map parameters = new HashMap<>(); - parameters.put("cassandra.contact.point", "127.0.0.1:9142"); - parameters.put("cassandra.keyspace", "geomesa"); - parameters.put("cassandra.catalog", "mycatalog"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - .. _cassandra_parameters: Cassandra Data Store Parameters -------------------------------- +=============================== -The data store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for a Cassandra data store (required parameters are marked with ``*``): ==================================== ======= ======================================================================================== Parameter Type Description @@ -40,5 +22,19 @@ Parameter Type Description ``geomesa.stats.enable`` Boolean Toggle collection of statistics (currently not implemented) ==================================== ======= ======================================================================================== -More information on using GeoTools can be found in the `GeoTools user guide -`__. +Programmatic Access +------------------- + +An instance of a Cassandra data store can be obtained through the normal GeoTools discovery methods, +assuming that the GeoMesa code is on the classpath. + +.. code-block:: java + + Map parameters = new HashMap<>(); + parameters.put("cassandra.contact.point", "127.0.0.1:9142"); + parameters.put("cassandra.keyspace", "geomesa"); + parameters.put("cassandra.catalog", "mycatalog"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/filesystem/usage.rst b/docs/user/filesystem/usage.rst index d4a491c2aedd..0af521143d77 100644 --- a/docs/user/filesystem/usage.rst +++ b/docs/user/filesystem/usage.rst @@ -1,27 +1,9 @@ -Using the FileSystem Data Store Programmatically -================================================ - -Creating a Data Store ---------------------- - -An instance of a FileSystem data store can be obtained through the normal GeoTools discovery methods, assuming that -the GeoMesa code is on the classpath: - -.. code-block:: java - - Map parameters = new HashMap<>; - parameters.put("fs.path", "hdfs://localhost:9000/fs-root/"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - -More information on using GeoTools can be found in the `GeoTools user guide `_. - .. _fsds_parameters: FileSystem Data Store Parameters --------------------------------- +================================ -The FileSystem data store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for a FileSystem data store (required parameters are marked with ``*``): =============================== ====== =================================================================================== Parameter Type Description @@ -37,3 +19,18 @@ Parameter Type Description ``fs.config.xml`` String Additional Hadoop configuration properties, as a standard XML ```` element =============================== ====== =================================================================================== + +Programmatic Access +------------------- + +An instance of a FileSystem data store can be obtained through the normal GeoTools discovery methods, assuming that +the GeoMesa code is on the classpath: + +.. code-block:: java + + Map parameters = new HashMap<>; + parameters.put("fs.path", "hdfs://localhost:9000/fs-root/"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/hbase/usage.rst b/docs/user/hbase/usage.rst index 35fa3964b209..dd642ee278ae 100644 --- a/docs/user/hbase/usage.rst +++ b/docs/user/hbase/usage.rst @@ -1,33 +1,9 @@ -Using the HBase Data Store Programmatically -=========================================== - -Creating a Data Store ---------------------- - -An instance of an HBase data store can be obtained through the normal GeoTools discovery methods, -assuming that the GeoMesa code is on the classpath. - -The HBase data store also requires that an appropriate ``hbase-site.xml`` file is available on the classpath; the -connection parameters for HBase are obtained from this file, including ``hbase.zookeeper.quorum`` and -``hbase.zookeeper.property.clientPort``. - -As an alternative to providing ``hbase-site.xml``, the Zookeeper connection can be specified through the -parameter ``hbase.zookeepers``. However, this method is not recommended, as other important configurations -(including security, if any) from ``hbase-site.xml`` may be required for correct operation. - -.. code-block:: java - - Map parameters = new HashMap<>(); - parameters.put("hbase.catalog", "geomesa"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - .. _hbase_parameters: HBase Data Store Parameters ---------------------------- +=========================== -The data store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for an HBase data store (required parameters are marked with ``*``): =========================================== ======= ======================================================================================== Parameter Type Description @@ -64,7 +40,30 @@ Parameter Type Description ``geomesa.partition.scan.parallel`` Boolean For partitioned schemas, execute scans in parallel instead of sequentially =========================================== ======= ======================================================================================== -Note: the ``hbase.coprocessor.*.enable`` parameters will be superseded by ``hbase.remote.filtering=false``. +.. note:: + + The HBase data store will also load any ``hbase-site.xml`` file that is available on the classpath. + +.. note:: + + The ``hbase.coprocessor.*.enable`` parameters will be superseded by ``hbase.remote.filtering=false``. + +The HBase data store needs an ``hbase-site.xml`` file in order to connect to HBase. This can be provided in multiple ways. +For simple use cases, the Zookeeper connection can be specified through the parameter ``hbase.zookeepers``. For normal use, +``hbase-site.xml`` can be provided at the root of the classpath, at an arbitrary location pointed to by ``hbase.config.paths``, +inline using ``hbase.config.xml``, or any combination of the above. + +Programmatic Access +------------------- + +An instance of an HBase data store can be obtained through the normal GeoTools discovery methods, +assuming that the GeoMesa code is on the classpath. + +.. code-block:: java + + Map parameters = new HashMap<>(); + parameters.put("hbase.catalog", "geomesa"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); -More information on using GeoTools can be found in the `GeoTools user guide -`__. +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/kafka/usage.rst b/docs/user/kafka/usage.rst index 04f841e5d3cc..e6569f91d91e 100644 --- a/docs/user/kafka/usage.rst +++ b/docs/user/kafka/usage.rst @@ -1,28 +1,7 @@ -Using the Kafka Data Store Programmatically -=========================================== - -Creating a Data Store ---------------------- - -An instance of a Kafka data store can be obtained through the normal GeoTools discovery methods, -assuming that the GeoMesa code is on the classpath. To create a ``KafkaDataStore`` there are two -required properties, one for the Apache Kafka connection, ``kafka.brokers``, and one for the Apache -Zookeeper connection, ``kafka.zookeepers``. An optional parameter, ``kafka.zk.path`` is -used to specify a path in Zookeeper under which schemas are stored. If no zk path is specified then -a default path will be used. Configuration parameters are described fully below. - -.. code-block:: java - - Map parameters = new HashMap<>(); - parameters.put("kafka.brokers", "localhost:9092"); - parameters.put("kafka.zookeepers", "localhost:2181"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - .. _kafka_parameters: Kafka Data Store Parameters ---------------------------- +=========================== The Kafka data store differs from most data stores in that the data set is kept entirely in memory. Because of this, the in-memory indexing can be configured at runtime through data store parameters. See :ref:`kafka_index_config` for @@ -31,7 +10,7 @@ more information on the available indexing options. Because configuration options can reference attributes from a particular SimpleFeatureType, it may be necessary to create multiple Kafka data store instances when dealing with multiple schemas. -The Kafka data store accepts the following parameters (required parameters are marked with ``*``): +Use the following parameters for a Kafka data store (required parameters are marked with ``*``): ==================================== ======= ==================================================================================================== Parameter Type Description @@ -83,5 +62,17 @@ Parameter Type Description ``geomesa.security.auths`` String Default authorizations used to query data, comma-separated ==================================== ======= ==================================================================================================== -More information on using GeoTools can be found in the `GeoTools user guide -`__. +Programmatic Access +------------------- + +An instance of a Kafka data store can be obtained through the normal GeoTools discovery methods, +assuming that the GeoMesa code is on the classpath. + +.. code-block:: java + + Map parameters = new HashMap<>(); + parameters.put("kafka.brokers", "localhost:9092"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/lambda/usage.rst b/docs/user/lambda/usage.rst index e53bd124466e..0fb515abb84c 100644 --- a/docs/user/lambda/usage.rst +++ b/docs/user/lambda/usage.rst @@ -1,34 +1,9 @@ -Using the Lambda Data Store Programmatically -============================================ - -Creating a Data Store ---------------------- - -An instance of a Lambda data store can be obtained through the normal GeoTools discovery methods, assuming -that the GeoMesa code is on the classpath: - -.. code-block:: java - - Map parameters = new HashMap<>; - parameters.put("lambda.accumulo.instance.name", "myInstance"); - parameters.put("lambda.accumulo.zookeepers", "zoo1,zoo2,zoo3"); - parameters.put("lambda.accumulo.user", "myUser"); - parameters.put("lambda.accumulo.password", "myPassword"); - parameters.put("lambda.accumulo.tableName", "my_table"); - parameters.put("lambda.kafka.brokers", "kafka1:9092,kafka2:9092"); - parameters.put("lambda.kafka.zookeepers", "zoo1,zoo2,zoo3"); - parameters.put("lambda.expiry", "10 minutes"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - -More information on using GeoTools can be found in the `GeoTools user guide `_. - .. _lambda_parameters: Lambda Data Store Parameters ----------------------------- +============================ -The data store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for a Lambda data store (required parameters are marked with ``*``): ====================================== ======= ================================================================================================== Parameter Type Description @@ -64,4 +39,28 @@ Parameter Type Description ``geomesa.stats.enable`` Boolean Toggle collection of statistics for newly created feature types ====================================== ======= ================================================================================================== -Note: one (but not both) of ``lambda.accumulo.password`` and ``lambda.accumulo.keytab.path`` must be provided. +.. note: + + One (but not both) of ``lambda.accumulo.password`` and ``lambda.accumulo.keytab.path`` must be provided. + +Programmatic Access +------------------- + +An instance of a Lambda data store can be obtained through the normal GeoTools discovery methods, assuming +that the GeoMesa code is on the classpath: + +.. code-block:: java + + Map parameters = new HashMap<>; + parameters.put("lambda.accumulo.instance.name", "myInstance"); + parameters.put("lambda.accumulo.zookeepers", "zoo1,zoo2,zoo3"); + parameters.put("lambda.accumulo.user", "myUser"); + parameters.put("lambda.accumulo.password", "myPassword"); + parameters.put("lambda.accumulo.tableName", "my_table"); + parameters.put("lambda.kafka.brokers", "kafka1:9092,kafka2:9092"); + parameters.put("lambda.kafka.zookeepers", "zoo1,zoo2,zoo3"); + parameters.put("lambda.expiry", "10 minutes"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/postgis/usage.rst b/docs/user/postgis/usage.rst index 8127f7d25398..b822bd2a13ff 100644 --- a/docs/user/postgis/usage.rst +++ b/docs/user/postgis/usage.rst @@ -1,28 +1,7 @@ -Using the Partitioned PostGIS Data Store Programmatically -========================================================= - -Creating a Data Store ---------------------- - -An instance of a PostGIS data store can be obtained through the normal GeoTools discovery methods, -assuming that the GeoMesa code is on the classpath. - -.. code-block:: java - - Map parameters = new HashMap<>(); - parameters.put("dbtype", "postgis-partitioned"); - parameters.put("host", "localhost"); - parameters.put("port", "5432"); - parameters.put("database", "geomesa"); - parameters.put("user", "postgres"); - parameters.put("passwd", "postgres"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - .. _pg_partition_parameters: Partitioned PostGIS Data Store Parameters ------------------------------------------ +========================================= The partitioned PostGIS data store uses the same parameters as the standard PostGIS data store, except that ``dbtype`` **must** be set to ``postgis-partitioned``. See the @@ -39,3 +18,23 @@ Parameter Type Description for more information. Setting this timeout may help prevent abandoned queries from slowing down database operations. ======================================= ======== =================================================================================================================================== + +Programmatic Access +------------------- + +An instance of a PostGIS data store can be obtained through the normal GeoTools discovery methods, +assuming that the GeoMesa code is on the classpath. + +.. code-block:: java + + Map parameters = new HashMap<>(); + parameters.put("dbtype", "postgis-partitioned"); + parameters.put("host", "localhost"); + parameters.put("port", "5432"); + parameters.put("database", "geomesa"); + parameters.put("user", "postgres"); + parameters.put("passwd", "postgres"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/docs/user/redis/usage.rst b/docs/user/redis/usage.rst index 35d4246599fc..03d5ff91f31d 100644 --- a/docs/user/redis/usage.rst +++ b/docs/user/redis/usage.rst @@ -1,26 +1,9 @@ -Using the Redis Data Store Programmatically -=========================================== - -Creating a Data Store ---------------------- - -An instance of a Redis data store can be obtained through the normal GeoTools discovery methods, -assuming that the GeoMesa code is on the classpath. - -.. code-block:: java - - Map parameters = new HashMap<>(); - parameters.put("redis.url", "redis://localhost:6379"); - parameters.put("redis.catalog", "geomesa"); - org.geotools.api.data.DataStore dataStore = - org.geotools.api.data.DataStoreFinder.getDataStore(parameters); - .. _redis_parameters: Redis Data Store Parameters ---------------------------- +=========================== -The data store takes several parameters (required parameters are marked with ``*``): +Use the following parameters for a Redis data store (required parameters are marked with ``*``): ====================================== ======= ==================================================================================== Parameter Type Description @@ -47,5 +30,18 @@ Parameter Type Description ``geomesa.partition.scan.parallel`` Boolean For partitioned schemas, execute scans in parallel instead of sequentially ====================================== ======= ==================================================================================== -More information on using GeoTools can be found in the `GeoTools user guide -`__. +Creating a Data Store +--------------------- + +An instance of a Redis data store can be obtained through the normal GeoTools discovery methods, +assuming that the GeoMesa code is on the classpath. + +.. code-block:: java + + Map parameters = new HashMap<>(); + parameters.put("redis.url", "redis://localhost:6379"); + parameters.put("redis.catalog", "geomesa"); + org.geotools.api.data.DataStore dataStore = + org.geotools.api.data.DataStoreFinder.getDataStore(parameters); + +More information on using GeoTools can be found in the `GeoTools user guide `_. diff --git a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/GeoMesaParam.scala b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/GeoMesaParam.scala index e0d2f94789a4..508c6051e14b 100644 --- a/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/GeoMesaParam.scala +++ b/geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/GeoMesaParam.scala @@ -57,7 +57,7 @@ class GeoMesaParam[T <: AnyRef]( desc, !optional, GeoMesaParam.sample(default), - GeoMesaParam.metadata(password, largeText, extension, enumerations, supportsNiFiExpressions) + GeoMesaParam.metadata(password, largeText, extension, enumerations, supportsNiFiExpressions, deprecatedKeys.contains(_key)) ) with LazyLogging { private val deprecated = deprecatedKeys ++ deprecatedParams.map(_.key) @@ -265,7 +265,8 @@ object GeoMesaParam { largeText: Boolean, extension: String, enumerations: Seq[AnyRef], - supportsNiFiExpressions: Boolean): java.util.Map[String, AnyRef] = { + supportsNiFiExpressions: Boolean, + deprecated: Boolean = false): java.util.Map[String, AnyRef] = { var opt: Option[java.util.Map[String, AnyRef]] = None lazy val map = { val m = new java.util.HashMap[String, AnyRef] @@ -290,6 +291,9 @@ object GeoMesaParam { if (supportsNiFiExpressions) { map.put(SupportsNiFiExpressions, java.lang.Boolean.TRUE) } + if (deprecated) { + map.put(Parameter.DEPRECATED, java.lang.Boolean.TRUE) + } opt.map(Collections.unmodifiableMap[String,AnyRef]).orNull }