default.cfg

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


[GCP]
# The project in which to scan for buckets and load object information into a table.
PROJECT=CONFIGURE_ME

# The project in which to scan for buckets / objects only. Use this (or BIGQUERY.JOB_PROJECT) if to span BQ and GCS across different projects.
# GCS_PROJECT=CONFIGURE_ME

# Inventories object ACLS for objects in buckets without uniform bucket-level access (UBLA) enabled. NOTE: This option can add considerable time to the scan. 
# ACLS=yes

[RUNTIME]
# Number of worker threads. Two threads will be reserved for listing buckets, and the remaining threads will be used to send list pages into BigQuery. Even on a single core machine, this should be set to at least 4 to allow for context switches during IO waits.
WORKERS=64

# Amount of work items (page listings) to store. More items will use more memory, but a larger work queue can improve performance if you see throughput stuttering.
WORK_QUEUE_SIZE=1000

# Log level for the inventory loader. Default is INFO.
# LOG_LEVEL=DEBUG


[BIGQUERY]
# The dataset to use for inventory data.
DATASET_NAME=CONFIGURE_ME

# A table in which to place the object inventory.
INVENTORY_TABLE=object_metadata

# How many rows to stream into BigQuery before starting a new stream.
# Default is 100, which is conservative, but most configurations can run much larger. Higher numbers use more memory, and an excessively high number may hit BQ limits.
BATCH_WRITE_SIZE=500

# Project to use for running BQ jobs. This is useful if you want to run the job in one project but store the data in another.
# Default is GCP.PROJECT
# JOB_PROJECT=

[PUBSUB]
# The topic to listen to for object updates. Just give the short name of the topic, not the fully qualified name.
TOPIC_SHORT_NAME=gcs_updates

# The subscription to listen to for object updates (will be created if not found)
SUBSCRIPTION_SHORT_NAME=gcs_updates_sub_01

# The message wait timeout in seconds. Defaults to 10.
# This value shouldn't need adjusting. During the 10 second wait, notifications that are enqueued to be written to BigQuery
# could be lost in the event of a KP/plug-pull. If you need to shorten this window, you probably should also shrink the batch write size.
# TIMEOUT=