Migrate configuration handling to the confuse lib (#364)

* WIP: Transform log parsing and handling to use the confuse lib

* Default log level to INFO, include stern warning with defaults

* Implement the confuse library handling in notifiers

I've tested and verified the following to function:
- Telegram
- Pushover

* Expect ConfigViews correctly in typing

* Simplify wallet mojo testing

* Amend documentation for new config handling.

* Explicitly handle daily_stats average times as floats

* Make non_skipped_signage_points checking typing explicit

* Type signage_point_stats init values explicitly

* Fix silly confuse import

* More explicit typing and minor cleanup

* Validate log consumer config via confuse templates

* Add more type safety to notifier parsing

* Fix handler enable checking

This ensures the backwards compatibility is still carried forward of
enabling misconfigured handlers.

* Also disable confuse.exceptions type checking since there are no stubs

* Fix template formatting with black

* Set config defaults to the conservative values that were in code

* Move notifier_defaults config block to top level

This prevents it getting interpreted as a valid notifier and causing a
warning.

* Add comments to config-example.yaml where the default differs

* Handle MQTT notifier type checking via confuse

* Improve testability of tests that use SHOWCASE_NOTIFICATIONS

Now only the showcases are run where other tokens are also present

* Dedupe test env variables of mqtt & smtp with prefixes

Also, switch SMTP testing to verify SMTP_HOST, not SMTP_USERNAME as
username is not a mandatory param for smtp to work, but a host is.

* Validate SMTP notifier credentials with template

Also amend SMTP testing to allow setting SMTP auth

.github/workflows/ci-workflow.yml

@@ -38,7 +38,7 @@ jobs:
         flake8 src tests --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Type Check with MyPy
       run: |
-        mypy src tests
+        mypy --install-types --non-interactive --check-untyped-defs src tests
     - name: Unit Tests
       run: |
         python -m unittest

INTEGRATIONS.md

@@ -38,7 +38,15 @@ verify the sender email.
 Test with:
 
 ```
-SENDER="[email protected]" SENDER_NAME="ChiaDog" RECIPIENT="[email protected]" HOST=smtp.example.com PORT=587 USERNAME_SMTP=username PASSWORD_SMTP=password python3 -m unittest tests.notifier.test_smtp_notifier
+SMTP_SENDER="[email protected]" \
+SMTP_SENDER_NAME="ChiaDog" \
+SMTP_RECIPIENT="[email protected]" \
+SMTP_HOST=smtp.example.com \
+SMTP_PORT=587 \
+SMTP_ENABLE_AUTH=true \
+SMTP_USERNAME=username \
+SMTP_PASSWORD=password \
+python3 -m unittest tests.notifier.test_smtp_notifier
 ```
 
 ## Slack
@@ -113,7 +121,7 @@ Messages sent to the MQTT topic look like this:
 Test with:
 
 ```
-HOST=<hostname> PORT=<port> TOPIC=<mqtt_topic> python3 -m unittest tests.notifier.test_mqtt_notifier 
+MQTT_HOST=<hostname> MQTT_PORT=<port> MQTT_TOPIC=<mqtt_topic> python3 -m unittest tests.notifier.test_mqtt_notifier 
 ```
 
 Or with full parameters:

README.md

@@ -125,7 +125,7 @@ cd chiadog
 cp config-example.yaml config.yaml
 ```
 
-4. Open up `config.yaml` in your editor and configure it to your preferences.
+4. Open up `config.yaml` in your editor and configure it to your preferences. The example is large, feel free to omit any portions where you're fine with the defaults!
 
 ### Updating to the latest release
 
@@ -141,8 +141,6 @@ git pull
 ./install.sh
 ```
 
-> Important: Automated migration of config is not supported. Please check that your `config.yaml` has all new fields introduced in `config-example.yaml` and add anything missing. If correctly migrated, you shouldn't get any ERROR logs.
-
 ## Monitoring a local harvester / farmer
 
 1. Open `config.yaml` and configure `file_log_consumer`:

config-example.yaml

@@ -1,5 +1,7 @@
 # Please copy this example config to config.yaml
 # and adjust it to your needs.
+# Most config values have sane defaults! This example is more verbose than it needs to be,
+# your config only needs to override what you want to change.
 
 # This is useful to differentiate multiple chiadog
 # instances monitoring multiple harvesters
@@ -29,7 +31,7 @@ keep_alive_monitor:
 # Enable this and you'll receive a daily summary notification
 # on your farm performance at the specified time of the day.
 daily_stats:
-  enable: true
+  enable: true # default: false
   time_of_day: "21:00"
   frequency_hours: 24
 
@@ -40,7 +42,7 @@ handlers:
     enable: true
     # Transactions with lower amount mojos will be ignored
     # Use this to avoid notification spam during dust storms
-    min_mojos_amount:  5
+    min_mojos_amount:  5 # default: 0
   # Checks for skipped signage points (full node)
   finished_signage_point_handler:
     enable: true
@@ -61,6 +63,9 @@ handlers:
 # notifications to each of them. E.g. enable daily_stats only to E-mail.
 # If you enable wallet_events you'll get notifications anytime your
 # wallet receives some XCH (e.g. farming reward).
+#
+# NOTE: No notifier is enabled by default, and all notification topics are disabled by default.
+# You'll need to enable the notifiers and topics that you want to see!
 notifier:
   pushover:
     enable: false
@@ -134,7 +139,7 @@ notifier:
     decreasing_plot_events: true
     increasing_plot_events: false
     topic: chia/chiadog/alert
-    qos: 1
+    qos: 1 # default: 0
     retain: false
     credentials:
       host: '192.168.0.10'

main.py

@@ -8,11 +8,14 @@
 from pathlib import Path
 from typing import Tuple
 
+# lib
+import confuse
+
 # project
 from src.chia_log.handlers.daily_stats.stats_manager import StatsManager
 from src.chia_log.log_consumer import create_log_consumer_from_config
 from src.chia_log.log_handler import LogHandler
-from src.config import Config, is_win_platform
+from src.util import is_win_platform
 from src.notifier.keep_alive_monitor import KeepAliveMonitor
 from src.notifier.notify_manager import NotifyManager
 
@@ -43,8 +46,8 @@ def get_log_level(log_level: str) -> int:
     return logging.INFO
 
 
-def init(config:Config):
-    log_level = get_log_level(config.get_log_level_config())
+def init(config: confuse.core.Configuration):
+    log_level = get_log_level(config["log_level"].get())
     logging.basicConfig(
         format="[%(asctime)s] [%(levelname)8s] --- %(message)s (%(filename)s:%(lineno)s)",
         level=log_level,
@@ -54,24 +57,24 @@ def init(config:Config):
     logging.info(f"Starting Chiadog ({version()})")
 
     # Create log consumer based on provided configuration
-    chia_logs_config = config.get_chia_logs_config()
+    chia_logs_config = config['chia_logs']
     log_consumer = create_log_consumer_from_config(chia_logs_config)
     if log_consumer is None:
         exit(0)
 
     # Keep a reference here so we can stop the thread
     # TODO: read keep-alive thresholds from config
-    keep_alive_monitor = KeepAliveMonitor(config=config.get_keep_alive_monitor_config())
+    keep_alive_monitor = KeepAliveMonitor(config=config['keep_alive_monitor'])
 
     # Notify manager is responsible for the lifecycle of all notifiers
     notify_manager = NotifyManager(config=config, keep_alive_monitor=keep_alive_monitor)
 
     # Stats manager accumulates stats over 24 hours and sends a summary each day
-    stats_manager = StatsManager(config=config.get_daily_stats_config(), notify_manager=notify_manager)
+    stats_manager = StatsManager(config=config['daily_stats'], notify_manager=notify_manager)
 
     # Link stuff up in the log handler
     # Pipeline: Consume -> Handle -> Notify
-    log_handler = LogHandler(config=config.get_handlers_config(), log_consumer=log_consumer, notify_manager=notify_manager,
+    log_handler = LogHandler(config=config, log_consumer=log_consumer, notify_manager=notify_manager,
                              stats_manager=stats_manager)
 
     def interrupt(signal_number, frame):
@@ -108,8 +111,15 @@ def version():
     # Parse config and configure logger
     argparse, args = parse_arguments()
 
+    # init sane config defaults
+    source_path = Path(__file__).resolve()
+    source_dir = source_path.parent
+    config = confuse.Configuration('chiadog', __name__)
+    config.set_file(source_dir / 'src/default_config.yaml')
+
+    # Override with given config
     if args.config:
-        conf = Config(Path(args.config))
-        init(conf)
+        config.set_file(Path(args.config))
+        init(config)
     elif args.version:
         print(version())

pyproject.toml

@@ -1,2 +1,7 @@
 [tool.black]
-line-length = 120
+line-length = 120
+
+# No type hints: https://github.com/beetbox/confuse/issues/121
+[[tool.mypy.overrides]]
+module = ["confuse", "confuse.exceptions"]
+ignore_missing_imports = true

requirements.txt

@@ -3,3 +3,4 @@ python-dateutil~=2.8.1
 PyYAML==5.4
 retry==0.9.2
 pygtail==0.11.1
+confuse==2.0.0

src/chia_log/handlers/__init__.py

@@ -11,6 +11,9 @@ class that analyses a specific part of the log
 from typing import List, Optional
 import logging
 
+# lib
+from confuse import ConfigView
+
 # project
 from .daily_stats.stats_manager import StatsManager
 from src.notifier import Event
@@ -24,7 +27,7 @@ class LogHandlerInterface(ABC):
     def config_name() -> str:
         pass
 
-    def __init__(self, config: Optional[dict] = None):
+    def __init__(self, config: ConfigView):
         logging.info(f"Initializing handler: {self.config_name()}")
 
     @abstractmethod

src/chia_log/handlers/condition_checkers/non_skipped_signage_points.py

@@ -1,6 +1,7 @@
 # std
 import logging
 from typing import Optional
+from datetime import datetime
 
 # project
 from . import FinishedSignageConditionChecker
@@ -17,11 +18,11 @@ class NonSkippedSignagePoints(FinishedSignageConditionChecker):
 
     def __init__(self):
         logging.info("Enabled check for finished signage points.")
-        self._last_signage_point_timestamp = None
-        self._last_signage_point = None
+        self._last_signage_point_timestamp: datetime = datetime.fromtimestamp(0)
+        self._last_signage_point: int = 0
 
     def check(self, obj: FinishedSignagePointMessage) -> Optional[Event]:
-        if self._last_signage_point is None:
+        if self._last_signage_point == 0:
             self._last_signage_point_timestamp = obj.timestamp
             self._last_signage_point = obj.signage_point
             return None

src/chia_log/handlers/daily_stats/stat_accumulators/search_time_stats.py

@@ -9,14 +9,14 @@ class SearchTimeStats(HarvesterActivityConsumer, StatAccumulator):
     def __init__(self):
         self._last_reset_time = datetime.now()
         self._num_measurements = 0
-        self._avg_time_seconds = 0
+        self._avg_time_seconds = 0.0
         self._over_5_seconds = 0
         self._over_15_seconds = 0
 
     def reset(self):
         self._last_reset_time = datetime.now()
         self._num_measurements = 0
-        self._avg_time_seconds = 0
+        self._avg_time_seconds = 0.0
         self._over_5_seconds = 0
         self._over_15_seconds = 0
 
@@ -29,7 +29,6 @@ def consume(self, obj: HarvesterActivityMessage):
             self._over_15_seconds += 1
 
     def get_summary(self) -> str:
-
         pct_over_5seconds: float = 0
         pct_over_15seconds: float = 0
 

src/chia_log/handlers/daily_stats/stat_accumulators/signage_point_stats.py

@@ -9,8 +9,8 @@
 class SignagePointStats(FinishedSignageConsumer, StatAccumulator):
     def __init__(self):
         self._last_reset_time = datetime.now()
-        self._last_signage_point_timestamp = None
-        self._last_signage_point = None
+        self._last_signage_point_timestamp: datetime = datetime.fromtimestamp(0)
+        self._last_signage_point: int = 0
         self._skips_total = 0
         self._total = 0
 
@@ -20,13 +20,16 @@ def reset(self):
         self._total = 0
 
     def consume(self, obj: FinishedSignagePointMessage):
-        if self._last_signage_point is None:
+        if self._last_signage_point == 0:
             self._last_signage_point_timestamp = obj.timestamp
             self._last_signage_point = obj.signage_point
             return
 
         valid, skips = calculate_skipped_signage_points(
-            self._last_signage_point_timestamp, self._last_signage_point, obj.timestamp, obj.signage_point
+            self._last_signage_point_timestamp,
+            self._last_signage_point,
+            obj.timestamp,
+            obj.signage_point,
         )
 
         if not valid:

src/chia_log/handlers/daily_stats/stats_manager.py

@@ -6,6 +6,9 @@
 from threading import Thread
 from time import sleep
 
+# lib
+from confuse import ConfigView
+
 # project
 from . import (
     HarvesterActivityConsumer,
@@ -36,10 +39,10 @@ class StatsManager:
     with a summary from all stats that have been collected for the past 24 hours.
     """
 
-    def __init__(self, config: dict, notify_manager: NotifyManager):
-        self._enable = config.get("enable", False)
-        self._notify_time = self._parse_notify_time(config.get("time_of_day", "21:00"))
-        self._frequency_hours = config.get("frequency_hours", 24)
+    def __init__(self, config: ConfigView, notify_manager: NotifyManager):
+        self._enable = config["enable"].get(bool)
+        self._notify_time = self._parse_notify_time(config["time_of_day"].get())
+        self._frequency_hours = config["frequency_hours"].get(int)
 
         if not self._enable:
             logging.warning("Disabled stats and daily notifications")

src/chia_log/handlers/wallet_added_coin_handler.py

@@ -2,6 +2,9 @@
 import logging
 from typing import List, Optional
 
+# lib
+from confuse import ConfigView
+
 # project
 from . import LogHandlerInterface
 from ..parsers.wallet_added_coin_parser import WalletAddedCoinParser
@@ -18,11 +21,10 @@ class WalletAddedCoinHandler(LogHandlerInterface):
     def config_name() -> str:
         return "wallet_added_coin_handler"
 
-    def __init__(self, config: Optional[dict] = None):
+    def __init__(self, config: ConfigView):
         super().__init__(config)
         self._parser = WalletAddedCoinParser()
-        config = config or {}
-        self.min_mojos_amount = config.get("min_mojos_amount", 0)
+        self.min_mojos_amount = config["min_mojos_amount"].get(int)
         logging.info(f"Filtering transaction with mojos less than {self.min_mojos_amount}")
 
     def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> List[Event]: