From aecd6873a78ff7ff5c45e54527bf8c8c26c46672 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Wed, 23 Aug 2023 12:23:56 +0800 Subject: [PATCH 01/13] Rebase vpbot based from telego --- mika/vpbot/Chart.yaml | 15 +- mika/vpbot/README.md | 62 +- mika/vpbot/templates/_helpers.tpl | 956 ++++++++++++++++++++++++++- mika/vpbot/templates/configmap.yaml | 112 +++- mika/vpbot/templates/deployment.yaml | 174 ++++- mika/vpbot/templates/pvc.yaml | 34 +- mika/vpbot/templates/secret.yaml | 21 +- mika/vpbot/templates/service.yaml | 10 + mika/vpbot/values.yaml | 167 ++--- 9 files changed, 1346 insertions(+), 205 deletions(-) diff --git a/mika/vpbot/Chart.yaml b/mika/vpbot/Chart.yaml index a7cd3b1e..b2ff0cce 100644 --- a/mika/vpbot/Chart.yaml +++ b/mika/vpbot/Chart.yaml @@ -2,17 +2,16 @@ apiVersion: v2 name: vpbot description: Vpbot is a Telegram bot with support for a number of useful features such as prayer time notifications, COVID-19 statistics, and more. type: application -version: 0.3.1 -appVersion: "1.5.6-stable" +version: 0.4.0 +appVersion: "0.1.0-stable-r1" keywords: - - "vpbot" - - "quarantine-bot" - - "telegram" - - "bot" -home: "https://github.com/irfanhakim-as/quarantine-bot" + - "Vpbot" + - "Telegram" + - "Bot" +home: "https://github.com/irfanhakim-as/vpbot" icon: "https://irfanhakim-as.github.io/charts/logos/vpbot.png" sources: - - "https://github.com/irfanhakim-as/quarantine-bot" + - "https://github.com/irfanhakim-as/vpbot" - "https://irfanhakim-as.github.io/charts/mika/vpbot" maintainers: - name: "Irfan Hakim" diff --git a/mika/vpbot/README.md b/mika/vpbot/README.md index 4264dd1d..c305c583 100644 --- a/mika/vpbot/README.md +++ b/mika/vpbot/README.md @@ -1,4 +1,4 @@ -# [`vpbot`](https://github.com/irfanhakim-as/quarantine-bot) 🔒 +# [`vpbot`](https://github.com/irfanhakim-as/vpbot) 🔒 ## Prerequisites @@ -69,12 +69,21 @@ Verify that your chart has been installed. Replace `$namespace` and `$release_na helm ls --namespace $namespace | grep "$release_name" ``` -## How to upgrade +### Add custom files -After making any necessary changes to the `values.yaml` file, upgrade the desired chart. Replace `$release_name` and `$namespace` accordingly. +To implement additional Telegram commands, create a custom `commands.py` file with a `Commands` class inheriting from `lib.telegram.BaseCommand`. Refer to `/base/lib/commands.py` from the container for more information. + +To add additional Telegram messages, create a custom `messages.py` file with both `MESSAGES` and `ICONS` dict. Refer to `/base/lib/messages.py` from the container for more information. + +Upgrade (or install) the chart while adding the `commands.py` and `messages.py files with the `--set-file` flag. ```sh -helm upgrade $release_name mika/vpbot --namespace $namespace --values values.yaml --wait +helm upgrade --install $release_name mika/vpbot \ +--namespace $namespace \ +--values values.yaml \ +--set-file vpbot.commands=commands.py \ +--set-file vpbot.messages=messages.py \ +--wait ``` ## How to uninstall @@ -110,33 +119,38 @@ Deploy [`mika/postgres-agent`](../postgres-agent/) with `postgres.mode.drop` set | image.vpbot.pullPolicy | string | `""` | The policy that determines when Kubernetes should pull the Vpbot container image. Default: `"IfNotPresent"`. | | image.vpbot.registry | string | `""` | The registry where the Vpbot container image is hosted. Default: `"ghcr.io"`. | | image.vpbot.repository | string | `""` | The name of the repository that contains the Vpbot container image used. Default: `"irfanhakim-as/vpbot"`. | -| image.vpbot.tag | string | `""` | The tag that specifies the version of the Vpbot container image used. Default: `"Chart appVersion"`. | +| image.vpbot.tag | string | `""` | The tag that specifies the version of the Vpbot container image used. Default: `Chart appVersion`. | | imagePullSecrets | list | `[]` | Credentials used to securely authenticate and authorise the pulling of container images from private registries. | | replicaCount | string | `""` | The desired number of running replicas for Vpbot. Default: `"1"`. | -| resources.redis.limits.cpu | string | `"15m"` | The maximum amount of CPU resources allowed for Redis. | -| resources.redis.limits.memory | string | `"60Mi"` | The maximum amount of memory allowed for Redis. | -| resources.redis.requests.cpu | string | `"5m"` | The minimum amount of CPU resources required by Redis. | -| resources.redis.requests.memory | string | `"30Mi"` | The minimum amount of memory required by Redis. | -| resources.vpbot.limits.cpu | string | `"50m"` | The maximum amount of CPU resources allowed for Vpbot. | +| resources.ngrok.limits.cpu | string | `"20m"` | The maximum amount of CPU resources allowed for Ngrok. | +| resources.ngrok.limits.memory | string | `"50Mi"` | The maximum amount of memory allowed for Ngrok. | +| resources.ngrok.requests.cpu | string | `"10m"` | The minimum amount of CPU resources required by Ngrok. | +| resources.ngrok.requests.memory | string | `"20Mi"` | The minimum amount of memory required by Ngrok. | +| resources.scheduler.limits.cpu | string | `"20m"` | The maximum amount of CPU resources allowed for Scheduler. | +| resources.scheduler.limits.memory | string | `"200Mi"` | The maximum amount of memory allowed for Scheduler. | +| resources.scheduler.requests.cpu | string | `"10m"` | The minimum amount of CPU resources required by Scheduler. | +| resources.scheduler.requests.memory | string | `"100Mi"` | The minimum amount of memory required by Scheduler. | +| resources.vpbot.limits.cpu | string | `"200m"` | The maximum amount of CPU resources allowed for Vpbot. | | resources.vpbot.limits.memory | string | `"500Mi"` | The maximum amount of memory allowed for Vpbot. | -| resources.vpbot.requests.cpu | string | `"10m"` | The minimum amount of CPU resources required by Vpbot. | -| resources.vpbot.requests.memory | string | `"250Mi"` | The minimum amount of memory required by Vpbot. | -| telegram.admin_id | string | `""` | The unique Telegram chat ID of the Vpbot administrator. | -| telegram.devel_id | string | `""` | The unique Telegram chat ID of the Vpbot developer. | -| telegram.token | string | `""` | The Telegram bot token used by Vpbot to communicate with Telegram. | -| telegram.webhook | string | `""` | The Telegram bot webhook path used by Vpbot to communicate with Telegram. Default: `"telegram/webhooks/"`. | -| vpbot.celery_timezone | string | `""` | The timezone for the task scheduler used by Vpbot to schedule time-dependent operations. Default: `"Asia/Kuala_Lumpur"`. | +| resources.vpbot.requests.cpu | string | `"50m"` | The minimum amount of CPU resources required by Vpbot. | +| resources.vpbot.requests.memory | string | `"300Mi"` | The minimum amount of memory required by Vpbot. | | vpbot.cloudflared.domain | string | `""` | Registered domain name on Cloudflare used for Vpbot. | | vpbot.cloudflared.enabled | bool | `false` | Specifies whether Vpbot should run using a Cloudflare tunnel. | +| vpbot.commands | file | `""` | Custom Telegram `commands.py` file for Vpbot. | | vpbot.debug | string | `""` | Specifies whether Vpbot should run in debug mode. Default: `false`. | -| vpbot.log_size_limit | string | `""` | The log file size limit in megabytes. Default: `"4"`. | +| vpbot.messages | file | `""` | Custom Telegram `messages.py` file for Vpbot. | | vpbot.name | string | `""` | The name of the Vpbot service. Default: `"Vpbot"`. | | vpbot.ngrok.enabled | bool | `false` | Specifies whether Vpbot should run using an Ngrok tunnel. | | vpbot.ngrok.token | string | `""` | Ngrok authentication token. | -| vpbot.persistence.enabled | bool | `false` | Specifies whether Vpbot should persist its storage. | -| vpbot.persistence.logs.storage | string | `""` | The amount of persistent storage allocated for Vpbot logs. Default: `"50Mi"`. | -| vpbot.persistence.migrations.storage | string | `""` | The amount of persistent storage allocated for Vpbot migration files. Default: `"50Mi"`. | -| vpbot.persistence.static.storage | string | `""` | The amount of persistent storage allocated for Vpbot static files. Default: `"50Mi"`. | -| vpbot.persistence.storageClassName | string | `""` | The storage class name used for dynamically provisioning a persistent volume for the Vpbot storage. Default: `"longhorn"`. | +| vpbot.persistence.enabled | bool | `false` | Specifies whether Vpbot should persist its logs. | +| vpbot.persistence.storage | string | `""` | The amount of persistent storage allocated for Vpbot logs. Default: `"10Mi"`. | +| vpbot.persistence.storageClassName | string | `""` | The storage class name used for dynamically provisioning a persistent volume for the Vpbot logs storage. Default: `"longhorn"`. | +| vpbot.scheduler.apscheduler | bool | `true` | Specifies whether APScheduler should be used by Vpbot as the task scheduler. | +| vpbot.scheduler.celery | bool | `false` | Specifies whether Celery should be used by Vpbot as the task scheduler. | +| vpbot.scheduler.schedule.clean_model | string | `""` | The hours at which the task scheduler cleans up the database. Default: `"0"`. | +| vpbot.scheduler.schedule.object_scheduler | string | `""` | The second intervals at which the task scheduler sends scheduled messages. Default: `"2"`. | +| vpbot.scheduler.timezone | string | `""` | The timezone for the task scheduler used by Vpbot to schedule time-dependent operations. Default: `"Asia/Kuala_Lumpur"`. | | vpbot.secret | string | `""` | A 50-character secret key used for secure session management and cryptographic operations within the Vpbot service. | -| vpbot.user_pass | string | `""` | The default user password for users created by Vpbot. | +| vpbot.telegram.api | string | `""` | API endpoint or URL for the Telegram bot. Default: `"https://api.telegram.org/bot"`. | +| vpbot.telegram.token | string | `""` | The Telegram bot token used by Vpbot to communicate with Telegram. | +| vpbot.telegram.webhook | string | `""` | The Telegram bot webhook path used by Vpbot to communicate with Telegram. Must contain a trailing slash. Default: `"webhook/telegram/"`. | diff --git a/mika/vpbot/templates/_helpers.tpl b/mika/vpbot/templates/_helpers.tpl index 3ea9949c..d8169b4d 100644 --- a/mika/vpbot/templates/_helpers.tpl +++ b/mika/vpbot/templates/_helpers.tpl @@ -69,12 +69,12 @@ Apache site-config.conf template ServerName DOMAIN:443 UseCanonicalName On ServerAdmin support@mikahomelab.com - DocumentRoot /quarantine-bot - WSGIScriptAlias / /quarantine-bot/quarantine_bot/wsgi.py - WSGIDaemonProcess DOMAIN python-path=/quarantine-bot + DocumentRoot /base + WSGIScriptAlias / /base/base/wsgi.py + WSGIDaemonProcess DOMAIN python-path=/base WSGIProcessGroup DOMAIN - + Require all granted @@ -85,12 +85,946 @@ Apache site-config.conf template Require all granted - Alias /media /quarantine-bot/media - - Require all granted - - - ErrorLog /quarantine-bot/logs/apache.error.log - CustomLog /quarantine-bot/logs/apache.access.log combined + ErrorLog /var/log/apache2/apache.error.log + CustomLog /var/log/apache2/apache.access.log combined {{- end }} + +{{/* +APScheduler /entrypoint.sh template +*/}} +{{- define "vpbot.apscheduler-entrypoint-sh" -}} +#!/bin/bash + +export APP_ROOT="base" + +# ================= DO NOT EDIT BEYOND THIS LINE ================= + +python3 manage.py makemigrations + +python3 manage.py migrate + +tail -f /dev/null +{{- end }} + +{{/* +APScheduler /base/base/apps.py template +*/}} +{{- define "vpbot.apscheduler-apps-py" -}} +from django.apps import AppConfig + +class BaseConfig(AppConfig): + name = "base" + + def ready(self): + from . import signals + from . import tasks + tasks.start() +{{- end }} + +{{/* +APScheduler /base/base/tasks.py template +*/}} +{{- define "vpbot.apscheduler-tasks-py" -}} +from django.conf import settings +from base.scheduler import object_scheduler +from lib.telegram import clean_model +from apscheduler.schedulers.blocking import BlockingScheduler + + +SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") + + +def start(): + scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) + + job_name = "clean_model" + scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) + + job_name = "object_scheduler" + scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) + + scheduler.start() +{{- end }} + +{{/* +Celery /etc/default/celeryd template +*/}} +{{- define "vpbot.default-celeryd" -}} +# Names of nodes to start +# most people will only start one node: +#CELERYD_NODES="worker1" +# but you can also start multiple and configure settings +# for each in CELERYD_OPTS +#CELERYD_NODES="worker1 worker2 worker3" +# alternatively, you can specify the number of nodes to start: +#CELERYD_NODES=10 +CELERYD_NODES="worker1" + +# Absolute or relative path to the 'celery' command: +#CELERY_BIN="/usr/local/bin/celery" +#CELERY_BIN="/virtualenvs/def/bin/celery" +CELERY_BIN="/usr/local/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +#CELERY_APP="proj" +# or fully qualified: +#CELERY_APP="proj.tasks:app" +CELERY_APP="base" + +# Where to chdir at start. +#CELERYD_CHDIR="/opt/Myproject/" +CELERYD_CHDIR="/base/" + +# Extra command-line arguments to the worker +#CELERYD_OPTS="--time-limit=300 --concurrency=8" +# Configure node-specific settings by appending node name to arguments: +#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" +CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" + +# Set logging level +#CELERYD_LOG_LEVEL="DEBUG" +CELERYD_LOG_LEVEL="WARNING" + +# %n will be replaced with the first part of the nodename. +#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +#CELERYD_PID_FILE="/var/run/celery/%n.pid" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +# Workers should run as an unprivileged user. +# You need to create this user manually (or you can choose +# a user/group combination that already exists (e.g., nobody). +#CELERYD_USER="celery" +#CELERYD_GROUP="celery" +CELERYD_USER="root" +CELERYD_GROUP="root" + +# If enabled pid and log directories will be created if missing, +# and owned by the userid/group configured. +#CELERY_CREATE_DIRS=1 +CELERY_CREATE_DIRS=1 +{{- end }} + +{{/* +Celery /etc/init.d/celerybeat template +*/}} +{{- define "vpbot.initd-celerybeat" -}} +#!/bin/sh -e +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e +VERSION=10.1 +echo "celery init v${VERSION}." + +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use 'celery beat --detach'" + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +scripts="" + +if test -f /etc/default/celeryd; then + scripts="/etc/default/celeryd" + _config_sanity /etc/default/celeryd + . /etc/default/celeryd +fi + +EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" +if test -f "$EXTRA_CONFIG"; then + scripts="$scripts, $EXTRA_CONFIG" + _config_sanity "$EXTRA_CONFIG" + . "$EXTRA_CONFIG" +fi + +echo "Using configuration: $scripts" + +CELERY_BIN=${CELERY_BIN:-"celery"} +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/beat.pid" +DEFAULT_LOG_FILE="/var/log/celery/beat.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_CELERYBEAT="$CELERY_BIN" + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} +CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $*" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + + +create_paths () { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + +is_running() { + pid=$1 + ps $pid > /dev/null 2>&1 +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + if ! is_running $pid; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping ${SCRIPT_NAME}... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +_chuid () { + ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ + "$CELERYBEAT_USER" -c "$CELERYBEAT $*" +} + +start_beat () { + echo "Starting ${SCRIPT_NAME}..." + _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" \ + $CELERYBEAT_OPTS +} + + +check_status () { + local failed= + local pid_file=$CELERYBEAT_PID_FILE + if [ ! -e $pid_file ]; then + echo "${SCRIPT_NAME} is down: no pid file found" + failed=true + elif [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." + failed=true + else + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "${SCRIPT_NAME}: bad pid file ($pid_file)" + failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" + failed=true + else + echo "${SCRIPT_NAME} (pid $pid) is up..." + fi + fi + fi + + [ "$failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_beat + ;; + stop) + check_paths + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + status) + check_status + ;; + restart) + echo "Restarting celery periodic task scheduler" + check_paths + stop_beat && check_dev_null && start_beat + ;; + create-paths) + check_dev_null + create_paths + ;; + check-paths) + check_dev_null + check_paths + ;; + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} + +{{/* +Celery /etc/init.d/celeryd template +*/}} +{{- define "vpbot.initd-celeryd" -}} +#!/bin/sh -e +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init-scripts, copy this script and give it a different +# name. That is, if your new application named "little-worker" needs an init, +# you should use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# +VERSION=10.1 +echo "celery init v${VERSION}." +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use the 'celery multi' utility, " + echo " or 'celery worker --detach'." + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery worker --detach" + +if [ -d "/etc/default" ]; then + CELERY_CONFIG_DIR="/etc/default" +else + CELERY_CONFIG_DIR="/usr/local/etc" +fi + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +if [ -f "$CELERY_DEFAULTS" ]; then + _config_sanity "$CELERY_DEFAULTS" + echo "Using config script: $CELERY_DEFAULTS" + . "$CELERY_DEFAULTS" +fi + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +# Options to su +# can be used to enable login shell (CELERYD_SU_ARGS="-l"), +# or even to use start-stop-daemon instead of su. +CELERYD_SU=${CELERY_SU:-"su"} +CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +_get_pidfiles () { + # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. + ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 +} + + +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pidfile in $(_get_pidfiles); do + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" +} + + +start_workers () { + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers () { + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +kill_workers() { + _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + echo "WARNING: Use with caution in production" + echo "The workers will attempt to restart, but they may not be able to." + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pidfile in $(_get_pidfiles); do + if [ ! -r $pidfile ]; then + echo "${SCRIPT_NAME} down: no pidfiles found" + one_failed=true + break + fi + + local node=`basename "$pidfile" .pid` + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_workers + ;; + + stop) + check_dev_null + check_paths + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + restart_workers + ;; + + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + + try-restart) + check_dev_null + check_paths + restart_workers + ;; + + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} + +{{/* +Celery /base/base/celery.py template +*/}} +{{- define "vpbot.celery-py" -}} +from __future__ import absolute_import, unicode_literals +import re +import os +from celery import Celery +from celery.schedules import crontab +from datetime import timedelta +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") +app = Celery("base") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + + +# make integer from object scheduler seconds string +object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) + + +app.conf.beat_schedule = { + # clean model + "clean_model" : { + "task" : "base.tasks.clean_model_task", + "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), + }, + # check for any objects that need to be sent + "object_scheduler" : { + "task" : "base.tasks.object_scheduler_task", + "schedule" : timedelta(seconds=object_scheduler_seconds), + }, +} + + +@app.task(bind=True) +def debug_task(self): + print("Request: {0!r}".format(self.request)) +{{- end }} + +{{/* +Celery /base/base/__init__.py template +*/}} +{{- define "vpbot.celery-init-py" -}} +from .celery import app as celery_app + +__all__ = ("celery_app",) +{{- end }} + +{{/* +Celery /base/base/tasks.py template +*/}} +{{- define "vpbot.celery-tasks-py" -}} +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from base.scheduler import object_scheduler +from lib.telegram import clean_model + + +# clean model +@shared_task +def clean_model_task(): + clean_model() + + +# check for any objects that need to be sent +@shared_task +def object_scheduler_task(): + object_scheduler() +{{- end }} diff --git a/mika/vpbot/templates/configmap.yaml b/mika/vpbot/templates/configmap.yaml index a1d027dd..027658bc 100644 --- a/mika/vpbot/templates/configmap.yaml +++ b/mika/vpbot/templates/configmap.yaml @@ -1,13 +1,18 @@ {{- $name := .Values.vpbot.name | default "Vpbot" | toString | quote }} {{- $debug := .Values.vpbot.debug | default "false" | toString | quote }} +{{- $api := .Values.vpbot.telegram.api | default "https://api.telegram.org/bot" | toString | quote }} +{{- $webhook := .Values.vpbot.telegram.webhook | default "webhook/telegram/" | toString | quote }} {{- $redis_service := printf "redis://%s-vpbot-svc" .Release.Name | quote }} -{{- $celery_timezone := .Values.vpbot.celery_timezone | default "Asia/Kuala_Lumpur" | toString | quote }} -{{- $webhook := .Values.telegram.webhook | default "telegram/webhooks/" | toString | quote }} -{{- $log_size_limit := .Values.vpbot.log_size_limit | default "4" | toString | quote }} -{{- $app_version := .Values.image.vpbot.tag | default .Chart.AppVersion | toString | quote }} +{{- $scheduler_timezone := .Values.vpbot.scheduler.timezone | default "Asia/Kuala_Lumpur" | toString | quote }} {{- $domain := .Values.vpbot.cloudflared.domain | default "localhost" | toString }} +{{- $clean_model := .Values.vpbot.scheduler.schedule.clean_model | default "0" | toString | quote }} +{{- $object_scheduler := .Values.vpbot.scheduler.schedule.object_scheduler | default "2" | toString | quote }} +{{- $apscheduler := .Values.vpbot.scheduler.apscheduler }} +{{- $celery := .Values.vpbot.scheduler.celery }} {{- $cloudflared := .Values.vpbot.cloudflared.enabled }} {{- $ngrok := .Values.vpbot.ngrok.enabled }} +{{- $commands := .Values.vpbot.commands }} +{{- $messages := .Values.vpbot.messages }} apiVersion: v1 kind: ConfigMap metadata: @@ -17,16 +22,19 @@ metadata: data: APP_NAME: {{ $name }} DEBUG: {{ $debug }} - CELERY_BROKER: {{ $redis_service }} - CELERY_BACKEND: {{ $redis_service }} - CELERY_TIMEZONE: {{ $celery_timezone }} + TELEGRAM_BOT_API_URL: {{ $api }} + TELEGRAM_WEBHOOK_PATH: {{ $webhook }} NGROK: {{ $ngrok | toString | quote }} - DJANGO_WEBHOOK_URI: {{ $webhook }} - LOG_SIZE_LIMIT: {{ $log_size_limit }} - APP_VERSION: {{ $app_version }} {{- if $cloudflared }} DOMAIN: {{ $domain | quote }} {{- end }} + {{- if or $apscheduler $celery }} + SCHEDULER_TIMEZONE: {{ $scheduler_timezone }} + {{- end }} + {{- if $celery }} + CELERY_BROKER: {{ $redis_service }} + CELERY_BACKEND: {{ $redis_service }} + {{- end }} --- apiVersion: v1 kind: ConfigMap @@ -37,3 +45,87 @@ metadata: data: site-config.conf: |- {{- include "vpbot.site-config-conf" . | replace "DOMAIN" $domain | nindent 4 }} +{{- if $apscheduler }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-apscheduler-config + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + entrypoint.sh: |- + {{- include "vpbot.apscheduler-entrypoint-sh" . | nindent 4 }} +{{- else if $celery }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-default-celery-config + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + celeryd: |- + {{- include "vpbot.default-celeryd" . | nindent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-initd-celery-config + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + celerybeat: |- + {{- include "vpbot.initd-celerybeat" . | nindent 4 }} + celeryd: |- + {{- include "vpbot.initd-celeryd" . | nindent 4 }} +{{- end }} +{{- if or $apscheduler $celery }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-scheduler-config + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + tasks.py: |- + {{- if $apscheduler }} + {{- include "vpbot.apscheduler-tasks-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | nindent 4 }} + {{- else if $celery }} + {{- include "vpbot.celery-tasks-py" . | nindent 4 }} + {{- end }} + {{- if $apscheduler }} + apps.py: |- + {{- include "vpbot.apscheduler-apps-py" . | nindent 4 }} + {{- else if $celery }} + celery.py: |- + {{- include "vpbot.celery-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | nindent 4 }} + init.py: |- + {{- include "vpbot.celery-init-py" . | nindent 4 }} + {{- end }} +{{- end }} +{{- if $commands }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-commands-cm + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + commands.py: |- + {{ $commands | toString | nindent 4 | trim }} +{{- end }} +{{- if $messages }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-vpbot-messages-cm + labels: + {{- include "vpbot.labels" . | nindent 4 }} +data: + messages.py: |- + {{ $messages | toString | nindent 4 | trim }} +{{- end }} diff --git a/mika/vpbot/templates/deployment.yaml b/mika/vpbot/templates/deployment.yaml index b6c063a6..64c051fe 100644 --- a/mika/vpbot/templates/deployment.yaml +++ b/mika/vpbot/templates/deployment.yaml @@ -1,8 +1,8 @@ {{- $replica_count := .Values.replicaCount | default "1" | toString }} -{{- $vpbot_registry := .Values.image.vpbot.registry | default "ghcr.io" | toString }} -{{- $vpbot_repository := .Values.image.vpbot.repository | default "irfanhakim-as/quarantine-bot" | toString }} -{{- $vpbot_tag := .Values.image.vpbot.tag | default .Chart.AppVersion | toString }} -{{- $vpbot_pullPolicy := .Values.image.vpbot.pullPolicy | default "IfNotPresent" | toString | quote }} +{{- $telego_registry := .Values.image.vpbot.registry | default "ghcr.io" | toString }} +{{- $telego_repository := .Values.image.vpbot.repository | default "irfanhakim-as/vpbot" | toString }} +{{- $telego_tag := .Values.image.vpbot.tag | default .Chart.AppVersion | toString }} +{{- $telego_pullPolicy := .Values.image.vpbot.pullPolicy | default "IfNotPresent" | toString | quote }} {{- $redis_registry := .Values.image.redis.registry | default "docker.io" | toString }} {{- $redis_repository := .Values.image.redis.repository | default "redis" | toString }} {{- $redis_tag := .Values.image.redis.tag | default "alpine" | toString }} @@ -11,9 +11,13 @@ {{- $ngrok_repository := .Values.image.ngrok.repository | default "wernight/ngrok" | toString }} {{- $ngrok_tag := .Values.image.ngrok.tag | default "latest" | toString }} {{- $ngrok_pullPolicy := .Values.image.ngrok.pullPolicy | default "IfNotPresent" | toString | quote }} +{{- $apscheduler := .Values.vpbot.scheduler.apscheduler }} +{{- $celery := .Values.vpbot.scheduler.celery }} {{- $cloudflared := .Values.vpbot.cloudflared.enabled }} {{- $ngrok := .Values.vpbot.ngrok.enabled }} {{- $persistence := .Values.vpbot.persistence.enabled }} +{{- $commands := .Values.vpbot.commands }} +{{- $messages := .Values.vpbot.messages }} apiVersion: apps/v1 kind: Deployment metadata: @@ -30,6 +34,7 @@ spec: {{- with .Values.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} + "helm.sh/hook-weight": "1" {{- end }} labels: {{- include "vpbot.selectorLabels" . | nindent 8 }} @@ -40,42 +45,102 @@ spec: {{- end }} containers: - name: vpbot - image: {{ printf "%s/%s:%s" $vpbot_registry $vpbot_repository $vpbot_tag | quote }} - imagePullPolicy: {{ $vpbot_pullPolicy }} + image: {{ printf "%s/%s:%s" $telego_registry $telego_repository $telego_tag | quote }} + imagePullPolicy: {{ $telego_pullPolicy }} ports: - name: vpbot containerPort: 80 protocol: TCP resources: {{- toYaml .Values.resources.vpbot | nindent 12 }} - {{- if $cloudflared }} - env: - - name: ALLOWED_HOSTS - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-vpbot-cm - key: DOMAIN - {{- end }} envFrom: - configMapRef: name: {{ .Release.Name }}-vpbot-cm - secretRef: name: {{ .Release.Name }}-vpbot-secret - - secretRef: - name: {{ .Release.Name }}-vpbot-postgres-secret volumeMounts: - name: {{ .Release.Name }}-vpbot-site-config mountPath: /etc/apache2/sites-available/000-default.conf subPath: site-config.conf readOnly: true + {{- if $celery }} + - name: {{ .Release.Name }}-vpbot-default-celery-config + mountPath: /etc/default/celeryd + subPath: celeryd + - name: {{ .Release.Name }}-vpbot-initd-celery-config + mountPath: /etc/init.d/celerybeat + subPath: celerybeat + - name: {{ .Release.Name }}-vpbot-initd-celery-config + mountPath: /etc/init.d/celeryd + subPath: celeryd + - name: {{ .Release.Name }}-vpbot-scheduler-config + mountPath: /base/base/celery.py + subPath: celery.py + - name: {{ .Release.Name }}-vpbot-scheduler-config + mountPath: /base/base/tasks.py + subPath: tasks.py + - name: {{ .Release.Name }}-vpbot-scheduler-config + mountPath: /base/base/__init__.py + subPath: init.py + {{- end }} + {{- if $persistence }} + - name: {{ .Release.Name }}-vpbot-logs + mountPath: /vpbot/logs + {{- end }} + {{- if $commands }} + - name: {{ .Release.Name }}-vpbot-commands-cm + mountPath: /base/lib/commands.py + subPath: commands.py + readOnly: true + {{- end }} + {{- if $messages }} + - name: {{ .Release.Name }}-vpbot-messages-cm + mountPath: /base/lib/messages.py + subPath: messages.py + readOnly: true + {{- end }} + {{- if $apscheduler }} + - name: apscheduler + image: {{ printf "%s/%s:%s" $telego_registry $telego_repository $telego_tag | quote }} + imagePullPolicy: {{ $telego_pullPolicy }} + ports: + - name: vpbot + containerPort: 80 + protocol: TCP + resources: + {{- toYaml .Values.resources.scheduler | nindent 12 }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-vpbot-cm + - secretRef: + name: {{ .Release.Name }}-vpbot-secret + volumeMounts: + - name: {{ .Release.Name }}-vpbot-apscheduler-config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + - name: {{ .Release.Name }}-vpbot-scheduler-config + mountPath: /base/base/apps.py + subPath: apps.py + - name: {{ .Release.Name }}-vpbot-scheduler-config + mountPath: /base/base/tasks.py + subPath: tasks.py {{- if $persistence }} - - name: {{ .Release.Name }}-vpbot-static - mountPath: /static - name: {{ .Release.Name }}-vpbot-logs - mountPath: /quarantine-bot/logs - - name: {{ .Release.Name }}-vpbot-migrations - mountPath: /quarantine-bot/quarantine_bot/migrations + mountPath: /var/log/apache2 + {{- end }} + {{- if $commands }} + - name: {{ .Release.Name }}-vpbot-commands-cm + mountPath: /base/lib/commands.py + subPath: commands.py + readOnly: true + {{- end }} + {{- if $messages }} + - name: {{ .Release.Name }}-vpbot-messages-cm + mountPath: /base/lib/messages.py + subPath: messages.py + readOnly: true {{- end }} + {{- else if $celery }} - name: redis image: {{ printf "%s/%s:%s" $redis_registry $redis_repository $redis_tag | quote }} imagePullPolicy: {{ $redis_pullPolicy }} @@ -84,7 +149,8 @@ spec: containerPort: 6379 protocol: TCP resources: - {{- toYaml .Values.resources.redis | nindent 12 }} + {{- toYaml .Values.resources.scheduler | nindent 12 }} + {{- end }} {{- if $ngrok }} - name: ngrok image: {{ printf "%s/%s:%s" $ngrok_registry $ngrok_repository $ngrok_tag | quote }} @@ -99,6 +165,8 @@ spec: - name: ngrok containerPort: 4040 protocol: TCP + resources: + {{- toYaml .Values.resources.ngrok | nindent 12 }} env: - name: NGROK_AUTH valueFrom: @@ -110,14 +178,64 @@ spec: - name: {{ .Release.Name }}-vpbot-site-config configMap: name: {{ .Release.Name }}-vpbot-site-config + {{- if $apscheduler }} + - name: {{ .Release.Name }}-vpbot-apscheduler-config + configMap: + name: {{ .Release.Name }}-vpbot-apscheduler-config + defaultMode: 0755 + items: + - key: entrypoint.sh + path: entrypoint.sh + {{- else if $celery }} + - name: {{ .Release.Name }}-vpbot-default-celery-config + configMap: + name: {{ .Release.Name }}-vpbot-default-celery-config + defaultMode: 0640 + items: + - key: celeryd + path: celeryd + - name: {{ .Release.Name }}-vpbot-initd-celery-config + configMap: + name: {{ .Release.Name }}-vpbot-initd-celery-config + defaultMode: 0755 + items: + - key: celerybeat + path: celerybeat + - key: celeryd + path: celeryd + {{- end }} + {{- if or $apscheduler $celery }} + - name: {{ .Release.Name }}-vpbot-scheduler-config + configMap: + name: {{ .Release.Name }}-vpbot-scheduler-config + defaultMode: 0775 + items: + - key: tasks.py + path: tasks.py + {{- if $apscheduler }} + - key: apps.py + path: apps.py + {{- else if $celery }} + - key: celery.py + path: celery.py + - key: init.py + path: init.py + {{- end }} + {{- end }} {{- if $persistence }} - - name: {{ .Release.Name }}-vpbot-static - persistentVolumeClaim: - claimName: {{ .Release.Name }}-vpbot-static-pvc - name: {{ .Release.Name }}-vpbot-logs persistentVolumeClaim: claimName: {{ .Release.Name }}-vpbot-logs-pvc - - name: {{ .Release.Name }}-vpbot-migrations - persistentVolumeClaim: - claimName: {{ .Release.Name }}-vpbot-migrations-pvc {{- end }} + {{- if $commands }} + - name: {{ .Release.Name }}-vpbot-commands-cm + configMap: + name: {{ .Release.Name }}-vpbot-commands-cm + {{- end }} + {{- if $messages }} + - name: {{ .Release.Name }}-vpbot-messages-cm + configMap: + name: {{ .Release.Name }}-vpbot-messages-cm + {{- end }} + securityContext: + fsGroup: 33 diff --git a/mika/vpbot/templates/pvc.yaml b/mika/vpbot/templates/pvc.yaml index ab6bb3e7..77c6eaea 100644 --- a/mika/vpbot/templates/pvc.yaml +++ b/mika/vpbot/templates/pvc.yaml @@ -1,6 +1,4 @@ -{{- $logs_storage := .Values.vpbot.persistence.logs.storage | default "50Mi" | toString | quote }} -{{- $static_storage := .Values.vpbot.persistence.static.storage | default "50Mi" | toString | quote }} -{{- $migrations_storage := .Values.vpbot.persistence.migrations.storage | default "50Mi" | toString | quote }} +{{- $storage := .Values.vpbot.persistence.storage | default "10Mi" | toString | quote }} {{- $storage_class_name := .Values.vpbot.persistence.storageClassName | default "longhorn" | toString | quote }} {{- $persistence := .Values.vpbot.persistence.enabled }} {{- if $persistence }} @@ -15,34 +13,6 @@ spec: - ReadWriteMany resources: requests: - storage: {{ $logs_storage }} - storageClassName: {{ $storage_class_name }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Release.Name }}-vpbot-static-pvc - labels: - {{- include "vpbot.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: {{ $static_storage }} - storageClassName: {{ $storage_class_name }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Release.Name }}-vpbot-migrations-pvc - labels: - {{- include "vpbot.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: {{ $migrations_storage }} + storage: {{ $storage }} storageClassName: {{ $storage_class_name }} {{- end }} diff --git a/mika/vpbot/templates/secret.yaml b/mika/vpbot/templates/secret.yaml index b0e53cb2..c7c00b2a 100644 --- a/mika/vpbot/templates/secret.yaml +++ b/mika/vpbot/templates/secret.yaml @@ -1,8 +1,5 @@ {{- $secret := .Values.vpbot.secret | toString | b64enc }} -{{- $telegram_token := .Values.telegram.token | toString | b64enc }} -{{- $admin_id := .Values.telegram.admin_id | toString | b64enc }} -{{- $devel_id := .Values.telegram.devel_id | toString | b64enc }} -{{- $user_pass := .Values.vpbot.user_pass | toString | b64enc }} +{{- $telegram_token := .Values.vpbot.telegram.token | toString | b64enc }} {{- $ngrok_token := .Values.vpbot.ngrok.token | toString | b64enc }} {{- $db_type := .Values.db.type | default "postgresql" | toString | b64enc }} {{- $db_name := .Values.db.name | toString | b64enc }} @@ -21,24 +18,12 @@ type: Opaque data: SECRET_KEY: {{ $secret }} TELEGRAM_BOT_TOKEN: {{ $telegram_token }} - TELEGRAM_ADMIN_ID: {{ $admin_id }} - TELEGRAM_DEVEL_ID: {{ $devel_id }} - DEFAULT_USER_PASS: {{ $user_pass }} {{- if $ngrok }} NGROK_AUTH_TOKEN: {{ $ngrok_token }} {{- end }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Release.Name }}-vpbot-postgres-secret - labels: - {{- include "vpbot.labels" . | nindent 4 }} -type: Opaque -data: - DB_HOST: {{ $db_host }} DB_TYPE: {{ $db_type }} - DB_PORT: {{ $db_port }} DB_NAME: {{ $db_name }} DB_USER: {{ $db_user }} DB_PASS: {{ $db_password }} + DB_HOST: {{ $db_host }} + DB_PORT: {{ $db_port }} diff --git a/mika/vpbot/templates/service.yaml b/mika/vpbot/templates/service.yaml index 8f22471a..fc4cbeed 100644 --- a/mika/vpbot/templates/service.yaml +++ b/mika/vpbot/templates/service.yaml @@ -1,3 +1,5 @@ +{{- $celery := .Values.vpbot.scheduler.celery }} +{{- $ngrok := .Values.vpbot.ngrok.enabled }} apiVersion: v1 kind: Service metadata: @@ -11,9 +13,17 @@ spec: targetPort: vpbot protocol: TCP name: {{ .Release.Name }}-vpbot + {{- if $celery }} - port: 6379 targetPort: redis protocol: TCP name: {{ .Release.Name }}-redis + {{- end }} + {{- if $ngrok }} + - port: 4040 + targetPort: ngrok + protocol: TCP + name: {{ .Release.Name }}-ngrok + {{- end }} selector: {{- include "vpbot.selectorLabels" . | nindent 4 }} diff --git a/mika/vpbot/values.yaml b/mika/vpbot/values.yaml index e225cbcf..f0f5541a 100644 --- a/mika/vpbot/values.yaml +++ b/mika/vpbot/values.yaml @@ -14,7 +14,7 @@ image: # Default: "ghcr.io" registry: "" # The name of the repository that contains the Vpbot container image used. - # Default: "irfanhakim-as/quarantine-bot" + # Default: "irfanhakim-as/vpbot" repository: "" # The tag that specifies the version of the Vpbot container image used. # Default: Chart appVersion @@ -69,56 +69,69 @@ vpbot: # Example: # debug: true debug: "" - # The log file size limit in megabytes. - # Default: "4" - # Example: - # log_size_limit: "3" - log_size_limit: "" # A 50-character secret key used for secure session management and cryptographic operations within the Vpbot service. # Example: # secret: "6&p4%t)-$8a14fmfh92py8j55+us51r6%e52dzy&=a3-6yd4#e" secret: "" - # The default user password for users created by Vpbot. - # Example: - # user_pass: "password" - user_pass: "" - # The timezone for the task scheduler used by Vpbot to schedule time-dependent operations. - # Default: "Asia/Kuala_Lumpur" - # Example: - # timezone: "Asia/Kuala_Lumpur" - celery_timezone: "" - # Vpbot storage persistence configurations. + # Vpbot scheduler configurations. + scheduler: + # Specifies whether APScheduler should be used by Vpbot as the task scheduler. + # Example: + # apscheduler: false + apscheduler: true + # Specifies whether Celery should be used by Vpbot as the task scheduler. + # Example: + # celery: true + celery: false + # The timezone for the task scheduler used by Vpbot to schedule time-dependent operations. + # Default: "Asia/Kuala_Lumpur" + # Example: + # timezone: "Asia/Kuala_Lumpur" + timezone: "" + # The schedule for the task scheduler used by Vpbot. + schedule: + # The hours at which the task scheduler cleans up the database. + # Default: "0" + # Example: + # clean_model: "0,6,12,18" + clean_model: "" + # The second intervals at which the task scheduler sends scheduled messages. + # Default: "2" + # Example: + # object_scheduler: "3" + object_scheduler: "" + # Vpbot Telegram configurations. + telegram: + # API endpoint or URL for the Telegram bot. + # Default: "https://api.telegram.org/bot" + # Example: + # api: "https://api.telegram.org/bot" + api: "" + # The Telegram bot token used by Vpbot to communicate with Telegram. + # Example: + # token: "1282813559:RrqoFIuWZwNfMLrx3Jd2mdFJnxrMObgQEvz" + token: "" + # The Telegram bot webhook path used by Vpbot to communicate with Telegram. Must contain a trailing slash. + # Default: "webhook/telegram/" + # Example: + # webhook: "telegram/jbbhb73f6s/" + webhook: "" + # Vpbot log persistence configurations. persistence: - # Specifies whether Vpbot should persist its storage. + # Specifies whether Vpbot should persist its logs. # Example: # enabled: true enabled: false - # The storage class name used for dynamically provisioning a persistent volume for the Vpbot storage. + # The amount of persistent storage allocated for Vpbot logs. + # Default: "10Mi" + # Example: + # storage: "1Gi" + storage: "" + # The storage class name used for dynamically provisioning a persistent volume for the Vpbot logs storage. # Default: "longhorn" # Example: # storageClassName: "longhorn" storageClassName: "" - # Vpbot logs persistence configurations. - logs: - # The amount of persistent storage allocated for Vpbot logs. - # Default: "50Mi" - # Example: - # storage: "1Gi" - storage: "" - # Vpbot static files persistence configurations. - static: - # The amount of persistent storage allocated for Vpbot static files. - # Default: "50Mi" - # Example: - # storage: "1Gi" - storage: "" - # Vpbot migration files persistence configurations. - migrations: - # The amount of persistent storage allocated for Vpbot migration files. - # Default: "50Mi" - # Example: - # storage: "1Gi" - storage: "" # Vpbot Ngrok configurations. ngrok: # Specifies whether Vpbot should run using an Ngrok tunnel. @@ -139,26 +152,10 @@ vpbot: # Example: # domain: "vpbot.example.com" domain: "" - -# Vpbot Telegram configurations. -telegram: - # The Telegram bot webhook path used by Vpbot to communicate with Telegram. - # Default: "telegram/webhooks/" - # Example: - # webhook: "bot/jbbhb73f6s/" - webhook: "" - # The Telegram bot token used by Vpbot to communicate with Telegram. - # Example: - # token: "1282813559:RrqoFIuWZwNfMLrx3Jd2mdFJnxrMObgQEvz" - token: "" - # The unique Telegram chat ID of the Vpbot administrator. - # Example: - # admin_id: "6649538406" - admin_id: "" - # The unique Telegram chat ID of the Vpbot developer. - # Example: - # devel_id: "4637142749" - devel_id: "" + # Custom Telegram commands.py file for Vpbot. + commands: "" + # Custom Telegram messages.py file for Vpbot. + messages: "" # Database configurations. db: @@ -198,40 +195,62 @@ resources: # The minimum amount of CPU resources required by Vpbot. # Example: # cpu: "20m" - cpu: "10m" + cpu: "50m" # The minimum amount of memory required by Vpbot. # Example: # memory: "250Mi" - memory: "250Mi" + memory: "300Mi" # The maximum amount of resources allowed for Vpbot. limits: # The maximum amount of CPU resources allowed for Vpbot. # Example: # cpu: "50m" - cpu: "50m" + cpu: "200m" # The maximum amount of memory allowed for Vpbot. # Example: # memory: "450Mi" memory: "500Mi" - # Redis container resources. - redis: - # The minimum amount of resources required by Redis to run. + # Scheduler container resources. + scheduler: + # The minimum amount of resources required by Scheduler to run. requests: - # The minimum amount of CPU resources required by Redis. + # The minimum amount of CPU resources required by Scheduler. # Example: # cpu: "5m" - cpu: "5m" - # The minimum amount of memory required by Redis. + cpu: "10m" + # The minimum amount of memory required by Scheduler. + # Example: + # memory: "30Mi" + memory: "100Mi" + # The maximum amount of resources allowed for Scheduler. + limits: + # The maximum amount of CPU resources allowed for Scheduler. + # Example: + # cpu: "15m" + cpu: "20m" + # The maximum amount of memory allowed for Scheduler. + # Example: + # memory: "50Mi" + memory: "200Mi" + # Ngrok container resources. + ngrok: + # The minimum amount of resources required by Ngrok to run. + requests: + # The minimum amount of CPU resources required by Ngrok. + # Example: + # cpu: "5m" + cpu: "10m" + # The minimum amount of memory required by Ngrok. # Example: # memory: "30Mi" - memory: "30Mi" - # The maximum amount of resources allowed for Redis. + memory: "20Mi" + # The maximum amount of resources allowed for Ngrok. limits: - # The maximum amount of CPU resources allowed for Redis. + # The maximum amount of CPU resources allowed for Ngrok. # Example: # cpu: "15m" - cpu: "15m" - # The maximum amount of memory allowed for Redis. + cpu: "20m" + # The maximum amount of memory allowed for Ngrok. # Example: # memory: "50Mi" - memory: "60Mi" + memory: "50Mi" From eabcef6d8668c8d3de4731dcb8d35168a16848e6 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Wed, 23 Aug 2023 12:34:03 +0800 Subject: [PATCH 02/13] Add default location value --- mika/vpbot/README.md | 1 + mika/vpbot/templates/configmap.yaml | 2 ++ mika/vpbot/values.yaml | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/mika/vpbot/README.md b/mika/vpbot/README.md index c305c583..53d93b85 100644 --- a/mika/vpbot/README.md +++ b/mika/vpbot/README.md @@ -138,6 +138,7 @@ Deploy [`mika/postgres-agent`](../postgres-agent/) with `postgres.mode.drop` set | vpbot.cloudflared.enabled | bool | `false` | Specifies whether Vpbot should run using a Cloudflare tunnel. | | vpbot.commands | file | `""` | Custom Telegram `commands.py` file for Vpbot. | | vpbot.debug | string | `""` | Specifies whether Vpbot should run in debug mode. Default: `false`. | +| vpbot.defaults.location | string | `""` | The default user location code used for personalised services. Default: `"wlp-0"`. | | vpbot.messages | file | `""` | Custom Telegram `messages.py` file for Vpbot. | | vpbot.name | string | `""` | The name of the Vpbot service. Default: `"Vpbot"`. | | vpbot.ngrok.enabled | bool | `false` | Specifies whether Vpbot should run using an Ngrok tunnel. | diff --git a/mika/vpbot/templates/configmap.yaml b/mika/vpbot/templates/configmap.yaml index 027658bc..1a659e44 100644 --- a/mika/vpbot/templates/configmap.yaml +++ b/mika/vpbot/templates/configmap.yaml @@ -1,5 +1,6 @@ {{- $name := .Values.vpbot.name | default "Vpbot" | toString | quote }} {{- $debug := .Values.vpbot.debug | default "false" | toString | quote }} +{{- $location := .Values.vpbot.defaults.location | default "wlp-0" | toString | quote }} {{- $api := .Values.vpbot.telegram.api | default "https://api.telegram.org/bot" | toString | quote }} {{- $webhook := .Values.vpbot.telegram.webhook | default "webhook/telegram/" | toString | quote }} {{- $redis_service := printf "redis://%s-vpbot-svc" .Release.Name | quote }} @@ -22,6 +23,7 @@ metadata: data: APP_NAME: {{ $name }} DEBUG: {{ $debug }} + DEFAULT_LOCATION_CODE: {{ $location }} TELEGRAM_BOT_API_URL: {{ $api }} TELEGRAM_WEBHOOK_PATH: {{ $webhook }} NGROK: {{ $ngrok | toString | quote }} diff --git a/mika/vpbot/values.yaml b/mika/vpbot/values.yaml index f0f5541a..78aef797 100644 --- a/mika/vpbot/values.yaml +++ b/mika/vpbot/values.yaml @@ -73,6 +73,13 @@ vpbot: # Example: # secret: "6&p4%t)-$8a14fmfh92py8j55+us51r6%e52dzy&=a3-6yd4#e" secret: "" + # Vpbot defaults configurations. + defaults: + # The default user location code used for personalised services. + # Default: "wlp-0" + # Example: + # location: "phg-0" + location: "" # Vpbot scheduler configurations. scheduler: # Specifies whether APScheduler should be used by Vpbot as the task scheduler. From ba8847c2d29d641070185e655764a3456a7cc896 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Wed, 23 Aug 2023 12:43:48 +0800 Subject: [PATCH 03/13] Add solat api configs --- mika/vpbot/README.md | 2 ++ mika/vpbot/templates/configmap.yaml | 4 ++++ mika/vpbot/values.yaml | 14 ++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/mika/vpbot/README.md b/mika/vpbot/README.md index 53d93b85..75b6748c 100644 --- a/mika/vpbot/README.md +++ b/mika/vpbot/README.md @@ -134,6 +134,8 @@ Deploy [`mika/postgres-agent`](../postgres-agent/) with `postgres.mode.drop` set | resources.vpbot.limits.memory | string | `"500Mi"` | The maximum amount of memory allowed for Vpbot. | | resources.vpbot.requests.cpu | string | `"50m"` | The minimum amount of CPU resources required by Vpbot. | | resources.vpbot.requests.memory | string | `"300Mi"` | The minimum amount of memory required by Vpbot. | +| vpbot.api.solat.id | string | `""` | The API endpoint for acquiring Indonesian prayer times. Default: `"https://api.myquran.com/v1/sholat/jadwal/%s"`. | +| vpbot.api.solat.my | string | `""` | The API endpoint for acquiring Malaysian prayer times. Default: `"https://mpt.i906.my/api/prayer/%s"`. | | vpbot.cloudflared.domain | string | `""` | Registered domain name on Cloudflare used for Vpbot. | | vpbot.cloudflared.enabled | bool | `false` | Specifies whether Vpbot should run using a Cloudflare tunnel. | | vpbot.commands | file | `""` | Custom Telegram `commands.py` file for Vpbot. | diff --git a/mika/vpbot/templates/configmap.yaml b/mika/vpbot/templates/configmap.yaml index 1a659e44..3270d9c0 100644 --- a/mika/vpbot/templates/configmap.yaml +++ b/mika/vpbot/templates/configmap.yaml @@ -1,5 +1,7 @@ {{- $name := .Values.vpbot.name | default "Vpbot" | toString | quote }} {{- $debug := .Values.vpbot.debug | default "false" | toString | quote }} +{{- $solat_api_id := .Values.vpbot.api.solat.id | default "https://api.myquran.com/v1/sholat/jadwal/%s" | toString | quote }} +{{- $solat_api_my := .Values.vpbot.api.solat.my | default "https://mpt.i906.my/api/prayer/%s" | toString | quote }} {{- $location := .Values.vpbot.defaults.location | default "wlp-0" | toString | quote }} {{- $api := .Values.vpbot.telegram.api | default "https://api.telegram.org/bot" | toString | quote }} {{- $webhook := .Values.vpbot.telegram.webhook | default "webhook/telegram/" | toString | quote }} @@ -23,6 +25,8 @@ metadata: data: APP_NAME: {{ $name }} DEBUG: {{ $debug }} + PRAY_API_ID_URI: {{ $solat_api_id }} + PRAY_API_MY_URI: {{ $solat_api_my }} DEFAULT_LOCATION_CODE: {{ $location }} TELEGRAM_BOT_API_URL: {{ $api }} TELEGRAM_WEBHOOK_PATH: {{ $webhook }} diff --git a/mika/vpbot/values.yaml b/mika/vpbot/values.yaml index 78aef797..a6080ea2 100644 --- a/mika/vpbot/values.yaml +++ b/mika/vpbot/values.yaml @@ -80,6 +80,20 @@ vpbot: # Example: # location: "phg-0" location: "" + # Vpbot API configurations. + api: + # Solat module API configurations. + solat: + # The API endpoint for acquiring Indonesian prayer times. + # Default: "https://api.myquran.com/v1/sholat/jadwal/%s" + # Example: + # id: "https://api.myquran.com/v1/sholat/jadwal/%s" + id: "" + # The API endpoint for acquiring Malaysian prayer times. + # Default: "https://mpt.i906.my/api/prayer/%s" + # Example: + # my: "https://mpt.i906.my/api/prayer/%s" + my: "" # Vpbot scheduler configurations. scheduler: # Specifies whether APScheduler should be used by Vpbot as the task scheduler. From 3836b95e0fe2c025ef510740530345dffccaf2ec Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 24 Aug 2023 09:41:32 +0800 Subject: [PATCH 04/13] Separate helper templates --- mika/vpbot/templates/__init__.py.tpl | 30 + mika/vpbot/templates/_helpers.tpl | 968 ---------------------- mika/vpbot/templates/apps.py.tpl | 14 + mika/vpbot/templates/celery.py.tpl | 38 + mika/vpbot/templates/celerybeat.tpl | 334 ++++++++ mika/vpbot/templates/celeryd.tpl | 476 +++++++++++ mika/vpbot/templates/entrypoint.sh.tpl | 16 + mika/vpbot/templates/site-config.conf.tpl | 28 + mika/vpbot/templates/tasks.py.tpl | 24 + 9 files changed, 960 insertions(+), 968 deletions(-) create mode 100644 mika/vpbot/templates/__init__.py.tpl create mode 100644 mika/vpbot/templates/apps.py.tpl create mode 100644 mika/vpbot/templates/celery.py.tpl create mode 100644 mika/vpbot/templates/celerybeat.tpl create mode 100644 mika/vpbot/templates/celeryd.tpl create mode 100644 mika/vpbot/templates/entrypoint.sh.tpl create mode 100644 mika/vpbot/templates/site-config.conf.tpl create mode 100644 mika/vpbot/templates/tasks.py.tpl diff --git a/mika/vpbot/templates/__init__.py.tpl b/mika/vpbot/templates/__init__.py.tpl new file mode 100644 index 00000000..c83ee734 --- /dev/null +++ b/mika/vpbot/templates/__init__.py.tpl @@ -0,0 +1,30 @@ +{{/* +Celery /base/base/__init__.py template +*/}} +{{- define "vpbot.celery-init-py" -}} +from .celery import app as celery_app + +__all__ = ("celery_app",) +{{- end }} + +{{/* +Celery /base/base/tasks.py template +*/}} +{{- define "vpbot.celery-tasks-py" -}} +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from base.scheduler import object_scheduler +from lib.telegram import clean_model + + +# clean model +@shared_task +def clean_model_task(): + clean_model() + + +# check for any objects that need to be sent +@shared_task +def object_scheduler_task(): + object_scheduler() +{{- end }} diff --git a/mika/vpbot/templates/_helpers.tpl b/mika/vpbot/templates/_helpers.tpl index d8169b4d..d50224b3 100644 --- a/mika/vpbot/templates/_helpers.tpl +++ b/mika/vpbot/templates/_helpers.tpl @@ -60,971 +60,3 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} - -{{/* -Apache site-config.conf template -*/}} -{{- define "vpbot.site-config-conf" -}} - - ServerName DOMAIN:443 - UseCanonicalName On - ServerAdmin support@mikahomelab.com - DocumentRoot /base - WSGIScriptAlias / /base/base/wsgi.py - WSGIDaemonProcess DOMAIN python-path=/base - WSGIProcessGroup DOMAIN - - - - Require all granted - - - - Alias /static /static - - Require all granted - - - ErrorLog /var/log/apache2/apache.error.log - CustomLog /var/log/apache2/apache.access.log combined - -{{- end }} - -{{/* -APScheduler /entrypoint.sh template -*/}} -{{- define "vpbot.apscheduler-entrypoint-sh" -}} -#!/bin/bash - -export APP_ROOT="base" - -# ================= DO NOT EDIT BEYOND THIS LINE ================= - -python3 manage.py makemigrations - -python3 manage.py migrate - -tail -f /dev/null -{{- end }} - -{{/* -APScheduler /base/base/apps.py template -*/}} -{{- define "vpbot.apscheduler-apps-py" -}} -from django.apps import AppConfig - -class BaseConfig(AppConfig): - name = "base" - - def ready(self): - from . import signals - from . import tasks - tasks.start() -{{- end }} - -{{/* -APScheduler /base/base/tasks.py template -*/}} -{{- define "vpbot.apscheduler-tasks-py" -}} -from django.conf import settings -from base.scheduler import object_scheduler -from lib.telegram import clean_model -from apscheduler.schedulers.blocking import BlockingScheduler - - -SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") - - -def start(): - scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) - - job_name = "clean_model" - scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) - - job_name = "object_scheduler" - scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) - - scheduler.start() -{{- end }} - -{{/* -Celery /etc/default/celeryd template -*/}} -{{- define "vpbot.default-celeryd" -}} -# Names of nodes to start -# most people will only start one node: -#CELERYD_NODES="worker1" -# but you can also start multiple and configure settings -# for each in CELERYD_OPTS -#CELERYD_NODES="worker1 worker2 worker3" -# alternatively, you can specify the number of nodes to start: -#CELERYD_NODES=10 -CELERYD_NODES="worker1" - -# Absolute or relative path to the 'celery' command: -#CELERY_BIN="/usr/local/bin/celery" -#CELERY_BIN="/virtualenvs/def/bin/celery" -CELERY_BIN="/usr/local/bin/celery" - -# App instance to use -# comment out this line if you don't use an app -#CELERY_APP="proj" -# or fully qualified: -#CELERY_APP="proj.tasks:app" -CELERY_APP="base" - -# Where to chdir at start. -#CELERYD_CHDIR="/opt/Myproject/" -CELERYD_CHDIR="/base/" - -# Extra command-line arguments to the worker -#CELERYD_OPTS="--time-limit=300 --concurrency=8" -# Configure node-specific settings by appending node name to arguments: -#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" -CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" - -# Set logging level -#CELERYD_LOG_LEVEL="DEBUG" -CELERYD_LOG_LEVEL="WARNING" - -# %n will be replaced with the first part of the nodename. -#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -#CELERYD_PID_FILE="/var/run/celery/%n.pid" -CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -CELERYD_PID_FILE="/var/run/celery/%n.pid" - -# Workers should run as an unprivileged user. -# You need to create this user manually (or you can choose -# a user/group combination that already exists (e.g., nobody). -#CELERYD_USER="celery" -#CELERYD_GROUP="celery" -CELERYD_USER="root" -CELERYD_GROUP="root" - -# If enabled pid and log directories will be created if missing, -# and owned by the userid/group configured. -#CELERY_CREATE_DIRS=1 -CELERY_CREATE_DIRS=1 -{{- end }} - -{{/* -Celery /etc/init.d/celerybeat template -*/}} -{{- define "vpbot.initd-celerybeat" -}} -#!/bin/sh -e -# ========================================================= -# celerybeat - Starts the Celery periodic task scheduler. -# ========================================================= -# -# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - -### BEGIN INIT INFO -# Provides: celerybeat -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery periodic task scheduler -### END INIT INFO - -# Cannot use set -e/bash -e since the kill -0 command will abort -# abnormally in the absence of a valid process ID. -#set -e -VERSION=10.1 -echo "celery init v${VERSION}." - -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use 'celery beat --detach'" - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -scripts="" - -if test -f /etc/default/celeryd; then - scripts="/etc/default/celeryd" - _config_sanity /etc/default/celeryd - . /etc/default/celeryd -fi - -EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" -if test -f "$EXTRA_CONFIG"; then - scripts="$scripts, $EXTRA_CONFIG" - _config_sanity "$EXTRA_CONFIG" - . "$EXTRA_CONFIG" -fi - -echo "Using configuration: $scripts" - -CELERY_BIN=${CELERY_BIN:-"celery"} -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/beat.pid" -DEFAULT_LOG_FILE="/var/log/celery/beat.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_CELERYBEAT="$CELERY_BIN" - -CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} -CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} - -CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} -CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYBEAT_PID_FILE" ]; then - CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYBEAT_LOG_FILE" ]; then - CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" -fi - -CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` -CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` - -# Extra start-stop-daemon options, like user/group. - -CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} -if [ -n "$CELERYBEAT_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" -fi - - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $*" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYBEAT_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" - chown "$CELERYBEAT_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYBEAT_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" - chgrp "$CELERYBEAT_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_PID_DIR" - fi -} - - -create_paths () { - create_default_dir "$CELERYBEAT_LOG_DIR" - create_default_dir "$CELERYBEAT_PID_DIR" -} - -is_running() { - pid=$1 - ps $pid > /dev/null 2>&1 -} - -wait_pid () { - pid=$1 - forever=1 - i=0 - while [ $forever -gt 0 ]; do - if ! is_running $pid; then - echo "OK" - forever=0 - else - kill -TERM "$pid" - i=$((i + 1)) - if [ $i -gt 60 ]; then - echo "ERROR" - echo "Timed out while stopping (30s)" - forever=0 - else - sleep 0.5 - fi - fi - done -} - - -stop_beat () { - echo -n "Stopping ${SCRIPT_NAME}... " - if [ -f "$CELERYBEAT_PID_FILE" ]; then - wait_pid $(cat "$CELERYBEAT_PID_FILE") - else - echo "NOT RUNNING" - fi -} - -_chuid () { - ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ - "$CELERYBEAT_USER" -c "$CELERYBEAT $*" -} - -start_beat () { - echo "Starting ${SCRIPT_NAME}..." - _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ - --pidfile="$CELERYBEAT_PID_FILE" \ - --logfile="$CELERYBEAT_LOG_FILE" \ - --loglevel="$CELERYBEAT_LOG_LEVEL" \ - $CELERYBEAT_OPTS -} - - -check_status () { - local failed= - local pid_file=$CELERYBEAT_PID_FILE - if [ ! -e $pid_file ]; then - echo "${SCRIPT_NAME} is down: no pid file found" - failed=true - elif [ ! -r $pid_file ]; then - echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." - failed=true - else - local pid=`cat "$pid_file"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "${SCRIPT_NAME}: bad pid file ($pid_file)" - failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" - failed=true - else - echo "${SCRIPT_NAME} (pid $pid) is up..." - fi - fi - fi - - [ "$failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_beat - ;; - stop) - check_paths - stop_beat - ;; - reload|force-reload) - echo "Use start+stop" - ;; - status) - check_status - ;; - restart) - echo "Restarting celery periodic task scheduler" - check_paths - stop_beat && check_dev_null && start_beat - ;; - create-paths) - check_dev_null - create_paths - ;; - check-paths) - check_dev_null - check_paths - ;; - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /etc/init.d/celeryd template -*/}} -{{- define "vpbot.initd-celeryd" -}} -#!/bin/sh -e -# ============================================ -# celeryd - Starts the Celery worker daemon. -# ============================================ -# -# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - - -### BEGIN INIT INFO -# Provides: celeryd -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery task worker daemon -### END INIT INFO -# -# -# To implement separate init-scripts, copy this script and give it a different -# name. That is, if your new application named "little-worker" needs an init, -# you should use: -# -# cp /etc/init.d/celeryd /etc/init.d/little-worker -# -# You can then configure this by manipulating /etc/default/little-worker. -# -VERSION=10.1 -echo "celery init v${VERSION}." -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use the 'celery multi' utility, " - echo " or 'celery worker --detach'." - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_NODES="celery" -DEFAULT_CELERYD="-m celery worker --detach" - -if [ -d "/etc/default" ]; then - CELERY_CONFIG_DIR="/etc/default" -else - CELERY_CONFIG_DIR="/usr/local/etc" -fi - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -if [ -f "$CELERY_DEFAULTS" ]; then - _config_sanity "$CELERY_DEFAULTS" - echo "Using config script: $CELERY_DEFAULTS" - . "$CELERY_DEFAULTS" -fi - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -# Options to su -# can be used to enable login shell (CELERYD_SU_ARGS="-l"), -# or even to use start-stop-daemon instead of su. -CELERYD_SU=${CELERY_SU:-"su"} -CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} - -CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYD_PID_FILE" ]; then - CELERYD_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYD_LOG_FILE" ]; then - CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERY_BIN=${CELERY_BIN:-"celery"} -CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} -CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYD_OPTS="$CELERYD_OPTS $2" -fi - -CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` -CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` - -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" -fi - - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $* (errno $?)" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYD_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYD_USER'" - chown "$CELERYD_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYD_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYD_GROUP'" - chgrp "$CELERYD_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYD_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYD_PID_DIR" - fi -} - -create_paths() { - create_default_dir "$CELERYD_LOG_DIR" - create_default_dir "$CELERYD_PID_DIR" -} - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - - -_get_pidfiles () { - # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. - ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 -} - - -_get_pids() { - found_pids=0 - my_exitcode=0 - - for pidfile in $(_get_pidfiles); do - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - my_exitcode=1 - else - found_pids=1 - echo "$pid" - fi - - if [ $found_pids -eq 0 ]; then - echo "${SCRIPT_NAME}: All nodes down" - exit $my_exitcode - fi - done -} - - -_chuid () { - ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" -} - - -start_workers () { - if [ ! -z "$CELERYD_ULIMIT" ]; then - ulimit $CELERYD_ULIMIT - fi - _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -dryrun () { - (C_FAKEFORK=1 start_workers --verbose) -} - - -stop_workers () { - _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers () { - _chuid restart $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -kill_workers() { - _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers_graceful () { - echo "WARNING: Use with caution in production" - echo "The workers will attempt to restart, but they may not be able to." - local worker_pids= - worker_pids=`_get_pids` - [ "$one_failed" ] && exit 1 - - for worker_pid in $worker_pids; do - local failed= - kill -HUP $worker_pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" - one_failed=true - else - echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -check_status () { - my_exitcode=0 - found_pids=0 - - local one_failed= - for pidfile in $(_get_pidfiles); do - if [ ! -r $pidfile ]; then - echo "${SCRIPT_NAME} down: no pidfiles found" - one_failed=true - break - fi - - local node=`basename "$pidfile" .pid` - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" - one_failed=true - else - echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." - fi - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_workers - ;; - - stop) - check_dev_null - check_paths - stop_workers - ;; - - reload|force-reload) - echo "Use restart" - ;; - - status) - check_status - ;; - - restart) - check_dev_null - check_paths - restart_workers - ;; - - graceful) - check_dev_null - restart_workers_graceful - ;; - - kill) - check_dev_null - kill_workers - ;; - - dryrun) - check_dev_null - dryrun - ;; - - try-restart) - check_dev_null - check_paths - restart_workers - ;; - - create-paths) - check_dev_null - create_paths - ;; - - check-paths) - check_dev_null - check_paths - ;; - - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /base/base/celery.py template -*/}} -{{- define "vpbot.celery-py" -}} -from __future__ import absolute_import, unicode_literals -import re -import os -from celery import Celery -from celery.schedules import crontab -from datetime import timedelta -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") -app = Celery("base") -app.config_from_object("django.conf:settings", namespace="CELERY") -app.autodiscover_tasks() - - -# make integer from object scheduler seconds string -object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) - - -app.conf.beat_schedule = { - # clean model - "clean_model" : { - "task" : "base.tasks.clean_model_task", - "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), - }, - # check for any objects that need to be sent - "object_scheduler" : { - "task" : "base.tasks.object_scheduler_task", - "schedule" : timedelta(seconds=object_scheduler_seconds), - }, -} - - -@app.task(bind=True) -def debug_task(self): - print("Request: {0!r}".format(self.request)) -{{- end }} - -{{/* -Celery /base/base/__init__.py template -*/}} -{{- define "vpbot.celery-init-py" -}} -from .celery import app as celery_app - -__all__ = ("celery_app",) -{{- end }} - -{{/* -Celery /base/base/tasks.py template -*/}} -{{- define "vpbot.celery-tasks-py" -}} -from __future__ import absolute_import, unicode_literals -from celery import shared_task -from base.scheduler import object_scheduler -from lib.telegram import clean_model - - -# clean model -@shared_task -def clean_model_task(): - clean_model() - - -# check for any objects that need to be sent -@shared_task -def object_scheduler_task(): - object_scheduler() -{{- end }} diff --git a/mika/vpbot/templates/apps.py.tpl b/mika/vpbot/templates/apps.py.tpl new file mode 100644 index 00000000..8532292c --- /dev/null +++ b/mika/vpbot/templates/apps.py.tpl @@ -0,0 +1,14 @@ +{{/* +APScheduler /base/base/apps.py template +*/}} +{{- define "vpbot.apscheduler-apps-py" -}} +from django.apps import AppConfig + +class BaseConfig(AppConfig): + name = "base" + + def ready(self): + from . import signals + from . import tasks + tasks.start() +{{- end }} diff --git a/mika/vpbot/templates/celery.py.tpl b/mika/vpbot/templates/celery.py.tpl new file mode 100644 index 00000000..f5ebed0b --- /dev/null +++ b/mika/vpbot/templates/celery.py.tpl @@ -0,0 +1,38 @@ +{{/* +Celery /base/base/celery.py template +*/}} +{{- define "vpbot.celery-py" -}} +from __future__ import absolute_import, unicode_literals +import re +import os +from celery import Celery +from celery.schedules import crontab +from datetime import timedelta +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") +app = Celery("base") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + + +# make integer from object scheduler seconds string +object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) + + +app.conf.beat_schedule = { + # clean model + "clean_model" : { + "task" : "base.tasks.clean_model_task", + "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), + }, + # check for any objects that need to be sent + "object_scheduler" : { + "task" : "base.tasks.object_scheduler_task", + "schedule" : timedelta(seconds=object_scheduler_seconds), + }, +} + + +@app.task(bind=True) +def debug_task(self): + print("Request: {0!r}".format(self.request)) +{{- end }} diff --git a/mika/vpbot/templates/celerybeat.tpl b/mika/vpbot/templates/celerybeat.tpl new file mode 100644 index 00000000..0dbb0969 --- /dev/null +++ b/mika/vpbot/templates/celerybeat.tpl @@ -0,0 +1,334 @@ +{{/* +Celery /etc/init.d/celerybeat template +*/}} +{{- define "vpbot.initd-celerybeat" -}} +#!/bin/sh -e +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e +VERSION=10.1 +echo "celery init v${VERSION}." + +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use 'celery beat --detach'" + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +scripts="" + +if test -f /etc/default/celeryd; then + scripts="/etc/default/celeryd" + _config_sanity /etc/default/celeryd + . /etc/default/celeryd +fi + +EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" +if test -f "$EXTRA_CONFIG"; then + scripts="$scripts, $EXTRA_CONFIG" + _config_sanity "$EXTRA_CONFIG" + . "$EXTRA_CONFIG" +fi + +echo "Using configuration: $scripts" + +CELERY_BIN=${CELERY_BIN:-"celery"} +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/beat.pid" +DEFAULT_LOG_FILE="/var/log/celery/beat.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_CELERYBEAT="$CELERY_BIN" + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} +CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $*" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + + +create_paths () { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + +is_running() { + pid=$1 + ps $pid > /dev/null 2>&1 +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + if ! is_running $pid; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping ${SCRIPT_NAME}... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +_chuid () { + ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ + "$CELERYBEAT_USER" -c "$CELERYBEAT $*" +} + +start_beat () { + echo "Starting ${SCRIPT_NAME}..." + _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" \ + $CELERYBEAT_OPTS +} + + +check_status () { + local failed= + local pid_file=$CELERYBEAT_PID_FILE + if [ ! -e $pid_file ]; then + echo "${SCRIPT_NAME} is down: no pid file found" + failed=true + elif [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." + failed=true + else + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "${SCRIPT_NAME}: bad pid file ($pid_file)" + failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" + failed=true + else + echo "${SCRIPT_NAME} (pid $pid) is up..." + fi + fi + fi + + [ "$failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_beat + ;; + stop) + check_paths + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + status) + check_status + ;; + restart) + echo "Restarting celery periodic task scheduler" + check_paths + stop_beat && check_dev_null && start_beat + ;; + create-paths) + check_dev_null + create_paths + ;; + check-paths) + check_dev_null + check_paths + ;; + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/vpbot/templates/celeryd.tpl b/mika/vpbot/templates/celeryd.tpl new file mode 100644 index 00000000..8d60608e --- /dev/null +++ b/mika/vpbot/templates/celeryd.tpl @@ -0,0 +1,476 @@ +{{/* +Celery /etc/default/celeryd template +*/}} +{{- define "vpbot.default-celeryd" -}} +# Names of nodes to start +# most people will only start one node: +#CELERYD_NODES="worker1" +# but you can also start multiple and configure settings +# for each in CELERYD_OPTS +#CELERYD_NODES="worker1 worker2 worker3" +# alternatively, you can specify the number of nodes to start: +#CELERYD_NODES=10 +CELERYD_NODES="worker1" + +# Absolute or relative path to the 'celery' command: +#CELERY_BIN="/usr/local/bin/celery" +#CELERY_BIN="/virtualenvs/def/bin/celery" +CELERY_BIN="/usr/local/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +#CELERY_APP="proj" +# or fully qualified: +#CELERY_APP="proj.tasks:app" +CELERY_APP="base" + +# Where to chdir at start. +#CELERYD_CHDIR="/opt/Myproject/" +CELERYD_CHDIR="/base/" + +# Extra command-line arguments to the worker +#CELERYD_OPTS="--time-limit=300 --concurrency=8" +# Configure node-specific settings by appending node name to arguments: +#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" +CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" + +# Set logging level +#CELERYD_LOG_LEVEL="DEBUG" +CELERYD_LOG_LEVEL="WARNING" + +# %n will be replaced with the first part of the nodename. +#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +#CELERYD_PID_FILE="/var/run/celery/%n.pid" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +# Workers should run as an unprivileged user. +# You need to create this user manually (or you can choose +# a user/group combination that already exists (e.g., nobody). +#CELERYD_USER="celery" +#CELERYD_GROUP="celery" +CELERYD_USER="root" +CELERYD_GROUP="root" + +# If enabled pid and log directories will be created if missing, +# and owned by the userid/group configured. +#CELERY_CREATE_DIRS=1 +CELERY_CREATE_DIRS=1 +{{- end }} + +{{/* +Celery /etc/init.d/celeryd template +*/}} +{{- define "vpbot.initd-celeryd" -}} +#!/bin/sh -e +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init-scripts, copy this script and give it a different +# name. That is, if your new application named "little-worker" needs an init, +# you should use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# +VERSION=10.1 +echo "celery init v${VERSION}." +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use the 'celery multi' utility, " + echo " or 'celery worker --detach'." + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery worker --detach" + +if [ -d "/etc/default" ]; then + CELERY_CONFIG_DIR="/etc/default" +else + CELERY_CONFIG_DIR="/usr/local/etc" +fi + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +if [ -f "$CELERY_DEFAULTS" ]; then + _config_sanity "$CELERY_DEFAULTS" + echo "Using config script: $CELERY_DEFAULTS" + . "$CELERY_DEFAULTS" +fi + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +# Options to su +# can be used to enable login shell (CELERYD_SU_ARGS="-l"), +# or even to use start-stop-daemon instead of su. +CELERYD_SU=${CELERY_SU:-"su"} +CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +_get_pidfiles () { + # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. + ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 +} + + +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pidfile in $(_get_pidfiles); do + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" +} + + +start_workers () { + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers () { + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +kill_workers() { + _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + echo "WARNING: Use with caution in production" + echo "The workers will attempt to restart, but they may not be able to." + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pidfile in $(_get_pidfiles); do + if [ ! -r $pidfile ]; then + echo "${SCRIPT_NAME} down: no pidfiles found" + one_failed=true + break + fi + + local node=`basename "$pidfile" .pid` + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_workers + ;; + + stop) + check_dev_null + check_paths + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + restart_workers + ;; + + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + + try-restart) + check_dev_null + check_paths + restart_workers + ;; + + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/vpbot/templates/entrypoint.sh.tpl b/mika/vpbot/templates/entrypoint.sh.tpl new file mode 100644 index 00000000..732410d9 --- /dev/null +++ b/mika/vpbot/templates/entrypoint.sh.tpl @@ -0,0 +1,16 @@ +{{/* +APScheduler /entrypoint.sh template +*/}} +{{- define "vpbot.apscheduler-entrypoint-sh" -}} +#!/bin/bash + +export APP_ROOT="base" + +# ================= DO NOT EDIT BEYOND THIS LINE ================= + +python3 manage.py makemigrations + +python3 manage.py migrate + +tail -f /dev/null +{{- end }} diff --git a/mika/vpbot/templates/site-config.conf.tpl b/mika/vpbot/templates/site-config.conf.tpl new file mode 100644 index 00000000..9acebc53 --- /dev/null +++ b/mika/vpbot/templates/site-config.conf.tpl @@ -0,0 +1,28 @@ +{{/* +Apache site-config.conf template +*/}} +{{- define "vpbot.site-config-conf" -}} + + ServerName DOMAIN:443 + UseCanonicalName On + ServerAdmin support@mikahomelab.com + DocumentRoot /base + WSGIScriptAlias / /base/base/wsgi.py + WSGIDaemonProcess DOMAIN python-path=/base + WSGIProcessGroup DOMAIN + + + + Require all granted + + + + Alias /static /static + + Require all granted + + + ErrorLog /var/log/apache2/apache.error.log + CustomLog /var/log/apache2/apache.access.log combined + +{{- end }} diff --git a/mika/vpbot/templates/tasks.py.tpl b/mika/vpbot/templates/tasks.py.tpl new file mode 100644 index 00000000..dd9f3e48 --- /dev/null +++ b/mika/vpbot/templates/tasks.py.tpl @@ -0,0 +1,24 @@ +{{/* +APScheduler /base/base/tasks.py template +*/}} +{{- define "vpbot.apscheduler-tasks-py" -}} +from django.conf import settings +from base.scheduler import object_scheduler +from lib.telegram import clean_model +from apscheduler.schedulers.blocking import BlockingScheduler + + +SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") + + +def start(): + scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) + + job_name = "clean_model" + scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) + + job_name = "object_scheduler" + scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) + + scheduler.start() +{{- end }} From 6f288c88253388b2037473ba0af5ef37eefbc10d Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 24 Aug 2023 09:44:55 +0800 Subject: [PATCH 05/13] Move celery tasks.py template --- mika/vpbot/templates/__init__.py.tpl | 22 ---------------------- mika/vpbot/templates/tasks.py.tpl | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mika/vpbot/templates/__init__.py.tpl b/mika/vpbot/templates/__init__.py.tpl index c83ee734..b91d2497 100644 --- a/mika/vpbot/templates/__init__.py.tpl +++ b/mika/vpbot/templates/__init__.py.tpl @@ -6,25 +6,3 @@ from .celery import app as celery_app __all__ = ("celery_app",) {{- end }} - -{{/* -Celery /base/base/tasks.py template -*/}} -{{- define "vpbot.celery-tasks-py" -}} -from __future__ import absolute_import, unicode_literals -from celery import shared_task -from base.scheduler import object_scheduler -from lib.telegram import clean_model - - -# clean model -@shared_task -def clean_model_task(): - clean_model() - - -# check for any objects that need to be sent -@shared_task -def object_scheduler_task(): - object_scheduler() -{{- end }} diff --git a/mika/vpbot/templates/tasks.py.tpl b/mika/vpbot/templates/tasks.py.tpl index dd9f3e48..3fe4019b 100644 --- a/mika/vpbot/templates/tasks.py.tpl +++ b/mika/vpbot/templates/tasks.py.tpl @@ -22,3 +22,25 @@ def start(): scheduler.start() {{- end }} + +{{/* +Celery /base/base/tasks.py template +*/}} +{{- define "vpbot.celery-tasks-py" -}} +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from base.scheduler import object_scheduler +from lib.telegram import clean_model + + +# clean model +@shared_task +def clean_model_task(): + clean_model() + + +# check for any objects that need to be sent +@shared_task +def object_scheduler_task(): + object_scheduler() +{{- end }} From afe6549a6a2c3e18303786701e4de8344fe6ac7b Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 24 Aug 2023 10:59:21 +0800 Subject: [PATCH 06/13] Add 2 new schedule values --- mika/vpbot/README.md | 2 ++ mika/vpbot/values.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/mika/vpbot/README.md b/mika/vpbot/README.md index 75b6748c..9730c0f5 100644 --- a/mika/vpbot/README.md +++ b/mika/vpbot/README.md @@ -152,6 +152,8 @@ Deploy [`mika/postgres-agent`](../postgres-agent/) with `postgres.mode.drop` set | vpbot.scheduler.celery | bool | `false` | Specifies whether Celery should be used by Vpbot as the task scheduler. | | vpbot.scheduler.schedule.clean_model | string | `""` | The hours at which the task scheduler cleans up the database. Default: `"0"`. | | vpbot.scheduler.schedule.object_scheduler | string | `""` | The second intervals at which the task scheduler sends scheduled messages. Default: `"2"`. | +| vpbot.scheduler.schedule.solat_clean_db | string | `""` | The hours at which the task scheduler cleans up the solat module database. Default: `"0"`. | +| vpbot.scheduler.schedule.solat_notification | string | `""` | The minute intervals at which the task scheduler sends prayer time notifications. Default: `"1"`. | | vpbot.scheduler.timezone | string | `""` | The timezone for the task scheduler used by Vpbot to schedule time-dependent operations. Default: `"Asia/Kuala_Lumpur"`. | | vpbot.secret | string | `""` | A 50-character secret key used for secure session management and cryptographic operations within the Vpbot service. | | vpbot.telegram.api | string | `""` | API endpoint or URL for the Telegram bot. Default: `"https://api.telegram.org/bot"`. | diff --git a/mika/vpbot/values.yaml b/mika/vpbot/values.yaml index a6080ea2..0eb31110 100644 --- a/mika/vpbot/values.yaml +++ b/mika/vpbot/values.yaml @@ -121,6 +121,16 @@ vpbot: # Example: # object_scheduler: "3" object_scheduler: "" + # The hours at which the task scheduler cleans up the solat module database. + # Default: "0" + # Example: + # solat_clean_db: "0,6,12,18" + solat_clean_db: "" + # The minute intervals at which the task scheduler sends prayer time notifications. + # Default: "1" + # Example: + # solat_notification: "3" + solat_notification: "" # Vpbot Telegram configurations. telegram: # API endpoint or URL for the Telegram bot. From 8c31148e03b094ae931a37862c56796cb01e62d5 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 24 Aug 2023 11:02:10 +0800 Subject: [PATCH 07/13] Add 2 new schedules for the solat module in apscheduler --- mika/vpbot/templates/configmap.yaml | 4 +++- mika/vpbot/templates/tasks.py.tpl | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mika/vpbot/templates/configmap.yaml b/mika/vpbot/templates/configmap.yaml index 3270d9c0..b4cbebfa 100644 --- a/mika/vpbot/templates/configmap.yaml +++ b/mika/vpbot/templates/configmap.yaml @@ -10,6 +10,8 @@ {{- $domain := .Values.vpbot.cloudflared.domain | default "localhost" | toString }} {{- $clean_model := .Values.vpbot.scheduler.schedule.clean_model | default "0" | toString | quote }} {{- $object_scheduler := .Values.vpbot.scheduler.schedule.object_scheduler | default "2" | toString | quote }} +{{- $solat_clean_db := .Values.vpbot.scheduler.schedule.solat_clean_db | default "0" | toString | quote }} +{{- $solat_notification := .Values.vpbot.scheduler.schedule.solat_notification | default "1" | toString | quote }} {{- $apscheduler := .Values.vpbot.scheduler.apscheduler }} {{- $celery := .Values.vpbot.scheduler.celery }} {{- $cloudflared := .Values.vpbot.cloudflared.enabled }} @@ -97,7 +99,7 @@ metadata: data: tasks.py: |- {{- if $apscheduler }} - {{- include "vpbot.apscheduler-tasks-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | nindent 4 }} + {{- include "vpbot.apscheduler-tasks-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | replace "SOLAT_CLEAN_DB_HOURS" $solat_clean_db | replace "SOLAT_NOTIF_MINUTES" $solat_notification | nindent 4 }} {{- else if $celery }} {{- include "vpbot.celery-tasks-py" . | nindent 4 }} {{- end }} diff --git a/mika/vpbot/templates/tasks.py.tpl b/mika/vpbot/templates/tasks.py.tpl index 3fe4019b..5add95a4 100644 --- a/mika/vpbot/templates/tasks.py.tpl +++ b/mika/vpbot/templates/tasks.py.tpl @@ -2,10 +2,13 @@ APScheduler /base/base/tasks.py template */}} {{- define "vpbot.apscheduler-tasks-py" -}} +from apscheduler.schedulers.blocking import BlockingScheduler from django.conf import settings from base.scheduler import object_scheduler -from lib.telegram import clean_model -from apscheduler.schedulers.blocking import BlockingScheduler +from lib import ( + solat, + telegram, +) SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") @@ -14,12 +17,18 @@ SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") def start(): scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) - job_name = "clean_model" - scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) + job_name = "telegram_clean_model" + scheduler.add_job(telegram.clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) job_name = "object_scheduler" scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) + job_name = "solat_clean_db" + scheduler.add_job(solat.clean_db, "cron", hour=SOLAT_CLEAN_DB_HOURS, id=job_name, replace_existing=True) + + job_name = "solat_notify_solat_times" + scheduler.add_job(solat.notify_solat_times, "cron", minute="*/" + SOLAT_NOTIF_MINUTES, id=job_name, replace_existing=True) + scheduler.start() {{- end }} From 9b89145c019bf345f5eb0dc3cb847fc14a9cd78e Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 24 Aug 2023 11:03:15 +0800 Subject: [PATCH 08/13] Add 2 new schedules for solat module to celery --- mika/vpbot/templates/celery.py.tpl | 16 +++++++++++++--- mika/vpbot/templates/configmap.yaml | 2 +- mika/vpbot/templates/tasks.py.tpl | 23 +++++++++++++++++++---- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/mika/vpbot/templates/celery.py.tpl b/mika/vpbot/templates/celery.py.tpl index f5ebed0b..3d548433 100644 --- a/mika/vpbot/templates/celery.py.tpl +++ b/mika/vpbot/templates/celery.py.tpl @@ -19,9 +19,9 @@ object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) app.conf.beat_schedule = { - # clean model - "clean_model" : { - "task" : "base.tasks.clean_model_task", + # clean telegram model + "telegram_clean_model" : { + "task" : "base.tasks.telegram_clean_model_task", "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), }, # check for any objects that need to be sent @@ -29,6 +29,16 @@ app.conf.beat_schedule = { "task" : "base.tasks.object_scheduler_task", "schedule" : timedelta(seconds=object_scheduler_seconds), }, + # clean solat db + "solat_clean_db" : { + "task" : "base.tasks.solat_clean_db_task", + "schedule" : crontab(hour=SOLAT_CLEAN_DB_HOURS, minute="0"), + }, + # send prayer time notifications + "solat_notify_solat_times" : { + "task" : "base.tasks.solat_notify_solat_times_task", + "schedule" : crontab(minute="*/" + SOLAT_NOTIF_MINUTES), + }, } diff --git a/mika/vpbot/templates/configmap.yaml b/mika/vpbot/templates/configmap.yaml index b4cbebfa..d527ff18 100644 --- a/mika/vpbot/templates/configmap.yaml +++ b/mika/vpbot/templates/configmap.yaml @@ -108,7 +108,7 @@ data: {{- include "vpbot.apscheduler-apps-py" . | nindent 4 }} {{- else if $celery }} celery.py: |- - {{- include "vpbot.celery-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | nindent 4 }} + {{- include "vpbot.celery-py" . | replace "CLEAN_MODEL_HOURS" $clean_model | replace "OBJECT_SCHEDULER_SECONDS" $object_scheduler | replace "SOLAT_CLEAN_DB_HOURS" $solat_clean_db | replace "SOLAT_NOTIF_MINUTES" $solat_notification | nindent 4 }} init.py: |- {{- include "vpbot.celery-init-py" . | nindent 4 }} {{- end }} diff --git a/mika/vpbot/templates/tasks.py.tpl b/mika/vpbot/templates/tasks.py.tpl index 5add95a4..7e8a6117 100644 --- a/mika/vpbot/templates/tasks.py.tpl +++ b/mika/vpbot/templates/tasks.py.tpl @@ -39,17 +39,32 @@ Celery /base/base/tasks.py template from __future__ import absolute_import, unicode_literals from celery import shared_task from base.scheduler import object_scheduler -from lib.telegram import clean_model +from lib import ( + solat, + telegram, +) -# clean model +# clean telegram model @shared_task -def clean_model_task(): - clean_model() +def telegram_clean_model_task(): + telegram.clean_model() # check for any objects that need to be sent @shared_task def object_scheduler_task(): object_scheduler() + + +# clean solat db +@shared_task +def solat_clean_db_task(): + solat.clean_db() + + +# send prayer time notifications +@shared_task +def solat_notify_solat_times_task(): + solat.notify_solat_times() {{- end }} From 830f21171d75b7650f6b579bd4af63f0f6df91f8 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Sat, 26 Aug 2023 00:17:22 +0800 Subject: [PATCH 09/13] Update chart and app version --- mika/telego/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mika/telego/Chart.yaml b/mika/telego/Chart.yaml index 26cb61e1..0b65d2e4 100644 --- a/mika/telego/Chart.yaml +++ b/mika/telego/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: telego description: Telego is an easy to use Telegram bot framework built on top of Django. type: application -version: 0.3.0 -appVersion: "0.1.0-stable-r1" +version: 0.4.0 +appVersion: "0.2.0-stable-r1" keywords: - "Telego" - "Telegram" From 07b404458a7a8cde1156c4ff3e164d977c1683ef Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Sat, 26 Aug 2023 00:18:27 +0800 Subject: [PATCH 10/13] Separate helper templates into their own files --- mika/telego/templates/__init__.py.tpl | 8 + mika/telego/templates/_helpers.tpl | 968 --------------------- mika/telego/templates/apps.py.tpl | 14 + mika/telego/templates/celery.py.tpl | 38 + mika/telego/templates/celerybeat.tpl | 334 +++++++ mika/telego/templates/celeryd.tpl | 476 ++++++++++ mika/telego/templates/entrypoint.sh.tpl | 16 + mika/telego/templates/site-config.conf.tpl | 28 + mika/telego/templates/tasks.py.tpl | 46 + 9 files changed, 960 insertions(+), 968 deletions(-) create mode 100644 mika/telego/templates/__init__.py.tpl create mode 100644 mika/telego/templates/apps.py.tpl create mode 100644 mika/telego/templates/celery.py.tpl create mode 100644 mika/telego/templates/celerybeat.tpl create mode 100644 mika/telego/templates/celeryd.tpl create mode 100644 mika/telego/templates/entrypoint.sh.tpl create mode 100644 mika/telego/templates/site-config.conf.tpl create mode 100644 mika/telego/templates/tasks.py.tpl diff --git a/mika/telego/templates/__init__.py.tpl b/mika/telego/templates/__init__.py.tpl new file mode 100644 index 00000000..072dcf4d --- /dev/null +++ b/mika/telego/templates/__init__.py.tpl @@ -0,0 +1,8 @@ +{{/* +Celery /base/base/__init__.py template +*/}} +{{- define "telego.celery-init-py" -}} +from .celery import app as celery_app + +__all__ = ("celery_app",) +{{- end }} diff --git a/mika/telego/templates/_helpers.tpl b/mika/telego/templates/_helpers.tpl index c22effda..5f664925 100644 --- a/mika/telego/templates/_helpers.tpl +++ b/mika/telego/templates/_helpers.tpl @@ -60,971 +60,3 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} - -{{/* -Apache site-config.conf template -*/}} -{{- define "telego.site-config-conf" -}} - - ServerName DOMAIN:443 - UseCanonicalName On - ServerAdmin support@mikahomelab.com - DocumentRoot /base - WSGIScriptAlias / /base/base/wsgi.py - WSGIDaemonProcess DOMAIN python-path=/base - WSGIProcessGroup DOMAIN - - - - Require all granted - - - - Alias /static /static - - Require all granted - - - ErrorLog /var/log/apache2/apache.error.log - CustomLog /var/log/apache2/apache.access.log combined - -{{- end }} - -{{/* -APScheduler /entrypoint.sh template -*/}} -{{- define "telego.apscheduler-entrypoint-sh" -}} -#!/bin/bash - -export APP_ROOT="base" - -# ================= DO NOT EDIT BEYOND THIS LINE ================= - -python3 manage.py makemigrations - -python3 manage.py migrate - -tail -f /dev/null -{{- end }} - -{{/* -APScheduler /base/base/apps.py template -*/}} -{{- define "telego.apscheduler-apps-py" -}} -from django.apps import AppConfig - -class BaseConfig(AppConfig): - name = "base" - - def ready(self): - from . import signals - from . import tasks - tasks.start() -{{- end }} - -{{/* -APScheduler /base/base/tasks.py template -*/}} -{{- define "telego.apscheduler-tasks-py" -}} -from django.conf import settings -from base.scheduler import object_scheduler -from lib.telegram import clean_model -from apscheduler.schedulers.blocking import BlockingScheduler - - -SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") - - -def start(): - scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) - - job_name = "clean_model" - scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) - - job_name = "object_scheduler" - scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) - - scheduler.start() -{{- end }} - -{{/* -Celery /etc/default/celeryd template -*/}} -{{- define "telego.default-celeryd" -}} -# Names of nodes to start -# most people will only start one node: -#CELERYD_NODES="worker1" -# but you can also start multiple and configure settings -# for each in CELERYD_OPTS -#CELERYD_NODES="worker1 worker2 worker3" -# alternatively, you can specify the number of nodes to start: -#CELERYD_NODES=10 -CELERYD_NODES="worker1" - -# Absolute or relative path to the 'celery' command: -#CELERY_BIN="/usr/local/bin/celery" -#CELERY_BIN="/virtualenvs/def/bin/celery" -CELERY_BIN="/usr/local/bin/celery" - -# App instance to use -# comment out this line if you don't use an app -#CELERY_APP="proj" -# or fully qualified: -#CELERY_APP="proj.tasks:app" -CELERY_APP="base" - -# Where to chdir at start. -#CELERYD_CHDIR="/opt/Myproject/" -CELERYD_CHDIR="/base/" - -# Extra command-line arguments to the worker -#CELERYD_OPTS="--time-limit=300 --concurrency=8" -# Configure node-specific settings by appending node name to arguments: -#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" -CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" - -# Set logging level -#CELERYD_LOG_LEVEL="DEBUG" -CELERYD_LOG_LEVEL="WARNING" - -# %n will be replaced with the first part of the nodename. -#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -#CELERYD_PID_FILE="/var/run/celery/%n.pid" -CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -CELERYD_PID_FILE="/var/run/celery/%n.pid" - -# Workers should run as an unprivileged user. -# You need to create this user manually (or you can choose -# a user/group combination that already exists (e.g., nobody). -#CELERYD_USER="celery" -#CELERYD_GROUP="celery" -CELERYD_USER="root" -CELERYD_GROUP="root" - -# If enabled pid and log directories will be created if missing, -# and owned by the userid/group configured. -#CELERY_CREATE_DIRS=1 -CELERY_CREATE_DIRS=1 -{{- end }} - -{{/* -Celery /etc/init.d/celerybeat template -*/}} -{{- define "telego.initd-celerybeat" -}} -#!/bin/sh -e -# ========================================================= -# celerybeat - Starts the Celery periodic task scheduler. -# ========================================================= -# -# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - -### BEGIN INIT INFO -# Provides: celerybeat -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery periodic task scheduler -### END INIT INFO - -# Cannot use set -e/bash -e since the kill -0 command will abort -# abnormally in the absence of a valid process ID. -#set -e -VERSION=10.1 -echo "celery init v${VERSION}." - -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use 'celery beat --detach'" - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -scripts="" - -if test -f /etc/default/celeryd; then - scripts="/etc/default/celeryd" - _config_sanity /etc/default/celeryd - . /etc/default/celeryd -fi - -EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" -if test -f "$EXTRA_CONFIG"; then - scripts="$scripts, $EXTRA_CONFIG" - _config_sanity "$EXTRA_CONFIG" - . "$EXTRA_CONFIG" -fi - -echo "Using configuration: $scripts" - -CELERY_BIN=${CELERY_BIN:-"celery"} -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/beat.pid" -DEFAULT_LOG_FILE="/var/log/celery/beat.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_CELERYBEAT="$CELERY_BIN" - -CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} -CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} - -CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} -CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYBEAT_PID_FILE" ]; then - CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYBEAT_LOG_FILE" ]; then - CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" -fi - -CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` -CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` - -# Extra start-stop-daemon options, like user/group. - -CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} -if [ -n "$CELERYBEAT_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" -fi - - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $*" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYBEAT_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" - chown "$CELERYBEAT_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYBEAT_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" - chgrp "$CELERYBEAT_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_PID_DIR" - fi -} - - -create_paths () { - create_default_dir "$CELERYBEAT_LOG_DIR" - create_default_dir "$CELERYBEAT_PID_DIR" -} - -is_running() { - pid=$1 - ps $pid > /dev/null 2>&1 -} - -wait_pid () { - pid=$1 - forever=1 - i=0 - while [ $forever -gt 0 ]; do - if ! is_running $pid; then - echo "OK" - forever=0 - else - kill -TERM "$pid" - i=$((i + 1)) - if [ $i -gt 60 ]; then - echo "ERROR" - echo "Timed out while stopping (30s)" - forever=0 - else - sleep 0.5 - fi - fi - done -} - - -stop_beat () { - echo -n "Stopping ${SCRIPT_NAME}... " - if [ -f "$CELERYBEAT_PID_FILE" ]; then - wait_pid $(cat "$CELERYBEAT_PID_FILE") - else - echo "NOT RUNNING" - fi -} - -_chuid () { - ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ - "$CELERYBEAT_USER" -c "$CELERYBEAT $*" -} - -start_beat () { - echo "Starting ${SCRIPT_NAME}..." - _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ - --pidfile="$CELERYBEAT_PID_FILE" \ - --logfile="$CELERYBEAT_LOG_FILE" \ - --loglevel="$CELERYBEAT_LOG_LEVEL" \ - $CELERYBEAT_OPTS -} - - -check_status () { - local failed= - local pid_file=$CELERYBEAT_PID_FILE - if [ ! -e $pid_file ]; then - echo "${SCRIPT_NAME} is down: no pid file found" - failed=true - elif [ ! -r $pid_file ]; then - echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." - failed=true - else - local pid=`cat "$pid_file"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "${SCRIPT_NAME}: bad pid file ($pid_file)" - failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" - failed=true - else - echo "${SCRIPT_NAME} (pid $pid) is up..." - fi - fi - fi - - [ "$failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_beat - ;; - stop) - check_paths - stop_beat - ;; - reload|force-reload) - echo "Use start+stop" - ;; - status) - check_status - ;; - restart) - echo "Restarting celery periodic task scheduler" - check_paths - stop_beat && check_dev_null && start_beat - ;; - create-paths) - check_dev_null - create_paths - ;; - check-paths) - check_dev_null - check_paths - ;; - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /etc/init.d/celeryd template -*/}} -{{- define "telego.initd-celeryd" -}} -#!/bin/sh -e -# ============================================ -# celeryd - Starts the Celery worker daemon. -# ============================================ -# -# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - - -### BEGIN INIT INFO -# Provides: celeryd -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery task worker daemon -### END INIT INFO -# -# -# To implement separate init-scripts, copy this script and give it a different -# name. That is, if your new application named "little-worker" needs an init, -# you should use: -# -# cp /etc/init.d/celeryd /etc/init.d/little-worker -# -# You can then configure this by manipulating /etc/default/little-worker. -# -VERSION=10.1 -echo "celery init v${VERSION}." -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use the 'celery multi' utility, " - echo " or 'celery worker --detach'." - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_NODES="celery" -DEFAULT_CELERYD="-m celery worker --detach" - -if [ -d "/etc/default" ]; then - CELERY_CONFIG_DIR="/etc/default" -else - CELERY_CONFIG_DIR="/usr/local/etc" -fi - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -if [ -f "$CELERY_DEFAULTS" ]; then - _config_sanity "$CELERY_DEFAULTS" - echo "Using config script: $CELERY_DEFAULTS" - . "$CELERY_DEFAULTS" -fi - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -# Options to su -# can be used to enable login shell (CELERYD_SU_ARGS="-l"), -# or even to use start-stop-daemon instead of su. -CELERYD_SU=${CELERY_SU:-"su"} -CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} - -CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYD_PID_FILE" ]; then - CELERYD_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYD_LOG_FILE" ]; then - CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERY_BIN=${CELERY_BIN:-"celery"} -CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} -CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYD_OPTS="$CELERYD_OPTS $2" -fi - -CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` -CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` - -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" -fi - - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $* (errno $?)" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYD_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYD_USER'" - chown "$CELERYD_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYD_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYD_GROUP'" - chgrp "$CELERYD_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYD_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYD_PID_DIR" - fi -} - -create_paths() { - create_default_dir "$CELERYD_LOG_DIR" - create_default_dir "$CELERYD_PID_DIR" -} - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - - -_get_pidfiles () { - # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. - ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 -} - - -_get_pids() { - found_pids=0 - my_exitcode=0 - - for pidfile in $(_get_pidfiles); do - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - my_exitcode=1 - else - found_pids=1 - echo "$pid" - fi - - if [ $found_pids -eq 0 ]; then - echo "${SCRIPT_NAME}: All nodes down" - exit $my_exitcode - fi - done -} - - -_chuid () { - ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" -} - - -start_workers () { - if [ ! -z "$CELERYD_ULIMIT" ]; then - ulimit $CELERYD_ULIMIT - fi - _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -dryrun () { - (C_FAKEFORK=1 start_workers --verbose) -} - - -stop_workers () { - _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers () { - _chuid restart $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -kill_workers() { - _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers_graceful () { - echo "WARNING: Use with caution in production" - echo "The workers will attempt to restart, but they may not be able to." - local worker_pids= - worker_pids=`_get_pids` - [ "$one_failed" ] && exit 1 - - for worker_pid in $worker_pids; do - local failed= - kill -HUP $worker_pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" - one_failed=true - else - echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -check_status () { - my_exitcode=0 - found_pids=0 - - local one_failed= - for pidfile in $(_get_pidfiles); do - if [ ! -r $pidfile ]; then - echo "${SCRIPT_NAME} down: no pidfiles found" - one_failed=true - break - fi - - local node=`basename "$pidfile" .pid` - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" - one_failed=true - else - echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." - fi - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_workers - ;; - - stop) - check_dev_null - check_paths - stop_workers - ;; - - reload|force-reload) - echo "Use restart" - ;; - - status) - check_status - ;; - - restart) - check_dev_null - check_paths - restart_workers - ;; - - graceful) - check_dev_null - restart_workers_graceful - ;; - - kill) - check_dev_null - kill_workers - ;; - - dryrun) - check_dev_null - dryrun - ;; - - try-restart) - check_dev_null - check_paths - restart_workers - ;; - - create-paths) - check_dev_null - create_paths - ;; - - check-paths) - check_dev_null - check_paths - ;; - - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /base/base/celery.py template -*/}} -{{- define "telego.celery-py" -}} -from __future__ import absolute_import, unicode_literals -import re -import os -from celery import Celery -from celery.schedules import crontab -from datetime import timedelta -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") -app = Celery("base") -app.config_from_object("django.conf:settings", namespace="CELERY") -app.autodiscover_tasks() - - -# make integer from object scheduler seconds string -object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) - - -app.conf.beat_schedule = { - # clean model - "clean_model" : { - "task" : "base.tasks.clean_model_task", - "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), - }, - # check for any objects that need to be sent - "object_scheduler" : { - "task" : "base.tasks.object_scheduler_task", - "schedule" : timedelta(seconds=object_scheduler_seconds), - }, -} - - -@app.task(bind=True) -def debug_task(self): - print("Request: {0!r}".format(self.request)) -{{- end }} - -{{/* -Celery /base/base/__init__.py template -*/}} -{{- define "telego.celery-init-py" -}} -from .celery import app as celery_app - -__all__ = ("celery_app",) -{{- end }} - -{{/* -Celery /base/base/tasks.py template -*/}} -{{- define "telego.celery-tasks-py" -}} -from __future__ import absolute_import, unicode_literals -from celery import shared_task -from base.scheduler import object_scheduler -from lib.telegram import clean_model - - -# clean model -@shared_task -def clean_model_task(): - clean_model() - - -# check for any objects that need to be sent -@shared_task -def object_scheduler_task(): - object_scheduler() -{{- end }} diff --git a/mika/telego/templates/apps.py.tpl b/mika/telego/templates/apps.py.tpl new file mode 100644 index 00000000..526e2ed0 --- /dev/null +++ b/mika/telego/templates/apps.py.tpl @@ -0,0 +1,14 @@ +{{/* +APScheduler /base/base/apps.py template +*/}} +{{- define "telego.apscheduler-apps-py" -}} +from django.apps import AppConfig + +class BaseConfig(AppConfig): + name = "base" + + def ready(self): + from . import signals + from . import tasks + tasks.start() +{{- end }} diff --git a/mika/telego/templates/celery.py.tpl b/mika/telego/templates/celery.py.tpl new file mode 100644 index 00000000..c6736ca7 --- /dev/null +++ b/mika/telego/templates/celery.py.tpl @@ -0,0 +1,38 @@ +{{/* +Celery /base/base/celery.py template +*/}} +{{- define "telego.celery-py" -}} +from __future__ import absolute_import, unicode_literals +import re +import os +from celery import Celery +from celery.schedules import crontab +from datetime import timedelta +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") +app = Celery("base") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + + +# make integer from object scheduler seconds string +object_scheduler_seconds = int(re.sub(r"\D", "", OBJECT_SCHEDULER_SECONDS)) + + +app.conf.beat_schedule = { + # clean model + "clean_model" : { + "task" : "base.tasks.clean_model_task", + "schedule" : crontab(hour=CLEAN_MODEL_HOURS, minute="0"), + }, + # check for any objects that need to be sent + "object_scheduler" : { + "task" : "base.tasks.object_scheduler_task", + "schedule" : timedelta(seconds=object_scheduler_seconds), + }, +} + + +@app.task(bind=True) +def debug_task(self): + print("Request: {0!r}".format(self.request)) +{{- end }} diff --git a/mika/telego/templates/celerybeat.tpl b/mika/telego/templates/celerybeat.tpl new file mode 100644 index 00000000..a6ce2487 --- /dev/null +++ b/mika/telego/templates/celerybeat.tpl @@ -0,0 +1,334 @@ +{{/* +Celery /etc/init.d/celerybeat template +*/}} +{{- define "telego.initd-celerybeat" -}} +#!/bin/sh -e +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e +VERSION=10.1 +echo "celery init v${VERSION}." + +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use 'celery beat --detach'" + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +scripts="" + +if test -f /etc/default/celeryd; then + scripts="/etc/default/celeryd" + _config_sanity /etc/default/celeryd + . /etc/default/celeryd +fi + +EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" +if test -f "$EXTRA_CONFIG"; then + scripts="$scripts, $EXTRA_CONFIG" + _config_sanity "$EXTRA_CONFIG" + . "$EXTRA_CONFIG" +fi + +echo "Using configuration: $scripts" + +CELERY_BIN=${CELERY_BIN:-"celery"} +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/beat.pid" +DEFAULT_LOG_FILE="/var/log/celery/beat.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_CELERYBEAT="$CELERY_BIN" + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} +CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $*" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + + +create_paths () { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + +is_running() { + pid=$1 + ps $pid > /dev/null 2>&1 +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + if ! is_running $pid; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping ${SCRIPT_NAME}... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +_chuid () { + ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ + "$CELERYBEAT_USER" -c "$CELERYBEAT $*" +} + +start_beat () { + echo "Starting ${SCRIPT_NAME}..." + _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" \ + $CELERYBEAT_OPTS +} + + +check_status () { + local failed= + local pid_file=$CELERYBEAT_PID_FILE + if [ ! -e $pid_file ]; then + echo "${SCRIPT_NAME} is down: no pid file found" + failed=true + elif [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." + failed=true + else + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "${SCRIPT_NAME}: bad pid file ($pid_file)" + failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" + failed=true + else + echo "${SCRIPT_NAME} (pid $pid) is up..." + fi + fi + fi + + [ "$failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_beat + ;; + stop) + check_paths + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + status) + check_status + ;; + restart) + echo "Restarting celery periodic task scheduler" + check_paths + stop_beat && check_dev_null && start_beat + ;; + create-paths) + check_dev_null + create_paths + ;; + check-paths) + check_dev_null + check_paths + ;; + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/telego/templates/celeryd.tpl b/mika/telego/templates/celeryd.tpl new file mode 100644 index 00000000..d6242277 --- /dev/null +++ b/mika/telego/templates/celeryd.tpl @@ -0,0 +1,476 @@ +{{/* +Celery /etc/default/celeryd template +*/}} +{{- define "telego.default-celeryd" -}} +# Names of nodes to start +# most people will only start one node: +#CELERYD_NODES="worker1" +# but you can also start multiple and configure settings +# for each in CELERYD_OPTS +#CELERYD_NODES="worker1 worker2 worker3" +# alternatively, you can specify the number of nodes to start: +#CELERYD_NODES=10 +CELERYD_NODES="worker1" + +# Absolute or relative path to the 'celery' command: +#CELERY_BIN="/usr/local/bin/celery" +#CELERY_BIN="/virtualenvs/def/bin/celery" +CELERY_BIN="/usr/local/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +#CELERY_APP="proj" +# or fully qualified: +#CELERY_APP="proj.tasks:app" +CELERY_APP="base" + +# Where to chdir at start. +#CELERYD_CHDIR="/opt/Myproject/" +CELERYD_CHDIR="/base/" + +# Extra command-line arguments to the worker +#CELERYD_OPTS="--time-limit=300 --concurrency=8" +# Configure node-specific settings by appending node name to arguments: +#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" +CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" + +# Set logging level +#CELERYD_LOG_LEVEL="DEBUG" +CELERYD_LOG_LEVEL="WARNING" + +# %n will be replaced with the first part of the nodename. +#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +#CELERYD_PID_FILE="/var/run/celery/%n.pid" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +# Workers should run as an unprivileged user. +# You need to create this user manually (or you can choose +# a user/group combination that already exists (e.g., nobody). +#CELERYD_USER="celery" +#CELERYD_GROUP="celery" +CELERYD_USER="root" +CELERYD_GROUP="root" + +# If enabled pid and log directories will be created if missing, +# and owned by the userid/group configured. +#CELERY_CREATE_DIRS=1 +CELERY_CREATE_DIRS=1 +{{- end }} + +{{/* +Celery /etc/init.d/celeryd template +*/}} +{{- define "telego.initd-celeryd" -}} +#!/bin/sh -e +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init-scripts, copy this script and give it a different +# name. That is, if your new application named "little-worker" needs an init, +# you should use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# +VERSION=10.1 +echo "celery init v${VERSION}." +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use the 'celery multi' utility, " + echo " or 'celery worker --detach'." + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery worker --detach" + +if [ -d "/etc/default" ]; then + CELERY_CONFIG_DIR="/etc/default" +else + CELERY_CONFIG_DIR="/usr/local/etc" +fi + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +if [ -f "$CELERY_DEFAULTS" ]; then + _config_sanity "$CELERY_DEFAULTS" + echo "Using config script: $CELERY_DEFAULTS" + . "$CELERY_DEFAULTS" +fi + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +# Options to su +# can be used to enable login shell (CELERYD_SU_ARGS="-l"), +# or even to use start-stop-daemon instead of su. +CELERYD_SU=${CELERY_SU:-"su"} +CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +_get_pidfiles () { + # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. + ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 +} + + +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pidfile in $(_get_pidfiles); do + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" +} + + +start_workers () { + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers () { + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +kill_workers() { + _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + echo "WARNING: Use with caution in production" + echo "The workers will attempt to restart, but they may not be able to." + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pidfile in $(_get_pidfiles); do + if [ ! -r $pidfile ]; then + echo "${SCRIPT_NAME} down: no pidfiles found" + one_failed=true + break + fi + + local node=`basename "$pidfile" .pid` + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_workers + ;; + + stop) + check_dev_null + check_paths + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + restart_workers + ;; + + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + + try-restart) + check_dev_null + check_paths + restart_workers + ;; + + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/telego/templates/entrypoint.sh.tpl b/mika/telego/templates/entrypoint.sh.tpl new file mode 100644 index 00000000..2f464d90 --- /dev/null +++ b/mika/telego/templates/entrypoint.sh.tpl @@ -0,0 +1,16 @@ +{{/* +APScheduler /entrypoint.sh template +*/}} +{{- define "telego.apscheduler-entrypoint-sh" -}} +#!/bin/bash + +export APP_ROOT="base" + +# ================= DO NOT EDIT BEYOND THIS LINE ================= + +python3 manage.py makemigrations + +python3 manage.py migrate + +tail -f /dev/null +{{- end }} diff --git a/mika/telego/templates/site-config.conf.tpl b/mika/telego/templates/site-config.conf.tpl new file mode 100644 index 00000000..7a76c0d2 --- /dev/null +++ b/mika/telego/templates/site-config.conf.tpl @@ -0,0 +1,28 @@ +{{/* +Apache site-config.conf template +*/}} +{{- define "telego.site-config-conf" -}} + + ServerName DOMAIN:443 + UseCanonicalName On + ServerAdmin support@mikahomelab.com + DocumentRoot /base + WSGIScriptAlias / /base/base/wsgi.py + WSGIDaemonProcess DOMAIN python-path=/base + WSGIProcessGroup DOMAIN + + + + Require all granted + + + + Alias /static /static + + Require all granted + + + ErrorLog /var/log/apache2/apache.error.log + CustomLog /var/log/apache2/apache.access.log combined + +{{- end }} diff --git a/mika/telego/templates/tasks.py.tpl b/mika/telego/templates/tasks.py.tpl new file mode 100644 index 00000000..c0ff9486 --- /dev/null +++ b/mika/telego/templates/tasks.py.tpl @@ -0,0 +1,46 @@ +{{/* +APScheduler /base/base/tasks.py template +*/}} +{{- define "telego.apscheduler-tasks-py" -}} +from django.conf import settings +from base.scheduler import object_scheduler +from lib.telegram import clean_model +from apscheduler.schedulers.blocking import BlockingScheduler + + +SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") + + +def start(): + scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) + + job_name = "clean_model" + scheduler.add_job(clean_model, "cron", hour=CLEAN_MODEL_HOURS, id=job_name, replace_existing=True) + + job_name = "object_scheduler" + scheduler.add_job(object_scheduler, "cron", second="*/" + OBJECT_SCHEDULER_SECONDS, id=job_name, replace_existing=True) + + scheduler.start() +{{- end }} + +{{/* +Celery /base/base/tasks.py template +*/}} +{{- define "telego.celery-tasks-py" -}} +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from base.scheduler import object_scheduler +from lib.telegram import clean_model + + +# clean model +@shared_task +def clean_model_task(): + clean_model() + + +# check for any objects that need to be sent +@shared_task +def object_scheduler_task(): + object_scheduler() +{{- end }} From b7365e26671dc59f09cdd2a49d030cdf2bdcbd95 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 7 Sep 2023 02:24:30 +0800 Subject: [PATCH 11/13] Update app and chart version --- mika/rizz/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mika/rizz/Chart.yaml b/mika/rizz/Chart.yaml index 3c8a4291..68cc5cb7 100644 --- a/mika/rizz/Chart.yaml +++ b/mika/rizz/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rizz description: Rizz is a simple web application that tracks and posts content from RSS Feeds to Mastodon. type: application -version: 0.0.5 -appVersion: "0.0.1-stable-r1" +version: 0.0.6 +appVersion: "0.0.1-stable-r2" keywords: - "rss" - "monitor" From 614476ae50eeed7e50dd371cb76f6cbccad3077d Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 7 Sep 2023 02:24:58 +0800 Subject: [PATCH 12/13] Separate helper templates --- mika/rizz/templates/__init__.py.tpl | 8 + mika/rizz/templates/_helpers.tpl | 982 ----------------------- mika/rizz/templates/apps.py.tpl | 14 + mika/rizz/templates/celery.py.tpl | 37 + mika/rizz/templates/celerybeat.tpl | 334 ++++++++ mika/rizz/templates/celeryd.tpl | 476 +++++++++++ mika/rizz/templates/entrypoint.sh.tpl | 16 + mika/rizz/templates/site-config.conf.tpl | 28 + mika/rizz/templates/tasks.py.tpl | 61 ++ 9 files changed, 974 insertions(+), 982 deletions(-) create mode 100644 mika/rizz/templates/__init__.py.tpl create mode 100644 mika/rizz/templates/apps.py.tpl create mode 100644 mika/rizz/templates/celery.py.tpl create mode 100644 mika/rizz/templates/celerybeat.tpl create mode 100644 mika/rizz/templates/celeryd.tpl create mode 100644 mika/rizz/templates/entrypoint.sh.tpl create mode 100644 mika/rizz/templates/site-config.conf.tpl create mode 100644 mika/rizz/templates/tasks.py.tpl diff --git a/mika/rizz/templates/__init__.py.tpl b/mika/rizz/templates/__init__.py.tpl new file mode 100644 index 00000000..77ea480b --- /dev/null +++ b/mika/rizz/templates/__init__.py.tpl @@ -0,0 +1,8 @@ +{{/* +Celery /base/base/__init__.py template +*/}} +{{- define "rizz.celery-init-py" -}} +from .celery import app as celery_app + +__all__ = ("celery_app",) +{{- end }} diff --git a/mika/rizz/templates/_helpers.tpl b/mika/rizz/templates/_helpers.tpl index aa05f820..52a0167f 100644 --- a/mika/rizz/templates/_helpers.tpl +++ b/mika/rizz/templates/_helpers.tpl @@ -60,985 +60,3 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} - -{{/* -Apache site-config.conf template -*/}} -{{- define "rizz.site-config-conf" -}} - - ServerName DOMAIN:443 - UseCanonicalName On - ServerAdmin support@mikahomelab.com - DocumentRoot /base - WSGIScriptAlias / /base/base/wsgi.py - WSGIDaemonProcess DOMAIN python-path=/base - WSGIProcessGroup DOMAIN - - - - Require all granted - - - - Alias /static /static - - Require all granted - - - ErrorLog /var/log/apache2/apache.error.log - CustomLog /var/log/apache2/apache.access.log combined - -{{- end }} - -{{/* -APScheduler /entrypoint.sh template -*/}} -{{- define "rizz.apscheduler-entrypoint-sh" -}} -#!/bin/bash - -export APP_ROOT="base" - -# ================= DO NOT EDIT BEYOND THIS LINE ================= - -python3 manage.py makemigrations - -python3 manage.py migrate - -tail -f /dev/null -{{- end }} - -{{/* -APScheduler /base/base/apps.py template -*/}} -{{- define "rizz.apscheduler-apps-py" -}} -from django.apps import AppConfig - -class BaseConfig(AppConfig): - name = "base" - - def ready(self): - from . import signals - from . import tasks - tasks.start() -{{- end }} - -{{/* -APScheduler /base/base/tasks.py template -*/}} -{{- define "rizz.apscheduler-tasks-py" -}} -from django.conf import settings -from base.scheduler import post_scheduler -from lib.rss import ( - clean_data, - update_data, -) -from apscheduler.schedulers.blocking import BlockingScheduler - - -SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") - - -def start(): - scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) - - job_name = "clean_data" - scheduler.add_job(clean_data, "cron", hour=CLEAN_DATA_HOURS, id=job_name, replace_existing=True) - - job_name = "update_data" - scheduler.add_job(update_data, "cron", hour=UPDATE_DATA_HOURS, id=job_name, replace_existing=True) - - job_name = "post_scheduler" - scheduler.add_job(post_scheduler, "cron", hour=POST_SCHEDULER_HOURS, id=job_name, replace_existing=True) - - scheduler.start() -{{- end }} - -{{/* -Celery /etc/default/celeryd template -*/}} -{{- define "rizz.default-celeryd" -}} -# Names of nodes to start -# most people will only start one node: -#CELERYD_NODES="worker1" -# but you can also start multiple and configure settings -# for each in CELERYD_OPTS -#CELERYD_NODES="worker1 worker2 worker3" -# alternatively, you can specify the number of nodes to start: -#CELERYD_NODES=10 -CELERYD_NODES="worker1" - -# Absolute or relative path to the 'celery' command: -#CELERY_BIN="/usr/local/bin/celery" -#CELERY_BIN="/virtualenvs/def/bin/celery" -CELERY_BIN="/usr/local/bin/celery" - -# App instance to use -# comment out this line if you don't use an app -#CELERY_APP="proj" -# or fully qualified: -#CELERY_APP="proj.tasks:app" -CELERY_APP="base" - -# Where to chdir at start. -#CELERYD_CHDIR="/opt/Myproject/" -CELERYD_CHDIR="/base/" - -# Extra command-line arguments to the worker -#CELERYD_OPTS="--time-limit=300 --concurrency=8" -# Configure node-specific settings by appending node name to arguments: -#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" -CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" - -# Set logging level -#CELERYD_LOG_LEVEL="DEBUG" -CELERYD_LOG_LEVEL="WARNING" - -# %n will be replaced with the first part of the nodename. -#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -#CELERYD_PID_FILE="/var/run/celery/%n.pid" -CELERYD_LOG_FILE="/var/log/celery/%n%I.log" -CELERYD_PID_FILE="/var/run/celery/%n.pid" - -# Workers should run as an unprivileged user. -# You need to create this user manually (or you can choose -# a user/group combination that already exists (e.g., nobody). -#CELERYD_USER="celery" -#CELERYD_GROUP="celery" -CELERYD_USER="root" -CELERYD_GROUP="root" - -# If enabled pid and log directories will be created if missing, -# and owned by the userid/group configured. -#CELERY_CREATE_DIRS=1 -CELERY_CREATE_DIRS=1 -{{- end }} - -{{/* -Celery /etc/init.d/celerybeat template -*/}} -{{- define "rizz.initd-celerybeat" -}} -#!/bin/sh -e -# ========================================================= -# celerybeat - Starts the Celery periodic task scheduler. -# ========================================================= -# -# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - -### BEGIN INIT INFO -# Provides: celerybeat -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery periodic task scheduler -### END INIT INFO - -# Cannot use set -e/bash -e since the kill -0 command will abort -# abnormally in the absence of a valid process ID. -#set -e -VERSION=10.1 -echo "celery init v${VERSION}." - -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use 'celery beat --detach'" - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -scripts="" - -if test -f /etc/default/celeryd; then - scripts="/etc/default/celeryd" - _config_sanity /etc/default/celeryd - . /etc/default/celeryd -fi - -EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" -if test -f "$EXTRA_CONFIG"; then - scripts="$scripts, $EXTRA_CONFIG" - _config_sanity "$EXTRA_CONFIG" - . "$EXTRA_CONFIG" -fi - -echo "Using configuration: $scripts" - -CELERY_BIN=${CELERY_BIN:-"celery"} -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/beat.pid" -DEFAULT_LOG_FILE="/var/log/celery/beat.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_CELERYBEAT="$CELERY_BIN" - -CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} -CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} - -CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} -CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYBEAT_PID_FILE" ]; then - CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYBEAT_LOG_FILE" ]; then - CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" -fi - -CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` -CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` - -# Extra start-stop-daemon options, like user/group. - -CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} -if [ -n "$CELERYBEAT_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" -fi - - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $*" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYBEAT_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" - chown "$CELERYBEAT_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYBEAT_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" - chgrp "$CELERYBEAT_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYBEAT_PID_DIR" - fi -} - - -create_paths () { - create_default_dir "$CELERYBEAT_LOG_DIR" - create_default_dir "$CELERYBEAT_PID_DIR" -} - -is_running() { - pid=$1 - ps $pid > /dev/null 2>&1 -} - -wait_pid () { - pid=$1 - forever=1 - i=0 - while [ $forever -gt 0 ]; do - if ! is_running $pid; then - echo "OK" - forever=0 - else - kill -TERM "$pid" - i=$((i + 1)) - if [ $i -gt 60 ]; then - echo "ERROR" - echo "Timed out while stopping (30s)" - forever=0 - else - sleep 0.5 - fi - fi - done -} - - -stop_beat () { - echo -n "Stopping ${SCRIPT_NAME}... " - if [ -f "$CELERYBEAT_PID_FILE" ]; then - wait_pid $(cat "$CELERYBEAT_PID_FILE") - else - echo "NOT RUNNING" - fi -} - -_chuid () { - ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ - "$CELERYBEAT_USER" -c "$CELERYBEAT $*" -} - -start_beat () { - echo "Starting ${SCRIPT_NAME}..." - _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ - --pidfile="$CELERYBEAT_PID_FILE" \ - --logfile="$CELERYBEAT_LOG_FILE" \ - --loglevel="$CELERYBEAT_LOG_LEVEL" \ - $CELERYBEAT_OPTS -} - - -check_status () { - local failed= - local pid_file=$CELERYBEAT_PID_FILE - if [ ! -e $pid_file ]; then - echo "${SCRIPT_NAME} is down: no pid file found" - failed=true - elif [ ! -r $pid_file ]; then - echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." - failed=true - else - local pid=`cat "$pid_file"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "${SCRIPT_NAME}: bad pid file ($pid_file)" - failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" - failed=true - else - echo "${SCRIPT_NAME} (pid $pid) is up..." - fi - fi - fi - - [ "$failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_beat - ;; - stop) - check_paths - stop_beat - ;; - reload|force-reload) - echo "Use start+stop" - ;; - status) - check_status - ;; - restart) - echo "Restarting celery periodic task scheduler" - check_paths - stop_beat && check_dev_null && start_beat - ;; - create-paths) - check_dev_null - create_paths - ;; - check-paths) - check_dev_null - check_paths - ;; - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /etc/init.d/celeryd template -*/}} -{{- define "rizz.initd-celeryd" -}} -#!/bin/sh -e -# ============================================ -# celeryd - Starts the Celery worker daemon. -# ============================================ -# -# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} -# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) -# -# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts - - -### BEGIN INIT INFO -# Provides: celeryd -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: celery task worker daemon -### END INIT INFO -# -# -# To implement separate init-scripts, copy this script and give it a different -# name. That is, if your new application named "little-worker" needs an init, -# you should use: -# -# cp /etc/init.d/celeryd /etc/init.d/little-worker -# -# You can then configure this by manipulating /etc/default/little-worker. -# -VERSION=10.1 -echo "celery init v${VERSION}." -if [ $(id -u) -ne 0 ]; then - echo "Error: This program can only be used by the root user." - echo " Unprivileged users must use the 'celery multi' utility, " - echo " or 'celery worker --detach'." - exit 1 -fi - -origin_is_runlevel_dir () { - set +e - dirname $0 | grep -q "/etc/rc.\.d" - echo $? -} - -# Can be a runlevel symlink (e.g., S02celeryd) -if [ $(origin_is_runlevel_dir) -eq 0 ]; then - SCRIPT_FILE=$(readlink "$0") -else - SCRIPT_FILE="$0" -fi -SCRIPT_NAME="$(basename "$SCRIPT_FILE")" - -DEFAULT_USER="celery" -DEFAULT_PID_FILE="/var/run/celery/%n.pid" -DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" -DEFAULT_LOG_LEVEL="WARNING" -DEFAULT_NODES="celery" -DEFAULT_CELERYD="-m celery worker --detach" - -if [ -d "/etc/default" ]; then - CELERY_CONFIG_DIR="/etc/default" -else - CELERY_CONFIG_DIR="/usr/local/etc" -fi - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} - -# Make sure executable configuration script is owned by root -_config_sanity() { - local path="$1" - local owner=$(ls -ld "$path" | awk '{print $3}') - local iwgrp=$(ls -ld "$path" | cut -b 6) - local iwoth=$(ls -ld "$path" | cut -b 9) - - if [ "$(id -u $owner)" != "0" ]; then - echo "Error: Config script '$path' must be owned by root!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with mailicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change ownership of the script:" - echo " $ sudo chown root '$path'" - exit 1 - fi - - if [ "$iwoth" != "-" ]; then # S_IWOTH - echo "Error: Config script '$path' cannot be writable by others!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi - if [ "$iwgrp" != "-" ]; then # S_IWGRP - echo "Error: Config script '$path' cannot be writable by group!" - echo - echo "Resolution:" - echo "Review the file carefully, and make sure it hasn't been " - echo "modified with malicious intent. When sure the " - echo "script is safe to execute with superuser privileges " - echo "you can change the scripts permissions:" - echo " $ sudo chmod 640 '$path'" - exit 1 - fi -} - -if [ -f "$CELERY_DEFAULTS" ]; then - _config_sanity "$CELERY_DEFAULTS" - echo "Using config script: $CELERY_DEFAULTS" - . "$CELERY_DEFAULTS" -fi - -# Sets --app argument for CELERY_BIN -CELERY_APP_ARG="" -if [ ! -z "$CELERY_APP" ]; then - CELERY_APP_ARG="--app=$CELERY_APP" -fi - -# Options to su -# can be used to enable login shell (CELERYD_SU_ARGS="-l"), -# or even to use start-stop-daemon instead of su. -CELERYD_SU=${CELERY_SU:-"su"} -CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} - -CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} - -# Set CELERY_CREATE_DIRS to always create log/pid dirs. -CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} -CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS -CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS -if [ -z "$CELERYD_PID_FILE" ]; then - CELERYD_PID_FILE="$DEFAULT_PID_FILE" - CELERY_CREATE_RUNDIR=1 -fi -if [ -z "$CELERYD_LOG_FILE" ]; then - CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" - CELERY_CREATE_LOGDIR=1 -fi - -CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERY_BIN=${CELERY_BIN:-"celery"} -CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} -CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} - -export CELERY_LOADER - -if [ -n "$2" ]; then - CELERYD_OPTS="$CELERYD_OPTS $2" -fi - -CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` -CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` - -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" -fi - - -check_dev_null() { - if [ ! -c /dev/null ]; then - echo "/dev/null is not a character device!" - exit 75 # EX_TEMPFAIL - fi -} - - -maybe_die() { - if [ $? -ne 0 ]; then - echo "Exiting: $* (errno $?)" - exit 77 # EX_NOPERM - fi -} - -create_default_dir() { - if [ ! -d "$1" ]; then - echo "- Creating default directory: '$1'" - mkdir -p "$1" - maybe_die "Couldn't create directory $1" - echo "- Changing permissions of '$1' to 02755" - chmod 02755 "$1" - maybe_die "Couldn't change permissions for $1" - if [ -n "$CELERYD_USER" ]; then - echo "- Changing owner of '$1' to '$CELERYD_USER'" - chown "$CELERYD_USER" "$1" - maybe_die "Couldn't change owner of $1" - fi - if [ -n "$CELERYD_GROUP" ]; then - echo "- Changing group of '$1' to '$CELERYD_GROUP'" - chgrp "$CELERYD_GROUP" "$1" - maybe_die "Couldn't change group of $1" - fi - fi -} - - -check_paths() { - if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then - create_default_dir "$CELERYD_LOG_DIR" - fi - if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then - create_default_dir "$CELERYD_PID_DIR" - fi -} - -create_paths() { - create_default_dir "$CELERYD_LOG_DIR" - create_default_dir "$CELERYD_PID_DIR" -} - -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - - -_get_pidfiles () { - # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. - ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 -} - - -_get_pids() { - found_pids=0 - my_exitcode=0 - - for pidfile in $(_get_pidfiles); do - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - my_exitcode=1 - else - found_pids=1 - echo "$pid" - fi - - if [ $found_pids -eq 0 ]; then - echo "${SCRIPT_NAME}: All nodes down" - exit $my_exitcode - fi - done -} - - -_chuid () { - ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" -} - - -start_workers () { - if [ ! -z "$CELERYD_ULIMIT" ]; then - ulimit $CELERYD_ULIMIT - fi - _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -dryrun () { - (C_FAKEFORK=1 start_workers --verbose) -} - - -stop_workers () { - _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers () { - _chuid restart $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - $CELERY_APP_ARG \ - $CELERYD_OPTS -} - - -kill_workers() { - _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" -} - - -restart_workers_graceful () { - echo "WARNING: Use with caution in production" - echo "The workers will attempt to restart, but they may not be able to." - local worker_pids= - worker_pids=`_get_pids` - [ "$one_failed" ] && exit 1 - - for worker_pid in $worker_pids; do - local failed= - kill -HUP $worker_pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" - one_failed=true - else - echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -check_status () { - my_exitcode=0 - found_pids=0 - - local one_failed= - for pidfile in $(_get_pidfiles); do - if [ ! -r $pidfile ]; then - echo "${SCRIPT_NAME} down: no pidfiles found" - one_failed=true - break - fi - - local node=`basename "$pidfile" .pid` - local pid=`cat "$pidfile"` - local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` - if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then - echo "bad pid file ($pidfile)" - one_failed=true - else - local failed= - kill -0 $pid 2> /dev/null || failed=true - if [ "$failed" ]; then - echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" - one_failed=true - else - echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." - fi - fi - done - - [ "$one_failed" ] && exit 1 || exit 0 -} - - -case "$1" in - start) - check_dev_null - check_paths - start_workers - ;; - - stop) - check_dev_null - check_paths - stop_workers - ;; - - reload|force-reload) - echo "Use restart" - ;; - - status) - check_status - ;; - - restart) - check_dev_null - check_paths - restart_workers - ;; - - graceful) - check_dev_null - restart_workers_graceful - ;; - - kill) - check_dev_null - kill_workers - ;; - - dryrun) - check_dev_null - dryrun - ;; - - try-restart) - check_dev_null - check_paths - restart_workers - ;; - - create-paths) - check_dev_null - create_paths - ;; - - check-paths) - check_dev_null - check_paths - ;; - - *) - echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" - exit 64 # EX_USAGE - ;; -esac - -exit 0 -{{- end }} - -{{/* -Celery /base/base/celery.py template -*/}} -{{- define "rizz.celery-py" -}} -from __future__ import absolute_import, unicode_literals -import os -from celery import Celery -from celery.schedules import crontab -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") -app = Celery("base") -app.config_from_object("django.conf:settings", namespace="CELERY") -app.autodiscover_tasks() - - -app.conf.beat_schedule = { - # clean data - "clean_data" : { - "task" : "base.tasks.clean_data_task", - "schedule" : crontab(hour=CLEAN_DATA_HOURS, minute="0"), - }, - # update data - "update_data" : { - "task" : "base.tasks.update_data_task", - "schedule" : crontab(hour=UPDATE_DATA_HOURS, minute="0"), - }, - # check for any posts that need to be posted - "post_scheduler" : { - "task" : "base.tasks.post_scheduler_task", - "schedule" : crontab(hour=POST_SCHEDULER_HOURS, minute="0"), - }, -} - - -@app.task(bind=True) -def debug_task(self): - print("Request: {0!r}".format(self.request)) -{{- end }} - -{{/* -Celery /base/base/__init__.py template -*/}} -{{- define "rizz.celery-init-py" -}} -from .celery import app as celery_app - -__all__ = ("celery_app",) -{{- end }} - -{{/* -Celery /base/base/tasks.py template -*/}} -{{- define "rizz.celery-tasks-py" -}} -from __future__ import absolute_import, unicode_literals -from celery import shared_task -from base.scheduler import post_scheduler -from lib.rss import ( - clean_data, - update_data, -) - - -# clean data -@shared_task -def clean_data_task(): - clean_data() - - -# update data -@shared_task -def update_data_task(): - update_data() - - -# check for any posts that need to be posted -@shared_task -def post_scheduler_task(): - post_scheduler() -{{- end }} diff --git a/mika/rizz/templates/apps.py.tpl b/mika/rizz/templates/apps.py.tpl new file mode 100644 index 00000000..6af98d79 --- /dev/null +++ b/mika/rizz/templates/apps.py.tpl @@ -0,0 +1,14 @@ +{{/* +APScheduler /base/base/apps.py template +*/}} +{{- define "rizz.apscheduler-apps-py" -}} +from django.apps import AppConfig + +class BaseConfig(AppConfig): + name = "base" + + def ready(self): + from . import signals + from . import tasks + tasks.start() +{{- end }} diff --git a/mika/rizz/templates/celery.py.tpl b/mika/rizz/templates/celery.py.tpl new file mode 100644 index 00000000..503f578f --- /dev/null +++ b/mika/rizz/templates/celery.py.tpl @@ -0,0 +1,37 @@ +{{/* +Celery /base/base/celery.py template +*/}} +{{- define "rizz.celery-py" -}} +from __future__ import absolute_import, unicode_literals +import os +from celery import Celery +from celery.schedules import crontab +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "base.settings") +app = Celery("base") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + + +app.conf.beat_schedule = { + # clean data + "clean_data" : { + "task" : "base.tasks.clean_data_task", + "schedule" : crontab(hour=CLEAN_DATA_HOURS, minute="0"), + }, + # update data + "update_data" : { + "task" : "base.tasks.update_data_task", + "schedule" : crontab(hour=UPDATE_DATA_HOURS, minute="0"), + }, + # check for any posts that need to be posted + "post_scheduler" : { + "task" : "base.tasks.post_scheduler_task", + "schedule" : crontab(hour=POST_SCHEDULER_HOURS, minute="0"), + }, +} + + +@app.task(bind=True) +def debug_task(self): + print("Request: {0!r}".format(self.request)) +{{- end }} diff --git a/mika/rizz/templates/celerybeat.tpl b/mika/rizz/templates/celerybeat.tpl new file mode 100644 index 00000000..e5316778 --- /dev/null +++ b/mika/rizz/templates/celerybeat.tpl @@ -0,0 +1,334 @@ +{{/* +Celery /etc/init.d/celerybeat template +*/}} +{{- define "rizz.initd-celerybeat" -}} +#!/bin/sh -e +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e +VERSION=10.1 +echo "celery init v${VERSION}." + +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use 'celery beat --detach'" + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +scripts="" + +if test -f /etc/default/celeryd; then + scripts="/etc/default/celeryd" + _config_sanity /etc/default/celeryd + . /etc/default/celeryd +fi + +EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" +if test -f "$EXTRA_CONFIG"; then + scripts="$scripts, $EXTRA_CONFIG" + _config_sanity "$EXTRA_CONFIG" + . "$EXTRA_CONFIG" +fi + +echo "Using configuration: $scripts" + +CELERY_BIN=${CELERY_BIN:-"celery"} +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/beat.pid" +DEFAULT_LOG_FILE="/var/log/celery/beat.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_CELERYBEAT="$CELERY_BIN" + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +CELERYBEAT_SU=${CELERYBEAT_SU:-"su"} +CELERYBEAT_SU_ARGS=${CELERYBEAT_SU_ARGS:-""} + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $*" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + + +create_paths () { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + +is_running() { + pid=$1 + ps $pid > /dev/null 2>&1 +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + if ! is_running $pid; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping ${SCRIPT_NAME}... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +_chuid () { + ${CELERYBEAT_SU} ${CELERYBEAT_SU_ARGS} \ + "$CELERYBEAT_USER" -c "$CELERYBEAT $*" +} + +start_beat () { + echo "Starting ${SCRIPT_NAME}..." + _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" \ + $CELERYBEAT_OPTS +} + + +check_status () { + local failed= + local pid_file=$CELERYBEAT_PID_FILE + if [ ! -e $pid_file ]; then + echo "${SCRIPT_NAME} is down: no pid file found" + failed=true + elif [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is in unknown state, user cannot read pid file." + failed=true + else + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "${SCRIPT_NAME}: bad pid file ($pid_file)" + failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (pid $pid) is down, but pid file exists!" + failed=true + else + echo "${SCRIPT_NAME} (pid $pid) is up..." + fi + fi + fi + + [ "$failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_beat + ;; + stop) + check_paths + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + status) + check_status + ;; + restart) + echo "Restarting celery periodic task scheduler" + check_paths + stop_beat && check_dev_null && start_beat + ;; + create-paths) + check_dev_null + create_paths + ;; + check-paths) + check_dev_null + check_paths + ;; + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths|status}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/rizz/templates/celeryd.tpl b/mika/rizz/templates/celeryd.tpl new file mode 100644 index 00000000..bfc81256 --- /dev/null +++ b/mika/rizz/templates/celeryd.tpl @@ -0,0 +1,476 @@ +{{/* +Celery /etc/default/celeryd template +*/}} +{{- define "rizz.default-celeryd" -}} +# Names of nodes to start +# most people will only start one node: +#CELERYD_NODES="worker1" +# but you can also start multiple and configure settings +# for each in CELERYD_OPTS +#CELERYD_NODES="worker1 worker2 worker3" +# alternatively, you can specify the number of nodes to start: +#CELERYD_NODES=10 +CELERYD_NODES="worker1" + +# Absolute or relative path to the 'celery' command: +#CELERY_BIN="/usr/local/bin/celery" +#CELERY_BIN="/virtualenvs/def/bin/celery" +CELERY_BIN="/usr/local/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +#CELERY_APP="proj" +# or fully qualified: +#CELERY_APP="proj.tasks:app" +CELERY_APP="base" + +# Where to chdir at start. +#CELERYD_CHDIR="/opt/Myproject/" +CELERYD_CHDIR="/base/" + +# Extra command-line arguments to the worker +#CELERYD_OPTS="--time-limit=300 --concurrency=8" +# Configure node-specific settings by appending node name to arguments: +#CELERYD_OPTS="--time-limit=300 -c 8 -c:worker2 4 -c:worker3 2 -Ofair:worker1" +CELERYD_OPTS="--time-limit=300 --concurrency=8 --without-gossip --without-mingle --without-heartbeat -Ofair --pool=solo" + +# Set logging level +#CELERYD_LOG_LEVEL="DEBUG" +CELERYD_LOG_LEVEL="WARNING" + +# %n will be replaced with the first part of the nodename. +#CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +#CELERYD_PID_FILE="/var/run/celery/%n.pid" +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +# Workers should run as an unprivileged user. +# You need to create this user manually (or you can choose +# a user/group combination that already exists (e.g., nobody). +#CELERYD_USER="celery" +#CELERYD_GROUP="celery" +CELERYD_USER="root" +CELERYD_GROUP="root" + +# If enabled pid and log directories will be created if missing, +# and owned by the userid/group configured. +#CELERY_CREATE_DIRS=1 +CELERY_CREATE_DIRS=1 +{{- end }} + +{{/* +Celery /etc/init.d/celeryd template +*/}} +{{- define "rizz.initd-celeryd" -}} +#!/bin/sh -e +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celeryd (or /usr/local/etc/celeryd on BSD) +# +# See https://docs.celeryq.dev/en/latest/userguide/daemonizing.html#generic-init-scripts + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init-scripts, copy this script and give it a different +# name. That is, if your new application named "little-worker" needs an init, +# you should use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# +VERSION=10.1 +echo "celery init v${VERSION}." +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use the 'celery multi' utility, " + echo " or 'celery worker --detach'." + exit 1 +fi + +origin_is_runlevel_dir () { + set +e + dirname $0 | grep -q "/etc/rc.\.d" + echo $? +} + +# Can be a runlevel symlink (e.g., S02celeryd) +if [ $(origin_is_runlevel_dir) -eq 0 ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" +DEFAULT_LOG_LEVEL="WARNING" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery worker --detach" + +if [ -d "/etc/default" ]; then + CELERY_CONFIG_DIR="/etc/default" +else + CELERY_CONFIG_DIR="/usr/local/etc" +fi + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"$CELERY_CONFIG_DIR/${SCRIPT_NAME}"} + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully, and make sure it hasn't been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +if [ -f "$CELERY_DEFAULTS" ]; then + _config_sanity "$CELERY_DEFAULTS" + echo "Using config script: $CELERY_DEFAULTS" + . "$CELERY_DEFAULTS" +fi + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +# Options to su +# can be used to enable login shell (CELERYD_SU_ARGS="-l"), +# or even to use start-stop-daemon instead of su. +CELERYD_SU=${CELERY_SU:-"su"} +CELERYD_SU_ARGS=${CELERYD_SU_ARGS:-""} + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +_get_pidfiles () { + # note: multi < 3.1.14 output to stderr, not stdout, hence the redirect. + ${CELERYD_MULTI} expand "${CELERYD_PID_FILE}" ${CELERYD_NODES} 2>&1 +} + + +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pidfile in $(_get_pidfiles); do + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + ${CELERYD_SU} ${CELERYD_SU_ARGS} "$CELERYD_USER" -c "$CELERYD_MULTI $*" +} + + +start_workers () { + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers () { + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +kill_workers() { + _chuid kill $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + echo "WARNING: Use with caution in production" + echo "The workers will attempt to restart, but they may not be able to." + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pidfile in $(_get_pidfiles); do + if [ ! -r $pidfile ]; then + echo "${SCRIPT_NAME} down: no pidfiles found" + one_failed=true + break + fi + + local node=`basename "$pidfile" .pid` + local pid=`cat "$pidfile"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pidfile)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is down, but pidfile exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is up..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_workers + ;; + + stop) + check_dev_null + check_paths + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + restart_workers + ;; + + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + + try-restart) + check_dev_null + check_paths + restart_workers + ;; + + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 +{{- end }} diff --git a/mika/rizz/templates/entrypoint.sh.tpl b/mika/rizz/templates/entrypoint.sh.tpl new file mode 100644 index 00000000..5049c808 --- /dev/null +++ b/mika/rizz/templates/entrypoint.sh.tpl @@ -0,0 +1,16 @@ +{{/* +APScheduler /entrypoint.sh template +*/}} +{{- define "rizz.apscheduler-entrypoint-sh" -}} +#!/bin/bash + +export APP_ROOT="base" + +# ================= DO NOT EDIT BEYOND THIS LINE ================= + +python3 manage.py makemigrations + +python3 manage.py migrate + +tail -f /dev/null +{{- end }} diff --git a/mika/rizz/templates/site-config.conf.tpl b/mika/rizz/templates/site-config.conf.tpl new file mode 100644 index 00000000..ec32f96e --- /dev/null +++ b/mika/rizz/templates/site-config.conf.tpl @@ -0,0 +1,28 @@ +{{/* +Apache site-config.conf template +*/}} +{{- define "rizz.site-config-conf" -}} + + ServerName DOMAIN:443 + UseCanonicalName On + ServerAdmin support@mikahomelab.com + DocumentRoot /base + WSGIScriptAlias / /base/base/wsgi.py + WSGIDaemonProcess DOMAIN python-path=/base + WSGIProcessGroup DOMAIN + + + + Require all granted + + + + Alias /static /static + + Require all granted + + + ErrorLog /var/log/apache2/apache.error.log + CustomLog /var/log/apache2/apache.access.log combined + +{{- end }} diff --git a/mika/rizz/templates/tasks.py.tpl b/mika/rizz/templates/tasks.py.tpl new file mode 100644 index 00000000..fecc711a --- /dev/null +++ b/mika/rizz/templates/tasks.py.tpl @@ -0,0 +1,61 @@ +{{/* +APScheduler /base/base/tasks.py template +*/}} +{{- define "rizz.apscheduler-tasks-py" -}} +from django.conf import settings +from base.scheduler import post_scheduler +from lib.rss import ( + clean_data, + update_data, +) +from apscheduler.schedulers.blocking import BlockingScheduler + + +SCHEDULER_TIMEZONE = getattr(settings, "SCHEDULER_TIMEZONE") + + +def start(): + scheduler = BlockingScheduler(timezone=SCHEDULER_TIMEZONE) + + job_name = "clean_data" + scheduler.add_job(clean_data, "cron", hour=CLEAN_DATA_HOURS, id=job_name, replace_existing=True) + + job_name = "update_data" + scheduler.add_job(update_data, "cron", hour=UPDATE_DATA_HOURS, id=job_name, replace_existing=True) + + job_name = "post_scheduler" + scheduler.add_job(post_scheduler, "cron", hour=POST_SCHEDULER_HOURS, id=job_name, replace_existing=True) + + scheduler.start() +{{- end }} + +{{/* +Celery /base/base/tasks.py template +*/}} +{{- define "rizz.celery-tasks-py" -}} +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from base.scheduler import post_scheduler +from lib.rss import ( + clean_data, + update_data, +) + + +# clean data +@shared_task +def clean_data_task(): + clean_data() + + +# update data +@shared_task +def update_data_task(): + update_data() + + +# check for any posts that need to be posted +@shared_task +def post_scheduler_task(): + post_scheduler() +{{- end }} From d0f22a0bd03bc6a23fcf5089424bde07193b0179 Mon Sep 17 00:00:00 2001 From: irfanhakim-as Date: Thu, 7 Sep 2023 02:47:19 +0800 Subject: [PATCH 13/13] Update app version --- mika/vpbot/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mika/vpbot/Chart.yaml b/mika/vpbot/Chart.yaml index b2ff0cce..7e213d94 100644 --- a/mika/vpbot/Chart.yaml +++ b/mika/vpbot/Chart.yaml @@ -3,7 +3,7 @@ name: vpbot description: Vpbot is a Telegram bot with support for a number of useful features such as prayer time notifications, COVID-19 statistics, and more. type: application version: 0.4.0 -appVersion: "0.1.0-stable-r1" +appVersion: "0.1.0-preview-r4-1" keywords: - "Vpbot" - "Telegram"