Merge pull request #104 from MrAnno/image-impr

Container image and CI improvements

.github/workflows/axosyslog-docker.yml

@@ -9,9 +9,6 @@ on:
       tarball-artifact:
         required: false
         type: string
-      axosyslog-modules-artifact:
-        required: true
-        type: string
       snapshot-version:
         required: false
         type: string
@@ -37,12 +34,6 @@ jobs:
           name: ${{ inputs.tarball-artifact }}
           path: docker/apkbuild/axoflow/axosyslog
 
-      - name: Download axosyslog-modules tarball artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.axosyslog-modules-artifact }}
-          path: docker/apkbuild/axoflow/axosyslog
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx

.github/workflows/nightly-release.yml

@@ -69,29 +69,9 @@ jobs:
   #   with:
   #     pkg-type: nightly
 
-  axosyslog-modules:
-    name: axosyslog-modules
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v4
-
-      - name: Create AxoSyslog modules tarball
-        working-directory: docker
-        run: |
-          tar -czvf python-modules.tar.gz python-modules/
-
-      - name: Store axosyslog-modules tarball as artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: axosyslog-modules-tarball
-          path: docker/python-modules.tar.gz
-
   build-and-test:
     runs-on: ubuntu-latest
-    needs:
-      - tarball
-      - axosyslog-modules
+    needs: tarball
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
@@ -102,12 +82,6 @@ jobs:
           name: source-tarball
           path: docker/apkbuild/axoflow/axosyslog
 
-      - name: Download axosyslog-modules tarball artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: axosyslog-modules-tarball
-          path: docker/apkbuild/axoflow/axosyslog
-
       - name: Build Docker image
         uses: docker/build-push-action@v5
         with:
@@ -125,15 +99,13 @@ jobs:
           export SYSLOG_NG_VERSION="${{ needs.tarball.outputs.snapshot-version }}"
           docker/tests/smoke.sh
 
-
   publish-image:
     if: github.ref == 'refs/heads/main'
     uses: ./.github/workflows/axosyslog-docker.yml
-    needs: [tarball, build-and-test, axosyslog-modules]
+    needs: [tarball, build-and-test]
     with:
       pkg-type: nightly
       tarball-artifact: source-tarball
-      axosyslog-modules-artifact: axosyslog-modules-tarball
       snapshot-version: ${{ needs.tarball.outputs.snapshot-version }}
 
   # https://github.com/actions/delete-package-versions/issues/90

.github/workflows/stable-release.yml

@@ -64,29 +64,8 @@ jobs:
   #   with:
   #     pkg-type: stable
 
-  # TODO integrate axosyslog-modules with the source tree and remove this
-  axosyslog-modules:
-    name: axosyslog-modules
-    needs: find-draft-release-run
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v4
-
-      - name: Create AxoSyslog modules tarball
-        working-directory: docker
-        run: |
-          tar -czvf python-modules.tar.gz python-modules/
-
-      - name: Store axosyslog-modules tarball as artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: axosyslog-modules-tarball
-          path: docker/python-modules.tar.gz
-
   publish-image:
     uses: ./.github/workflows/axosyslog-docker.yml
-    needs: axosyslog-modules
+    needs: find-draft-release-run
     with:
       pkg-type: stable
-      axosyslog-modules-artifact: axosyslog-modules-tarball

docker/Makefile.am

@@ -1,8 +1,4 @@
 EXTRA_DIST += \
-    docker/python-modules/webhook/__init__.py \
-    docker/python-modules/webhook/scl/webhook.conf \
-    docker/python-modules/webhook/source.py \
-    docker/python-modules/requirements.txt \
     docker/tests/smoke.sh \
     docker/alpine.dockerfile \
     docker/apkbuild/axoflow/rootbld-repositories \

docker/apkbuild/axoflow/axosyslog/APKBUILD

@@ -52,7 +52,6 @@ subpackages="
 	$pkgname-grpc:_grpc
 	"
 source="https://github.com/axoflow/axosyslog/releases/download/axosyslog-$pkgver/axosyslog-$pkgver.tar.gz
-	python-modules.tar.gz
 	"
 builddir="$srcdir/axosyslog-$pkgver"
 
@@ -147,9 +146,6 @@ _python3() {
 	depends="$pkgname=$pkgver-r$pkgrel python3"
 	install="$subpkgname.post-install"
 
-	cat $srcdir/python-modules/requirements.txt >> $pkgdir/usr/lib/syslog-ng/python/requirements.txt
-	rm $srcdir/python-modules/requirements.txt
-	mv $srcdir/python-modules/* $pkgdir/usr/lib/syslog-ng/python/syslogng/modules
 	_submv usr/lib/syslog-ng/libmod-python.so usr/lib/syslog-ng/python usr/bin/syslog-ng-update-virtualenv
 }
 
@@ -178,6 +174,5 @@ _submv() {
 }
 
 sha512sums="
-2f1e0dea4c0ecfc3c77df7e6ac231ee8436c9c78fcb4df8ccdc417fea7d56791fdeb0844ac35f0342ce7c2bea5618d8723b6b54319c556120099eb809873082e  axosyslog-4.7.1.tar.gz
-SKIP  python-modules.tar.gz
+SKIP  axosyslog-4.7.1.tar.gz
 "

docker/apkbuild/axoflow/axosyslog/APKBUILD-debug.patch

@@ -9,7 +9,7 @@
  makedepends="
  	bison
  	bpftool
-@@ -86,6 +86,7 @@
+@@ -85,6 +85,7 @@
  build() {
  	CFLAGS="$CFLAGS -flto=auto" \
  	./configure \

modules/python-modules/Makefile.am

@@ -1,67 +1,74 @@
 EXTRA_DIST += \
-	modules/python-modules/syslogng/__init__.py				\
-	modules/python-modules/syslogng/confgen.py				\
-	modules/python-modules/syslogng/dest.py					\
-	modules/python-modules/syslogng/logger.py				\
-	modules/python-modules/syslogng/message.py				\
-	modules/python-modules/syslogng/parser.py				\
-	modules/python-modules/syslogng/persist.py				\
-	modules/python-modules/syslogng/reloc.py				\
-	modules/python-modules/syslogng/source.py				\
-	modules/python-modules/syslogng/template.py				\
-	modules/python-modules/syslogng/debuggercli/__init__.py			\
-	modules/python-modules/syslogng/debuggercli/choicecompleter.py		\
-	modules/python-modules/syslogng/debuggercli/commandlinelexer.py		\
-	modules/python-modules/syslogng/debuggercli/completer.py		\
-	modules/python-modules/syslogng/debuggercli/completerlang.py		\
-	modules/python-modules/syslogng/debuggercli/debuglang.py		\
-	modules/python-modules/syslogng/debuggercli/debuggercli.py		\
-	modules/python-modules/syslogng/debuggercli/editline.py 		\
-	modules/python-modules/syslogng/debuggercli/getoptlexer.py		\
-	modules/python-modules/syslogng/debuggercli/langcompleter.py		\
-	modules/python-modules/syslogng/debuggercli/lexer.py			\
-	modules/python-modules/syslogng/debuggercli/lexertoken.py		\
-	modules/python-modules/syslogng/debuggercli/macrocompleter.py		\
-	modules/python-modules/syslogng/debuggercli/readline.py			\
-	modules/python-modules/syslogng/debuggercli/syslognginternals.py	\
-	modules/python-modules/syslogng/debuggercli/tablexer.py			\
-	modules/python-modules/syslogng/debuggercli/templatelang.py		\
-	modules/python-modules/syslogng/debuggercli/templatelexer.py		\
-	modules/python-modules/syslogng/debuggercli/tflang.py			\
-	modules/python-modules/syslogng/debuggercli/tests/__init__.py		\
-	modules/python-modules/syslogng/debuggercli/tests/test_completer.py    	\
-	modules/python-modules/syslogng/debuggercli/tests/test_commandlinelexer.py\
-	modules/python-modules/syslogng/debuggercli/tests/test_completerlang.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_choicecompleter.py\
-	modules/python-modules/syslogng/debuggercli/tests/test_debuglang.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_debuggercli.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_getoptlexer.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_langcompleter.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_lexer.py		\
-	modules/python-modules/syslogng/debuggercli/tests/test_macrocompleter.py\
-	modules/python-modules/syslogng/debuggercli/tests/test_tablexer.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_templatelexer.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_templatelang.py	\
-	modules/python-modules/syslogng/debuggercli/tests/test_tflang.py	\
-	modules/python-modules/pylintrc						\
-	modules/python-modules/test_pymodules.sh				\
-	modules/python-modules/CMakeLists.txt					\
-	\
-	modules/python-modules/syslogng/modules/example/scl/example.conf	\
-	modules/python-modules/syslogng/modules/example/__init__.py		\
-	modules/python-modules/syslogng/modules/kubernetes/scl/kubernetes.conf	\
-	modules/python-modules/syslogng/modules/kubernetes/__init__.py		\
-	modules/python-modules/syslogng/modules/hypr/tests/test_confgen.py	\
-	modules/python-modules/syslogng/modules/hypr/tests/test_hypr_audit_source.py	\
-	modules/python-modules/syslogng/modules/hypr/scl/hypr.conf		\
-	modules/python-modules/syslogng/modules/hypr/__init__.py		\
-	modules/python-modules/syslogng/modules/s3/__init__.py		\
-	modules/python-modules/syslogng/modules/s3/compressable_file_buffer.py		\
-	modules/python-modules/syslogng/modules/s3/s3_object.py		\
-	modules/python-modules/syslogng/modules/s3/s3_destination.py		\
-	modules/python-modules/syslogng/modules/s3/scl/s3.conf		\
-	modules/python-modules/setup.py						\
-	modules/python-modules/README.md
+	modules/python-modules/.gitignore \
+	modules/python-modules/CMakeLists.txt \
+	modules/python-modules/Makefile.am \
+	modules/python-modules/pylintrc \
+	modules/python-modules/README.md \
+	modules/python-modules/setup.py \
+	modules/python-modules/syslogng/__init__.py \
+	modules/python-modules/syslogng/confgen.py \
+	modules/python-modules/syslogng/debuggercli/__init__.py \
+	modules/python-modules/syslogng/debuggercli/choicecompleter.py \
+	modules/python-modules/syslogng/debuggercli/commandlinelexer.py \
+	modules/python-modules/syslogng/debuggercli/completer.py \
+	modules/python-modules/syslogng/debuggercli/completerlang.py \
+	modules/python-modules/syslogng/debuggercli/debuggercli.py \
+	modules/python-modules/syslogng/debuggercli/debuglang.py \
+	modules/python-modules/syslogng/debuggercli/editline.py \
+	modules/python-modules/syslogng/debuggercli/getoptlexer.py \
+	modules/python-modules/syslogng/debuggercli/langcompleter.py \
+	modules/python-modules/syslogng/debuggercli/lexer.py \
+	modules/python-modules/syslogng/debuggercli/lexertoken.py \
+	modules/python-modules/syslogng/debuggercli/macrocompleter.py \
+	modules/python-modules/syslogng/debuggercli/readline.py \
+	modules/python-modules/syslogng/debuggercli/syslognginternals.py \
+	modules/python-modules/syslogng/debuggercli/tablexer.py \
+	modules/python-modules/syslogng/debuggercli/templatelang.py \
+	modules/python-modules/syslogng/debuggercli/templatelexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/__init__.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_choicecompleter.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_commandlinelexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_completer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_completerlang.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_debuggercli.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_debuglang.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_getoptlexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_langcompleter.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_lexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_macrocompleter.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_tablexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_templatelang.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_templatelexer.py \
+	modules/python-modules/syslogng/debuggercli/tests/test_tflang.py \
+	modules/python-modules/syslogng/debuggercli/tflang.py \
+	modules/python-modules/syslogng/dest.py \
+	modules/python-modules/syslogng/logger.py \
+	modules/python-modules/syslogng/message.py \
+	modules/python-modules/syslogng/modules/example/__init__.py \
+	modules/python-modules/syslogng/modules/example/scl/example.conf \
+	modules/python-modules/syslogng/modules/hypr/__init__.py \
+	modules/python-modules/syslogng/modules/hypr/README.md \
+	modules/python-modules/syslogng/modules/hypr/scl/hypr.conf \
+	modules/python-modules/syslogng/modules/hypr/tests/test_confgen.py \
+	modules/python-modules/syslogng/modules/hypr/tests/test_hypr_audit_source.py \
+	modules/python-modules/syslogng/modules/kubernetes/__init__.py \
+	modules/python-modules/syslogng/modules/kubernetes/scl/kubernetes.conf \
+	modules/python-modules/syslogng/modules/kubernetes/tests/test_kubernetes_api_enrichment.py \
+	modules/python-modules/syslogng/modules/s3/__init__.py \
+	modules/python-modules/syslogng/modules/s3/compressable_file_buffer.py \
+	modules/python-modules/syslogng/modules/s3/s3_destination.py \
+	modules/python-modules/syslogng/modules/s3/s3_object.py \
+	modules/python-modules/syslogng/modules/s3/scl/s3.conf \
+	modules/python-modules/syslogng/modules/webhook/__init__.py \
+	modules/python-modules/syslogng/modules/webhook/scl/webhook.conf \
+	modules/python-modules/syslogng/modules/webhook/source.py \
+	modules/python-modules/syslogng/parser.py \
+	modules/python-modules/syslogng/persist.py \
+	modules/python-modules/syslogng/reloc.py \
+	modules/python-modules/syslogng/source.py \
+	modules/python-modules/syslogng/template.py \
+	modules/python-modules/test_pymodules.sh
+
 
 if ENABLE_PYTHON
 

modules/python-modules/setup.py

@@ -38,6 +38,7 @@
   "syslogng.modules.kubernetes",
   "syslogng.modules.hypr",
   "syslogng.modules.s3",
+  "syslogng.modules.webhook",
 ]
 
 requires_addons=[
@@ -47,6 +48,8 @@
   "requests",
   # s3
   "boto3",
+  # webhook
+  "tornado",
 ]
 
 packages = packages_builtin

modules/python-modules/syslogng/modules/webhook/__init__.py

@@ -0,0 +1,27 @@
+#############################################################################
+# Copyright (c) 2024 László Várady
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 as published
+# by the Free Software Foundation, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+# As an additional exemption you are allowed to compile & link against the
+# OpenSSL libraries as published by the OpenSSL project. See the file
+# COPYING for details.
+#
+#############################################################################
+
+from .source import HTTPSource
+
+__all__ = [
+    "HTTPSource"
+]

docker/python-modules/webhook/source.py → modules/python-modules/syslogng/modules/webhook/source.py

@@ -34,6 +34,7 @@
 signal.signal(signal.SIGTERM, signal.SIG_IGN)
 WEBHOOK_QUERY_NV_PREFIX = "webhook.query."
 
+
 class Handler(tornado.web.RequestHandler):
     def initialize(self, source) -> None:
         self.source = source
@@ -66,7 +67,7 @@ def _construct_msg(self, request, path_arguments) -> LogMessage:
         return msg
 
     def set_default_headers(self) -> None:
-        self.set_header("Server", "axosyslog");
+        self.set_header("Server", "axosyslog")
 
     def get_current_user(self):
         if not self.source.auth_token:
@@ -83,6 +84,7 @@ def get_current_user(self):
     def write_error(self, status_code: int, **kwargs: Any) -> None:
         self.set_status(status_code)
 
+
 class HTTPSource(LogSource):
     def init(self, options: dict[str, Any]) -> bool:
         self.logger = logging.getLogger("http")
@@ -139,7 +141,7 @@ def suspend(self) -> None:
     def wakeup(self) -> None:
         self.suspended.clear()
 
-    def request_exit(self) -> None :
+    def request_exit(self) -> None:
         asyncio.run_coroutine_threadsafe(self.stopServer(), self.event_loop)
         pass
 

packaging/debian/axosyslog-mod-python.install

@@ -5,5 +5,6 @@ usr/lib/syslog-ng/*/python/syslogng/debuggercli/*
 usr/lib/syslog-ng/*/python/syslogng/modules/kubernetes/*
 usr/lib/syslog-ng/*/python/syslogng/modules/hypr/*
 usr/lib/syslog-ng/*/python/syslogng/modules/s3/*
+usr/lib/syslog-ng/*/python/syslogng/modules/webhook/*
 usr/lib/syslog-ng/*/python/syslogng*.egg-info/*
 usr/bin/syslog-ng-update-virtualenv

requirements.txt

@@ -18,3 +18,4 @@ six==1.16.0
 urllib3==1.26.18
 websocket-client==1.3.1
 ply==3.11
+tornado==6.4