Skip to content

Commit

Permalink
snapshot
Browse files Browse the repository at this point in the history
  • Loading branch information
ctheune committed Mar 18, 2024
1 parent af9ef7b commit 499126a
Show file tree
Hide file tree
Showing 65 changed files with 960 additions and 116 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ devenv.local.nix
.pre-commit-config.yaml

__pycache__
work/
1 change: 1 addition & 0 deletions development.ini
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pyramid.includes =
pyramid_debugtoolbar

sqlalchemy.url = postgresql://localhost/aramaki?host=%(here)s/.devenv/state/postgres
redis.url = redis://localhost

retry.attempts = 3

Expand Down
20 changes: 10 additions & 10 deletions devenv.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
"devenv": {
"locked": {
"dir": "src/modules",
"lastModified": 1706018268,
"narHash": "sha256-d24+re0t8b6HYGzAPZCIJed85n23RUFXQa2yuHoW0uQ=",
"lastModified": 1710144971,
"narHash": "sha256-CjTOdoBvT/4AQncTL20SDHyJNgsXZjtGbz62yDIUYnM=",
"owner": "cachix",
"repo": "devenv",
"rev": "ad0ae333b210e31237e1fc4a7ddab71a01785add",
"rev": "6c0bad0045f1e1802f769f7890f6a59504825f4d",
"type": "github"
},
"original": {
Expand Down Expand Up @@ -74,17 +74,17 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1705916986,
"narHash": "sha256-iBpfltu6QvN4xMpen6jGGEb6jOqmmVQKUrXdOJ32u8w=",
"lastModified": 1710565619,
"narHash": "sha256-xu/EnZCNdIj7m/QjCNIG5vrCA4TYg5uwFReb9XDxET0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d7f206b723e42edb09d9d753020a84b3061a79d8",
"rev": "8ac30a39abc5ea67037dfbf090d6e89f187c6e50",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"rev": "8ac30a39abc5ea67037dfbf090d6e89f187c6e50",
"type": "github"
}
},
Expand Down Expand Up @@ -115,11 +115,11 @@
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1705757126,
"narHash": "sha256-Eksr+n4Q8EYZKAN0Scef5JK4H6FcHc+TKNHb95CWm+c=",
"lastModified": 1708018599,
"narHash": "sha256-M+Ng6+SePmA8g06CmUZWi1AjG2tFBX9WCXElBHEKnyM=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "f56597d53fd174f796b5a7d3ee0b494f9e2285cc",
"rev": "5df5a70ad7575f6601d91f0efec95dd9bc619431",
"type": "github"
},
"original": {
Expand Down
19 changes: 8 additions & 11 deletions devenv.nix
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,8 @@
alembic upgrade head
'';

scripts.aramaki-web.exec = ''
pserve development.ini
'';

enterShell = ''
echo
echo
echo 🦾 Helper scripts you can run to make your development richer:
echo 🦾
${pkgs.gnused}/bin/sed -e 's| |••|g' -e 's|=| |' <<EOF | ${pkgs.util-linuxMinimal}/bin/column -t | ${pkgs.gnused}/bin/sed -e 's|^|🦾 |' -e 's|••| |g'
Expand All @@ -47,14 +43,15 @@
echo
'';

processes = {
# aramaki-server-web.exec = "pserve development.ini";
# aramaki-server-processing.exec = "aramaki-server processing";
# aramaki-server-federation.exec = "aramaki-server federation";
# aramaki-server-agent-manager.exec = "aramaki-server agent-manager";
# aramaki-agent.exec = "aramaki-agent";
process-managers.overmind.enable = true;
process-managers.honcho.enable = false;

processes = {
aramaki-web.exec = "aramaki web development.ini --reload";
aramaki-processing.exec = "aramaki processing development.ini";
aramaki-federation.exec = "aramaki federation development.ini";
tailwindcss.exec = "cd tailwind; (while true; do sleep 10; done) | tailwindcss -i aramaki.css -o ../src/aramaki/server/web/static/aramaki.css --minify --watch";
agent-forward.exec = "ssh -R 8764:localhost:8764 test38.fcio.net";
};

services.postgres = {
Expand Down
3 changes: 2 additions & 1 deletion devenv.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
inputs:
nixpkgs:
url: github:NixOS/nixpkgs/nixos-23.11
# 23.11
url: github:NixOS/nixpkgs?rev=8ac30a39abc5ea67037dfbf090d6e89f187c6e50
149 changes: 148 additions & 1 deletion poetry.lock

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ zope-sqlalchemy = "^3.0"

# peppercorn = "^0.6"
peppercorn = {git = "https://github.com/ctheune/peppercorn.git", branch = "htmx-workaround-1686-wrong-form-dataset-construction" }

aiofiles = "^23.2.1"
fswatch = "^0.1.1"
zstd = "^1.5.5.1"
redis = "^5.0.1"

[tool.poetry.group.dev.dependencies]
webtest = ">=3"
Expand All @@ -52,12 +55,10 @@ pytest-patterns = ">=0.1"
"zope.testbrowser" = ">=6.0"

[tool.poetry.plugins."paste.app_factory"]
main = "aramaki.server.web.main:main"
main = "aramaki.web.main:main"

[tool.poetry.plugins."console_scripts"]
aramaki-initialize-db = "aramaki.scripts.initialize_db:main"
aramaki-server = "aramaki.server.main:main"
aramaki-agent = "aramaki.agent.main:main"
aramaki = "aramaki.scripts:main"

[project.urls]
Homepage = "https://github.com/flyingcircusio/aramaki"
Expand All @@ -70,7 +71,6 @@ line-length = 79
profile = "black"
line_length = 79


[[tool.mypy.overrides]]
module = "aramaki.*"
check_untyped_defs = true
Expand Down
23 changes: 23 additions & 0 deletions src/aramaki/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import redis
import redis.asyncio


def includeme(config):
"""
Initialize the model for a Pyramid app.
"""

config.include("aramaki.models")

def get_redis(request):
return redis.Redis.from_url(request.registry.settings["redis.url"])

config.add_request_method(get_redis, "redis", reify=True)

def get_async_redis(request):
return redis.asyncio.Redis.from_url(
request.registry.settings["redis.url"]
)

config.add_request_method(get_async_redis, "aredis", reify=True)
7 changes: 0 additions & 7 deletions src/aramaki/agent/main.py

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Pyramid bootstrap environment. """

from alembic import context
from pyramid.paster import get_appsettings, setup_logging
from sqlalchemy import engine_from_config

from aramaki.server.models.meta import Base
from aramaki.models.meta import Base

config = context.config

Expand Down
File renamed without changes.
72 changes: 72 additions & 0 deletions src/aramaki/alembic/versions/20240317_5362472a2276.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""add observations
Revision ID: 5362472a2276
Revises: c9618fad3eb8
Create Date: 2024-03-17 09:33:27.491993
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "5362472a2276"
down_revision = "c9618fad3eb8"
branch_labels = None
depends_on = None


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"observation",
sa.Column("system_id", sa.Uuid(), nullable=False),
sa.Column("name", sa.String(), nullable=False),
sa.Column(
"labels", postgresql.JSONB(astext_type=sa.Text()), nullable=False
),
sa.Column(
"data", postgresql.JSONB(astext_type=sa.Text()), nullable=True
),
sa.ForeignKeyConstraint(
["system_id"],
["system.id"],
name=op.f("fk_observation_system_id_system"),
),
sa.PrimaryKeyConstraint(
"system_id", "name", "labels", name=op.f("pk_observation")
),
)
op.create_index(
"ix_labels",
"observation",
["labels"],
unique=False,
postgresql_using="gin",
)
op.drop_index("idxgin", table_name="test", postgresql_using="gin")
op.drop_table("test")
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"test",
sa.Column("a", sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column(
"b",
postgresql.JSONB(astext_type=sa.Text()),
autoincrement=False,
nullable=True,
),
sa.UniqueConstraint("a", "b", name="test_a_b_key"),
)
op.create_index(
"idxgin", "test", ["b"], unique=False, postgresql_using="gin"
)
op.drop_index(
"ix_labels", table_name="observation", postgresql_using="gin"
)
op.drop_table("observation")
# ### end Alembic commands ###
51 changes: 51 additions & 0 deletions src/aramaki/alembic/versions/20240317_a01c1b272031.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""simplify labels
Revision ID: a01c1b272031
Revises: 5362472a2276
Create Date: 2024-03-17 11:24:05.048918
"""

import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "a01c1b272031"
down_revision = "5362472a2276"
branch_labels = None
depends_on = None


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(
"ix_labels", table_name="observation", postgresql_using="gin"
)
op.alter_column(
"observation",
"labels",
existing_type=postgresql.JSONB(astext_type=sa.Text()),
type_=sa.String(),
existing_nullable=False,
)
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"observation",
"labels",
existing_type=sa.String(),
type_=postgresql.JSONB(astext_type=sa.Text()),
existing_nullable=False,
)
op.create_index(
"ix_labels",
"observation",
["labels"],
unique=False,
postgresql_using="gin",
)
# ### end Alembic commands ###
File renamed without changes.
11 changes: 4 additions & 7 deletions src/aramaki/server/conftest.py → src/aramaki/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import zope.testbrowser.wsgi
from pyramid.paster import get_appsettings

from aramaki.server import models
from aramaki.server.models.meta import Base
from aramaki.server.web.main import main
from aramaki import models
from aramaki.models.meta import Base
from aramaki.web.main import main


def pytest_addoption(parser):
Expand All @@ -34,11 +34,8 @@ def dbengine(app_settings, ini_file):

alembic_cfg = alembic.config.Config(ini_file)
Base.metadata.drop_all(bind=engine)
# https://github.com/sqlalchemy/alembic/issues/1420
alembic.command.stamp(alembic_cfg, None, purge=True) # type: ignore

alembic.command.stamp(alembic_cfg, "base", purge=True)
alembic.command.upgrade(alembic_cfg, "head")

yield engine


Expand Down
45 changes: 45 additions & 0 deletions src/aramaki/federation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import asyncio

import websockets

from aramaki.processing import MessageBus


async def server(env):
async def handle_federation(websocket):
"""Handle processing of a single websocket connection.
On the incoming side this basically places things into the federation
queue.
On the outgoing side this basically pulls things from the federation
queue and sends them out over the appropriate federation sockets.
We want to scale out as much processing onto the job queue as possible
as that is something we'd like to be able to scale horizontally,
potentially over multiple nodes.
Scaling the federation handler would be possible behind a load balancer
and/or using the SO_REUSE socket option.
To reduce processing requirements in this process there is no
verification, no decompression or anything going on, just simply
placing the messages into a redis queue.
"""
bus = MessageBus(
None, # no database here
None, # no syncronous redis here
env["request"].aredis,
)
while True:
message = await websocket.recv()
# XXX Security: we need some kind of authorization and potentially
# rate limiting. However, this would only be abuse/DOS protection,
# not anything CPU intensive like validating with the database,
# which should happen in the processing environment.
asyncio.create_task(bus.a_record_external(message))

# XXX make configurable
async with websockets.serve(handle_federation, "0.0.0.0", 8764):
await asyncio.Future() # run forever
Loading

0 comments on commit 499126a

Please sign in to comment.