-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
65 changed files
with
960 additions
and
116 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -35,3 +35,4 @@ devenv.local.nix | |
.pre-commit-config.yaml | ||
|
||
__pycache__ | ||
work/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
inputs: | ||
nixpkgs: | ||
url: github:NixOS/nixpkgs/nixos-23.11 | ||
# 23.11 | ||
url: github:NixOS/nixpkgs?rev=8ac30a39abc5ea67037dfbf090d6e89f187c6e50 |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
import redis | ||
import redis.asyncio | ||
|
||
|
||
def includeme(config): | ||
""" | ||
Initialize the model for a Pyramid app. | ||
""" | ||
|
||
config.include("aramaki.models") | ||
|
||
def get_redis(request): | ||
return redis.Redis.from_url(request.registry.settings["redis.url"]) | ||
|
||
config.add_request_method(get_redis, "redis", reify=True) | ||
|
||
def get_async_redis(request): | ||
return redis.asyncio.Redis.from_url( | ||
request.registry.settings["redis.url"] | ||
) | ||
|
||
config.add_request_method(get_async_redis, "aredis", reify=True) |
This file was deleted.
Oops, something went wrong.
3 changes: 2 additions & 1 deletion
3
src/aramaki/server/alembic/env.py → src/aramaki/alembic/env.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
"""add observations | ||
Revision ID: 5362472a2276 | ||
Revises: c9618fad3eb8 | ||
Create Date: 2024-03-17 09:33:27.491993 | ||
""" | ||
import sqlalchemy as sa | ||
from alembic import op | ||
from sqlalchemy.dialects import postgresql | ||
|
||
# revision identifiers, used by Alembic. | ||
revision = "5362472a2276" | ||
down_revision = "c9618fad3eb8" | ||
branch_labels = None | ||
depends_on = None | ||
|
||
|
||
def upgrade(): | ||
# ### commands auto generated by Alembic - please adjust! ### | ||
op.create_table( | ||
"observation", | ||
sa.Column("system_id", sa.Uuid(), nullable=False), | ||
sa.Column("name", sa.String(), nullable=False), | ||
sa.Column( | ||
"labels", postgresql.JSONB(astext_type=sa.Text()), nullable=False | ||
), | ||
sa.Column( | ||
"data", postgresql.JSONB(astext_type=sa.Text()), nullable=True | ||
), | ||
sa.ForeignKeyConstraint( | ||
["system_id"], | ||
["system.id"], | ||
name=op.f("fk_observation_system_id_system"), | ||
), | ||
sa.PrimaryKeyConstraint( | ||
"system_id", "name", "labels", name=op.f("pk_observation") | ||
), | ||
) | ||
op.create_index( | ||
"ix_labels", | ||
"observation", | ||
["labels"], | ||
unique=False, | ||
postgresql_using="gin", | ||
) | ||
op.drop_index("idxgin", table_name="test", postgresql_using="gin") | ||
op.drop_table("test") | ||
# ### end Alembic commands ### | ||
|
||
|
||
def downgrade(): | ||
# ### commands auto generated by Alembic - please adjust! ### | ||
op.create_table( | ||
"test", | ||
sa.Column("a", sa.INTEGER(), autoincrement=False, nullable=True), | ||
sa.Column( | ||
"b", | ||
postgresql.JSONB(astext_type=sa.Text()), | ||
autoincrement=False, | ||
nullable=True, | ||
), | ||
sa.UniqueConstraint("a", "b", name="test_a_b_key"), | ||
) | ||
op.create_index( | ||
"idxgin", "test", ["b"], unique=False, postgresql_using="gin" | ||
) | ||
op.drop_index( | ||
"ix_labels", table_name="observation", postgresql_using="gin" | ||
) | ||
op.drop_table("observation") | ||
# ### end Alembic commands ### |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
"""simplify labels | ||
Revision ID: a01c1b272031 | ||
Revises: 5362472a2276 | ||
Create Date: 2024-03-17 11:24:05.048918 | ||
""" | ||
|
||
import sqlalchemy as sa | ||
from alembic import op | ||
from sqlalchemy.dialects import postgresql | ||
|
||
# revision identifiers, used by Alembic. | ||
revision = "a01c1b272031" | ||
down_revision = "5362472a2276" | ||
branch_labels = None | ||
depends_on = None | ||
|
||
|
||
def upgrade(): | ||
# ### commands auto generated by Alembic - please adjust! ### | ||
op.drop_index( | ||
"ix_labels", table_name="observation", postgresql_using="gin" | ||
) | ||
op.alter_column( | ||
"observation", | ||
"labels", | ||
existing_type=postgresql.JSONB(astext_type=sa.Text()), | ||
type_=sa.String(), | ||
existing_nullable=False, | ||
) | ||
# ### end Alembic commands ### | ||
|
||
|
||
def downgrade(): | ||
# ### commands auto generated by Alembic - please adjust! ### | ||
op.alter_column( | ||
"observation", | ||
"labels", | ||
existing_type=sa.String(), | ||
type_=postgresql.JSONB(astext_type=sa.Text()), | ||
existing_nullable=False, | ||
) | ||
op.create_index( | ||
"ix_labels", | ||
"observation", | ||
["labels"], | ||
unique=False, | ||
postgresql_using="gin", | ||
) | ||
# ### end Alembic commands ### |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
import asyncio | ||
|
||
import websockets | ||
|
||
from aramaki.processing import MessageBus | ||
|
||
|
||
async def server(env): | ||
async def handle_federation(websocket): | ||
"""Handle processing of a single websocket connection. | ||
On the incoming side this basically places things into the federation | ||
queue. | ||
On the outgoing side this basically pulls things from the federation | ||
queue and sends them out over the appropriate federation sockets. | ||
We want to scale out as much processing onto the job queue as possible | ||
as that is something we'd like to be able to scale horizontally, | ||
potentially over multiple nodes. | ||
Scaling the federation handler would be possible behind a load balancer | ||
and/or using the SO_REUSE socket option. | ||
To reduce processing requirements in this process there is no | ||
verification, no decompression or anything going on, just simply | ||
placing the messages into a redis queue. | ||
""" | ||
bus = MessageBus( | ||
None, # no database here | ||
None, # no syncronous redis here | ||
env["request"].aredis, | ||
) | ||
while True: | ||
message = await websocket.recv() | ||
# XXX Security: we need some kind of authorization and potentially | ||
# rate limiting. However, this would only be abuse/DOS protection, | ||
# not anything CPU intensive like validating with the database, | ||
# which should happen in the processing environment. | ||
asyncio.create_task(bus.a_record_external(message)) | ||
|
||
# XXX make configurable | ||
async with websockets.serve(handle_federation, "0.0.0.0", 8764): | ||
await asyncio.Future() # run forever |
Oops, something went wrong.