From ea1ce33660ecc840b9509386069a4dfaada72eb1 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Mon, 15 Jan 2024 16:12:13 -0300 Subject: [PATCH 1/2] added EXTERNAL_LISTEN_HOST added fileLimit --- .env | 2 ++ Dockerfile-generateconfig | 4 ++-- Makefile | 2 ++ README.md | 13 +++++++++++-- docker-compose-generateconfig.yml | 10 ---------- docker-generateconfig/etc/coordinator.yml | 5 +++++ docker-generateconfig/generate_config.sh | 1 + docker-generateconfig/processing.sh | 6 ++++++ docker-generateconfig/setListenIp.py | 21 +++++++++++++++++++++ 9 files changed, 50 insertions(+), 14 deletions(-) delete mode 100644 docker-compose-generateconfig.yml create mode 100755 docker-generateconfig/setListenIp.py diff --git a/.env b/.env index c59a188..bb9fd08 100644 --- a/.env +++ b/.env @@ -11,6 +11,8 @@ MONGO_VERSION=7.0.2 REDIS_VERSION=7.2.0-v6 S3_EMULATOR_VERSION=0.0.1 +EXTERNAL_LISTEN_HOST="127.0.0.1" + ANY_SYNC_NODE_1_HOST=any-sync-node-1 ANY_SYNC_NODE_1_PORT=1001 ANY_SYNC_NODE_1_ADDRESSES=${ANY_SYNC_NODE_1_HOST}:${ANY_SYNC_NODE_1_PORT} diff --git a/Dockerfile-generateconfig b/Dockerfile-generateconfig index b35dd78..9631d34 100644 --- a/Dockerfile-generateconfig +++ b/Dockerfile-generateconfig @@ -6,11 +6,11 @@ COPY --chmod=777 docker-generateconfig/generate_config.sh .env . RUN ./generate_config.sh FROM alpine:3.18.4 -RUN apk add --no-cache bash perl +RUN apk add --no-cache bash yq perl python3 py3-yaml WORKDIR /opt/processing COPY docker-generateconfig/etc/ tmp-etc/ COPY --chmod=777 docker-generateconfig/processing.sh . +COPY --chmod=777 docker-generateconfig/setListenIp.py . COPY --from=generator /opt/generateconfig/ generateconfig/ CMD ./processing.sh - diff --git a/Makefile b/Makefile index 1e7bb52..c4a1e23 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ pull: down: docker compose down +logs: + docker compose logs -f # build with "plain" log for debug build: diff --git a/README.md b/README.md index 281ca08..b30dccd 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,18 @@ Self-host for any-sync, designed for review and testing purposes. db.getMongo().setReadPref('primaryPreferred'); db.nodeConf.find().sort( { _id: -1 } ).limit(1) ``` -## Set specific versions +* run client + ``` + # macos example + ANYTYPE_LOG_LEVEL="*=DEBUG" ANYPROF=:6060 ANY_SYNC_NETWORK=/any-sync-dockercompose/etc/client.yml /Applications/Anytype.app/Contents/MacOS/Anytype + ``` + +## configuration Use file .env -### Compatible versions +* Set specific versions: find and edit variables with suffix "_VERSION" +* Set external listen host: default 127.0.0.1, for change you need edit variable "EXTERNAL_LISTEN_HOST" + +## Compatible versions You can find compatible versions on these pages: * stable versions, used in [production](https://puppetdoc.anytype.io/api/v1/prod-any-sync-compatible-versions/) * unstable versions, used in [test stand](https://puppetdoc.anytype.io/api/v1/stage1-any-sync-compatible-versions/) diff --git a/docker-compose-generateconfig.yml b/docker-compose-generateconfig.yml deleted file mode 100644 index 9cae9cf..0000000 --- a/docker-compose-generateconfig.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.9" -services: - generateconfig: - image: generateconfig - build: - context: . - dockerfile: Dockerfile-generateconfig - volumes: - - ./tmp/generateconfig/:/generateconfig/ - command: /gen.sh diff --git a/docker-generateconfig/etc/coordinator.yml b/docker-generateconfig/etc/coordinator.yml index 8881133..2ce93a4 100644 --- a/docker-generateconfig/etc/coordinator.yml +++ b/docker-generateconfig/etc/coordinator.yml @@ -24,3 +24,8 @@ quic: - %ANY_SYNC_COORDINATOR_QUIC_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 + +fileLimit: + limitDefault: 1099511627776 + limitAlphaUsers: 1099511627776 + limitNightlyUsers: 1099511627776 diff --git a/docker-generateconfig/generate_config.sh b/docker-generateconfig/generate_config.sh index a3ebfa4..ce35464 100755 --- a/docker-generateconfig/generate_config.sh +++ b/docker-generateconfig/generate_config.sh @@ -39,6 +39,7 @@ if ! [[ -s account0.yml ]]; then fi fi +yq --indent 4 --inplace 'del(.creationTime)' nodes.yml yq --indent 4 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml diff --git a/docker-generateconfig/processing.sh b/docker-generateconfig/processing.sh index 1e0dd63..c8f7eef 100755 --- a/docker-generateconfig/processing.sh +++ b/docker-generateconfig/processing.sh @@ -16,6 +16,12 @@ for node_type in filenode coordinator consensusnode; do mkdir -p "${dest_path}/any-sync-${node_type}" done +# add external listen host +./setListenIp.py "${EXTERNAL_LISTEN_HOST}" "generateconfig/nodes.yml" + +# create config for clients +cp "generateconfig/nodes.yml" "${dest_path}/client.yml" + # Generate network file sed 's|^| |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${network_file}" diff --git a/docker-generateconfig/setListenIp.py b/docker-generateconfig/setListenIp.py new file mode 100755 index 0000000..6ce6502 --- /dev/null +++ b/docker-generateconfig/setListenIp.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import yaml + +listenHost = sys.argv[1] +yamlFile = sys.argv[2] + +with open(yamlFile, 'r') as file: + config = yaml.load(file,Loader=yaml.Loader) + +for index, nodes in enumerate(config['nodes']): + addresses = nodes['addresses'] + port = addresses[0].split(':')[1] + listenAddress = listenHost +':'+ port + if listenAddress not in addresses: + addresses.append(listenAddress) + +with open(yamlFile, 'w') as file: + yaml.dump(config, file) From ebf56e9190ef89d7abaa617bae862b082445cbe2 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Mon, 15 Jan 2024 16:58:17 -0300 Subject: [PATCH 2/2] fixed redis-servr data dir and fixed s3 emulator dns name --- docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 297d6de..f0563da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: container_name: redis image: "redis/redis-stack-server:${REDIS_VERSION}" restart: always - command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction --protected-mode no --loadmodule /opt/redis-stack/lib/redisbloom.so + command: redis-server --dir /data/ --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction --protected-mode no --loadmodule /opt/redis-stack/lib/redisbloom.so ports: - "${REDIS_PORT}:${REDIS_PORT}" volumes: @@ -45,6 +45,10 @@ services: - "${AWS_PORT}:4569" volumes: - ./storage/s3_root:/s3_root + networks: + default: + aliases: + - s3-emulator.s3-emulator any-sync-coordinator_bootstrap: image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"