diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 0000000..23fb004 --- /dev/null +++ b/fabfile.py @@ -0,0 +1,232 @@ +import tempfile +import secrets +import pathlib +import string + +from fabric import task + + + +PYTHON_VERSION = "3.7.9" + + +@task +def provision(conn, domain): + postgres_password = "".join(( + secrets.choice(string.ascii_letters + string.digits) + for _ in range(12) + )) + django_secret_key = "".join(( + secrets.choice(string.ascii_letters + string.digits) + for _ in range(64) + )) + + # Install apt deps + apt_deps = " ".join(APT_DEPENDENCIES) + conn.run(f"apt install -y {apt_deps}") + + # Setup PyEnv + conn.run("curl https://pyenv.run | bash") + _append_bashrc(conn, PYENV_BASHRC) + conn.run(f"pyenv install {PYTHON_VERSION}") + conn.run(f"pyenv global {PYTHON_VERSION}") + + # Create the `web` user with their own home director and group + conn.run("useradd --create-home --user-group web") + + # Clone the repository + conn.run("git clone https://github.com/pipermerriam/ethereum-function-signature-registry.git /home/web/ethereum-function-signature-registry") + + # + # Setup and Install project dependencies + # + conn.run("pip install virtualenv") + conn.run("python -m virtualenv /home/web/venv") + + conn.run("/home/web/venv/bin/pip install -r /home/web/ethereum-function-signature-registry/requirements.txt") + + with tempfile.TemporaryDirectory() as base_path: + dotenv_file_path = pathlib.Path(base_path) / '.env' + dotenv_file_path.write_text(DOTENV.format( + DOMAIN=domain, + POSTGRES_PASSWORD=postgres_password, + SECRET_KEY=django_secret_key, + )) + + conn.put(str(dotenv_file_path), remote='/home/web/ethereum-function-signature-registry/.env') + + + Setup Postgres User and Database + + with tempfile.TemporaryDirectory() as base_path: + # systemd service for worker + pgpass_file_path = pathlib.Path(base_path) / '.pgpass' + pgpass_file_path.write_text(f"*.*.*.bytes4.{postgres_password}") + + conn.put(str(pgpass_file_path), remote='/root/.pgpass') + conn.run("chmod 600 /root/.pgpass") + + conn.run(f"sudo -u postgres psql -c \"CREATE ROLE bytes4 PASSWORD '{postgres_password}' SUPERUSER LOGIN;\"") + conn.run("sudo -u postgres createdb --no-password bytes4") + + # + # Setup Redis + # + conn.run('sed -i "s/supervised no/supervised systemd/g" /etc/redis/redis.conf') + conn.run("service redis restart") + + # + # Setup config files for uwsgi/nginx/4byte-worker + # + with tempfile.TemporaryDirectory() as base_path: + # systemd service for worker + worker_service_file_path = pathlib.Path(base_path) / '4byte.service' + worker_service_file_path.write_text(SYSTEMD_WORKER_SERVICE) + + conn.put(str(worker_service_file_path), remote='/etc/systemd/system/4byte.service') + + # nginx configuration file + nginx_4byte_conf = pathlib.Path(base_path) / '4byte' + nginx_4byte_conf.write_text(NGINX_4BYTE.format(DOMAIN=domain)) + + conn.put(str(nginx_4byte_conf), remote='/etc/nginx/sites-available/4byte') + conn.run('ln -s /etc/nginx/sites-available/4byte /etc/nginx/sites-enabled/') + + # uwsgi configuration file + uwsgi_4byte_conf = pathlib.Path(base_path) / '4byte.ini' + uwsgi_4byte_conf.write_text(UWSGI_CONF) + + conn.put(str(uwsgi_4byte_conf), remote='/etc/uwsgi/apps-available/4byte.ini') + conn.run('ln -s /etc/uwsgi/apps-available/4byte.ini /etc/uwsgi/apps-enabled/') + + + + +def _append_bashrc(conn, content: str) -> None: + for line in content.splitlines(): + if not line: + continue + conn.run(line) + conn.run(f"echo '{line}' >> /root/.bashrc") + + +APT_DEPENDENCIES = ( + # build + "automake", + "build-essential", + "curl", + "gcc", + "git", + "gpg", + "software-properties-common", + "pkg-config", + "zlib1g", + "zlib1g-dev", + "libbz2-dev", + "libreadline-dev", + "libssl-dev", + "libsqlite3-dev", + "libffi-dev", + # application + "nginx", + "uwsgi", + "uwsgi-plugin-python3", + "redis-server", + "postgresql", + "postgresql-contrib", + "postgresql-server-dev-11", + # convenience + "htop", + "tmux", +) + + +SYSTEMD_WORKER_SERVICE = """[Unit] +Description=4byte worker +After=network.target +StartLimitIntervalSec=0 + +[Service] +WorkingDirectory=/home/web/ethereum-function-signature-registry +Type=simple +Restart=always +RestartSec=1 +User=web +ExecStartPre= +ExecStart=/home/web/venv/bin/python /home/web/ethereum-function-signature-registry/manage.py run_huey --verbosity 3 +""" + + +PYENV_BASHRC = """export PATH="/root/.pyenv/bin:$PATH" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +""" + +NGINX_4BYTE = """server {{ + server_name {DOMAIN}; + + listen 80; + listen [::]:80; + + if ($host = {DOMAIN}) {{ + return 301 https://$host$request_uri; + }} # managed by Certbot + + return 404; # managed by Certbot +}} + + +server {{ + server_name {DOMAIN}; # customize with your domain name + + location / {{ + # django running in uWSGI + uwsgi_pass unix:///run/uwsgi/app/4byte/socket; + include uwsgi_params; + uwsgi_read_timeout 300s; + client_max_body_size 32m; + }} + + # location /static/ {{ + # # static files + # alias /home/web/static/; # ending slash is required + # }} + + # location /media/ {{ + # # media files, uploaded by users + # alias /home/web/media/; # ending slash is required + # }} + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/{DOMAIN}/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/{DOMAIN}/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + +}} +""" + + +UWSGI_CONF = """[uwsgi] +plugin=python3 +uid=web +chdir=/home/web/ethereum-function-signature-registry +module=func_sig_registry.wsgi:application +master=True +vacuum=True +max-requests=5000 +processes=4 +virtualenv=/home/web/venv +""" + + +DOTENV = """DATABASE_URL=postgres://bytes4:{POSTGRES_PASSWORD}@127.0.0.1:5432/bytes4 +DJANGO_ALLOWED_HOSTS={DOMAIN} +DJANGO_DEBUG=False +DJANGO_DEBUG_TOOLBAR_ENABLED=False +DJANGO_SECRET_KEY={SECRET_KEY} +DJANGO_SECURE_SSL_REDIRECT=False +HUEY_WORKER_TYPE=thread +REDIS_URL=redis://127.0.0.1:6379 +""" diff --git a/func_sig_registry/settings.py b/func_sig_registry/settings.py index 8d21385..c8f3ed2 100644 --- a/func_sig_registry/settings.py +++ b/func_sig_registry/settings.py @@ -46,8 +46,6 @@ 'func_sig_registry.registry', 'rest_framework', 'django_tables2', - 'storages', - 's3_folder_storage', 'huey.contrib.djhuey', 'corsheaders', ] @@ -183,31 +181,6 @@ ) -# AWS Configuration -DEFAULT_S3_PATH = "media" -STATIC_S3_PATH = "static" - -AWS_ACCESS_KEY_ID = env.get('AWS_ACCESS_KEY_ID', type=str, default=None) -AWS_SECRET_ACCESS_KEY = env.get('AWS_SECRET_ACCESS_KEY', type=str, default=None) -AWS_STORAGE_BUCKET_NAME = env.get('AWS_STORAGE_BUCKET_NAME', type=str, default=None) -AWS_DEFAULT_REGION = env.get('AWS_DEFAULT_REGION', type=str, default=None) - -# Boto config -AWS_REDUCED_REDUNDANCY = True -AWS_QUERYSTRING_AUTH = False -AWS_S3_FILE_OVERWRITE = True -AWS_S3_SECURE_URLS = True -AWS_IS_GZIPPED = False -AWS_PRELOAD_METADATA = True -AWS_HEADERS = { - "Cache-Control": "public, max-age=86400", -} - -if AWS_DEFAULT_REGION: - # Fix for https://github.com/boto/boto/issues/621 - AWS_S3_HOST = "s3-{0}.amazonaws.com".format(AWS_DEFAULT_REGION) - - # DRF REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': [ diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b7df37..d8ca5ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ factory-boy==2.7.0 fake-factory==0.7.4 hypothesis==3.5.3 tox==2.4.1 +fabric==2.5.0 diff --git a/requirements.txt b/requirements.txt index 5fc8767..c1eb83c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,6 @@ django-tables2>=1.2.3,<1.3.0 djangorestframework>=3.3.3,<3.4.0 ipython>=6.2.1 pysha3==1.0.2 -django-s3-folder-storage>=0.3,<0.4 -django-storages>=1.4.1,<1.5.0 -boto>=2.41.0,<3.0.0 env-excavator>=1.5.0,<1.6.0 python-dotenv>=0.5.1,<0.6.0 psycopg2-binary>=2.7.4,<2.8.0