From 7119491838ecfa1b9d7889f1d7e3b555fda1b77e Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sun, 12 Jul 2020 17:41:00 +0000 Subject: [PATCH] performance boost with meinheld and flask-minify, fixes, admin can now change user password (#76) --- .dockerignore | 9 ++++- .env | 17 +++++++++ .env_postgres | 5 +++ .../CODE_OF_CONDUCT.md | 0 CONTRIBUTING.md => .github/CONTRIBUTING.md | 0 INSTALLATION.md => .github/INSTALLATION.md | 35 +++++++++++++------ .gitignore | 1 + Dockerfile | 8 ++--- README.md | 4 +-- app.json | 2 +- docker-compose-for-tests.yml | 20 +++-------- docker-compose.yml | 20 +++-------- src/FlaskRTBCTF/__init__.py | 14 +++++++- src/FlaskRTBCTF/admin/views.py | 12 +++++-- src/FlaskRTBCTF/config.py | 4 +-- src/FlaskRTBCTF/templates/home.html | 16 ++++----- src/FlaskRTBCTF/users/models.py | 14 ++++++-- src/FlaskRTBCTF/users/routes.py | 10 +++--- src/FlaskRTBCTF/utils/__init__.py | 2 ++ src/FlaskRTBCTF/utils/helpers.py | 4 +-- src/FlaskRTBCTF/utils/migrate.py | 3 ++ src/FlaskRTBCTF/utils/minify.py | 3 ++ src/docker-entrypoint.sh | 9 ++--- src/init_db.sh | 7 ++++ src/{create_db.py => populate_db.prod.py} | 19 ++++++---- ..._db_for_testing.py => populate_db.test.py} | 8 ++--- src/requirements.txt | 5 ++- src/run.py | 3 +- src/runserver.sh | 7 ++++ 29 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 .env create mode 100644 .env_postgres rename CODE_OF_CONDUCT.md => .github/CODE_OF_CONDUCT.md (100%) rename CONTRIBUTING.md => .github/CONTRIBUTING.md (100%) rename INSTALLATION.md => .github/INSTALLATION.md (56%) create mode 100644 src/FlaskRTBCTF/utils/migrate.py create mode 100644 src/FlaskRTBCTF/utils/minify.py create mode 100644 src/init_db.sh rename src/{create_db.py => populate_db.prod.py} (85%) rename src/{populate_db_for_testing.py => populate_db.test.py} (98%) create mode 100644 src/runserver.sh diff --git a/.dockerignore b/.dockerignore index e6d3665..94e4aa6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,11 @@ venv/ .vscode/ *.db .idea/ -.github/ \ No newline at end of file +.github/ +.lgtm.yml +Procfile +app.json +setup.cfg +runtime.txt +migrations/ +*.test.py \ No newline at end of file diff --git a/.env b/.env new file mode 100644 index 0000000..f0d1e15 --- /dev/null +++ b/.env @@ -0,0 +1,17 @@ +##### RTB-CTF-Framework environment variables configuration ##### + +# Generate a psuedo-random secret with: .. +#... python3 -c "print(__import__('secrets').token_hex(16))" +SECRET_KEY=DontForgetToChangeMe +# If serving over HTTPS, mark this True for added security +SSL_ENABLED=False +# about 2 x number of CPU cores +WORKERS=4 +# Choose a strong password for administrator! +ADMIN_PASS=admin +# These values should be the same as specified in .env_postgres +DB_USER=user +DB_PASSWORD=password +DB_NAME=rtbctf +# Don't change +DB_PORT=5432 diff --git a/.env_postgres b/.env_postgres new file mode 100644 index 0000000..99cbc4b --- /dev/null +++ b/.env_postgres @@ -0,0 +1,5 @@ +# Please choose a strong password !! + +POSTGRES_USER=user +POSTGRES_PASSWORD=password +POSTGRES_DB=rtbctf diff --git a/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to .github/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/INSTALLATION.md b/.github/INSTALLATION.md similarity index 56% rename from INSTALLATION.md rename to .github/INSTALLATION.md index 4146a66..525b5bd 100644 --- a/INSTALLATION.md +++ b/.github/INSTALLATION.md @@ -2,11 +2,11 @@ ### Requirements -* Tested on `Python 3.8.2` +* Tested on `Python 3.8.3` * Python Packages: [`src/requirements.txt`](src/requirements.txt). * OS Packages: PostgreSQL version 11 or greater, `libpq-dev`, `python3-dev` packages. Please refer [here](https://tutorials.technology/solved_errors/9-Error-pg_config-executable-not-found.html). -### Build locally and run +### Build locally and run (Development) 1. Git clone the repo and `cd ` into it @@ -22,20 +22,33 @@ $ source venv/bin/activate $ cd src/ ``` -3. With `virtual environment` activated, install requirements, init db and run ! +3. With `virtual environment` activated, install requirements, init db, ```bash [venv]$ pip install -r requirements.txt -[venv]$ python create_db.py # Only required on first run -[venv]$ python run.py +[venv]$ chmod +x init_db.sh && ./init_db.sh # Only required on first run ``` -> Warning: If you make any change to [`config.py`](https://github.com/abs0lut3pwn4g3/RTB-CTF-Framework/blob/master/src/FlaskRTBCTF/config.py) logging/config class/score settings. It's highly recommended to create a new DB instance. +4. Now we can run our application, -### Docker + - For development server, -> Note: The Docker support is not tested for production yet. It's recommended to use Heroku for production. + ```bash + [venv]$ python run.py + ``` -```bash -$ docker-compose up -``` + - Production server + + ```bash + [venv]$ ./runserver.sh + ``` + +### Docker (Production) + +1. Define certain environment variables present in files `.env` and `.env_postgres`. + +2. After having configured these environment variables, just execute, + + ```bash + $ docker-compose up + ``` diff --git a/.gitignore b/.gitignore index 5258d0b..c237938 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ venv/ +migrations/ *.pyc .vscode/ *.db diff --git a/Dockerfile b/Dockerfile index 85dfcdc..b3c3c66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:3.8.2-alpine3.11 +FROM python:3.8.3-alpine3.12 -MAINTAINER eshaan7bansal@gmail.com +LABEL maintainer="eshaan7bansal@gmail.com" # Env RUN export DATABASE_URL="postgres://${DB_USER}:${DB_PASSWORD}@postgres:${DB_PORT}/${DB_NAME}" \ @@ -16,8 +16,8 @@ RUN adduser --shell /sbin/login www-data -DH # Install RTB-CTF-Framework WORKDIR /usr/src/app COPY src ./ -RUN pip install --no-cache-dir -r requirements.txt \ - && chown -R www-data ./ +RUN chown -R www-data ./ +RUN pip install --no-cache-dir -r requirements.txt USER www-data diff --git a/README.md b/README.md index 1494402..3d53f8c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The 100 second elevator-pitch is that: A Capture The Flag framework; one that is ## Build locally -Please see [INSTALLATION.md](INSTALLATION.md). +Please see [INSTALLATION.md](.github/INSTALLATION.md). ## Host a customized CTF with Heroku for free in under a minute @@ -83,7 +83,7 @@ The main purpose of this project is to serve as a scoring engine and CTF manager - [#rtb-ctf-framework on slack](https://rtb-ctf-framework.slack.com) -Please refer to [CONTRIBUTING.md](CONTRIBUTING.md) +Please refer to [CONTRIBUTING.md](.github/CONTRIBUTING.md) ## Live Demo diff --git a/app.json b/app.json index 08d33e1..1c9e822 100644 --- a/app.json +++ b/app.json @@ -33,6 +33,6 @@ } }, "scripts": { - "postdeploy": "python3 src/create_db.py" + "postdeploy": "bash src/init_db.sh" } } diff --git a/docker-compose-for-tests.yml b/docker-compose-for-tests.yml index 364eea9..32c309b 100644 --- a/docker-compose-for-tests.yml +++ b/docker-compose-for-tests.yml @@ -7,15 +7,8 @@ services: restart: unless-stopped expose: - "8000" - environment: - - DEBUG=False - - SECRET_KEY=changeme - - DB_USER=eshaan - - DB_PASSWORD=eshaan - - DB_NAME=rtbctf - - DB_PORT=5432 - - WORKERS=4 - - ADMIN_PASS=admin + env_file: + - .env depends_on: - postgres - redis @@ -26,18 +19,15 @@ services: restart: unless-stopped expose: - "5432" - environment: - - POSTGRES_USER=eshaan - - POSTGRES_PASSWORD=eshaan - - POSTGRES_DB=rtbctf + env_file: + - .env_postgres redis: - image: redis:6.0-rc4-alpine + image: redis:alpine3.12 container_name: rtb_redis restart: unless-stopped expose: - "6379" - nginx: image: library/nginx:1.16.1-alpine diff --git a/docker-compose.yml b/docker-compose.yml index ed77992..b681d93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,15 +6,8 @@ services: restart: unless-stopped expose: - "8000" - environment: - - DEBUG=False - - SSL_ENABLED=False - - DB_USER=eshaan - - DB_PASSWORD=eshaan - - DB_NAME=rtbctf - - DB_PORT=5432 - - WORKERS=4 - - ADMIN_PASS=admin + env_file: + - .env depends_on: - postgres - redis @@ -25,18 +18,15 @@ services: restart: unless-stopped expose: - "5432" - environment: - - POSTGRES_USER=eshaan - - POSTGRES_PASSWORD=eshaan - - POSTGRES_DB=rtbctf + env_file: + - .env_postgres redis: - image: redis:6.0-rc4-alpine + image: redis:alpine3.12 container_name: rtb_redis restart: unless-stopped expose: - "6379" - nginx: image: library/nginx:1.16.1-alpine diff --git a/src/FlaskRTBCTF/__init__.py b/src/FlaskRTBCTF/__init__.py index 8f35f04..4f2aded 100644 --- a/src/FlaskRTBCTF/__init__.py +++ b/src/FlaskRTBCTF/__init__.py @@ -12,6 +12,8 @@ mail, inject_app_context, inject_security_headers, + static_minify, + migrate, ) from FlaskRTBCTF.users.routes import users from FlaskRTBCTF.ctf.routes import ctf @@ -20,7 +22,15 @@ _blueprints = (users, ctf, main) -_extensions = (db, bcrypt, cache, login_manager, admin_manager, mail) +_extensions = ( + db, + bcrypt, + cache, + login_manager, + admin_manager, + mail, + static_minify, +) def create_app(config_class=Config): @@ -32,6 +42,8 @@ def create_app(config_class=Config): for _ext in _extensions: _ext.init_app(app) + migrate.init_app(app, db) + for _bp in _blueprints: app.register_blueprint(_bp) diff --git a/src/FlaskRTBCTF/admin/views.py b/src/FlaskRTBCTF/admin/views.py index 07fd979..686859b 100644 --- a/src/FlaskRTBCTF/admin/views.py +++ b/src/FlaskRTBCTF/admin/views.py @@ -37,8 +37,16 @@ def _handle_view(self, name, **kwargs): class UserAdminView(BaseModelView): - column_exclude_list = ("password",) - form_exclude_list = ("password",) + column_exclude_list = ("password", "_password") + column_details_exclude_list = column_exclude_list + column_descriptions = { + "_password": """ + you can change the password here manually, + it will be automatically hashed on save + """, + "isAdmin": "Think twice before checking this field.", + } + form_columns = ("username", "email", "isAdmin", "password") column_searchable_list = ("username", "email") @expose("/new/") diff --git a/src/FlaskRTBCTF/config.py b/src/FlaskRTBCTF/config.py index 81855ff..8dd22d0 100644 --- a/src/FlaskRTBCTF/config.py +++ b/src/FlaskRTBCTF/config.py @@ -1,13 +1,11 @@ import os -import secrets # Flask related Configurations -# Note: DO NOT FORGET TO CHANGE 'SECRET_KEY' ! class Config: DEBUG = False # Turn DEBUG OFF before deployment - SECRET_KEY = secrets.token_hex(16) + SECRET_KEY = os.environ.get("SECRET_KEY", "you-will-never-guess") SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or "sqlite:///site.db" # For local use, one can simply use SQLlite with: 'sqlite:///site.db' # For deployment on Heroku use: `os.environ.get('DATABASE_URL')` diff --git a/src/FlaskRTBCTF/templates/home.html b/src/FlaskRTBCTF/templates/home.html index 6cb3356..9471ffd 100644 --- a/src/FlaskRTBCTF/templates/home.html +++ b/src/FlaskRTBCTF/templates/home.html @@ -7,8 +7,7 @@

Welcome to {{ settings.ctf_name }}

{% if current_user.is_authenticated %}

- If you owned the box then you can submit the hashes - here. + Read the rules and have fun!

{% else %}

You need to login first.

@@ -22,17 +21,14 @@

Rules

diff --git a/src/FlaskRTBCTF/users/models.py b/src/FlaskRTBCTF/users/models.py index b50bf30..b53ee57 100644 --- a/src/FlaskRTBCTF/users/models.py +++ b/src/FlaskRTBCTF/users/models.py @@ -4,10 +4,12 @@ from flask import current_app from flask_login import UserMixin +from sqlalchemy.ext.hybrid import hybrid_property from ..ctf.models import UserChallenge, Challenge, UserMachine, Machine from ..utils.models import db from ..utils.cache import cache +from ..utils.bcrypt import bcrypt from ..utils.login_manager import login_manager @@ -22,10 +24,18 @@ class User(db.Model, UserMixin): id = db.Column(db.Integer, primary_key=True, index=True) username = db.Column(db.String(24), unique=True, nullable=False) email = db.Column(db.String(88), unique=True, nullable=False) - password = db.Column(db.String(64), nullable=False) + _password = db.Column(db.String(64), nullable=False) isAdmin = db.Column(db.Boolean, default=False) logs = db.relationship("Logs", backref="user", lazy=True, uselist=False) + @hybrid_property + def password(self): + return self._password + + @password.setter + def password(self, pwd): + self._password = bcrypt.generate_password_hash(pwd).decode("utf-8") + def get_reset_token(self, expires_sec=1800): s = Serializer(current_app.config["SECRET_KEY"], expires_sec) return s.dumps({"user_id": self.id}).decode("utf-8") @@ -94,4 +104,4 @@ class Logs(db.Model): rootSubmissionIP = db.Column(db.String, nullable=True) def __repr__(self): - return f"Logs('{self.user_id}','{self.visitedMachine}')" + return f"Logs('{self.user_id}', vistedMachine: '{self.visitedMachine}')" diff --git a/src/FlaskRTBCTF/users/routes.py b/src/FlaskRTBCTF/users/routes.py index 0492157..7cd0f43 100644 --- a/src/FlaskRTBCTF/users/routes.py +++ b/src/FlaskRTBCTF/users/routes.py @@ -29,11 +29,13 @@ def register(): form = RegistrationForm() if form.validate_on_submit(): - hashed_password = bcrypt.generate_password_hash(form.password.data).decode( - "utf-8" - ) + # hashed_password = bcrypt.generate_password_hash(form.password.data).decode( + # "utf-8" + # ) user = User( - username=form.username.data, email=form.email.data, password=hashed_password + username=form.username.data, + email=form.email.data, + password=form.password.data, ) log = Logs( user=user, diff --git a/src/FlaskRTBCTF/utils/__init__.py b/src/FlaskRTBCTF/utils/__init__.py index d1fc4dc..5735d0f 100644 --- a/src/FlaskRTBCTF/utils/__init__.py +++ b/src/FlaskRTBCTF/utils/__init__.py @@ -14,3 +14,5 @@ from .login_manager import login_manager, admin_only from .mail import mail, send_reset_email from .models import db +from .minify import static_minify +from .migrate import migrate diff --git a/src/FlaskRTBCTF/utils/helpers.py b/src/FlaskRTBCTF/utils/helpers.py index 91bece4..a283d46 100644 --- a/src/FlaskRTBCTF/utils/helpers.py +++ b/src/FlaskRTBCTF/utils/helpers.py @@ -14,13 +14,11 @@ def handle_admin_pass(default="admin"): passwd = os.environ.get("ADMIN_PASS", default) if not passwd: passwd = secrets.token_hex(16) - os.environ["ADMIN_PASS"] = passwd return passwd def handle_admin_email(default="admin@admin.com"): - em = os.environ.get("ADMIN_EMAIL", default) - return em + return os.environ.get("ADMIN_EMAIL", default) def inject_app_context(): diff --git a/src/FlaskRTBCTF/utils/migrate.py b/src/FlaskRTBCTF/utils/migrate.py new file mode 100644 index 0000000..8296ff9 --- /dev/null +++ b/src/FlaskRTBCTF/utils/migrate.py @@ -0,0 +1,3 @@ +from flask_migrate import Migrate + +migrate = Migrate() diff --git a/src/FlaskRTBCTF/utils/minify.py b/src/FlaskRTBCTF/utils/minify.py new file mode 100644 index 0000000..301945e --- /dev/null +++ b/src/FlaskRTBCTF/utils/minify.py @@ -0,0 +1,3 @@ +from flask_minify import minify + +static_minify = minify(html=True, js=True, cssless=True, caching_limit=0) diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh index ac6c150..77a4ba1 100644 --- a/src/docker-entrypoint.sh +++ b/src/docker-entrypoint.sh @@ -1,6 +1,7 @@ #!/bin/sh -python create_db.py -exec gunicorn "FlaskRTBCTF:create_app()" \ - --bind "0.0.0.0:8000" \ - --workers $WORKERS \ No newline at end of file +chmod +x init_db.sh runserver.sh +# init/ migrate DB +./init_db.sh +# run gunicorn production server +./runserver.sh \ No newline at end of file diff --git a/src/init_db.sh b/src/init_db.sh new file mode 100644 index 0000000..77c7035 --- /dev/null +++ b/src/init_db.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +export FLASK_APP="FlaskRTBCTF:create_app()" +flask db init +flask db migrate +flask db upgrade +python populate_db.prod.py diff --git a/src/create_db.py b/src/populate_db.prod.py similarity index 85% rename from src/create_db.py rename to src/populate_db.prod.py index 73eee9c..54e0086 100644 --- a/src/create_db.py +++ b/src/populate_db.prod.py @@ -1,7 +1,7 @@ import pytz from datetime import datetime -from FlaskRTBCTF import db, bcrypt, create_app +from FlaskRTBCTF import db, create_app from FlaskRTBCTF.main.models import Settings from FlaskRTBCTF.ctf.models import Machine, Challenge, Tag, Category from FlaskRTBCTF.users.models import User, Logs @@ -81,24 +81,27 @@ def populate_challs(): with app.app_context(): - db.create_all() + s = Settings.query.get(1) + a = User.query.filter_by(username="admin").first() + if s or a: + print("populate_db.prod: Skipping since, database is already populated!") + exit() - default_time = datetime.now(pytz.utc) + now_time = datetime.now(pytz.utc) - passwd = handle_admin_pass() admin_user = User( username="admin", email=handle_admin_email(), - password=bcrypt.generate_password_hash(passwd).decode("utf-8"), + password=handle_admin_pass(), isAdmin=True, ) db.session.add(admin_user) admin_log = Logs( user=admin_user, - accountCreationTime=default_time, + accountCreationTime=now_time, visitedMachine=True, - machineVisitTime=default_time, + machineVisitTime=now_time, ) db.session.add(admin_log) @@ -112,4 +115,6 @@ def populate_challs(): populate_challs() + print("populate_db.prod: Database populated with some default data!") + db.session.commit() diff --git a/src/populate_db_for_testing.py b/src/populate_db.test.py similarity index 98% rename from src/populate_db_for_testing.py rename to src/populate_db.test.py index 3485322..717d1e9 100644 --- a/src/populate_db_for_testing.py +++ b/src/populate_db.test.py @@ -2,7 +2,7 @@ import random from datetime import datetime -from FlaskRTBCTF import db, bcrypt, create_app +from FlaskRTBCTF import db, create_app from FlaskRTBCTF.ctf.models import ( Machine, UserMachine, @@ -1048,11 +1048,7 @@ def populate_users(): used.append(name) x += 1 try: - user = User( - username=name, - email=name + gen_email(), - password=bcrypt.generate_password_hash(name).decode("utf-8"), - ) + user = User(username=name, email=name + gen_email(), password=name,) log = Logs(user=user) db.session.add(user) db.session.add(log) diff --git a/src/requirements.txt b/src/requirements.txt index 8133403..5334ff4 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -23,4 +23,7 @@ Werkzeug==1.0.1 WTForms==2.2.1 tablib==1.1.0 Flask-Caching==1.9.0 -redis==3.5.3 \ No newline at end of file +redis==3.5.3 +meinheld==1.0.2 +Flask-Minify==0.25 +Flask-Migrate==2.5.3 \ No newline at end of file diff --git a/src/run.py b/src/run.py index ff3783a..a49109a 100644 --- a/src/run.py +++ b/src/run.py @@ -1,6 +1,7 @@ +import os from FlaskRTBCTF import create_app app = create_app() if __name__ == "__main__": - app.run(debug=app.config.get("DEBUG", False)) + app.run(debug=os.environ.get("DEBUG", False)) diff --git a/src/runserver.sh b/src/runserver.sh new file mode 100644 index 0000000..60f7ba4 --- /dev/null +++ b/src/runserver.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +export FLASK_APP="FlaskRTBCTF:create_app()" +exec gunicorn ${FLASK_APP} \ + --bind "0.0.0.0:8000" \ + --workers ${WORKERS:-4} + --worker-class egg:meinheld#gunicorn_worker \ No newline at end of file