Merge pull request #13 from gnosischain/feature/add-database

Feature/add database

.github/workflows/publish-api.yaml

@@ -78,6 +78,8 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
   eks-deployment-restart:
+    # Run job on branch dev only
+    if: github.ref == 'refs/heads/dev'
     runs-on: ubuntu-latest
     needs:  build-and-push-image
     permissions:

.github/workflows/publish-ui.yaml

@@ -71,6 +71,8 @@ jobs:
             "REACT_APP_FAUCET_API_URL=${{ secrets.PROD_REACT_APP_FAUCET_API_URL}}"
         
   eks-deployment-restart:
+    # Run job on branch dev only
+    if: github.ref == 'refs/heads/dev'
     runs-on: ubuntu-latest
     needs:  build-and-push-image
     permissions:

api/.env.example

@@ -2,7 +2,7 @@ FAUCET_AMOUNT=0.1
 FAUCET_PRIVATE_KEY=0x0000000000000000000000000000000000000000000000000000000000000000
 FAUCET_RPC_URL=https://rpc.chiadochain.net
 FAUCET_CHAIN_ID=10200
+FAUCET_DATABASE_URI=sqlite://
 FAUCET_ENABLED_TOKENS="[{\"address\": \"0x19C653Da7c37c66208fbfbE8908A5051B57b4C70\", \"name\":\"GNO\", \"maximumAmount\": 0.5}]"
-# FAUCET_ENABLED_TOKENS=
 CAPTCHA_VERIFY_ENDPOINT=https://api.hcaptcha.com/siteverify
 CAPTCHA_SECRET_KEY=0x0000000000000000000000000000000000000000

api/Dockerfile

@@ -1,9 +1,20 @@
-FROM python:3.8-alpine
+FROM python:3.10-alpine
 
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-
-COPY . /api
+ENV PYTHONUNBUFFERED 1
 WORKDIR /api
 
-CMD ["gunicorn", "--bind", "0.0.0.0:8000", "api:create_app()"]
+COPY requirements.txt ./
+
+# Signal handling for PID1 https://github.com/krallin/tini
+RUN apk add --update --no-cache tini libpq && \
+    apk add --no-cache --virtual .build-dependencies alpine-sdk libffi-dev && \
+    pip install --no-cache-dir -r requirements.txt && \
+    apk del .build-dependencies && \
+    find /usr/local \
+        \( -type d -a -name test -o -name tests \) \
+        -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \
+        -exec rm -rf '{}' +
+
+COPY . .
+
+ENTRYPOINT ["/sbin/tini", "--"]

api/api/api.py

@@ -2,10 +2,12 @@
 
 from flask import Flask
 from flask_cors import CORS
+from flask_migrate import Migrate
 
+from .manage import create_access_keys_cmd
 from .routes import apiv1
-from .services import Cache, Web3Singleton
-from .services.database import db, migrate
+from .services import Web3Singleton
+from .services.database import db
 
 
 def setup_logger(log_level):
@@ -32,17 +34,16 @@ def create_app():
     app = Flask(__name__)
     # Initialize main settings
     app.config.from_object('api.settings')
-    # Initialize Cache
-    app.config['FAUCET_CACHE'] = Cache(app.config['FAUCET_RATE_LIMIT_TIME_LIMIT_SECONDS'])
     # Initialize API Routes
     app.register_blueprint(apiv1, url_prefix="/api/v1")
+    # Add cli commands
+    app.cli.add_command(create_access_keys_cmd)
 
     with app.app_context():
         db.init_app(app)
-        migrate.init_app(app, db)
-        db.create_all()  # Create database tables for our data models
+        Migrate(app, db)
 
-    # Initialize Web3 class
+    # Initialize Web3 class for latter usage
     w3 = Web3Singleton(app.config['FAUCET_RPC_URL'], app.config['FAUCET_PRIVATE_KEY'])
 
     setup_cors(app)

api/api/const.py

@@ -1 +1,16 @@
-NATIVE_TOKEN_ADDRESS='native'
+from enum import Enum
+
+NATIVE_TOKEN_ADDRESS = 'native'
+DEFAULT_NATIVE_MAX_AMOUNT_PER_DAY = 0.01
+DEFAULT_ERC20_MAX_AMOUNT_PER_DAY = 0.01
+
+CHAIN_NAMES = {
+    1: 'ETHEREUM MAINNET',
+    100: 'GNOSIS CHAIN',
+    10200: 'CHIADO CHAIN'
+}
+
+
+class FaucetRequestType(Enum):
+    web = 'web'
+    cli = 'cli'

api/api/manage.py

@@ -0,0 +1,26 @@
+import logging
+
+import click
+from flask import current_app
+from flask.cli import with_appcontext
+
+from .services.database import AccessKey, AccessKeyConfig
+from .utils import generate_access_key
+
+
+@click.command(name='create_access_keys')
+@with_appcontext
+def create_access_keys_cmd():
+    access_key_id, secret_access_key = generate_access_key()
+    access_key = AccessKey()
+    access_key.access_key_id = access_key_id
+    access_key.secret_access_key = secret_access_key
+    access_key.save()
+
+    config = AccessKeyConfig()
+    config.access_key_id = access_key.access_key_id
+    config.chain_id = current_app.config["FAUCET_CHAIN_ID"]
+    config.save()
+
+    logging.info(f'Access Key ID    : ${access_key_id}')
+    logging.info(f'Secret access key: ${secret_access_key}')

api/api/routes.py

@@ -1,10 +1,12 @@
+from datetime import datetime
+
 from flask import Blueprint, current_app, jsonify, request
 from web3 import Web3
 
+from .const import FaucetRequestType
 from .services import (Strategy, Web3Singleton, captcha_verify, claim_native,
                        claim_token)
-from .services.database import AccessKey
-from .utils import is_amount_valid, is_token_enabled
+from .services.database import AccessKey, Token, Transaction
 
 apiv1 = Blueprint("version1", "version1")
 
@@ -16,21 +18,42 @@ def status():
 
 @apiv1.route("/info")
 def info():
+    enabled_tokens = Token.enabled_tokens()
+    enabled_tokens_json = [
+        {
+            'address': t.address,
+            'name': t.name,
+            'maximumAmount': t.max_amount_day
+        } for t in enabled_tokens
+    ]
     return jsonify(
-        enabledTokens=current_app.config['FAUCET_ENABLED_TOKENS'],
+        enabledTokens=enabled_tokens_json,
         chainId=current_app.config['FAUCET_CHAIN_ID'],
         chainName=current_app.config['FAUCET_CHAIN_NAME'],
         faucetAddress=current_app.config['FAUCET_ADDRESS']
     ), 200
 
 
-def _ask_route_validation(request_data, validate_captcha):
+def _ask_route_validation(request_data, validate_captcha=True):
+    """Validate `ask/` endpoint request data
+
+    Args:
+        request_data (object): request object
+        validate_captcha (bool, optional): True if captcha must be validated, False otherwise. Defaults to True.
+
+    Returns:
+        tuple: validation errors, amount, recipient, token address
+    """
     validation_errors = []
 
     # Captcha validation
     if validate_captcha:
         # check hcatpcha
-        catpcha_verified = captcha_verify(request_data.get('captcha'), current_app.config['CAPTCHA_VERIFY_ENDPOINT'], current_app.config['CAPTCHA_SECRET_KEY'])
+        catpcha_verified = captcha_verify(
+            request_data.get('captcha'),
+            current_app.config['CAPTCHA_VERIFY_ENDPOINT'], current_app.config['CAPTCHA_SECRET_KEY']
+        )
+
         if not catpcha_verified:
             validation_errors.append('captcha: validation failed')
 
@@ -44,52 +67,78 @@ def _ask_route_validation(request_data, validate_captcha):
     if not recipient or recipient.lower() == current_app.config['FAUCET_ADDRESS']:
         validation_errors.append('recipient: address cant\'t be the Faucet address itself')
 
+    amount = request_data.get('amount', None)
     token_address = request_data.get('tokenAddress', None)
-    if not token_address:
+
+    if token_address:
+        try:
+            # Clean up Token address
+            if token_address.lower() != 'native':
+                token_address = Web3.to_checksum_address(token_address)
+
+            token = Token.query.with_entities(Token.enabled,Token.max_amount_day).filter_by(
+                address=token_address,
+                chain_id=request_data.get('chainId')).first()
+
+            if token and token.enabled is True:
+                if not amount:
+                    validation_errors.append('amount: is required')
+                if amount and amount > token.max_amount_day:
+                    validation_errors.append('amount: a valid amount must be specified and must be less or equals to %s' % token[1])
+                # except ValueError as e:
+                #     message = "".join([arg for arg in e.args])
+                #     validation_errors.append(message)
+            else:
+                validation_errors.append('tokenAddress: %s is not enabled' % token_address)
+        except:
+            validation_errors.append('tokenAddress: invalid token address'), 400
+    else:
         validation_errors.append('tokenAddress: A valid token address or string \"native\" must be specified')
+    return validation_errors, amount, recipient, token_address
 
-    try:
-        if not is_token_enabled(token_address, current_app.config['FAUCET_ENABLED_TOKENS']):
-            validation_errors.append('tokenAddress: Token %s is not enabled' % token_address)
-    except:
-        validation_errors.append('tokenAddress: invalid token address'), 400
 
-    amount = request_data.get('amount', None)
+def _ask(request_data, validate_captcha=True, access_key=None):
+    """Process /ask request
 
-    try:
-        amount_valid, amount_limit = is_amount_valid(amount, token_address, current_app.config['FAUCET_ENABLED_TOKENS'])
-        if not amount_valid:
-            validation_errors.append('amount: a valid amount must be specified and must be less or equals to %s' % amount_limit)
-    except ValueError as e:
-        message = "".join([arg for arg in e.args])
-        validation_errors.append(message)
-
-    return validation_errors, amount, recipient, token_address
+    Args:
+        request_data (object): request object
+        validate_captcha (bool, optional): True if captcha must be validated, False otherwise. Defaults to True.
+        access_key (object, optional): AccessKey instance. Defaults to None.
 
+    Raises:
+        NotImplementedError:
 
-def _ask(request_data, validate_captcha):
+    Returns:
+        tuple: json content, status code
+    """
     validation_errors, amount, recipient, token_address = _ask_route_validation(request_data, validate_captcha)
 
     if len(validation_errors) > 0:
         return jsonify(errors=validation_errors), 400
 
-    # Cache
-    cache = current_app.config['FAUCET_CACHE']
+    ip_address = request.environ.get('HTTP_X_FORWARDED_FOR', request.remote_addr)
+
     if current_app.config['FAUCET_RATE_LIMIT_STRATEGY'].strategy == Strategy.address.value:
-        # Check last claim
-        if cache.limit_by_address(recipient):
-            return jsonify(errors=['recipient: you have exceeded the limit for today. Try again in %s hours' % cache.ttl(hours=True)]), 429
+        # Check last claim by recipient
+        transaction = Transaction.last_by_recipient(recipient)
     elif current_app.config['FAUCET_RATE_LIMIT_STRATEGY'].strategy == Strategy.ip.value:
-        ip_address = request.environ.get('HTTP_X_FORWARDED_FOR', request.remote_addr)
-        # Check last claim for the IP address
-        if cache.limit_by_ip(ip_address):
-            return jsonify(errors=['recipient: you have exceeded the limit for today. Try again in %s hours' % cache.ttl(hours=True)]), 429
+        # Check last claim by IP
+        transaction = Transaction.last_by_ip(ip_address)
     elif current_app.config['FAUCET_RATE_LIMIT_STRATEGY'].strategy == Strategy.ip_and_address:
         raise NotImplementedError
 
+    # Check if the recipient can claim funds, they must not have claimed any tokens 
+    # in the period of time defined by FAUCET_RATE_LIMIT_TIME_LIMIT_SECONDS
+    if transaction:
+        time_diff_seconds = (datetime.utcnow() - transaction.created).total_seconds()
+        if time_diff_seconds < current_app.config['FAUCET_RATE_LIMIT_TIME_LIMIT_SECONDS']:
+            time_diff_hours = time_diff_seconds/(24*60)
+            return jsonify(errors=['recipient: you have exceeded the limit for today. Try again in %d hours' % time_diff_hours]), 429
+
+    # convert amount to wei format
     amount_wei = Web3.to_wei(amount, 'ether')
     try:
-        # convert to checksum address
+        # convert recipient address to checksum address
         recipient = Web3.to_checksum_address(recipient)
 
         w3 = Web3Singleton(current_app.config['FAUCET_RPC_URL'], current_app.config['FAUCET_PRIVATE_KEY'])
@@ -98,6 +147,21 @@ def _ask(request_data, validate_captcha):
             tx_hash = claim_native(w3, current_app.config['FAUCET_ADDRESS'], recipient, amount_wei)
         else:
             tx_hash = claim_token(w3, current_app.config['FAUCET_ADDRESS'], recipient, amount_wei, token_address)
+
+        # save transaction data on DB
+        transaction = Transaction()
+        transaction.hash = tx_hash
+        transaction.recipient = recipient
+        transaction.amount = amount
+        transaction.token = token_address
+        transaction.requester_ip = ip_address
+        if access_key:
+            transaction.type = FaucetRequestType.cli.value
+            transaction.access_key_id = access_key.access_key_id
+        else:
+            transaction.type = FaucetRequestType.web.value
+        transaction.save()
+
         return jsonify(transactionHash=tx_hash), 200
     except ValueError as e:
         message = "".join([arg['message'] for arg in e.args])
@@ -106,13 +170,14 @@ def _ask(request_data, validate_captcha):
 
 @apiv1.route("/ask", methods=["POST"])
 def ask():
-    return _ask(request.get_json(), validate_captcha=True)
+    data, status_code = _ask(request.get_json(), validate_captcha=True, access_key=None)
+    return data, status_code
 
 
 @apiv1.route("/cli/ask", methods=["POST"])
 def cli_ask():
-    access_key_id = request.headers.get('FAUCET_ACCESS_KEY_ID', None)
-    secret_access_key = request.headers.get('FAUCET_SECRET_ACCESS_KEY', None)
+    access_key_id = request.headers.get('X-faucet-access-key-id', None)
+    secret_access_key = request.headers.get('X-faucet-secret-access-key', None)
 
     validation_errors = []
 
@@ -127,4 +192,5 @@ def cli_ask():
         validation_errors.append('Access denied')
         return jsonify(errors=validation_errors), 403
 
-    return _ask(request.get_json(), validate_captcha=False)
+    data, status_code = _ask(request.get_json(), validate_captcha=False, access_key=access_key)
+    return data, status_code

api/api/services/__init__.py

@@ -1,4 +1,3 @@
-from .cache import Cache
 from .captcha import captcha_verify
 from .database import DatabaseSingleton
 from .rate_limit import RateLimitStrategy, Strategy