diff --git a/codalab/lib/bundle_cli.py b/codalab/lib/bundle_cli.py index 0df4cf5a6..1156cb656 100644 --- a/codalab/lib/bundle_cli.py +++ b/codalab/lib/bundle_cli.py @@ -676,7 +676,7 @@ def print_table( def parse_spec(self, spec): """ Parse a global spec, which includes the instance and either a bundle or worksheet spec. - Example: https://worksheets.codalab.org/bundleservice::wine + Example: https://worksheets.codalab.org::wine Return (client, spec) """ tokens = spec.split(INSTANCE_SEPARATOR) @@ -927,7 +927,7 @@ def do_logout_command(self, args): Commands.Argument('name', help='Name of the alias (e.g., main).', nargs='?'), Commands.Argument( 'instance', - help='Instance to bind the alias to (e.g., https://codalab.org/bundleservice).', + help='Instance to bind the alias to (e.g., https://worksheets.codalab.org).', nargs='?', ), Commands.Argument('-r', '--remove', help='Remove this alias.', action='store_true'), @@ -965,7 +965,7 @@ def do_alias_command(self, args): Commands.Argument('key', help='key to set (e.g., cli/verbose).'), Commands.Argument( 'value', - help='Instance to bind the alias to (e.g., https://codalab.org/bundleservice).', + help='Instance to bind the alias to (e.g., https://worksheets.codalab.org).', nargs='?', ), Commands.Argument('-r', '--remove', help='Remove this key.', action='store_true'), diff --git a/codalab/lib/bundle_store.py b/codalab/lib/bundle_store.py index 92705c918..dcf19824e 100644 --- a/codalab/lib/bundle_store.py +++ b/codalab/lib/bundle_store.py @@ -7,58 +7,7 @@ from codalabworker.bundle_state import State -class BundleStoreCleanupMixin(object): - """A mixin for BundleStores that wish to support a cleanup operation - """ - - def cleanup(self, uuid, dry_run): - """ - Cleanup a given bundle. If dry_run is True, do not actually - delete the bundle from storage. - """ - pass - - -class BundleStoreHealthCheckMixin(object): - """ - This mixin defines functionality on a BundleStore that supports some sort of health-check mechanism. - - Health check is an intentionally broad term that leaves its definition up to the interpretation of each - BundleStore. Note that this method IS allowed to perform operations destructive to objects stored in the bundle - store, i.e. this is not an idempotent operation, and calling this method should be done with care. - """ - - def health_check(self, model, force): - pass - - -class BaseBundleStore(object): - """ - BaseBundleStore defines the basic interface that all subclasses are *required* to implement. Concrete subtypes of - this class my introduce new functionality, but they must all support at least these interfaces. - """ - - def __init__(self): - """ - Create and initialize a new instance of the bundle store. - """ - self.initialize_store() - - def initialize_store(self): - """ - Initialize the bundle store with whatever structure is needed for use. - """ - pass - - def get_bundle_location(self, data_hash): - """ - Gets the location of the bundle with cryptographic hash digest data_hash. Returns the location in the method - that makes the most sense for the storage mechanism being used. - """ - pass - - -class MultiDiskBundleStore(BaseBundleStore, BundleStoreCleanupMixin, BundleStoreHealthCheckMixin): +class MultiDiskBundleStore(object): """ Responsible for taking a set of locations and load-balancing the placement of bundle data between the locations. @@ -71,9 +20,6 @@ class MultiDiskBundleStore(BaseBundleStore, BundleStoreCleanupMixin, BundleStore # Location where MultiDiskBundleStore data and temp data is kept relative to CODALAB_HOME DATA_SUBDIRECTORY = 'bundles' CACHE_SIZE = 1 * 1000 * 1000 # number of entries to cache - MISC_TEMP_SUBDIRECTORY = ( - 'misc_temp' - ) # BundleServer writes out to here, so should have a different name def require_partitions(f): """Decorator added to MultiDiskBundleStore methods that require a disk to @@ -100,14 +46,17 @@ def __init__(self, codalab_home): self.codalab_home = path_util.normalize(codalab_home) self.partitions = os.path.join(self.codalab_home, 'partitions') - self.mtemp = os.path.join(self.codalab_home, MultiDiskBundleStore.MISC_TEMP_SUBDIRECTORY) + path_util.make_directory(self.partitions) + + self.refresh_partitions() + if self.__get_num_partitions() == 0: # Ensure at least one partition exists. + self.add_partition(None, 'default') + + self.lru_cache = OrderedDict() - # Perform initialization first to ensure that directories will be populated - super(MultiDiskBundleStore, self).__init__() + def refresh_partitions(self): nodes, _ = path_util.ls(self.partitions) self.nodes = nodes - self.lru_cache = OrderedDict() - super(MultiDiskBundleStore, self).__init__() def get_node_avail(self, node): # get absolute free space @@ -147,33 +96,24 @@ def get_bundle_location(self, uuid): self.lru_cache[uuid] = disk return os.path.join(self.partitions, disk, MultiDiskBundleStore.DATA_SUBDIRECTORY, uuid) - def initialize_store(self): - """ - Initializes the multi-disk bundle store. - """ - path_util.make_directory(self.partitions) - path_util.make_directory(self.mtemp) - - # Create the default partition, if there are no partitions currently - if self.__get_num_partitions() == 0: - # Create a default partition that links to the codalab_home - path_util.make_directory( - os.path.join(self.codalab_home, MultiDiskBundleStore.DATA_SUBDIRECTORY) - ) - default_partition = os.path.join(self.partitions, 'default') - path_util.soft_link(self.codalab_home, default_partition) - def add_partition(self, target, new_partition_name): """ - MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to - the target directory, which the user has configured as the mountpoint for some desired partition. + MultiDiskBundleStore specific method. Add a new partition to the bundle + store, which is actually a symlink to the target directory, which the + user has configured as the mountpoint for some desired partition. + If `target` is None, then make the `new_partition_name` the actual directory. """ - target = os.path.abspath(target) + if target is not None: + target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) print >>sys.stderr, "Adding new partition as %s..." % new_partition_location - path_util.soft_link(target, new_partition_location) + if target is None: + path_util.make_directory(new_partition_location) + else: + path_util.soft_link(target, new_partition_location) + # Where the bundles are stored mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) try: @@ -186,7 +126,7 @@ def add_partition(self, target, new_partition_name): ) sys.exit(1) - self.nodes.append(new_partition_name) + self.refresh_partitions() print >>sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name @@ -222,8 +162,7 @@ def rm_partition(self, partition): print >>sys.stderr, "Unlinking partition %s from CodaLab deployment..." % partition path_util.remove(partition_abs_path) - nodes, _ = path_util.ls(self.partitions) - self.nodes = nodes + self.refresh_partitions() print >>sys.stderr, "Partition removed successfully from bundle store pool" print >>sys.stdout, "Warning: this does not affect the bundles in the removed partition or any entries in the bundle database" self.lru_cache = OrderedDict() diff --git a/codalab/lib/codalab_manager.py b/codalab/lib/codalab_manager.py index bb009a6b2..6adcb5192 100644 --- a/codalab/lib/codalab_manager.py +++ b/codalab/lib/codalab_manager.py @@ -194,7 +194,7 @@ def init_config(self, dry_run=False): 'auth': {'class': 'RestOAuthHandler'}, 'verbose': 1, }, - 'aliases': {'main': MAIN_BUNDLE_SERVICE, 'localhost': 'http://localhost:2900'}, + 'aliases': {'main': MAIN_BUNDLE_SERVICE, 'localhost': 'http://localhost'}, 'workers': { 'default_cpu_image': 'codalab/default-cpu:latest', 'default_gpu_image': 'codalab/default-gpu:latest', @@ -393,7 +393,7 @@ def emailer(self): # Default to authless SMTP (supported by some servers) if user/password is unspecified. return SMTPEmailer( host=self.config['email']['host'], - user=self.config['email'].get('user', 'noreply@codalab.org'), + user=self.config['email'].get('username', 'noreply@codalab.org'), password=self.config['email'].get('password', None), use_tls=self.config['email'].get('use_tls', True), port=self.config['email'].get('port', 587), diff --git a/codalab/model/bundle_model.py b/codalab/model/bundle_model.py index b6b2ee8a5..6b800735b 100644 --- a/codalab/model/bundle_model.py +++ b/codalab/model/bundle_model.py @@ -2331,9 +2331,6 @@ def get_user_time_quota_left(self, user_id): def get_user_parallel_run_quota_left(self, user_id): user_info = self.get_user_info(user_id) parallel_run_quota = user_info['parallel_run_quota'] - if user_id == self.root_user_id: - # Root user has no parallel run quota - return parallel_run_quota with self.engine.begin() as connection: # Get all the runs belonging to this user whose workers are not personal workers # of the user themselves diff --git a/codalab/rest/workers.py b/codalab/rest/workers.py index 11e7ef470..647b2c3a7 100644 --- a/codalab/rest/workers.py +++ b/codalab/rest/workers.py @@ -24,14 +24,16 @@ def checkin(worker_id): """ WAIT_TIME_SECS = 3.0 + # Old workers might not have all the fields, so allow subsets to be + # missing. socket_id = local.worker_model.worker_checkin( request.user.user_id, worker_id, - request.json["tag"], - request.json["cpus"], - request.json["gpus"], - request.json["memory_bytes"], - request.json["free_disk_bytes"], + request.json.get("tag"), + request.json.get("cpus"), + request.json.get("gpus"), + request.json.get("memory_bytes"), + request.json.get("free_disk_bytes"), request.json["dependencies"], ) diff --git a/codalab/worker/default_bundle_manager.py b/codalab/worker/default_bundle_manager.py index ce4ba714c..b3650d7fa 100644 --- a/codalab/worker/default_bundle_manager.py +++ b/codalab/worker/default_bundle_manager.py @@ -29,7 +29,7 @@ def _schedule_run_bundles(self): self._cleanup_dead_workers(workers) self._restage_stuck_starting_bundles(workers) self._bring_offline_stuck_running_bundles(workers) - self._fail_on_too_many_resources(workers) + self._fail_on_too_many_resources() self._acknowledge_recently_finished_bundles(workers) # Schedule, preferring user-owned workers. @@ -59,36 +59,15 @@ def _check_resource_failure( return global_fail_string % (pretty_print(value), pretty_print(global_max)) return None - def _fail_on_too_many_resources(self, workers): + def _fail_on_too_many_resources(self): """ - Fails bundles that request more resources than available on any worker. + Fails bundles that request more resources than available for the given user. + Note: allow more resources than available on any worker because new + workers might get spun up in response to the presence of this run. """ for bundle in self._model.batch_get_bundles(state=State.STAGED, bundle_type='run'): - workers_list = workers.user_owned_workers(bundle.owner_id) + workers.user_owned_workers( - self._model.root_user_id - ) - - if len(workers_list) == 0: - return - failures = [] - failures.append( - self._check_resource_failure( - self._compute_request_cpus(bundle), - global_fail_string='No workers available with %s CPUs, max available: %s', - global_max=max(map(lambda worker: worker['cpus'], workers_list)), - ) - ) - - failures.append( - self._check_resource_failure( - self._compute_request_gpus(bundle), - global_fail_string='No workers available with %s GPUs, max available: %s', - global_max=max(map(lambda worker: worker['gpus'], workers_list)), - ) - ) - failures.append( self._check_resource_failure( self._compute_request_disk(bundle), diff --git a/codalab_service.py b/codalab_service.py index a5abeb557..c2f01490b 100755 --- a/codalab_service.py +++ b/codalab_service.py @@ -21,6 +21,7 @@ import argparse import errno import os +import socket import subprocess DEFAULT_SERVICES = ['mysql', 'nginx', 'frontend', 'rest-server', 'bundle-manager', 'worker', 'init'] @@ -116,6 +117,7 @@ def has_callable_default(self): CODALAB_ARGUMENTS = [ + # Basic settings CodalabArg( name='version', help='Version of CodaLab (usually the branch name)', @@ -150,36 +152,52 @@ def has_callable_default(self): ), ### MySQL CodalabArg(name='mysql_host', help='MySQL hostname', default='mysql'), # Inside Docker - CodalabArg(name='mysql_port', help='MySQL hostname', default=3306, type=int), + CodalabArg(name='mysql_port', help='MySQL port', default=3306, type=int), CodalabArg(name='mysql_database', help='MySQL database name', default='codalab_bundles'), CodalabArg(name='mysql_username', help='MySQL username', default='codalab'), CodalabArg(name='mysql_password', help='MySQL password', default='codalab'), CodalabArg(name='mysql_root_password', help='MySQL root password', default='codalab'), CodalabArg( - 'uid', + name='uid', help='UID:GID to run everything inside Docker and owns created files', default='%s:%s' % (os.getuid(), os.getgid()), ), CodalabArg( - 'codalab_home', + name='codalab_home', env_var='CODALAB_HOME', help='Path to store things like config.json for the REST server', default=var_path('home'), ), - CodalabArg('bundle_store', help='Path to store bundle data files', default=var_path('bundles')), - CodalabArg('mysql_mount', help='Path to store MySQL data files', default=var_path('mysql')), CodalabArg( - 'monitor_dir', help='Path to store monitor logs and DB backups', default=var_path('monitor') + name='bundle_mount', + help='Path to bundle data (just for mounting into Docker)', + default=var_path('home'), + ), + CodalabArg(name='mysql_mount', help='Path to store MySQL data', default=var_path('mysql')), + CodalabArg( + name='monitor_dir', + help='Path to store monitor logs and DB backups', + default=var_path('monitor'), ), CodalabArg( - 'worker_dir', + name='worker_dir', help='Path to store worker state / cached dependencies', default=var_path('worker'), ), - CodalabArg('http_port', help='Port for nginx', type=int, default=80), - CodalabArg('https_port', help='Port for nginx (when using SSL)', type=int, default=443), - CodalabArg('frontend_port', help='Port for frontend', type=int, default=2700), + CodalabArg(name='http_port', help='Port for nginx', type=int, default=80), + CodalabArg(name='https_port', help='Port for nginx (when using SSL)', type=int, default=443), + CodalabArg(name='frontend_port', help='Port for frontend', type=int, default=2700), CodalabArg(name='rest_port', help='Port for REST server', type=int, default=2900), + CodalabArg(name='rest_num_processes', help='Number of processes', type=int, default=1), + ### User + CodalabArg(name='user_disk_quota', help='How much space a user can use', default='100g'), + CodalabArg(name='user_time_quota', help='How much total time a user can use', default='100y'), + CodalabArg( + name='user_parallel_run_quota', + help='How many simultaneous runs a user can have', + type=int, + default=100, + ), ### Email CodalabArg(name='admin_email', help='Email to send admin notifications to (e.g., monitoring)'), CodalabArg(name='email_host', help='Send email by logging into this SMTP server'), @@ -230,8 +248,9 @@ def _get_parser(): unnamed.append(arg.flag) # Named parameters to add_argument named = {'help': arg.help} - if arg.has_constant_default(): - named['default'] = arg.default + # Don't set defaults here or else we won't know downstream + # whether a value was a default or passed in on the + # command-line. if arg.type == bool: named['action'] = 'store_true' else: @@ -362,6 +381,7 @@ def resolve_env_vars(args): for env_var in ['PATH', 'DOCKER_HOST']: if env_var in os.environ: environment[env_var] = os.environ[env_var] + environment['HOSTNAME'] = socket.gethostname() # Sometimes not available return environment def __init__(self, args): @@ -379,7 +399,6 @@ def __init__(self, args): ensure_directory_exists(self.args.monitor_dir) ensure_directory_exists(self.args.worker_dir) ensure_directory_exists(self.args.mysql_mount) - ensure_directory_exists(self.args.bundle_store) def execute(self): command = self.args.command @@ -545,6 +564,13 @@ def wait_rest_server(cmd): ('server/engine_url', mysql_url), ('server/rest_host', '0.0.0.0'), ('server/admin_email', self.args.admin_email), + ('server/support_email', self.args.admin_email), # Use same email + ('server/default_user_info/disk_quota', self.args.user_disk_quota), + ('server/default_user_info/time_quota', self.args.user_time_quota), + ( + 'server/default_user_info/parallel_run_quota', + self.args.user_parallel_run_quota, + ), ('email/host', self.args.email_host), ('email/username', self.args.email_username), ('email/password', self.args.email_password), @@ -603,7 +629,9 @@ def build_images(self): if self.args.push: self._run_docker_cmd( - 'login -u %s -p %s' % (self.args.docker_username, self.args.docker_password) + 'login --username {} --password {}'.format( + self.args.docker_username, self.args.docker_password + ) ) for image in images_to_build: self.push_image(image) diff --git a/docker/compose_files/docker-compose.ssl.yml b/docker/compose_files/docker-compose.ssl.yml index 4a6267f34..74c6c5501 100644 --- a/docker/compose_files/docker-compose.ssl.yml +++ b/docker/compose_files/docker-compose.ssl.yml @@ -7,4 +7,4 @@ services: volumes: - ./files/nginx.conf.ssl:/etc/nginx/nginx.conf:ro - ${CODALAB_SSL_KEY_FILE}:/opt/ssl/codalab.key - - ${CODALAB_SSL_CERT_FILE}:/opt/ssl/codalab.pem + - ${CODALAB_SSL_CERT_FILE}:/opt/ssl/codalab.crt diff --git a/docker/compose_files/docker-compose.yml b/docker/compose_files/docker-compose.yml index 0baeb4008..7f62df243 100644 --- a/docker/compose_files/docker-compose.yml +++ b/docker/compose_files/docker-compose.yml @@ -14,10 +14,14 @@ x-codalab-env: &codalab-env - CODALAB_MYSQL_DATABASE=${CODALAB_MYSQL_DATABASE} - CODALAB_MYSQL_USERNAME=${CODALAB_MYSQL_USERNAME} - CODALAB_MYSQL_PASSWORD=${CODALAB_MYSQL_PASSWORD} + - CODALAB_USER_DISK_QUOTA=${CODALAB_USER_DISK_QUOTA} + - CODALAB_USER_TIME_QUOTA=${CODALAB_USER_TIME_QUOTA} + - CODALAB_USER_PARALLEL_RUN_QUOTA=${CODALAB_USER_PARALLEL_RUN_QUOTA} - CODALAB_ADMIN_EMAIL=${CODALAB_ADMIN_EMAIL} - CODALAB_EMAIL_HOST=${CODALAB_EMAIL_HOST} - CODALAB_EMAIL_USERNAME=${CODALAB_EMAIL_USERNAME} - CODALAB_EMAIL_PASSWORD=${CODALAB_EMAIL_PASSWORD} + - HOSTNAME=${HOSTNAME} # Properties that every service inherits. x-codalab-base: &codalab-base @@ -38,7 +42,7 @@ x-codalab-root: &codalab-root x-codalab-server: &codalab-server volumes: - "${CODALAB_HOME}:${CODALAB_HOME}" - - "${CODALAB_BUNDLE_STORE}:${CODALAB_BUNDLE_STORE}" + - "${CODALAB_BUNDLE_MOUNT}:${CODALAB_BUNDLE_MOUNT}" - "${CODALAB_MONITOR_DIR}:${CODALAB_MONITOR_DIR}" ############################################################ @@ -86,7 +90,7 @@ services: worker: image: codalab/worker:${CODALAB_VERSION} - command: cl-worker --server http://rest-server:${CODALAB_REST_PORT} --verbose --work-dir ${CODALAB_WORKER_DIR} --network-prefix ${CODALAB_WORKER_NETWORK_NAME} + command: cl-worker --server http://rest-server:${CODALAB_REST_PORT} --verbose --work-dir ${CODALAB_WORKER_DIR} --network-prefix ${CODALAB_WORKER_NETWORK_NAME} --id ${HOSTNAME} <<: *codalab-base <<: *codalab-root # Not ideal since worker files saved as root, but without it, can't use docker volumes: @@ -105,6 +109,7 @@ services: volumes: - /var/lib/docker:/var/lib/docker - ${CODALAB_HOME}:${CODALAB_HOME} + - ${CODALAB_BUNDLE_MOUNT}:${CODALAB_BUNDLE_MOUNT} - ${CODALAB_MONITOR_DIR}:${CODALAB_MONITOR_DIR} networks: diff --git a/docker/compose_files/files/nginx.conf b/docker/compose_files/files/nginx.conf index 4c76786aa..5b9dedb80 100644 --- a/docker/compose_files/files/nginx.conf +++ b/docker/compose_files/files/nginx.conf @@ -68,7 +68,6 @@ http { server rest-server:2900; } - server { set $maintenance 0; @@ -99,19 +98,6 @@ http { #proxy_read_timeout 1200; #send_timeout 1200; - location /bundleservice { - if ($maintenance = 1) { - return 503; - } - proxy_pass http://rest/bundleservice; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_connect_timeout 1200; - proxy_send_timeout 1200; - proxy_read_timeout 1200; - send_timeout 1200; - } - location /rest { if ($maintenance = 1) { return 503; @@ -125,7 +111,6 @@ http { send_timeout 1200; } - location / { if ($maintenance = 1) { return 503; @@ -146,4 +131,3 @@ http { } } } - diff --git a/docker/compose_files/files/nginx.conf.ssl b/docker/compose_files/files/nginx.conf.ssl index 68cbdbdea..0940282a2 100644 --- a/docker/compose_files/files/nginx.conf.ssl +++ b/docker/compose_files/files/nginx.conf.ssl @@ -68,14 +68,6 @@ http { server rest-server:2900; } - server { - listen 80; - server_name 127.0.0.1 localhost; - location / { - rewrite ^ https://$server_name$request_uri permanent; - } - } - server { set $maintenance 0; @@ -85,10 +77,9 @@ http { ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:5m; ssl_session_timeout 10m; - ssl_certificate /opt/ssl/codalab.pem; + ssl_certificate /opt/ssl/codalab.crt; ssl_certificate_key /opt/ssl/codalab.key; - proxy_http_version 1.1; gzip on; @@ -114,19 +105,6 @@ http { #proxy_read_timeout 1200; #send_timeout 1200; - location /bundleservice { - if ($maintenance = 1) { - return 503; - } - proxy_pass http://rest/bundleservice; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_connect_timeout 1200; - proxy_send_timeout 1200; - proxy_read_timeout 1200; - send_timeout 1200; - } - location /rest { if ($maintenance = 1) { return 503; @@ -140,7 +118,6 @@ http { send_timeout 1200; } - location / { if ($maintenance = 1) { return 503; @@ -161,4 +138,3 @@ http { } } } - diff --git a/docs/CLI-Reference.md b/docs/CLI-Reference.md index acec1c0a0..648d4b733 100644 --- a/docs/CLI-Reference.md +++ b/docs/CLI-Reference.md @@ -488,7 +488,7 @@ This file is auto-generated from the output of `cl help -v` and provides the lis alias : Binds to . Arguments: name Name of the alias (e.g., main). - instance Instance to bind the alias to (e.g., https://codalab.org/bundleservice). + instance Instance to bind the alias to (e.g., https://worksheets.codalab.org). -r, --remove Remove this alias. config: @@ -497,7 +497,7 @@ This file is auto-generated from the output of `cl help -v` and provides the lis config : Sets to . Arguments: key key to set (e.g., cli/verbose). - value Instance to bind the alias to (e.g., https://codalab.org/bundleservice). + value Instance to bind the alias to (e.g., https://worksheets.codalab.org). -r, --remove Remove this key. logout: diff --git a/docs/Server-Setup.md b/docs/Server-Setup.md index 18818aed4..1f2e97025 100644 --- a/docs/Server-Setup.md +++ b/docs/Server-Setup.md @@ -133,29 +133,17 @@ You can also start an instance and run tests on it: ./codalab_service.py start -bd -s default test -To fix any style issues for the Python code: - - virtualenv -p python3.6 venv3.6 - venv3.6/bin/pip install black - venv3.6/bin/black codalab worker scripts *.py --diff - These must pass before you submit a PR. -## Other - -To auto-generate the REST reference and CLI references: +## Pre-commit - virtualenv -p python2.7 venv2.7 - venv2.7/bin/pip install -r requirements-server.txt - venv2.7/bin/python scripts/gen-rest-docs.py - venv2.7/bin/python scripts/gen-cli-docs.py +Before you commit, you should run the following script that makes automated +changes on your PR: -To generate the readthedocs documentation to preview locally: + ./pre-commit.sh - virtualenv -p python2.7 venv2.7 - venv2.7/bin/pip install -r requirements.docs.txt - venv2.7/bin/mkdocs build # Outputs to `site` - venv2.7/bin/mkdocs serve # Does a live preview +This script reformats your code to match our style conventions and +auto-generates documentation. ## Debugging @@ -192,7 +180,7 @@ If you want to modify the database schema, use `alembic` to create a migration. docker cp codalab_rest-server_1:/opt/codalab-worksheets/alembic/versions/ alembic/versions -1. Modify the migration script as necessary. +1. Modify the migration script `` as necessary. 1. Rebuild the Docker image: @@ -209,21 +197,14 @@ If you want to modify the database schema, use `alembic` to create a migration. # Production -If you want to make the CodaLab instance more permanent and exposed to a larger set of users, there are a couple -details to pay attention to. - -## Persistent Storage - -By default data files are stored in ephemeral Docker volumes. It's a good idea -to store Codalab data on a persistent location on your host machine's disk if -you're running real workflows on it. Here are a few configuration options you -might want to set for a real-use persistent instance: +If you want to make the CodaLab instance more permanent and exposed to a larger +set of users, there are a couple details to pay attention to. -- `--codalab-home`: Path to store server configuration and bundle data files. -- `--mysql-mount`: Path to store DB configuration and data files. -- `--external-db-url`: If you want to run your DB on another machine, this is the URL to connect to that database. You can set the user and password for this database using the `--mysql-user` and `--mysql-password` arguments. -- `--worker-dir`: Path to store worker configuration and temporary data files. -- `--bundle-store`: [EXPERIMENTAL] Another path to store bundle data. You can add as many of these as possible and bundle data will be distributed evenly across these paths. Good for when you mount multiple disks to distribute bundle data. WARNING: This is not fully supported and tested yet, but support is under development. +The preferred way to set CodaLab service options is via environment variables +instead of command-line options. This is useful for sensitive options like +passwords, and also useful for setting local defaults instead of reusing long +argument lists. For the list of environment variables, look at +`codalab_service.py`. Below, we provide the command-line variants. ## Security and Credentials @@ -231,11 +212,11 @@ By default, a lot of credentials are set to unsafe defaults (`"codalab"`). You should override these with more secure options. Here's a list of all credential options: -* `--codalab-username` [codalab]: Username of the admin account on the CodaLab platform -* `--codalab-password` [codalab]: Password of the admin account on the CodaLab platform -* `--mysql-root-password` [codalab]: Root password for the MYSQL database. -* `--mysql-user` [codalab]: MYSQL username for the CodaLab account on the MYSQL database -* `--mysql-password` [codalab]: MYSQL password for the CodaLab account on the MYSQL database +* `--codalab-username`: Username of the admin account on the CodaLab platform +* `--codalab-password`: Password of the admin account on the CodaLab platform +* `--mysql-root-password`: Root password for the MYSQL database. +* `--mysql-username`: MYSQL username for the CodaLab account on the MYSQL database +* `--mysql-password`: MYSQL password for the CodaLab account on the MYSQL database ### SSL @@ -250,19 +231,6 @@ your domain, you can serve over HTTPS as well. To do so: * `--ssl-cert-file`: Path to the certificate file * `--ssl-key-file`: Path to the key file -## Ports to be exposed - -Normally the service exposes a minimal number of port outside the Docker -network. if for whatever reason you want direct access to the individual ports -of the services, you can expose these at host ports of your choosing. - -* `--rest-port` [2900]: Port for REST API -* `--http-port` [80]: Port to serve HTTP from (nginx) -* `--mysql-port` [3306]: Port to expose the MySQL database -* `--frontend-port`: Port to serve the React frontend - -# Advanced customization - ## Multiple instances If for some reason you need to start more than one instance of the CodaLab @@ -276,20 +244,3 @@ service on the same machine, be careful about the following: * Avoid port clashing: If you're exposing ports, make sure you set different ports for different instances, at the very least you need to configure the `http-port` of later instances to something other than `80`. - -## Custom docker compose file - -For less common use cases, you might want to get your feet wet in Docker and -docker-compose and provide a custom docker-compose file to override our -configurations. You can use the `--user-compose-file` option to include a -custom docker-compose file that will override any configuration you want. To -understand our `compose` setup, please look into the source of -`codalab_service.py` and the docker-related files in the `./docker/` directory. - -## Configuration with environment variables - -Some of the CodaLab service options can be set via environment variables -instead of command-line options. This is useful for sensitive options like -passwords, and also useful for setting local defaults instead of reusing long -argument lists. For the list of environment variables, look at -`ARG_TO_ENV_VAR` in `codalab_service.py`. diff --git a/monitor.py b/monitor.py index d965ac64e..95a17266b 100644 --- a/monitor.py +++ b/monitor.py @@ -62,7 +62,7 @@ ) ) -hostname = socket.gethostname() +hostname = os.environ['HOSTNAME'] # Email admin_email = os.environ['CODALAB_ADMIN_EMAIL'] diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index 3cf461fb3..000000000 --- a/nginx.conf +++ /dev/null @@ -1,149 +0,0 @@ -user www-data; -worker_processes auto; -pid /run/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; -daemon off; # runs in Docker container so doesn't need daemon mode - -events { - worker_connections 768; - # multi_accept on; -} - -http { - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - # server_tokens off; - - # server_names_hash_bucket_size 64; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ## - # SSL Settings - ## - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - - ## - # Logging Settings - ## - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - ## - # Gzip Settings - ## - - gzip on; - gzip_disable "msie6"; - - # gzip_vary on; - # gzip_proxied any; - # gzip_comp_level 6; - # gzip_buffers 16 8k; - # gzip_http_version 1.1; - # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - - ## - # Virtual Host Configs - ## - - upstream frontend { - server http://localhost:2700; - } - - upstream rest { - server http://localhost:2900; - } - - - server { - set $maintenance 0; - - listen 80; - - proxy_http_version 1.1; - - gzip on; - gzip_min_length 4096; - gzip_buffers 4 32k; - gzip_types application/x-javascript text/css; - gzip_vary on; - - server_name localhost 127.0.0.1; - charset utf-8; - client_max_body_size 64000m; - client_body_buffer_size 64m; - - # Turn off request body buffering to allow direct streaming uploads. - # Note that the request body will be buffered regardless of this directive - # value unless HTTP/1.1 is enabled for proxying (configured above). - proxy_request_buffering off; - - #keepalive_timeout 10; - #proxy_buffering off; - #proxy_connect_timeout 1200; - #proxy_send_timeout 1200; - #proxy_read_timeout 1200; - #send_timeout 1200; - - location /bundleservice { - if ($maintenance = 1) { - return 503; - } - proxy_pass http://rest/bundleservice; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_connect_timeout 1200; - proxy_send_timeout 1200; - proxy_read_timeout 1200; - send_timeout 1200; - } - - location /rest { - if ($maintenance = 1) { - return 503; - } - proxy_pass http://rest; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_connect_timeout 1200; - proxy_send_timeout 1200; - proxy_read_timeout 1200; - send_timeout 1200; - } - - - location / { - if ($maintenance = 1) { - return 503; - } - proxy_pass http://frontend/; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_connect_timeout 1200; - proxy_send_timeout 1200; - proxy_read_timeout 1200; - send_timeout 1200; - } - - error_page 503 /error/503.html; - error_page 502 /error/50x.html; - location ^~ /error/ { - internal; - } - } -} - diff --git a/pre-commit.sh b/pre-commit.sh new file mode 100755 index 000000000..901461dd3 --- /dev/null +++ b/pre-commit.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Run this script before you commit. + +# TODO: merge into one once we have Python 3. + +if ! [ -e venv2.7 ]; then + virtualenv -p python2.7 venv2.7 || exit 1 + venv2.7/bin/pip install -r requirements-server.txt || exit 1 + venv2.7/bin/pip install -r requirements.docs.txt || exit 1 + rm -rf worker/codalabworker.egg-info # Need to clear because of different Python versions + # Install for generating docs. + venv2.7/bin/pip install -e worker # Not sure why this is necessary + venv2.7/bin/pip install -e . +fi + +if ! [ -e venv3.6 ]; then + virtualenv -p python3.6 venv3.6 || exit 1 + venv3.6/bin/pip install black==18.9b0 || exit 1 +fi + +# Generate docs +venv2.7/bin/python scripts/gen-rest-docs.py || exit 1 # Outputs to `docs` +venv2.7/bin/python scripts/gen-cli-docs.py || exit 1 # Outputs to `docs` +venv2.7/bin/mkdocs build || exit 1 # Outputs to `site` +# Note: run `venv2.7/bin/mkdocs serve` for a live preview + +# Fix style (mutates code!) +venv3.6/bin/black codalab worker scripts *.py || exit