Spool_up_EMR_Cluster.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "foster-cattle",
   "metadata": {},
   "source": [
    "# Spool up EMR Cluster to run HAIL with bootstrapping only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "analyzed-diary",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from botocore.exceptions import ClientError\n",
    "import time\n",
    "import sys\n",
    "import botocore\n",
    "import paramiko\n",
    "import re\n",
    "import os\n",
    "import yaml, re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "laden-monaco",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/hail-on-AWS-spot-instances'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PATH = os.path.abspath(os.getcwd())\n",
    "PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "minor-retailer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021021016\n"
     ]
    }
   ],
   "source": [
    "#Setup logging\n",
    "import logging, bluebee\n",
    "from bluebee import bgp\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n",
    "# create file handler which logs even debug messages\n",
    "logFile = f'{PATH}/spoolup_EMR_cluster.log'\n",
    "fh = logging.FileHandler(logFile)\n",
    "fh.setLevel(logging.INFO)\n",
    "# create console handler with a higher log level\n",
    "ch = logging.StreamHandler()\n",
    "ch.setLevel(logging.INFO)\n",
    "# create formatter and add it to the handlers\n",
    "formatter = logging.Formatter('%(asctime)-18s-%(levelname)-8s %(message)s', datefmt='%d%b%Y %H:%M:%S')\n",
    "ch.setFormatter(formatter)\n",
    "fh.setFormatter(formatter)\n",
    "# add the handlers to logger\n",
    "logger.addHandler(ch)\n",
    "logger.addHandler(fh)\n",
    "# bgp.api.dump_curl = True\n",
    "# bluebee.logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "drawn-operations",
   "metadata": {},
   "outputs": [],
   "source": [
    "def upload_file(file_name, bucket, object_name=None):\n",
    "    \"\"\"Upload a file to an S3 bucket\n",
    "\n",
    "    :param file_name: File to upload\n",
    "    :param bucket: Bucket to upload to\n",
    "    :param object_name: S3 object name. If not specified then file_name is used\n",
    "    :return: True if file was uploaded, else False\n",
    "    \"\"\"\n",
    "\n",
    "    # If S3 object_name was not specified, use file_name\n",
    "    if object_name is None:\n",
    "        object_name = file_name\n",
    "\n",
    "    # Upload the file\n",
    "    s3_client = boto3.client('s3')\n",
    "    try:\n",
    "        response = s3_client.upload_file(file_name, bucket, object_name)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "opposed-vermont",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30Mar2021 18:53:11-INFO     Configuration settings: {'config': {'EMR_RELEASE': 'emr-6.2.0', 'EMR_CLUSTER_NAME': 'tdeboer-hail', 'EC2_NAME_TAG': 'tdeboer-hail-EMR', 'OWNER_TAG': 'tdeboer-ilmn', 'PROJECT_TAG': 'GRE_on_ICA', 'REGION': 'us-east-1', 'MASTER_INSTANCE_TYPE': 'm4.large', 'WORKER_INSTANCE_TYPE': 'r4.4xlarge', 'WORKER_COUNT': '1', 'WORKER_BID_PRICE': '0.90', 'MASTER_HD_SIZE': '250', 'WORKER_HD_SIZE': '500', 'SUBNET_ID': '', 'S3_BUCKET': 's3://ilmn-hail/', 'BOOTSTRAP_S3_URI': 's3://ilmn-hail/bootstrap_hail/bootstrap.sh', 'KEY_NAME': 'hail-ES-GRE', 'PATH_TO_KEY': '/data/', 'WORKER_SECURITY_GROUP': 'sg-0df1e5704ca2a8196', 'MASTER_SECURITY_GROUP': 'sg-0bab1202c0aa453b3'}}\n"
     ]
    }
   ],
   "source": [
    "#Get the configuration as a yaml object\n",
    "c=yaml.load(open(PATH+\"/config_EMR_spot.yaml\"),Loader=yaml.SafeLoader)\n",
    "conf = c['config']\n",
    "logger.info(f'Configuration settings: {c}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "familiar-genealogy",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30Mar2021 18:53:12-INFO     Found credentials in shared credentials file: ~/.aws/credentials\n"
     ]
    }
   ],
   "source": [
    "#Store or update the bootstrapping file into the S3 bucket (same as used for the logs, for now)\n",
    "s = conf['BOOTSTRAP_S3_URI'].split('/',3)\n",
    "bucket = s[2]\n",
    "object = s[3]\n",
    "assert upload_file('bootstrap.sh',bucket,object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "female-finnish",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30Mar2021 19:03:02-INFO     Executing following command: \n",
      "aws emr create-cluster \\\n",
      "--name \"tdeboer-hail\" \\\n",
      "--release-label emr-6.2.0 \\\n",
      "--applications Name=Spark Name=Hadoop \\\n",
      "--tags 'project=GRE_on_ICA' 'Owner=tdeboer-ilmn' 'Name=tdeboer-hail-EMR' \\\n",
      "--ec2-attributes '{\"KeyName\":\"hail-ES-GRE\",\"InstanceProfile\":\"EMR_EC2_DefaultRole\",\"SubnetId\":\"\",  \"EmrManagedMasterSecurityGroup\":\"sg-0bab1202c0aa453b3\",\"EmrManagedSlaveSecurityGroup\":\"sg-0df1e5704ca2a8196\"}' \\\n",
      "--service-role EMR_DefaultRole \\\n",
      "--log-uri 's3://ilmn-hail/' \\\n",
      "--instance-groups '[{\"InstanceCount\":1,  \"EbsConfiguration\":{\"EbsBlockDeviceConfigs\":[{\"VolumeSpecification\":{\"SizeInGB\":250,\"VolumeType\":\"gp2\"},  \"VolumesPerInstance\":1}]},\"InstanceGroupType\":\"MASTER\",\"InstanceType\":\"m4.large\",\"Name\":\"Master-Instance\"},  {\"InstanceCount\":1,\"BidPrice\":\"0.90\",  \"EbsConfiguration\":{\"EbsBlockDeviceConfigs\":[{\"VolumeSpecification\":{\"SizeInGB\":500,  \"VolumeType\":\"gp2\"},\"VolumesPerInstance\":1}]},\"InstanceGroupType\":\"CORE\",\"InstanceType\":\"r4.4xlarge\",  \"Name\":\"Core-Group\"}]' \\\n",
      "--configurations file://./spark-configs.json \\\n",
      "--auto-scaling-role EMR_AutoScaling_DefaultRole \\\n",
      "--ebs-root-volume-size 32 \\\n",
      "--scale-down-behavior TERMINATE_AT_TASK_COMPLETION \\\n",
      "--region us-east-1 \\\n",
      "--bootstrap-actions Path='s3://ilmn-hail/bootstrap_hail/bootstrap.sh'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "command = f'''aws emr create-cluster \\\\\n",
    "--name \"{conf['EMR_CLUSTER_NAME']}\" \\\\\n",
    "--release-label {conf['EMR_RELEASE']} \\\\\n",
    "--applications Name=Spark Name=Hadoop \\\\\n",
    "--tags 'project={conf['PROJECT_TAG']}' 'Owner={conf['OWNER_TAG']}' 'Name={conf['EC2_NAME_TAG']}' \\\\\n",
    "--ec2-attributes '{{\"KeyName\":\"{conf['KEY_NAME']}\",\"InstanceProfile\":\"EMR_EC2_DefaultRole\",\"SubnetId\":\"{conf['SUBNET_ID']}\",\\\n",
    "  \"EmrManagedMasterSecurityGroup\":\"{conf['MASTER_SECURITY_GROUP']}\",\"EmrManagedSlaveSecurityGroup\":\"{conf['WORKER_SECURITY_GROUP']}\"}}' \\\\\n",
    "--service-role EMR_DefaultRole \\\\\n",
    "--log-uri '{conf['S3_BUCKET']}' \\\\\n",
    "--instance-groups '[{{\"InstanceCount\":{conf['WORKER_COUNT']},\\\n",
    "  \"EbsConfiguration\":{{\"EbsBlockDeviceConfigs\":[{{\"VolumeSpecification\":{{\"SizeInGB\":{conf['MASTER_HD_SIZE']},\"VolumeType\":\"gp2\"}},\\\n",
    "  \"VolumesPerInstance\":1}}]}},\"InstanceGroupType\":\"MASTER\",\"InstanceType\":\"{conf['MASTER_INSTANCE_TYPE']}\",\"Name\":\"Master-Instance\"}},\\\n",
    "  {{\"InstanceCount\":{conf['WORKER_COUNT']},\"BidPrice\":\"{conf['WORKER_BID_PRICE']}\",\\\n",
    "  \"EbsConfiguration\":{{\"EbsBlockDeviceConfigs\":[{{\"VolumeSpecification\":{{\"SizeInGB\":{conf['WORKER_HD_SIZE']},\\\n",
    "  \"VolumeType\":\"gp2\"}},\"VolumesPerInstance\":1}}]}},\"InstanceGroupType\":\"CORE\",\"InstanceType\":\"{conf['WORKER_INSTANCE_TYPE']}\",\\\n",
    "  \"Name\":\"Core-Group\"}}]' \\\\\n",
    "--configurations file://./spark-configs.json \\\\\n",
    "--auto-scaling-role EMR_AutoScaling_DefaultRole \\\\\n",
    "--ebs-root-volume-size 32 \\\\\n",
    "--scale-down-behavior TERMINATE_AT_TASK_COMPLETION \\\\\n",
    "--region {conf['REGION']} \\\\\n",
    "--bootstrap-actions Path='{conf['BOOTSTRAP_S3_URI']}'\n",
    "'''\n",
    "logger.info(f'Executing following command: \\n{command}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "promising-blend",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30Mar2021 19:03:07-WARNING  Created cluster with ID: \"j-FQUMTSF8ZHVQ\"\n"
     ]
    }
   ],
   "source": [
    "cluster_id_json=os.popen(command).read()\n",
    "#My default profile exports TEXT\n",
    "cluster_id=re.split('\\s',cluster_id_json)[1]\n",
    "# Gives EMR cluster information\n",
    "client_EMR = boto3.client('emr', region_name=c['config']['REGION'])\n",
    "logger.warning(f'Created cluster with ID: \"{cluster_id}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "animated-yugoslavia",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30Mar2021 19:14:28-INFO     Creating EMR...\n",
      "30Mar2021 19:14:28-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:14:58-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:15:28-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:15:58-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:16:28-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:16:59-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:17:29-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:17:59-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:18:29-INFO     Cluster status: BOOTSTRAPPING\n",
      "30Mar2021 19:18:59-INFO     Cluster status: BOOTSTRAPPING\n"
     ]
    }
   ],
   "source": [
    "# Cluster state update\n",
    "status_EMR='STARTING'\n",
    "tic = time.time()\n",
    "# Wait until the cluster is created\n",
    "logger.info('Creating EMR...')\n",
    "\n",
    "while (status_EMR!='EMPTY'):\n",
    "    details_EMR=client_EMR.describe_cluster(ClusterId=cluster_id)\n",
    "    status_EMR=details_EMR.get('Cluster').get('Status').get('State')\n",
    "    logger.info('Cluster status: '+status_EMR)\n",
    "    if (status_EMR=='WAITING'):\n",
    "        logger.warning('Cluster successfully created! Starting HAIL installation...')\n",
    "        toc=time.time()-tic\n",
    "        logger.warning(\"Total time to provision your cluster: %.2f \"%(toc/60)+\" minutes\")\n",
    "        break\n",
    "    if (status_EMR=='TERMINATED_WITH_ERRORS' or status_EMR=='TERMINATING'):\n",
    "        err = \"Cluster un-successfully created. Ending installation...\"\n",
    "        logger.error(err)\n",
    "        sys.exit(err)\n",
    "    time.sleep(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prospective-monitor",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get public DNS from master node\n",
    "master_dns=details_EMR.get('Cluster').get('MasterPublicDnsName')\n",
    "master_IP=re.sub(\"-\",\".\",master_dns.split(\".\")[0].split(\"ec2-\")[1])\n",
    "logger.info(f'Master IP address: {master_IP}')\n",
    "logger.warning(f'This is the public JupyterLab link: \"http://{master_IP}:8192\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "inside-parade",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.warning(f'Successfully started ENR cluster \"{cluster_id}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "complete-necessity",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}