From eb9cc2710eded799ae71ce0c95c06a97b2701c3b Mon Sep 17 00:00:00 2001 From: Tam Huynh Date: Thu, 1 Nov 2018 11:17:42 -0700 Subject: [PATCH] Initial checkin --- .gitignore | 330 ++++++++++++++++++++++++++++++++++++++ README.md | 80 ++++++++++ azuredeploy-centos.sh | 91 +++++++++++ azuredeploy-docker.sh | 106 +++++++++++++ azuredeploy.json | 362 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 969 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 azuredeploy-centos.sh create mode 100644 azuredeploy-docker.sh create mode 100644 azuredeploy.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7023fb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,330 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f8a553a --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +# Install OmniSci (MapD) on Microsoft Azure + +It's easy to deploy OmniSci (MapD) to Microsoft Azure! The Azure template on this page will install and configure OmniSci Community Edition (CE) onto a Linux virtual machine running CentOS 7.4 or Ubuntu 16.04-LTS. +* CentOS 7.4 environments will make use of Yum packages +* Ubuntu 16.04-LTS environments will be deployed in a Docker environment +* GPU or CPU VM? It's up to you! Our installation process will detect the type of VM you've chosen and deploy OmniSci CE GPU or CPU based upon that. + +Let's get started! + +## Identify your working environment +Identify an Azure Environment and User Account + +* You'll need an Azure account that has privileges to create and configure the Azure resources and services we've described + +Load the Azure Resource Manager template into the Azure Portal so you can enter the configuration parameters described below. You can click the "Deploy to Azure" link to begin this process. Don't worry - nothing will be deployed until you agree. The Visualize link will map out the resources associated with this deployment. + + Deploy to Azure    + Visualize + +New to Azure? Here's a description of the fields in the template. + +### Operating System, VM Series, and Data Disks +The OS Platform drop-down will let you select between CentOS 7.4 or Ubuntu 16.04-LTS +* CentOS 7.4 environments will make use of Yum packages +* Ubuntu 16.04-LTS environments will be deployed in a Docker environment + +We've pre-selected the lowest-cost GPU-powered VM series available: Standard_NC6 +* Can't afford a GPU? + * No worries! Remember, we'll automatically install the GPU or CPU version of OmniSci CE based on your VM selection. +* Deployment failures? You may need to increase your vCPU quota as described here: + * https://docs.microsoft.com/en-us/azure/azure-supportability/resource-manager-core-quotas-request +* Make the VM Series you enter is available in the Azure region you target. + * GPU-powered VMs are not available in all Azure regions +* Need help? The Azure VM Comparision website will show you the VMs available in a given region https://azureprice.net/ + +Azure supports data disks of up to 4TB (4095 GB). We've defaulted you to 512 GB. + * The number of data disks you can attach varies based on the VM series you select + * If you create multiple data disks, we'll consolidate them into a single RAID 0 partition + * We'll create a partition called /mapd-storage using your data disk configuration and configure OmniSci to make use of it + +### Hostnames and Networking +Public IP DNS is the public-facing hostname of the machine. It must be unique to the Azure region you are deploying to. + + * If you deploy to West US 2, for example, the fully-qualified hostname will be: your-hostname.westus2.cloudapp.azure.com + * Creatively challenged? + * Just leave it blank. We'll generate a unique one for you. You can change it later. + * Picking one yourself? + * Unfortunately, we're unable to determine if the value you enter is already being used at this time. + * We suggest you append the month, day, year to achieve uniqueness. For example: your-hostname-01012018 + * After the machine has been created, you can go back and change it through the Azure Portal + +### Deployment and Connecting to the OmniSci VM + +Before you deploy, you'll need to Agree to the terms and click Purchase to begin deployment. As a reminder, you're not actually paying to use this free template. However, the resources that you deploy and make use of will be billed to your subscription. + +* Once you've begun your deployment, you can remain in the Azure Portal or navigate away and come back. Either way, you'll receive a notification in the Azure Portal upon completion. Once this has occured: + * Navigate to the Azure Resource Group you targeted + * Look for a virtual machine called "mapdvm" (assuming you haven't changed the default value). Click it. + * On the "Overview" Pane for "mapdvm", you can: + * Click DNS Name if you don't like the unique value we generated for the public-facing hostname + * Click the Connect icon to see the username@hostname value you can supply to your SSH client. + * The DNS Name associated with your VM can then be used to access the OmniSci Immerse interface. + * In your web browser, navigate to http://hostname:9092 + +### Loading Data Into OmniSci + +Using OmniSci's installation directions, you can load sample data into your OmniSci environment. +* You'll need to connect to your VM using the SSH credentials described above. + +Navigate to the "Checkpoint" instructions underneath the Activation section. +* CentOS + * "su" to the user "mapd" after login to run the steps listed + * Environment issues? $MAPD_PATH will be /opt/mapd + * CPU - https://www.omnisci.com/docs/latest/4_centos7-yum-cpu-ce-recipe.html + * GPU - https://www.omnisci.com/docs/latest/4_centos7-yum-gpu-ce-recipe.html +* Ubuntu + Docker + * CPU - https://www.omnisci.com/docs/latest/4_docker_cpu_ce_recipe.html + * GPU - https://www.omnisci.com/docs/latest/4_docker_gpu_ce_recipe.html + + diff --git a/azuredeploy-centos.sh b/azuredeploy-centos.sh new file mode 100644 index 0000000..beee476 --- /dev/null +++ b/azuredeploy-centos.sh @@ -0,0 +1,91 @@ +# Log activity +LOG_PATH=/tmp/azuredeploy.log + +# Log parameters passed to this script. +echo $@ >> $LOG_PATH 2>&1 + +# Store parameters passed to this script +NUM_OF_DATA_DISKS=${1} + +# Basic info +date > $LOG_PATH 2>&1 +whoami >> $LOG_PATH 2>&1 + +# Set the storage area for MapD +MAPD_STORAGE=/mapd-storage + +# Setup the mapd user +MAPD_USERNAME=mapd +useradd -U $MAPD_USERNAME >> $LOG_PATH 2>&1 + +# Check if we have a NVIDIA GPU +NVIDIA_GPU=`lspci | grep NVIDIA` +if [ ! -z "$NVIDIA_GPU" ] +then + echo NVIDIA GPU Detected $NVIDIA_GPU >> $LOG_PATH 2>&1 +else + echo No NVIDIA GPU Detected >> $LOG_PATH 2>&1 +fi + +# Create the /mapd-storage data disk as a RAID 0 +mkdir $MAPD_STORAGE >> $LOG_PATH 2>&1 +if [ $NUM_OF_DATA_DISKS -eq 1 ]; then + mkfs -F -t ext4 /dev/sdc >> $LOG_PATH 2>&1 + echo "UUID=`blkid -s UUID /dev/sdc | cut -d '"' -f2` $MAPD_STORAGE ext4 defaults,discard 0 0" | tee -a /etc/fstab >> $LOG_PATH 2>&1 +else + apt-get install lsscsi -y >> $LOG_PATH 2>&1 + DEVICE_NAME_STRING= + for device in `lsscsi |grep -v "/dev/sda \|/dev/sdb \|/dev/sr0 " | cut -d "/" -f3`; do + DEVICE_NAME_STRING_TMP=`echo /dev/$device` + DEVICE_NAME_STRING=`echo $DEVICE_NAME_STRING $DEVICE_NAME_STRING_TMP` + done + mdadm --create /dev/md0 --level 0 --raid-devices=$NUM_OF_DATA_DISKS $DEVICE_NAME_STRING >> $LOG_PATH 2>&1 + mkfs -F -t ext4 /dev/md0 >> $LOG_PATH 2>&1 + echo "UUID=`blkid -s UUID /dev/md0 | cut -d '"' -f2` $MAPD_STORAGE ext4 defaults,discard 0 0" | tee -a /etc/fstab >> $LOG_PATH 2>&1 +fi + +mount $MAPD_STORAGE >> $LOG_PATH 2>&1 +chown -R $MAPD_USERNAME $MAPD_STORAGE >> $LOG_PATH 2>&1 +chgrp -R $MAPD_USERNAME $MAPD_STORAGE >> $LOG_PATH 2>&1 + +if [ ! -z "$NVIDIA_GPU" ] +then + echo NVIDIA GPU Detected $NVIDIA_GPU >> $LOG_PATH 2>&1 + curl https://releases.mapd.com/ce/mapd-ce-cuda.repo | tee /etc/yum.repos.d/mapd.repo >> $LOG_PATH 2>&1 + yum install mapd -y >> $LOG_PATH 2>&1 +else + echo No NVIDIA GPU Detected >> $LOG_PATH 2>&1 + curl https://releases.mapd.com/ce/mapd-ce-cpu.repo | tee /etc/yum.repos.d/mapd.repo >> $LOG_PATH 2>&1 + yum install mapd -y >> $LOG_PATH 2>&1 +fi + +MAPD_USERNAME_HOME=`eval echo ~$MAPD_USERNAME` +echo "export MAPD_USER=$MAPD_USERNAME" >> $MAPD_USERNAME_HOME/.bashrc +echo "export MAPD_GROUP=$MAPD_USERNAME" >> $MAPD_USERNAME_HOME/.bashrc +echo "export MAPD_PATH=/opt/mapd" >> $MAPD_USERNAME_HOME/.bashrc +echo "export MAPD_STORAGE=$MAPD_STORAGE" >> $MAPD_USERNAME_HOME/.bashrc +echo "export MAPD_LOG=$MAPD_STORAGE/data/mapd_log" >> $MAPD_USERNAME_HOME/.bashrc + +rm /tmp/install_mapd_systemd +echo /opt/mapd >> /tmp/install_mapd_systemd +echo $MAPD_STORAGE >> /tmp/install_mapd_systemd +echo $MAPD_USERNAME >> /tmp/install_mapd_systemd +echo $MAPD_USERNAME >> /tmp/install_mapd_systemd + +cd /opt/mapd/systemd >> $LOG_PATH 2>&1 +./install_mapd_systemd.sh < /tmp/install_mapd_systemd >> $LOG_PATH 2>&1 +rm /tmp/install_mapd_systemd + +cd $MAPD_PATH +systemctl start mapd_server >> $LOG_PATH 2>&1 +systemctl start mapd_web_server >> $LOG_PATH 2>&1 +systemctl enable mapd_server >> $LOG_PATH 2>&1 +systemctl enable mapd_web_server >> $LOG_PATH 2>&1 + +# Reboot to make sure updates register +# +shutdown -r +1 >> $LOG_PATH 2>&1 +echo done >> $LOG_PATH 2>&1 + +# Exit script with 0 code to tell Azure that the deployment is done +exit 0 >> $LOG_PATH 2>&1 \ No newline at end of file diff --git a/azuredeploy-docker.sh b/azuredeploy-docker.sh new file mode 100644 index 0000000..8816237 --- /dev/null +++ b/azuredeploy-docker.sh @@ -0,0 +1,106 @@ +# Credit to OmniSci for deployment script located here: +# https://github.com/omnisci/mapd_on_azure + +# Log activity +LOG_PATH=/tmp/azuredeploy.log + +# Log parameters passed to this script. +echo $@ >> $LOG_PATH 2>&1 + +# Store parameters passed to this script +NUM_OF_DATA_DISKS=${1} + +# Basic info +date > $LOG_PATH 2>&1 +whoami >> $LOG_PATH 2>&1 + +# Set the storage area for MapD +MAPD_STORAGE=/mapd-storage + +# Setup the mapd user +MAPD_USERNAME=mapd +useradd -U $MAPD_USERNAME >> $LOG_PATH 2>&1 + +# Check if we have a NVIDIA GPU +NVIDIA_GPU=`lspci | grep NVIDIA` +if [ ! -z "$NVIDIA_GPU" ] +then + echo NVIDIA GPU Detected $NVIDIA_GPU >> $LOG_PATH 2>&1 +else + echo No NVIDIA GPU Detected >> $LOG_PATH 2>&1 +fi + +apt-get install -y apt-transport-https ca-certificates curl software-properties-common + +# Install Docker CE +# +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - >> $LOG_PATH 2>&1 + +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" >> $LOG_PATH 2>&1 + +apt-get update >> $LOG_PATH 2>&1 +apt-get install -y docker-ce=18.03.0~ce-0~ubuntu >> $LOG_PATH 2>&1 +usermod -aG docker $MAPD_USERNAME >> $LOG_PATH 2>&1 + +# Install nvidia-docker if necessary +# +if [ ! -z "$NVIDIA_GPU" ] +then + curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \ + apt-key add - >> $LOG_PATH 2>&1 + + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) >> $LOG_PATH 2>&1 + + curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ + tee /etc/apt/sources.list.d/nvidia-docker.list >> $LOG_PATH 2>&1 + + apt-get update >> $LOG_PATH 2>&1 + apt-get install -y nvidia-docker2 >> $LOG_PATH 2>&1 + pkill -SIGHUP dockerd >> $LOG_PATH 2>&1 +fi + +# Create the /mapd-storage data disk as a RAID 0 +mkdir $MAPD_STORAGE >> $LOG_PATH 2>&1 +if [ $NUM_OF_DATA_DISKS -eq 1 ]; then + mkfs -t ext4 /dev/sdc >> $LOG_PATH 2>&1 + echo "UUID=`blkid -s UUID /dev/sdc | cut -d '"' -f2` $MAPD_STORAGE ext4 defaults,discard 0 0" | tee -a /etc/fstab >> $LOG_PATH 2>&1 +else + apt-get install lsscsi -y >> $LOG_PATH 2>&1 + DEVICE_NAME_STRING= + for device in `lsscsi |grep -v "/dev/sda \|/dev/sdb \|/dev/sr0 " | cut -d "/" -f3`; do + DEVICE_NAME_STRING_TMP=`echo /dev/$device` + DEVICE_NAME_STRING=`echo $DEVICE_NAME_STRING $DEVICE_NAME_STRING_TMP` + done + mdadm --create /dev/md0 --level 0 --raid-devices=$NUM_OF_DATA_DISKS $DEVICE_NAME_STRING >> $LOG_PATH 2>&1 + mkfs -t ext4 /dev/md0 >> $LOG_PATH 2>&1 + echo "UUID=`blkid -s UUID /dev/md0 | cut -d '"' -f2` $MAPD_STORAGE ext4 defaults,discard 0 0" | tee -a /etc/fstab >> $LOG_PATH 2>&1 +fi + +mount $MAPD_STORAGE >> $LOG_PATH 2>&1 +chown -R $MAPD_USERNAME $MAPD_STORAGE >> $LOG_PATH 2>&1 +chgrp -R $MAPD_USERNAME $MAPD_STORAGE >> $LOG_PATH 2>&1 + +# Setup startup in the user's crontab +sudo -u $MAPD_USERNAME crontab -l > $MAPD_STORAGE/mapd_start >> $LOG_PATH 2>&1 +if [ ! -z "$NVIDIA_GPU" ] +then + sudo -u $MAPD_USERNAME echo "docker run --runtime=nvidia -v $MAPD_STORAGE/mapd-docker-storage:/mapd-storage -p 9090-9092:9090-9092 mapd/mapd-ce-cuda" >> $MAPD_STORAGE/StartMapD.sh +else + sudo -u $MAPD_USERNAME echo "docker run -d -v $MAPD_STORAGE/mapd-docker-storage:/mapd-storage -p 9090-9092:9090-9092 mapd/mapd-ce-cpu" >> $MAPD_STORAGE/StartMapD.sh +fi +sudo -u $MAPD_USERNAME echo "@reboot bash $MAPD_STORAGE/StartMapD.sh &" >> $MAPD_STORAGE/mapd_start +sudo -u $MAPD_USERNAME crontab $MAPD_STORAGE/mapd_start >> $LOG_PATH 2>&1 +sudo -u $MAPD_USERNAME rm $MAPD_STORAGE/mapd_start >> $LOG_PATH 2>&1 + +# Reboot to make sure updates register +# +shutdown -r +1 >> $LOG_PATH 2>&1 +echo done >> $LOG_PATH 2>&1 + +# Exit script with 0 code to tell Azure that the deployment is done +exit 0 >> $LOG_PATH 2>&1 + + diff --git a/azuredeploy.json b/azuredeploy.json new file mode 100644 index 0000000..2abaabf --- /dev/null +++ b/azuredeploy.json @@ -0,0 +1,362 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "osPlatform": { + "type": "string", + "defaultValue": "ubuntu-docker", + "allowedValues": [ "ubuntu-docker","centos" ], + "metadata": { + "description": "OS for the Virtual Machine." + } + }, + "vmName": { + "type": "string", + "defaultValue": "mapdvm", + "metadata": { + "description": "Name for the Virtual Machine." + } + }, + "adminUsername": { + "type": "string", + "defaultValue": "mapdadmin", + "metadata": { + "description": "User name for the Virtual Machine." + } + }, + "adminPassword": { + "type": "securestring", + "metadata": { + "description": "Password or ssh key for the Virtual Machine." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_NC6", + "metadata": { + "description": "Size for the Virtual Machine." + } + }, + "dataDiskNumber" : { + "type" : "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 4, + "metadata": { + "description": "Number of data disk(s)." + } + }, + "dataDiskSize" : { + "type" : "int", + "defaultValue": 512, + "minValue": 32, + "maxValue": 4095, + "metadata": { + "description": "Size of each data disk(s)." + } + }, + "publicIpDns": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional: DNS of the public ip address for the VM" + } + }, + "_artifactsLocation": { + "type": "string", + "metadata": { + "description": "The base URI where artifacts required by this template are located. When the template is deployed using the accompanying scripts, a private location in the subscription will be used and this value will be automatically generated." + }, + "defaultValue": "https://raw.githubusercontent.com/tamhinsf/OmniSci4Azure/" + }, + "_artifactsLocationSasToken": { + "type": "securestring", + "metadata": { + "description": "The sasToken required to access _artifactsLocation. When the template is deployed using the accompanying scripts, a sasToken will be automatically generated." + }, + "defaultValue": "" + } + }, + "variables": { + "storageNewOrExisting" : "new", + "storageAccountName" : "[concat('storage', uniqueString(resourceGroup().id))]", + "virtualNetworkNewOrExisting": "new", + "storageAccountType": "Standard_LRS", + "storageAccountResourceGroupName" : "[resourceGroup().name]", + "virtualNetworkResourceGroupName" : "[resourceGroup().name]", + "publicIpResourceGroupName" : "[resourceGroup().name]", + "virtualNetworkName" : "[concat(parameters('vmName'), '-vnet')]", + "location": "[resourceGroup().location]", + "subnetName" : "default", + "subnetPrefix" : "10.0.0.0/24", + "addressPrefixes" : ["10.0.0.0/16"], + "publicIpNewOrExisting" : "new", + "publicIpName" : "[concat(parameters('vmName'), '-ip')]", + "nicName": "[concat(parameters('vmName'), '-nic')]", + "publicIpAddressId": { + "id": "[resourceId(variables('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', variables('publicIpName'))]" + }, + "networkSecurityGroupName": "[concat(parameters('vmName'), '-nsg')]", + "scriptFileName" : "[if(equals(parameters('osPlatform'),'ubuntu-docker'),'azuredeploy-docker.sh','azuredeploy-centos.sh')]", + "scriptArgs" : "[concat(' ', parameters('dataDiskNumber'), ' ' , parameters('adminUsername'))]", + "extensionForGpu" : "nvidiaExt", + "extensionForScript" : "scriptExt" + }, + "resources": [ + { + "condition": "[equals(variables('storageNewOrExisting'), 'new')]", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2018-02-01", + "name": "[variables('storageAccountName')]", + "location": "[variables('location')]", + "kind": "Storage", + "sku": { + "name": "[variables('storageAccountType')]" + } + }, + { + "condition": "[equals(variables('publicIpNewOrExisting'), 'new')]", + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2017-06-01", + "name": "[variables('publicIpName')]", + "location": "[variables('location')]", + "properties": { + "publicIPAllocationMethod": "Dynamic", + "dnsSettings": { + "domainNameLabel": "[if(equals(parameters('publicIpDns'), ''), concat(parameters('vmName'),'-',uniqueString(resourceGroup().id)), parameters('publicIpDns'))]" + } + } + }, + { + "condition": "[equals(variables('virtualNetworkNewOrExisting'), 'new')]", + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2017-06-01", + "name": "[variables('virtualNetworkName')]", + "location": "[variables('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": "[variables('addressPrefixes')]" + }, + "subnets": [ + { + "name": "[variables('subnetName')]", + "properties": { + "addressPrefix": "[variables('subnetPrefix')]" + } + } + ] + } + }, + { + "name": "[variables('networkSecurityGroupName')]", + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2017-06-01", + "location": "[variables('location')]", + "properties": { + "securityRules": [ + { + "name": "default-allow-ssh", + "properties": { + "priority": 1000, + "sourceAddressPrefix": "*", + "protocol": "Tcp", + "destinationPortRange": "22", + "access": "Allow", + "direction": "Inbound", + "sourcePortRange": "*", + "destinationAddressPrefix": "*" + } + }, + { + "name": "default-allow-http", + "properties": { + "priority": 1100, + "sourceAddressPrefix": "*", + "protocol": "Tcp", + "destinationPortRange": "80", + "access": "Allow", + "direction": "Inbound", + "sourcePortRange": "*", + "destinationAddressPrefix": "*" + } + }, + { + "name": "default-allow-https", + "properties": { + "priority": 1120, + "sourceAddressPrefix": "*", + "protocol": "Tcp", + "destinationPortRange": "443", + "access": "Allow", + "direction": "Inbound", + "sourcePortRange": "*", + "destinationAddressPrefix": "*" + } + }, + { + "name": "default-allow-mapd_ports", + "properties": { + "priority": 1200, + "sourceAddressPrefix": "*", + "protocol": "Tcp", + "destinationPortRange": "9090-9093", + "access": "Allow", + "direction": "Inbound", + "sourcePortRange": "*", + "destinationAddressPrefix": "*" + } + }, + { + "name": "default-allow-jupyternotebook", + "properties": { + "priority": 1210, + "sourceAddressPrefix": "*", + "protocol": "Tcp", + "destinationPortRange": "8888", + "access": "Allow", + "direction": "Inbound", + "sourcePortRange": "*", + "destinationAddressPrefix": "*" + } + } + ] + } + }, + { + "apiVersion": "2017-06-01", + "type": "Microsoft.Network/networkInterfaces", + "name": "[variables('nicName')]", + "location": "[variables('location')]", + "dependsOn": [ + "[variables('publicIpName')]", + "[variables('virtualNetworkName')]", + "[variables('networkSecurityGroupName')]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "subnet": { + "id": "[resourceId(variables('virtualNetworkResourceGroupName'), 'Microsoft.Network/virtualNetworks/subnets/', variables('virtualNetworkName'), variables('subnetName'))]" + }, + "publicIPAddress": "[if(not(equals(variables('publicIpNewOrExisting'), 'none')), variables('publicIpAddressId') , json('null'))]" + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]" + } + } + }, + { + "apiVersion": "2017-03-30", + "type": "Microsoft.Compute/virtualMachines", + "name": "[parameters('vmName')]", + "location": "[variables('location')]", + "dependsOn": [ + "[variables('storageAccountName')]", + "[variables('nicName')]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "osProfile": { + "computerName": "[parameters('vmName')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]" + }, + "storageProfile": { + "imageReference": { + "publisher": "[if(equals(parameters('osPlatform'),'ubuntu-docker'),'Canonical','OpenLogic')]", + "offer": "[if(equals(parameters('osPlatform'),'ubuntu-docker'),'UbuntuServer','CentOS')]", + "sku": "[if(equals(parameters('osPlatform'),'ubuntu-docker'),'16.04-LTS','7.4')]", + "version": "latest" + }, + "copy": [ + { + "name": "dataDisks", + "count": "[parameters('dataDiskNumber')]", + "input": { + "diskSizeGB": "[parameters('dataDiskSize')]", + "lun": "[copyIndex('dataDisks')]", + "createOption": "Empty" + } + } + ], + "osDisk": { + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('nicName'))]" + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": true, + "storageUri": "[reference(resourceId(variables('storageAccountResourceGroupName'), 'Microsoft.Storage/storageAccounts/', variables('storageAccountName')), '2018-02-01').primaryEndpoints.blob]" + } + } + }, + "resources": [ + { + "condition" : "[contains(toUpper(parameters('vmSize')),'_N')]", + "name": "[variables('extensionForGpu')]", + "type": "extensions", + "location": "[variables('location')]", + "apiVersion": "2017-03-30", + "dependsOn": [ + "[parameters('vmName')]" + ], + "properties": { + "publisher": "Microsoft.HpcCompute", + "type": "NvidiaGpuDriverLinux", + "typeHandlerVersion": "1.1", + "autoUpgradeMinorVersion": true, + "settings": { + } + } + }, + { + "name": "[variables('extensionForScript')]", + "type": "extensions", + "apiVersion": "2017-03-30", + "location": "[variables('location')]", + "dependsOn": [ + "[parameters('vmName')]", + "[variables('extensionForGpu')]" + ], + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.0", + "autoUpgradeMinorVersion": true, + "settings": { + "fileUris": [ + "[concat(parameters('_artifactsLocation'), concat('/', variables('scriptFileName'), parameters('_artifactsLocationSasToken')))]" + ] + }, + "protectedSettings": { + "commandToExecute": "[concat('bash ', variables('scriptFileName'), ' ', variables('scriptArgs'))]" + } + } + } + + ] + } + ], + "outputs": { + "ssh-command": { + "type": "string", + "value": "[if(equals(variables('publicIpNewOrExisting'), 'none'), 'no public ip', concat('ssh ', parameters('adminUsername'), '@', reference(resourceId(variables('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', variables('publicIpName')), '2018-04-01').dnsSettings.fqdn))]" + } + } + } + \ No newline at end of file