dev_menu.py

#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# -*- coding: utf-8 -*-
"""Tool to ease working with the build system and reproducing test results"""

import argparse
import os
import sys
from subprocess import check_call
import shlex
from ci.util import retry, remember_cwd
from typing import List
from collections import OrderedDict
import logging
import yaml
import shutil

DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv')
DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3')
DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml')

class Confirm(object):
    def __init__(self, cmds):
        self.cmds = cmds

    def __call__(self):
        resp = input("This will run the following command(s) '{}' are you sure? yes / no: ".format(self.cmds))
        while True:
            if resp.lower() == 'yes':
                handle_commands(self.cmds)
                return
            elif resp.lower() == 'no':
                return
            else:
                resp = input("Please answer yes or no: ")

class CMake(object):
    def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'):
        if os.path.exists(cmake_options_yaml):
            self.cmake_options_yaml = cmake_options_yaml
        else:
            self.cmake_options_yaml = cmake_options_yaml_default
        logging.info('Using {} for CMake configuration'.format(self.cmake_options_yaml))
        self.cmake_options = None
        self.read_config()

    def read_config(self):
        assert os.path.isfile(self.cmake_options_yaml)
        with open(self.cmake_options_yaml, 'r') as f:
            self.cmake_options = yaml.load(f)

    def _cmdlineflags(self):
        res = []
        for opt,v in self.cmake_options.items():
            res.append('-D{}={}'.format(opt,v))
        return res

    def cmake_command(self) -> str:
        """
        :return: Cmake command to run given the options
        """
        cmd_lst = ['cmake']
        cmd_lst.extend(self._cmdlineflags())
        return cmd_lst

    def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
        logging.info("CMake / {} build in directory {}".format(
            generator, os.path.abspath(build_dir)))
        cmd_lst = self.cmake_command()
        os.makedirs(build_dir, exist_ok=True)
        with remember_cwd():
            os.chdir(build_dir)
            cmd_lst.extend(['-G{}'.format(generator), '..'])
            logging.info('Executing: {}'.format('\t\n'.join(cmd_lst)))
            check_call(cmd_lst)
            logging.info('Now building')
            check_call(shlex.split(build_cmd))

def create_virtualenv(venv_exe, pyexe, venv) -> None:
    logging.info("Creating virtualenv in %s with python %s", venv, pyexe)
    if not (venv_exe and pyexe and venv):
        logging.warn("Skipping creation of virtualenv")
        return
    check_call([venv_exe, '-p', pyexe, venv])
    activate_this_py = os.path.join(venv, 'bin', 'activate_this.py')
    # Activate virtualenv in this interpreter
    exec(open(activate_this_py).read(), dict(__file__=activate_this_py))
    check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python'])
    check_call(['pip', 'install', '-r', 'tests/requirements.txt'])

def create_virtualenv_default():
    create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV)
    logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV)

COMMANDS = OrderedDict([
    ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV),
    [
        CMake(),
        create_virtualenv_default,
    ]),
    ('[Local] Python Unit tests',
        "./py3_venv/bin/nosetests -v tests/python/unittest/"
    ),
    ('[Docker] Website and docs build outputs to "docs/_build/html/"',
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"),
    ('[Docker] sanity_check. Check for linting and code formatting and licenses.',
    [
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check",
        "ci/build.py --platform ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check",
    ]),
    ('[Docker] Python3 CPU unittests',
    [
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas",
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu",
    ]),
    ('[Docker] Python3 GPU unittests',
    [
        "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu",
        "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu",
    ]),
    ('[Docker] Python3 GPU+MKLDNN unittests',
    [
        "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn",
        "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu",
    ]),
    ('[Docker] Python3 CPU Intel MKLDNN unittests',
    [
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn",
        "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu",
    ]),
    ('[Docker] Python3 ARMv7 unittests (QEMU)',
    [
        "ci/build.py -p armv7",
        "ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
    ]),
    ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',
       Confirm("ci/docker/runtime_functions.sh clean_repo"))
])

def clip(x, mini, maxi):
    return min(max(x,mini), maxi)

@retry((ValueError, RuntimeError), 3, delay_s = 0)
def show_menu(items: List[str], header=None) -> int:
    print('\n-- MXNet dev menu --\n')
    def hr():
        print(''.join(['-']*30))
    if header:
        print(header)
    hr()
    for i,x in enumerate(items,1):
        print('{}. {}'.format(i,x))
    hr()
    choice = int(input('Choose option> ')) - 1
    if choice < 0 or choice >= len(items):
        raise RuntimeError('Choice must be between {} and {}'.format(1, len(items)))
    return choice

def handle_commands(cmds) -> None:
    def handle_command(cmd):
        logging.info("Executing command: %s",cmd)
        check_call(shlex.split(cmd))

    if type(cmds) is list:
        for cmd in cmds:
            handle_commands(cmd)
    elif type(cmds) is str:
        handle_command(cmds)
    elif callable(cmds):
        cmds()
    else:
        raise RuntimeError("handle_commands(cmds): argument should be str or List[str] but is %s", type(cmds))

def use_menu_ui(args) -> None:
    command_list = list(COMMANDS.keys())
    if hasattr(args, 'choice') and args.choice and args.choice[0].isdigit():
        choice = int(args.choice[0]) - 1
    else:
        choice = show_menu(command_list, 'Available actions')
    handle_commands(COMMANDS[command_list[choice]])

def build(args) -> None:
    """Build using CMake"""
    venv_exe = shutil.which('virtualenv')
    pyexe = shutil.which(args.pyexe)
    if not venv_exe:
        logging.warn("virtualenv wasn't found in path, it's recommended to install virtualenv to manage python environments")
    if not pyexe:
        logging.warn("Python executable %s not found in path", args.pyexe)
    if args.cmake_options:
        cmake = CMake(args.cmake_options)
    else:
        cmake = CMake()
    cmake()
    create_virtualenv(venv_exe, pyexe, args.venv)

def main():
    logging.getLogger().setLevel(logging.INFO)
    parser = argparse.ArgumentParser(description="""Utility for compiling and testing MXNet easily""")
    parser.set_defaults(command='use_menu_ui')

    subparsers = parser.add_subparsers(help='sub-command help')
    build_parser = subparsers.add_parser('build', help='build with the specified flags from file')
    build_parser.add_argument('cmake_options', nargs='?',
        help='File containing CMake options in YAML')
    build_parser.add_argument('-v', '--venv',
        type=str,
        default=DEFAULT_PYENV,
        help='virtualenv dir')
    build_parser.add_argument('-p', '--pyexe',
        type=str,
        default=DEFAULT_PYTHON,
        help='python executable')
    build_parser.set_defaults(command='build')

    menu_parser = subparsers.add_parser('menu', help='jump to menu option #')
    menu_parser.set_defaults(command='use_menu_ui')
    menu_parser.add_argument('choice', nargs=1)

    args = parser.parse_args()
    globals()[args.command](args)
    return 0

if __name__ == '__main__':
    sys.exit(main())