From 66b74f86680804b40edcd70ce1a59ff5f85e6869 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Thu, 2 Feb 2017 00:07:41 -0500 Subject: [PATCH 1/7] Frozen executables work better now --- Jenkinsfile | 8 ++-- proscli/conductor.py | 8 ++-- prosconductor/providers/local.py | 63 +++++++++++++++++++++++++++++-- prosflasher/bootloader.py | 2 +- version | Bin 5 -> 7 bytes 5 files changed, 70 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d1cfd56..f67afc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,9 @@ stage('Build') { stage('Clone') { checkout scm bat 'git describe --tags > version' - build_ver = readFile 'verison' + build_ver = readFile 'version' + bat 'git describe --tags > inst_version' + inst_ver = readFile 'inst_version' } stage('Build') { venv.run 'pip3 install --upgrade -r requirements.txt' @@ -60,12 +62,12 @@ stage('Build') { bat 'if exist .\\exe.win del /s /q .\\exe.win' bat 'if exist .\\exe.win rmdir /s /q .\\exe.win' bat 'mkdir .\\exe.win' - for(file in unarchive(mapping: ['pros_cli-*-win-32bit.zip': '.'])) { + for(file in unarchive(mapping: ['**pros_cli-*-win-32bit.zip': '.'])) { file.unzip(file.getParent().child('exe.win')) } def advinst = "\"${tool 'Advanced Installer'}\\AdvancedInstaller.com\"" bat """ - ${advinst} /edit pros-windows.aip /SetVersion ${build_ver} + ${advinst} /edit pros-windows.aip /SetVersion ${inst_ver} ${advinst} /edit pros-windows.aip /ResetSync APPDIR\\cli -clearcontent ${advinst} /edit pros-windows.aip /NewSync APPDIR\\cli exe.win -existingfiles delete ${advinst} /build pros-windows.aip diff --git a/proscli/conductor.py b/proscli/conductor.py index 9d9f061..95478eb 100644 --- a/proscli/conductor.py +++ b/proscli/conductor.py @@ -17,7 +17,7 @@ # from typing import List -def first_run(ctx: proscli.utils.State, force=False, defaults=False, download=True): +def first_run(ctx: proscli.utils.State, force=False, defaults=False, doDownload=True): if len(utils.get_depot_configs(ctx.pros_cfg)) == 0: click.echo('You don\'t currently have any depots configured.') if len(utils.get_depot_configs(ctx.pros_cfg)) == 0 or force: @@ -28,7 +28,7 @@ def first_run(ctx: proscli.utils.State, force=False, defaults=False, download=Tr configure=False) click.echo('Added pros-mainline to available depots. ' 'Add more depots in the future by running `pros conduct add-depot`\n') - if (defaults or click.confirm('Download the latest kernel?', default=True)) and download: + if (defaults or click.confirm('Download the latest kernel?', default=True)) and doDownload: click.get_current_context().invoke(download, name='kernel', depot='pros-mainline') @@ -577,6 +577,6 @@ def upgradelib(cfg, location, library, version, depot): @click.option('--no-download', is_flag=True, default=True) @default_cfg def first_run_cmd(cfg, no_force, use_defaults, no_download): - first_run(cfg, force=no_force, defaults=use_defaults, download=no_download) + first_run(cfg, force=no_force, defaults=use_defaults, doDownload=no_download) -import proscli.conductor_management \ No newline at end of file +import proscli.conductor_management diff --git a/prosconductor/providers/local.py b/prosconductor/providers/local.py index 0909e29..30707e7 100644 --- a/prosconductor/providers/local.py +++ b/prosconductor/providers/local.py @@ -1,5 +1,5 @@ import click -import distutils.dir_util +# import distutils.dir_util import fnmatch import os.path from proscli.utils import debug, verbose @@ -11,6 +11,63 @@ import sys # from typing import Set, List +def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2, + ignore_dangling_symlinks=False): + """Modified shutil.copytree, but with exist_ok=True + """ + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst, exist_ok=True) + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if os.path.islink(srcname): + linkto = os.readlink(srcname) + if symlinks: + # We can't just leave it to `copy_function` because legacy + # code with a custom `copy_function` may rely on copytree + # doing the right thing. + os.symlink(linkto, dstname) + shutil.copystat(srcname, dstname, follow_symlinks=not symlinks) + else: + # ignore dangling symlink if the flag is on + if not os.path.exists(linkto) and ignore_dangling_symlinks: + continue + # otherwise let the copy occurs. copy2 will raise an error + if os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore, + copy_function) + else: + copy_function(srcname, dstname) + elif os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore, copy_function) + else: + # Will raise a SpecialFileError for unsupported file types + copy_function(srcname, dstname) + # catch the Error from the recursive copytree so that we can + # continue with other files + except shutil.Error as err: + errors.extend(err.args[0]) + except OSError as why: + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) + except OSError as why: + # Copying file access times may fail on Windows + if getattr(why, 'winerror', None) is None: + errors.append((src, dst, str(why))) + if errors: + raise Error(errors) + return dst + def get_local_templates(pros_cfg=None, filters=[], template_types=None): @@ -57,7 +114,7 @@ def create_project(identifier, dest, pros_cli=None): click.get_current_context().abort() sys.exit() config = TemplateConfig(file=filename) - distutils.dir_util.copy_tree(config.directory, dest) + copytree(config.directory, dest) for root, dirs, files in os.walk(dest): for d in dirs: if any([fnmatch.fnmatch(d, p) for p in config.template_ignore]): @@ -124,7 +181,7 @@ def install_lib(identifier, dest, pros_cli): sys.exit() proj_config = prosconfig.ProjectConfig(dest) config = TemplateConfig(file=filename) - distutils.dir_util.copy_tree(config.directory, dest) + copytree(config.directory, dest) for root, dirs, files in os.walk(dest): for d in dirs: if any([fnmatch.fnmatch(d, p) for p in config.template_ignore]): diff --git a/prosflasher/bootloader.py b/prosflasher/bootloader.py index 26c539f..d4caa53 100644 --- a/prosflasher/bootloader.py +++ b/prosflasher/bootloader.py @@ -46,8 +46,8 @@ def compute_address_commandable(address): def prepare_bootloader(port): click.echo('Preparing bootloader...', nl=False) prosflasher.upload.configure_port(port, serial.PARITY_EVEN) - port.write([0x7f]) port.flush() + port.write([0x7f]) response = port.read(1) debug_response(0x07f, response) if response is None or len(response) != 1 or response[0] != ACK: diff --git a/version b/version index acdc3f1b0b25a1106bf556dc03402b145d079a92..5abc8bd29a7e1367365723ac3eeb327c8c80a019 100644 GIT binary patch literal 7 OcmXrgGto0*-~s>y)d2qh literal 5 McmXrgGtn~w00Q9v^#A|> From 116f0afaf1cd6439c8de7c425b6bff0e70dd661f Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 13:51:57 -0500 Subject: [PATCH 2/7] Update Jenkinsfile --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f67afc0..eacbdb8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,8 +30,8 @@ stage('Build') { node('win&&x64') { def venv = new edu.purdue.pros.venv() stage('Clean') { - bat "${tool name: 'Default', type: 'git'} init" - bat "${tool name: 'Default', type: 'git'} clean -d -x -f" + bat "${tool name: 'Default', type: 'git'}\\git.exe init" + bat "${tool name: 'Default', type: 'git'}\\git.exe clean -d -x -f" } stage('Dependenices') { tool 'MSBuild' @@ -45,8 +45,8 @@ stage('Build') { checkout scm bat 'git describe --tags > version' build_ver = readFile 'version' - bat 'git describe --tags > inst_version' - inst_ver = readFile 'inst_version' + bat 'git describe --tags > inst_version' + inst_ver = readFile 'inst_version' } stage('Build') { venv.run 'pip3 install --upgrade -r requirements.txt' From 6db758d0cc5130795ec296b57c8c2124309a9d9e Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 13:56:02 -0500 Subject: [PATCH 3/7] Fix git calls --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eacbdb8..af14b66 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,8 +30,8 @@ stage('Build') { node('win&&x64') { def venv = new edu.purdue.pros.venv() stage('Clean') { - bat "${tool name: 'Default', type: 'git'}\\git.exe init" - bat "${tool name: 'Default', type: 'git'}\\git.exe clean -d -x -f" + bat "${tool name: 'Default', type: 'git'} init" + bat "${tool name: 'Default', type: 'git'} clean -d -x -f" } stage('Dependenices') { tool 'MSBuild' From f6ed081b6fe866ae0ed2c3372a7675aebac9e24b Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 15:07:41 -0500 Subject: [PATCH 4/7] Let first-run reapply default providers --- proscli/conductor.py | 12 ++++++++---- prosconfig/cliconfig.py | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/proscli/conductor.py b/proscli/conductor.py index 95478eb..dd45ad8 100644 --- a/proscli/conductor.py +++ b/proscli/conductor.py @@ -17,7 +17,7 @@ # from typing import List -def first_run(ctx: proscli.utils.State, force=False, defaults=False, doDownload=True): +def first_run(ctx: proscli.utils.State, force=False, defaults=False, doDownload=True, reapplyProviders=False): if len(utils.get_depot_configs(ctx.pros_cfg)) == 0: click.echo('You don\'t currently have any depots configured.') if len(utils.get_depot_configs(ctx.pros_cfg)) == 0 or force: @@ -30,7 +30,9 @@ def first_run(ctx: proscli.utils.State, force=False, defaults=False, doDownload= 'Add more depots in the future by running `pros conduct add-depot`\n') if (defaults or click.confirm('Download the latest kernel?', default=True)) and doDownload: click.get_current_context().invoke(download, name='kernel', depot='pros-mainline') - + if reapplyProviders: + click.echo('Applying default providers') + ctx.pros_cfg.applyDefaultProviders() @click.group(cls=AliasGroup) @default_options @@ -575,8 +577,10 @@ def upgradelib(cfg, location, library, version, depot): @click.option('--no-force', is_flag=True, default=True) @click.option('--use-defaults', is_flag=True, default=False) @click.option('--no-download', is_flag=True, default=True) +@click.option('--apply-providers', is_flag=True, default=False) @default_cfg -def first_run_cmd(cfg, no_force, use_defaults, no_download): - first_run(cfg, force=no_force, defaults=use_defaults, doDownload=no_download) +def first_run_cmd(cfg, no_force, use_defaults, no_download, apply_providers): + first_run(cfg, force=no_force, defaults=use_defaults, + doDownload=no_download, reapplyProviders=apply_providers) import proscli.conductor_management diff --git a/prosconfig/cliconfig.py b/prosconfig/cliconfig.py index 581c92f..321e4b6 100644 --- a/prosconfig/cliconfig.py +++ b/prosconfig/cliconfig.py @@ -10,10 +10,12 @@ def __init__(self, file=None, ctx=None): if not file: file = os.path.join(click.get_app_dir('PROS'), 'cli.pros') self.default_libraries = [] # type: list(str) + self.providers = [] + self.applyDefaultProviders() + super(CliConfig, self).__init__(file, ctx=ctx) + + def applyDefaultProviders(self): if os.path.isfile(prosconductor.providers.githubreleases.__file__): - self.providers = [prosconductor.providers.githubreleases.__file__] + self.providers.append(prosconductor.providers.githubreleases.__file__) elif hasattr(sys, 'frozen'): - self.providers = [os.path.join(os.path.dirname(sys.executable), 'githubreleases.pyc')] - else: - self.providers = [] - super(CliConfig, self).__init__(file, ctx=ctx) + self.providers.append(os.path.join(os.path.dirname(sys.executable), 'githubreleases.pyc')) From f058fdee437dc7fd9156b7dfb2bdeb3ff1eafbfa Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 16:05:17 -0500 Subject: [PATCH 5/7] Make registrar loading more robust --- Jenkinsfile | 5 ++++- prosconductor/providers/utils.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index af14b66..c075318 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ stage('Build') { checkout scm bat 'git describe --tags > version' build_ver = readFile 'version' - bat 'git describe --tags > inst_version' + bat 'git describe --tags --abbrev=0 > inst_version' inst_ver = readFile 'inst_version' } stage('Build') { @@ -72,6 +72,9 @@ stage('Build') { ${advinst} /edit pros-windows.aip /NewSync APPDIR\\cli exe.win -existingfiles delete ${advinst} /build pros-windows.aip """ + bat """ + ${advinst} /edit pros-updates.aip /NewUpdate ${file.getRemote()} -name "PROS${inst_ver}" -display_name "PROS ${build_ver}" -url "${BUILD_URL}artifact/output/windows_updates.txt" + """ archiveArtifacts artifacts: 'output/*', fingerprint: true } } diff --git a/prosconductor/providers/utils.py b/prosconductor/providers/utils.py index 8233afb..17d54ff 100644 --- a/prosconductor/providers/utils.py +++ b/prosconductor/providers/utils.py @@ -14,8 +14,9 @@ def get_all_provider_types(pros_cfg=None): pros_cfg = CliConfig() for provider_file in pros_cfg.providers: - spec = importlib.util.spec_from_file_location('prosconductor.providers.{}'.format(os.path.basename(provider_file).split('.')[0]), provider_file) - spec.loader.load_module() + if os.path.isfile(provider_file): + spec = importlib.util.spec_from_file_location('prosconductor.providers.{}'.format(os.path.basename(provider_file).split('.')[0]), provider_file) + spec.loader.load_module() return {x.registrar: x for x in DepotProvider.__subclasses__()} From d787d2be215f3e6c9ea4cc15883aa1e898ad5b0d Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 16:20:24 -0500 Subject: [PATCH 6/7] Update automatic creation of windows_updates --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c075318..32847cb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,8 @@ stage('Build') { ${advinst} /build pros-windows.aip """ bat """ - ${advinst} /edit pros-updates.aip /NewUpdate ${file.getRemote()} -name "PROS${inst_ver}" -display_name "PROS ${build_ver}" -url "${BUILD_URL}artifact/output/windows_updates.txt" + ${advinst} /edit pros-updates.aip /NewUpdate output\\pros-win.exe -name "PROS${inst_ver}" -display_name "PROS ${build_ver}" -url "${BUILD_URL}artifact/output/windows_updates.txt" + ${advinst} /build pros-updates.aip """ archiveArtifacts artifacts: 'output/*', fingerprint: true } From f81d404ea5a9c4e84411762c7fe66a93e5743b35 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 7 Feb 2017 17:27:10 -0500 Subject: [PATCH 7/7] Modify how BUILD_URL is used --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 32847cb..0abd0ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,7 @@ stage('Build') { checkout scm sh 'git describe --tags > version' build_ver = readFile 'version' + build_ver = build_ver.replaceAll("\\s","") println "Building CLI at version ${build_ver}" } stage('Build') { @@ -45,8 +46,10 @@ stage('Build') { checkout scm bat 'git describe --tags > version' build_ver = readFile 'version' + build_ver = build_ver.replaceAll("\\s","") bat 'git describe --tags --abbrev=0 > inst_version' inst_ver = readFile 'inst_version' + inst_ver = inst_ver.replaceAll("\\s","") } stage('Build') { venv.run 'pip3 install --upgrade -r requirements.txt' @@ -73,7 +76,7 @@ stage('Build') { ${advinst} /build pros-windows.aip """ bat """ - ${advinst} /edit pros-updates.aip /NewUpdate output\\pros-win.exe -name "PROS${inst_ver}" -display_name "PROS ${build_ver}" -url "${BUILD_URL}artifact/output/windows_updates.txt" + ${advinst} /edit pros-updates.aip /NewUpdate output\\pros-win.exe -name "PROS${inst_ver}" -display_name "PROS ${build_ver}" -url "${env.BUILD_URL}artifact/output/pros-win.exe" ${advinst} /build pros-updates.aip """ archiveArtifacts artifacts: 'output/*', fingerprint: true