From ddc5d58b8fe9d72bb7801c8975ff88ad2c92dd82 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Mon, 17 Aug 2020 22:19:51 +0100 Subject: [PATCH 1/6] cleanup --- ipython2cwl/cwltoolextractor.py | 58 ----------------------- ipython2cwl/requirements_manager.py | 16 ------- ipython2cwl/templates/template.dockerfile | 3 -- ipython2cwl/templates/template.setup | 12 ----- tests/test_cwltoolextractor.py | 27 +---------- tests/test_requirements_manager.py | 25 ---------- 6 files changed, 1 insertion(+), 140 deletions(-) delete mode 100644 ipython2cwl/requirements_manager.py delete mode 100644 ipython2cwl/templates/template.dockerfile delete mode 100644 ipython2cwl/templates/template.setup delete mode 100644 tests/test_requirements_manager.py diff --git a/ipython2cwl/cwltoolextractor.py b/ipython2cwl/cwltoolextractor.py index abe9962..0195a40 100644 --- a/ipython2cwl/cwltoolextractor.py +++ b/ipython2cwl/cwltoolextractor.py @@ -1,27 +1,15 @@ import ast import os -import platform -import shutil -import tarfile -import tempfile from collections import namedtuple from copy import deepcopy -from pathlib import Path from typing import Dict, Any, List, Tuple import astor # type: ignore import nbconvert # type: ignore -import yaml from nbformat.notebooknode import NotebookNode # type: ignore from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput, \ CWLDumpableFile, CWLDumpableBinaryFile, CWLDumpable, CWLPNGPlot, CWLPNGFigure -from .requirements_manager import RequirementsManager - -with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f: - DOCKERFILE_TEMPLATE = f.read() -with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.setup'])) as f: - SETUP_TEMPLATE = f.read() _VariableNameTypePair = namedtuple( 'VariableNameTypePair', @@ -314,49 +302,3 @@ def cwl_command_line_tool(self, docker_image_id: str = 'jn2cwl:latest') -> Dict: }, } return cwl_tool - - def compile(self, filename: Path = Path('notebookAsCWLTool.tar')) -> str: - """ - That method generates a tar file which includes the following files: - notebookTool - the python script - tool.cwl - the cwl description file - Dockerfile - the dockerfile to create the docker image - :param: filename - :return: The absolute path of the tar file - """ - workdir = tempfile.mkdtemp() - script_path = os.path.join(workdir, 'notebookTool') - cwl_path: str = os.path.join(workdir, 'tool.cwl') - dockerfile_path = os.path.join(workdir, 'Dockerfile') - setup_path = os.path.join(workdir, 'setup.py') - requirements_path = os.path.join(workdir, 'requirements.txt') - with open(script_path, 'wb') as script_fd: - script_fd.write(self._wrap_script_to_method(self._tree, self._variables).encode()) - with open(cwl_path, 'w') as cwl_fd: - yaml.safe_dump( - self.cwl_command_line_tool(), - cwl_fd, - encoding='utf-8' - ) - dockerfile = DOCKERFILE_TEMPLATE.format( - python_version=f'python:{".".join(platform.python_version_tuple())}' - ) - with open(dockerfile_path, 'w') as f: - f.write(dockerfile) - with open(setup_path, 'w') as f: - f.write(SETUP_TEMPLATE) - - with open(requirements_path, 'w') as f: - f.write(os.linesep.join(RequirementsManager.get_all())) - - with tarfile.open(str(filename.absolute()), 'w') as tar_fd: - def add_tar(file_to_add): tar_fd.add(file_to_add, arcname=os.path.basename(file_to_add)) - - add_tar(script_path) - add_tar(cwl_path) - add_tar(dockerfile_path) - add_tar(setup_path) - add_tar(requirements_path) - - shutil.rmtree(workdir) - return str(filename.absolute()) diff --git a/ipython2cwl/requirements_manager.py b/ipython2cwl/requirements_manager.py deleted file mode 100644 index 1dee5a7..0000000 --- a/ipython2cwl/requirements_manager.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import List - -from pip._internal.operations import freeze # type: ignore - - -class RequirementsManager: - """ - That class is responsible for generating the requirements.txt file of the activated python environment. - """ - - @classmethod - def get_all(cls) -> List[str]: - return [ - str(package.as_requirement()) for package in freeze.get_installed_distributions() - if package.project_name != 'ipython2cwl' - ] diff --git a/ipython2cwl/templates/template.dockerfile b/ipython2cwl/templates/template.dockerfile deleted file mode 100644 index 30f19de..0000000 --- a/ipython2cwl/templates/template.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM {python_version} -COPY . /app -RUN cd /app && pip install -r requirements.txt && python setup.py install diff --git a/ipython2cwl/templates/template.setup b/ipython2cwl/templates/template.setup deleted file mode 100644 index 3e55950..0000000 --- a/ipython2cwl/templates/template.setup +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - -name = 'JupyterAsATool' - -setup( - name=name, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - ], - scripts=['notebookTool'] -) diff --git a/tests/test_cwltoolextractor.py b/tests/test_cwltoolextractor.py index 724a030..fc089ea 100644 --- a/tests/test_cwltoolextractor.py +++ b/tests/test_cwltoolextractor.py @@ -1,7 +1,4 @@ import os -import tarfile -import tempfile -from pathlib import Path from unittest import TestCase import nbformat @@ -69,28 +66,6 @@ def test_AnnotatedIPython2CWLToolConverter_cwl_command_line_tool(self): cwl_tool ) - def test_AnnotatedIPython2CWLToolConverter_compile(self): - annotated_python_script = os.linesep.join([ - "import csv", - "input_filename: CWLFilePathInput = 'data.csv'", - "with open(input_filename) as f:", - "\tcsv_reader = csv.reader(f)", - "\tdata = [line for line in csv_reader]", - "print(data)" - ]) - compiled_tar_file = os.path.join(tempfile.mkdtemp(), 'file.tar') - extracted_dir = tempfile.mkdtemp() - print('compiled at tarfile:', - AnnotatedIPython2CWLToolConverter(annotated_python_script) - .compile(Path(compiled_tar_file))) - with tarfile.open(compiled_tar_file, 'r') as tar: - tar.extractall(path=extracted_dir) - print(compiled_tar_file) - self.assertSetEqual( - {'notebookTool', 'tool.cwl', 'Dockerfile', 'requirements.txt', 'setup.py'}, - set(os.listdir(extracted_dir)) - ) - def test_AnnotatedIPython2CWLToolConverter_optional_arguments(self): annotated_python_script = os.linesep.join([ "import csv", @@ -557,4 +532,4 @@ def test_AnnotatedIPython2CWLToolConverter_CWLPNGFigure(self): }, }, tool - ) \ No newline at end of file + ) diff --git a/tests/test_requirements_manager.py b/tests/test_requirements_manager.py deleted file mode 100644 index 15fd94e..0000000 --- a/tests/test_requirements_manager.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import shutil -from unittest import TestCase - -from ipython2cwl.requirements_manager import RequirementsManager - - -class TestRequirementsManager(TestCase): - here = os.path.abspath(os.path.dirname(__file__)) - maxDiff = None - - @classmethod - def tearDownClass(cls): - venvs_to_delete = [venv for venv in os.listdir(cls.here) if - venv.startswith('venv_') and os.path.isdir(os.sep.join([cls.here, venv]))] - for venv in venvs_to_delete: - venv = os.sep.join([cls.here, venv]) - print(f'Deleting venv: {venv}') - shutil.rmtree(venv) - - def test_get_all(self): - requirements = RequirementsManager.get_all() - requirements_without_version = [r.split('==')[0] for r in requirements] - self.assertIn('nbformat', requirements_without_version) - self.assertNotIn('ipython2cwl', requirements_without_version) From ee8db956ac02c7a9f8806130ef3b1422e02ba13f Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Mon, 17 Aug 2020 22:20:22 +0100 Subject: [PATCH 2/6] add makefile for coverage --- Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Makefile b/Makefile index b920674..d37dc19 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,18 @@ +COVERAGE_DIR=htmlcov + mypy: mypy $$(find ipython2cwl -name '*.py') + +coverage: coverage-run coverage-html coverage-pdf + +coverage-run: + coverage run --source ipython2cwl -m unittest discover tests + +coverage-html: + coverage html + +coverage-pdf: + wkhtmltopdf --title 'Coverage Report' --enable-local-file-access $(COVERAGE_DIR)/index.html $(COVERAGE_DIR)/ipython2cwl*.html $(COVERAGE_DIR)/ipython2cwl_coverage.pdf + +clean: + rm -rf $(COVERAGE_DIR) \ No newline at end of file From 39821c6fcb90cb4fc2f067e42ea5f65397813eb6 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Mon, 17 Aug 2020 22:25:56 +0100 Subject: [PATCH 3/6] increase version --- ipython2cwl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/__init__.py b/ipython2cwl/__init__.py index 2992519..78f31aa 100644 --- a/ipython2cwl/__init__.py +++ b/ipython2cwl/__init__.py @@ -1,2 +1,2 @@ """Compile IPython Jupyter Notebooks as CWL CommandLineTools""" -__version__ = "0.0.4" +__version__ = "0.0.5" From 0cdcf6a57acea8fe81d71f2f6dbd31268a6491ef Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Mon, 17 Aug 2020 22:45:53 +0100 Subject: [PATCH 4/6] change docker api version for travis --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8afa0bd..acd0c9c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ coverage>=5.1 coveralls>=2.0.0 virtualenv>=3.1.0 gitpython>=3.1.3 -docker>=4.2.1 +docker>=4.2.1,<4.3 cwltool==3.0.20200706173533 pandas==1.0.5 mypy From e7b8f3ae94c3ede5b18a0421a86f0a102dd3bb37 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Tue, 18 Aug 2020 18:27:14 +0100 Subject: [PATCH 5/6] filter out non python notebooks --- ipython2cwl/repo2cwl.py | 11 +++++- tests/repo-like/error.ipynb | 5 +++ tests/repo-like/non-python.ipynb | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/repo-like/error.ipynb create mode 100644 tests/repo-like/non-python.ipynb diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 78a89a2..dd0d950 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -25,8 +25,15 @@ def _get_notebook_paths_from_dir(dir_path: str): notebooks_paths = [] for path, _, files in os.walk(dir_path): for name in files: - if name.endswith('.ipynb'): - notebooks_paths.append(os.path.join(path, name)) + try: + if name.endswith('.ipynb'): + fn = os.path.join(path, name) + with open(fn) as fd: + notebook = nbformat.read(fd, as_version=4) + if notebook['metadata']['kernelspec']['language'] == "python": + notebooks_paths.append(fn) + except Exception as e: + logger.error(f"Failed parse document '{name}' because of exception: {e}") return notebooks_paths diff --git a/tests/repo-like/error.ipynb b/tests/repo-like/error.ipynb new file mode 100644 index 0000000..f00e531 --- /dev/null +++ b/tests/repo-like/error.ipynb @@ -0,0 +1,5 @@ +{ + "metadata": { + "this is not a Jupyter Document": 1 + } +} diff --git a/tests/repo-like/non-python.ipynb b/tests/repo-like/non-python.ipynb new file mode 100644 index 0000000..f5e594d --- /dev/null +++ b/tests/repo-like/non-python.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# just an non-ipython notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "List of Available Magic commands\n", + "\t- compile\n", + "\t- data\n", + "\t- displayData\n", + "\t- displayDataCSV\n", + "\t- displayDataImage\n", + "\t- edit\n", + "\t- execute\n", + "\t- executeWithProvenance\n", + "\t- githubImport\n", + "\t- logs\n", + "\t- magics\n", + "\t- newWorkflow\n", + "\t- newWorkflowAddInput\n", + "\t- newWorkflowAddOutputSource\n", + "\t- newWorkflowAddStep\n", + "\t- newWorkflowAddStepIn\n", + "\t- newWorkflowBuild\n", + "\t- sampleCSV\n", + "\t- scatter\n", + "\t- snippet\n", + "\t- system\n", + "\t- view\n", + "\t- viewTool" + ] + } + ], + "source": [ + "% magics" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Common Workflow Languages", + "language": "cwl", + "name": "cwlkernel" + }, + "language_info": { + "file_extension": ".cwl", + "mimetype": "text/x-cwl", + "name": "yaml" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 1c489b96b043f15d89c46be6d64d2c4cb93b7f8a Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Tue, 18 Aug 2020 22:34:38 +0100 Subject: [PATCH 6/6] accept coveralls failures --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a060130..2b8d4ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: script: - pycodestyle --max-line-length=119 $(find ipython2cwl -name '*.py') - coverage run --source ipython2cwl -m unittest discover tests - - coveralls + - coveralls || true - make mypy matrix: include: