Cleanup

.travis.yml

@@ -18,7 +18,7 @@ install:
 script:
   - pycodestyle --max-line-length=119 $(find ipython2cwl -name '*.py')
   - coverage run --source ipython2cwl -m unittest discover tests
-  - coveralls
+  - coveralls || true
   - make mypy
 matrix:
   include:

Makefile

@@ -1,2 +1,18 @@
+COVERAGE_DIR=htmlcov
+
 mypy:
 	 mypy $$(find ipython2cwl -name '*.py')
+
+coverage: coverage-run coverage-html coverage-pdf
+
+coverage-run:
+	coverage run --source ipython2cwl -m unittest discover tests
+
+coverage-html:
+	coverage html
+
+coverage-pdf:
+	wkhtmltopdf --title 'Coverage Report' --enable-local-file-access $(COVERAGE_DIR)/index.html $(COVERAGE_DIR)/ipython2cwl*.html $(COVERAGE_DIR)/ipython2cwl_coverage.pdf
+
+clean:
+	rm -rf $(COVERAGE_DIR)

ipython2cwl/__init__.py

@@ -1,2 +1,2 @@
 """Compile IPython Jupyter Notebooks as CWL CommandLineTools"""
-__version__ = "0.0.4"
+__version__ = "0.0.5"

ipython2cwl/cwltoolextractor.py

@@ -1,27 +1,15 @@
 import ast
 import os
-import platform
-import shutil
-import tarfile
-import tempfile
 from collections import namedtuple
 from copy import deepcopy
-from pathlib import Path
 from typing import Dict, Any, List, Tuple
 
 import astor  # type: ignore
 import nbconvert  # type: ignore
-import yaml
 from nbformat.notebooknode import NotebookNode  # type: ignore
 
 from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput, \
     CWLDumpableFile, CWLDumpableBinaryFile, CWLDumpable, CWLPNGPlot, CWLPNGFigure
-from .requirements_manager import RequirementsManager
-
-with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f:
-    DOCKERFILE_TEMPLATE = f.read()
-with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.setup'])) as f:
-    SETUP_TEMPLATE = f.read()
 
 _VariableNameTypePair = namedtuple(
     'VariableNameTypePair',
@@ -314,49 +302,3 @@ def cwl_command_line_tool(self, docker_image_id: str = 'jn2cwl:latest') -> Dict:
             },
         }
         return cwl_tool
-
-    def compile(self, filename: Path = Path('notebookAsCWLTool.tar')) -> str:
-        """
-        That method generates a tar file which includes the following files:
-        notebookTool - the python script
-        tool.cwl - the cwl description file
-        Dockerfile - the dockerfile to create the docker image
-        :param: filename
-        :return: The absolute path of the tar file
-        """
-        workdir = tempfile.mkdtemp()
-        script_path = os.path.join(workdir, 'notebookTool')
-        cwl_path: str = os.path.join(workdir, 'tool.cwl')
-        dockerfile_path = os.path.join(workdir, 'Dockerfile')
-        setup_path = os.path.join(workdir, 'setup.py')
-        requirements_path = os.path.join(workdir, 'requirements.txt')
-        with open(script_path, 'wb') as script_fd:
-            script_fd.write(self._wrap_script_to_method(self._tree, self._variables).encode())
-        with open(cwl_path, 'w') as cwl_fd:
-            yaml.safe_dump(
-                self.cwl_command_line_tool(),
-                cwl_fd,
-                encoding='utf-8'
-            )
-        dockerfile = DOCKERFILE_TEMPLATE.format(
-            python_version=f'python:{".".join(platform.python_version_tuple())}'
-        )
-        with open(dockerfile_path, 'w') as f:
-            f.write(dockerfile)
-        with open(setup_path, 'w') as f:
-            f.write(SETUP_TEMPLATE)
-
-        with open(requirements_path, 'w') as f:
-            f.write(os.linesep.join(RequirementsManager.get_all()))
-
-        with tarfile.open(str(filename.absolute()), 'w') as tar_fd:
-            def add_tar(file_to_add): tar_fd.add(file_to_add, arcname=os.path.basename(file_to_add))
-
-            add_tar(script_path)
-            add_tar(cwl_path)
-            add_tar(dockerfile_path)
-            add_tar(setup_path)
-            add_tar(requirements_path)
-
-        shutil.rmtree(workdir)
-        return str(filename.absolute())

ipython2cwl/repo2cwl.py

@@ -25,8 +25,15 @@ def _get_notebook_paths_from_dir(dir_path: str):
     notebooks_paths = []
     for path, _, files in os.walk(dir_path):
         for name in files:
-            if name.endswith('.ipynb'):
-                notebooks_paths.append(os.path.join(path, name))
+            try:
+                if name.endswith('.ipynb'):
+                    fn = os.path.join(path, name)
+                    with open(fn) as fd:
+                        notebook = nbformat.read(fd, as_version=4)
+                        if notebook['metadata']['kernelspec']['language'] == "python":
+                            notebooks_paths.append(fn)
+            except Exception as e:
+                logger.error(f"Failed parse document '{name}' because of exception: {e}")
     return notebooks_paths
 
 

test-requirements.txt

@@ -3,7 +3,7 @@ coverage>=5.1
 coveralls>=2.0.0
 virtualenv>=3.1.0
 gitpython>=3.1.3
-docker>=4.2.1
+docker>=4.2.1,<4.3
 cwltool==3.0.20200706173533
 pandas==1.0.5
 mypy

tests/repo-like/error.ipynb

@@ -0,0 +1,5 @@
+{
+    "metadata": {
+        "this is not a Jupyter Document": 1
+    }
+}

tests/repo-like/non-python.ipynb

@@ -0,0 +1,65 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# just an non-ipython notebook"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "List of Available Magic commands\n",
+      "\t- compile\n",
+      "\t- data\n",
+      "\t- displayData\n",
+      "\t- displayDataCSV\n",
+      "\t- displayDataImage\n",
+      "\t- edit\n",
+      "\t- execute\n",
+      "\t- executeWithProvenance\n",
+      "\t- githubImport\n",
+      "\t- logs\n",
+      "\t- magics\n",
+      "\t- newWorkflow\n",
+      "\t- newWorkflowAddInput\n",
+      "\t- newWorkflowAddOutputSource\n",
+      "\t- newWorkflowAddStep\n",
+      "\t- newWorkflowAddStepIn\n",
+      "\t- newWorkflowBuild\n",
+      "\t- sampleCSV\n",
+      "\t- scatter\n",
+      "\t- snippet\n",
+      "\t- system\n",
+      "\t- view\n",
+      "\t- viewTool"
+     ]
+    }
+   ],
+   "source": [
+    "% magics"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Common Workflow Languages",
+   "language": "cwl",
+   "name": "cwlkernel"
+  },
+  "language_info": {
+   "file_extension": ".cwl",
+   "mimetype": "text/x-cwl",
+   "name": "yaml"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

tests/test_cwltoolextractor.py

@@ -1,7 +1,4 @@
 import os
-import tarfile
-import tempfile
-from pathlib import Path
 from unittest import TestCase
 
 import nbformat
@@ -69,28 +66,6 @@ def test_AnnotatedIPython2CWLToolConverter_cwl_command_line_tool(self):
             cwl_tool
         )
 
-    def test_AnnotatedIPython2CWLToolConverter_compile(self):
-        annotated_python_script = os.linesep.join([
-            "import csv",
-            "input_filename: CWLFilePathInput = 'data.csv'",
-            "with open(input_filename) as f:",
-            "\tcsv_reader = csv.reader(f)",
-            "\tdata = [line for line in csv_reader]",
-            "print(data)"
-        ])
-        compiled_tar_file = os.path.join(tempfile.mkdtemp(), 'file.tar')
-        extracted_dir = tempfile.mkdtemp()
-        print('compiled at tarfile:',
-              AnnotatedIPython2CWLToolConverter(annotated_python_script)
-              .compile(Path(compiled_tar_file)))
-        with tarfile.open(compiled_tar_file, 'r') as tar:
-            tar.extractall(path=extracted_dir)
-        print(compiled_tar_file)
-        self.assertSetEqual(
-            {'notebookTool', 'tool.cwl', 'Dockerfile', 'requirements.txt', 'setup.py'},
-            set(os.listdir(extracted_dir))
-        )
-
     def test_AnnotatedIPython2CWLToolConverter_optional_arguments(self):
         annotated_python_script = os.linesep.join([
             "import csv",
@@ -557,4 +532,4 @@ def test_AnnotatedIPython2CWLToolConverter_CWLPNGFigure(self):
                 },
             },
             tool
-        )
+        )