[BACKPORT] Add separate check pipeline (#2299) (#2302)

.github/workflows/cancel-prev.yml

@@ -9,4 +9,4 @@ jobs:
       - uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml
+          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml,checks.yml

.github/workflows/checks.yml

@@ -0,0 +1,70 @@
+name: Pre-commit Checks
+
+on: [push, pull_request_target]
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Set up conda ${{ matrix.python-version }}
+        env:
+          PYTHON: ${{ matrix.python-version }}
+        shell: bash
+        run: |
+          source ./ci/install-conda.sh
+          python -m pip install --upgrade pip setuptools wheel coverage;
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          export DEFAULT_VENV=$VIRTUAL_ENV
+
+          pip install numpy scipy cython
+          pip install -e ".[dev,extra]"
+          pip install virtualenv flake8 codespell sphinx sphinx-intl
+          conda list -n test
+
+      - name: Lint with flake8
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 mars --count --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 mars --config="default" --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Check spellings
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          codespell
+
+      - name: Check copyright headers
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          python ci/copycheck.py
+
+      - name: Check JS with ESLint
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          pushd mars/services/web/ui
+          npm install
+          npm run lint
+          popd
+
+      - name: Check documentation build
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          pushd docs
+          pip install -r requirements-doc.txt
+          make html
+          popd

.github/workflows/core-ci.yml

@@ -12,7 +12,7 @@ jobs:
         python-version: [3.6, 3.7, 3.8, 3.9, 3.8-cython]
         include:
           - { os: ubuntu-latest, python-version: 3.8-cython, no-common-tests: 1,
-              no-deploy: 1, with-cython: 1, with-flake8: 1 }
+              no-deploy: 1, with-cython: 1 }
 
     steps:
     - name: Check out code
@@ -53,7 +53,7 @@ jobs:
         pip install git+https://github.com/mars-project/pytest-asyncio.git
         pip install numpy scipy cython
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
@@ -79,16 +79,6 @@ jobs:
         retry ./.github/workflows/download-etcd.sh
         conda list -n test
 
-    - name: Lint with flake8
-      shell: bash
-      if: success() && matrix.with-flake8 == 1
-      run: |
-        source ./ci/reload-env.sh
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 mars --count --select=E9,E111,E225,E302,E303,E901,E999,F7,F63,F82,F401,F811,F821,F822,F823,F841,W291,W292,W391,W605 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 mars --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
     - name: Test with pytest
       env:
         WITH_HADOOP: ${{ matrix.with-hadoop }}

.github/workflows/os-compat-ci.yml

@@ -40,7 +40,7 @@ jobs:
         pip install git+https://github.com/mars-project/pytest-asyncio.git
         pip install numpy scipy cython
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
         conda list -n test
 
     - name: Test with pytest

.github/workflows/platform-ci.yml

@@ -59,9 +59,9 @@ jobs:
           pip install -e ".[dev,extra]"
 
           if [[ $UNAME == "windows" ]]; then
-            pip install virtualenv flaky flake8
+            pip install virtualenv flaky
           else
-            pip install virtualenv flaky flake8 ray
+            pip install virtualenv flaky ray
             if [ -n "$WITH_KUBERNETES" ]; then
               ./.github/workflows/install-minikube.sh
               pip install kubernetes

azure-pipelines.yml

@@ -70,7 +70,7 @@ jobs:
           pip install numpy scipy cython
         fi
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then

ci/copycheck.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from pathlib import PurePath
+
+_MATCH_FILES = [
+    '*.py',
+    '*.pyx',
+]
+_IGNORES = [
+    'mars/learn/**/*.pyx',
+    'mars/lib/**/*.py',
+]
+
+
+def main():
+    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    miss_files = []
+    for root, _dirs, files in os.walk(os.path.join(root_path, 'mars')):
+        for fn in files:
+            rel_path = os.path.relpath(os.path.join(root, fn), root_path)
+            if any(PurePath(rel_path).match(patt) for patt in _IGNORES):
+                continue
+            if all(not PurePath(rel_path).match(patt) for patt in _MATCH_FILES):
+                continue
+
+            file_path = os.path.join(root, fn)
+            with open(file_path, 'rb') as input_file:
+                file_lines = [line for line in input_file.read().split(b'\n')
+                              if line.startswith(b'#')]
+            comments = b'\n'.join(file_lines)
+            if b'Copyright' not in comments:
+                miss_files.append(rel_path)
+    if miss_files:
+        file_list = '\n    '.join(miss_files)
+        sys.stderr.write(f'Please add missing copyright header for files:\n'
+                         f'    {file_list}\n')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

docs/source/development/contributing.rst

@@ -74,6 +74,16 @@ the root of Mars project:
 
     python setup.py build_ext -i
 
+Rebuilding Frontend Code
+````````````````````````
+Mars uses `React <https://reactjs.org>`_ to build its frontend. You need to
+install `nodejs <https://nodejs.org>`_ to build it from source. After all
+dependencies installed, simply use command below to build your frontend code:
+
+.. code-block:: bash
+
+    python setup.py build_web
+
 Running Tests
 -------------
 It is recommended to use ``pytest`` to run Mars tests. A simple command below
@@ -102,6 +112,16 @@ before running the pytest command mentioned above. After report is generated,
 it it recommended to remove all generated C files and binaries and rebuild
 without ``CYTHON_TRACE``, as this option will reduce the performance of Mars.
 
+Check Code Styles
+-----------------
+Before proposing changes to Mars, you need to make sure your code style meets
+our requirements. Mars uses `flake8 <https://flake8.pycqa.org>`_ to check
+Python code style.  Simply run command below to check your code style:
+
+.. code-block:: bash
+
+    flake8 mars
+
 Building Documentations
 -----------------------
 Mars uses ``sphinx`` to build documents. You need to install necessary packages

docs/source/locale/zh_CN/LC_MESSAGES/development/contributing.po

@@ -8,14 +8,14 @@ msgid ""
 msgstr ""
 "Project-Id-Version: mars 0.5.0a2\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-08-03 18:42+0800\n"
+"POT-Creation-Date: 2021-08-05 17:47+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Generated-By: Babel 2.7.0\n"
+"Generated-By: Babel 2.9.1\n"
 
 #: ../../source/development/contributing.rst:4
 msgid "Contributing to Mars"
@@ -65,7 +65,6 @@ msgid "Install in Conda"
 msgstr "在 Conda 中安装"
 
 #: ../../source/development/contributing.rst:25
-#, fuzzy
 msgid ""
 "It is recommended to develop Mars with conda. When you want to install "
 "Mars for development, use the following steps to create an environment "
@@ -133,34 +132,49 @@ msgstr ""
 "命令将它们重新编译为二进制文件："
 
 #: ../../source/development/contributing.rst:78
+msgid "Rebuilding Frontend Code"
+msgstr "重新编译前端代码"
+
+#: ../../source/development/contributing.rst:79
+msgid ""
+"Mars uses `React <https://reactjs.org>`_ to build its frontend. You need "
+"to install `nodejs <https://nodejs.org>`_ to build it from source. After "
+"all dependencies installed, simply use command below to build your "
+"frontend code:"
+msgstr ""
+"Mars 使用 `React <https://reactjs.org>`_ 编写前端，你需要安装 `nodejs <"
+"https://nodejs.org>`_ 完成 Mars 前端代码的构建。安装完成所有依赖后，可以"
+"使用下面的代码进行构建："
+
+#: ../../source/development/contributing.rst:88
 msgid "Running Tests"
 msgstr "运行测试用例"
 
-#: ../../source/development/contributing.rst:79
+#: ../../source/development/contributing.rst:89
 msgid ""
 "It is recommended to use ``pytest`` to run Mars tests. A simple command "
 "below will run all the tests of Mars:"
 msgstr ""
 "我们推荐使用 ``pytest`` 运行 Mars 测试用例。只需要一行命令即可运行 Mars "
 "的所有用例："
 
-#: ../../source/development/contributing.rst:86
+#: ../../source/development/contributing.rst:96
 msgid "If you want to generate a coverage report as well, you can run:"
 msgstr "如果你希望在运行的同时生成代码覆盖报告，可以运行："
 
-#: ../../source/development/contributing.rst:92
+#: ../../source/development/contributing.rst:102
 msgid "Coverage report will be put into the directory ``htmlcov``."
 msgstr "代码覆盖报告将会被放在 ``htmlcov`` 目录下。"
 
-#: ../../source/development/contributing.rst:94
+#: ../../source/development/contributing.rst:104
 msgid ""
 "The command above does not contain coverage data for Cython files by "
 "default. To obtain coverage data about Cython files, you can run"
 msgstr ""
 "上面的命令默认不包含 Cython 文件的覆盖率。为获得 Cython 文件的覆盖数据，"
 "可以运行"
 
-#: ../../source/development/contributing.rst:101
+#: ../../source/development/contributing.rst:111
 msgid ""
 "before running the pytest command mentioned above. After report is "
 "generated, it it recommended to remove all generated C files and binaries"
@@ -171,11 +185,26 @@ msgstr ""
 "生成的 C 文件和二进制文件，此后重新使用不带 ``CYTHON_TRACE`` 环境变量的"
 "命令编译，因为该选项会降低 Mars 的性能。"
 
-#: ../../source/development/contributing.rst:106
+#: ../../source/development/contributing.rst:116
+msgid "Check Code Styles"
+msgstr "检查代码样式"
+
+#: ../../source/development/contributing.rst:117
+msgid ""
+"Before proposing changes to Mars, you need to make sure your code style "
+"meets our requirements. Mars uses `flake8 <https://flake8.pycqa.org>`_ to"
+" check Python code style.  Simply run command below to check your code "
+"style:"
+msgstr ""
+"在向 Mars 提交代码前，你需要保证你的代码样式符合 Mars 项目的要求。Mars "
+"使用 `flake8 <https://flake8.pycqa.org>`_ 检查代码样式。你可以使用下面的"
+"代码运行检查："
+
+#: ../../source/development/contributing.rst:126
 msgid "Building Documentations"
 msgstr "编译文档"
 
-#: ../../source/development/contributing.rst:107
+#: ../../source/development/contributing.rst:127
 msgid ""
 "Mars uses ``sphinx`` to build documents. You need to install necessary "
 "packages with the command below to install these dependencies and build "
@@ -184,11 +213,11 @@ msgstr ""
 "Mars 使用 ``sphinx`` 编译文档。在编译 HTML 格式的文档前，你需要使用下面的"
 "命令安装必要的依赖包"
 
-#: ../../source/development/contributing.rst:117
+#: ../../source/development/contributing.rst:137
 msgid "The built documents are in ``docs/build/html`` directory."
 msgstr "编译完成的 HTML 文档位于 ``docs/build/html`` 目录下。"
 
-#: ../../source/development/contributing.rst:119
+#: ../../source/development/contributing.rst:139
 msgid ""
 "When you want to create translations of Mars documents, you may append "
 "``-l <locale>`` after the ``I18NSPHINXLANGS`` variable in ``Makefile``. "
@@ -201,7 +230,7 @@ msgstr ""
 "命令为文档生成翻译文件（``*.po``），生成后的文件位于 ``docs/source/locale"
 "/<locale>/LC_MESSAGES`` 下："
 
-#: ../../source/development/contributing.rst:130
+#: ../../source/development/contributing.rst:150
 msgid ""
 "After that you can translate Mars documents into your language. Note that"
 " when you run ``make gettext`` again, translations will be broken into a "
@@ -212,11 +241,11 @@ msgstr ""
 "make gettext``，翻译后的文字将被转为固定宽度文字。对于中文译者，为达到该"
 "效果，需要安装 ``jieba`` 分词包。"
 
-#: ../../source/development/contributing.rst:135
+#: ../../source/development/contributing.rst:155
 msgid "When you finish translation, you can run"
 msgstr "在完成翻译后，你可以执行"
 
-#: ../../source/development/contributing.rst:143
+#: ../../source/development/contributing.rst:163
 msgid "to build the document in the language you just translated into."
 msgstr "以使用你刚才翻译的语言编译文档。"
 

mars/learn/cluster/_k_means_elkan.pyx

@@ -194,7 +194,7 @@ def update_chunk_dense(
 
 cdef void _update_chunk_dense(
         floating *X,                             # IN
-        # expecting C alinged 2D array. XXX: Can be
+        # expecting C aligned 2D array. XXX: Can be
         # replaced by const memoryview when cython min
         # version is >= 0.3
         floating[::1] sample_weight,             # IN

mars/learn/cluster/_k_means_lloyd.pyx

@@ -46,7 +46,7 @@ def update_chunk_dense(
 
 cdef void _update_chunk_dense(
         floating *X,                          # IN
-        # expecting C alinged 2D array. XXX: Can be
+        # expecting C aligned 2D array. XXX: Can be
         # replaced by const memoryview when cython min
         # version is >= 0.3
         floating[::1] sample_weight,          # IN