From 7c4e404c17724e7b43066c3a6fa8a2ae4d173fca Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Thu, 30 May 2024 15:22:46 +0700 Subject: [PATCH 01/13] Cache dependencies on github actions --- .github/actions/install/action.yml | 26 ++++++++------ .../build-and-release-single-package.yml | 2 +- .github/workflows/build-and-release.yml | 2 +- .github/workflows/frontend.yml | 5 +++ .github/workflows/partial-tests.yml | 36 +++++++++++++++++-- .github/workflows/publish.yml | 4 +-- 6 files changed, 57 insertions(+), 18 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 1d24d34ccc..c4dc37e2c3 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -22,28 +22,27 @@ outputs: runs: using: "composite" steps: + - name: Replace Pipfile if pipfile-version = max + if: inputs.pipfile-version == 'max' + run: | + # Override the current Pipfile with the Pipfile containing the max versions + rm Pipfile + mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile + - uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} + cache: 'pipenv' + cache-dependency-path: '**/Pipfile' - name: Install pipenv run: pip install pipenv --upgrade shell: bash - - name: Install minimum supported dependencies - if: inputs.pipfile-version == 'min' + - name: Install pipenv dependencies run: pipenv install --dev --python=${{ inputs.python-version }} shell: bash - - name: Install maximum version supported dependencies - if: inputs.pipfile-version == 'max' - run: | - # Override the current Pipfile with the Pipfile containing the max versions - rm Pipfile - mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile - pipenv install --dev --python=${{ inputs.python-version }} - shell: bash - - name: Setup LibMagic (MacOS) if: inputs.os == 'macos-13' && inputs.install-gui run: brew install libmagic @@ -53,6 +52,11 @@ runs: if: inputs.install-gui with: node-version: 20 + cache: 'npm' + cache-dependency-path: | + frontend/taipy/package-lock.json + frontend/taipy-gui/package-lock.json + frontend/taipy-gui/dom/package-lock.json - name: Frontend Bundle Build if: inputs.install-gui diff --git a/.github/workflows/build-and-release-single-package.yml b/.github/workflows/build-and-release-single-package.yml index 43a80e1431..9dd9c19622 100644 --- a/.github/workflows/build-and-release-single-package.yml +++ b/.github/workflows/build-and-release-single-package.yml @@ -53,7 +53,7 @@ jobs: - uses: actions/checkout@v4 with: ssh-key: ${{secrets.DEPLOY_KEY}} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - uses: actions/setup-node@v4 diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b0b9cdb4f1..d49b2f9013 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -50,7 +50,7 @@ jobs: - uses: actions/checkout@v4 with: ssh-key: ${{secrets.DEPLOY_KEY}} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - uses: actions/setup-node@v4 diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 0f269a4c3a..651c2264f0 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -28,6 +28,11 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: | + frontend/taipy/package-lock.json + frontend/taipy-gui/package-lock.json + frontend/taipy-gui/dom/package-lock.json - name: Install dom dependencies working-directory: ./frontend/taipy-gui/dom diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index ec87bbc208..2ca7e17bbb 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -51,6 +51,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{matrix.python-version}} + cache: 'pipenv' + cache-dependency-path: '**/Pipfile' - name: Install Setuptools and wheel run: pip install --upgrade setuptools wheel @@ -64,9 +66,17 @@ jobs: if: matrix.os == 'macos-13' run: brew install libmagic - - uses: actions/setup-node@v4 + - name: Setup Node + if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' + uses: actions/setup-node@v4 with: node-version: 20 + cache: 'npm' + cache-dependency-path: | + frontend/taipy/package-lock.json + frontend/taipy-gui/package-lock.json + frontend/taipy-gui/dom/package-lock.json + - name: Frontend Bundle Build if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' run: pipenv run python tools/frontend/bundle_build.py @@ -128,7 +138,17 @@ jobs: core: - 'taipy/core/**' - - uses: actions/setup-python@v5 + - name: Setup Python with cache + if: steps.changes.outputs.core == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python-version}} + cache: 'pipenv' + cache-dependency-path: '**/Pipfile' + + - name: Setup Python without cache + if: steps.changes.outputs.core != 'true' + uses: actions/setup-python@v5 with: python-version: ${{matrix.python-version}} @@ -167,7 +187,17 @@ jobs: core: - 'taipy/core/**' - - uses: actions/setup-python@v5 + - name: Setup Python with cache + if: steps.changes.outputs.core == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python-version}} + cache: 'pipenv' + cache-dependency-path: '**/Pipfile' + + - name: Setup Python without cache + if: steps.changes.outputs.core != 'true' + uses: actions/setup-python@v5 with: python-version: ${{matrix.python-version}} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 516d4b42ba..6153be7fe0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.8 @@ -109,7 +109,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-versions }} From 9ecf9b9575ca5b48e0ad43619ff3a90055d05f29 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Thu, 30 May 2024 20:31:00 +0700 Subject: [PATCH 02/13] add missing shell property in action yml --- .github/actions/install/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index c4dc37e2c3..04f3849a3f 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -28,6 +28,7 @@ runs: # Override the current Pipfile with the Pipfile containing the max versions rm Pipfile mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile + shell: bash - uses: actions/setup-python@v5 with: From 3a604d4ba32e9cfc72ad193c366fc628db0426d9 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Thu, 30 May 2024 23:16:12 +0700 Subject: [PATCH 03/13] Per JR --- .github/workflows/partial-tests.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 2ca7e17bbb..29953feb35 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -146,12 +146,6 @@ jobs: cache: 'pipenv' cache-dependency-path: '**/Pipfile' - - name: Setup Python without cache - if: steps.changes.outputs.core != 'true' - uses: actions/setup-python@v5 - with: - python-version: ${{matrix.python-version}} - - name: Install pipenv if: steps.changes.outputs.core == 'true' run: pip install --upgrade pipenv @@ -195,12 +189,6 @@ jobs: cache: 'pipenv' cache-dependency-path: '**/Pipfile' - - name: Setup Python without cache - if: steps.changes.outputs.core != 'true' - uses: actions/setup-python@v5 - with: - python-version: ${{matrix.python-version}} - - name: Install pipenv if: steps.changes.outputs.core == 'true' run: pip install --upgrade pipenv From 50ddcc6da0f641608c3ca2dc94108197c8dc4887 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 00:24:08 +0700 Subject: [PATCH 04/13] Using wildcard patterns to cache npm dependencies --- .github/actions/install/action.yml | 5 +---- .github/workflows/frontend.yml | 5 +---- .github/workflows/partial-tests.yml | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 04f3849a3f..ba2a7b89ec 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -54,10 +54,7 @@ runs: with: node-version: 20 cache: 'npm' - cache-dependency-path: | - frontend/taipy/package-lock.json - frontend/taipy-gui/package-lock.json - frontend/taipy-gui/dom/package-lock.json + cache-dependency-path: '**/package-lock.json' - name: Frontend Bundle Build if: inputs.install-gui diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 651c2264f0..78ec2ff947 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -29,10 +29,7 @@ jobs: with: node-version: ${{ matrix.node-version }} cache: 'npm' - cache-dependency-path: | - frontend/taipy/package-lock.json - frontend/taipy-gui/package-lock.json - frontend/taipy-gui/dom/package-lock.json + cache-dependency-path: '**/package-lock.json' - name: Install dom dependencies working-directory: ./frontend/taipy-gui/dom diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 29953feb35..f7a29cc004 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -72,10 +72,7 @@ jobs: with: node-version: 20 cache: 'npm' - cache-dependency-path: | - frontend/taipy/package-lock.json - frontend/taipy-gui/package-lock.json - frontend/taipy-gui/dom/package-lock.json + cache-dependency-path: '**/package-lock.json' - name: Frontend Bundle Build if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' From f1c854dff06b975d1b315c786c766e886ab1d373 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 00:30:05 +0700 Subject: [PATCH 05/13] Only use the root pipfile as cache --- .github/actions/install/action.yml | 2 +- .github/workflows/partial-tests.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index ba2a7b89ec..255a9c9946 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -34,7 +34,7 @@ runs: with: python-version: ${{ inputs.python-version }} cache: 'pipenv' - cache-dependency-path: '**/Pipfile' + cache-dependency-path: 'Pipfile' - name: Install pipenv run: pip install pipenv --upgrade diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index f7a29cc004..a9c7eb5698 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -52,7 +52,7 @@ jobs: with: python-version: ${{matrix.python-version}} cache: 'pipenv' - cache-dependency-path: '**/Pipfile' + cache-dependency-path: 'Pipfile' - name: Install Setuptools and wheel run: pip install --upgrade setuptools wheel @@ -141,7 +141,7 @@ jobs: with: python-version: ${{matrix.python-version}} cache: 'pipenv' - cache-dependency-path: '**/Pipfile' + cache-dependency-path: 'Pipfile' - name: Install pipenv if: steps.changes.outputs.core == 'true' @@ -184,7 +184,7 @@ jobs: with: python-version: ${{matrix.python-version}} cache: 'pipenv' - cache-dependency-path: '**/Pipfile' + cache-dependency-path: 'Pipfile' - name: Install pipenv if: steps.changes.outputs.core == 'true' From 494579b05b5d4489e5618748d0197fff2edfdfd2 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 03:41:23 +0700 Subject: [PATCH 06/13] add support for frontend build cache --- .github/actions/install/action.yml | 31 +++++++++++- .github/workflows/frontend.yml | 24 +++++++++ .github/workflows/partial-tests.yml | 30 ++++++++++- tools/frontend/hash_source.py | 77 +++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tools/frontend/hash_source.py diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 255a9c9946..48288b9ddc 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -56,11 +56,40 @@ runs: cache: 'npm' cache-dependency-path: '**/package-lock.json' - - name: Frontend Bundle Build + - name: Hash frontend source code + if: inputs.install-gui + id: hash-frontend + run: | + pipenv run python tools/frontend/hash_frontend.py + echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + rm hash.txt + shell: bash + + - name: Restore cached frontend build if: inputs.install-gui + id: cache-fe-build-restore + uses: actions/cache@v4 + with: + path: | + taipy/gui/webapp + taipy/gui_core/lib + key: ${{ inputs.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + + - name: Frontend Bundle Build + if: inputs.install-gui && steps.cache-fe-build-restore.outputs.cache-hit != 'true' run: pipenv run python tools/frontend/bundle_build.py shell: bash + - name: Save frontend build as cache + if: inputs.install-gui + id: cache-fe-build-save + uses: actions/cache/save@v4 + with: + path: | + taipy/gui/webapp + taipy/gui_core/lib + key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }} + - name: Install Playwright if: inputs.install-gui run: pipenv run playwright install chromium --with-deps diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 78ec2ff947..9a441ad32f 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -24,6 +24,9 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: npm build and test with node ${{ matrix.node-version }} on ${{ matrix.os }} uses: actions/setup-node@v4 with: @@ -31,12 +34,33 @@ jobs: cache: 'npm' cache-dependency-path: '**/package-lock.json' + - name: Hash taipy-gui source code + id: hash-taipy-gui-fe + run: | + python tools/frontend/hash_frontend.py --taipy-gui-only + echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + rm hash.txt + - name: Restore cached frontend build + id: cache-taipy-gui-fe-build-restore + uses: actions/cache@v4 + with: + path: taipy/gui/webapp + key: ${{ matrix.os }}-taipy-gui-build-${{ steps.hash-taipy-gui-fe.outputs.HASH }} + - name: Install dom dependencies working-directory: ./frontend/taipy-gui/dom run: npm ci - name: Install dependencies run: npm ci --omit=optional - run: npm run build --if-present + + - name: Save frontend build as cache + id: cache-fe-build-save + uses: actions/cache/save@v4 + with: + path: taipy/gui/webapp + key: ${{ steps.cache-taipy-gui-fe-build-restore.outputs.cache-primary-key }} + - run: npm test - name: Code coverage diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index a9c7eb5698..56792ef5cc 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -74,10 +74,38 @@ jobs: cache: 'npm' cache-dependency-path: '**/package-lock.json' - - name: Frontend Bundle Build + - name: Hash frontend source code + if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' + id: hash-frontend + run: | + pipenv run python tools/frontend/hash_frontend.py + echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + rm hash.txt + + - name: Restore cached frontend build if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' + id: cache-fe-build-restore + uses: actions/cache@v4 + with: + path: | + taipy/gui/webapp + taipy/gui_core/lib + key: ${{ matrix.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + + - name: Frontend Bundle Build + if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build-restore.outputs.cache-hit != 'true' run: pipenv run python tools/frontend/bundle_build.py + - name: Save frontend build as cache + if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' + id: cache-fe-build-save + uses: actions/cache/save@v4 + with: + path: | + taipy/gui/webapp + taipy/gui_core/lib + key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }} + - name: Install Playwright if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' run: pipenv run playwright install chromium --with-deps diff --git a/tools/frontend/hash_source.py b/tools/frontend/hash_source.py new file mode 100644 index 0000000000..b504883993 --- /dev/null +++ b/tools/frontend/hash_source.py @@ -0,0 +1,77 @@ +# Copyright 2021-2024 Avaiga Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +import hashlib +import os +import sys + +ignore_folder = ["node_modules", "dist", "coverage", "build", "scripts", "test-config"] +ignore_file_name = [ + "DS_Store", + "env.local", + "env.development.local", + "env.test.local", + "env.production.local", + ".eslintrc.js", + ".gitignore", + "jest.config.js", + "typedoc-mkdocs.json", + ".env", + "dev.env", +] +ignore_file_extension = [".md"] + + +def hash_file(file_path): + hasher = hashlib.sha256() + with open(file_path, "rb") as f: + buf = f.read() + hasher.update(buf) + return hasher.hexdigest() + + +def hash_files_in_frontend_folder(frontend_folder): + combined_hasher = hashlib.sha256() + file_hashes = {} + lookup_fe_folder = [f"{frontend_folder}{os.sep}taipy", f"{frontend_folder}{os.sep}taipy-gui"] + if len(sys.argv) > 1 and sys.argv[1] == "--taipy-gui-only": + lookup_fe_folder.pop(0) + for root_folder in lookup_fe_folder: + # Sort before looping to ensure consistent cache key + for root, _, files in sorted(os.walk(root_folder)): + if any(ignore in root for ignore in ignore_folder): + continue + # Sort before looping to ensure consistent cache key + for file in sorted(files): + for ignore in ignore_file_name: + if ignore in file: + continue + for ignore in ignore_file_extension: + if file.endswith(ignore): + continue + file_path = os.path.join(root, file) + file_hash = hash_file(file_path) + file_hashes[file_path] = file_hash + combined_hasher.update(file_hash.encode()) + + combined_hash = combined_hasher.hexdigest() + return file_hashes, combined_hash + + +if __name__ == "__main__": + frontend_folder = "frontend" + file_hashes, combined_hash = hash_files_in_frontend_folder(frontend_folder) + # for path, file_hash in sorted(file_hashes.items()): + # print(f"{path}: {file_hash}") + print(f"Taipy Frontend hash {combined_hash}") # noqa: T201 + # write combined hash to file + with open("hash.txt", "w") as f: + f.write(combined_hash) From 2d5c103268de88b87dd7ebc460ae26dd9a892f0b Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 03:45:05 +0700 Subject: [PATCH 07/13] fix hash file name --- .github/actions/install/action.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/partial-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 48288b9ddc..d2bb1d7b4c 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -60,7 +60,7 @@ runs: if: inputs.install-gui id: hash-frontend run: | - pipenv run python tools/frontend/hash_frontend.py + pipenv run python tools/frontend/hash_source.py echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT rm hash.txt shell: bash diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 9a441ad32f..3f4c0d7453 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -37,7 +37,7 @@ jobs: - name: Hash taipy-gui source code id: hash-taipy-gui-fe run: | - python tools/frontend/hash_frontend.py --taipy-gui-only + python tools/frontend/hash_source.py --taipy-gui-only echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT rm hash.txt - name: Restore cached frontend build diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 56792ef5cc..47bb7e46c9 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -78,7 +78,7 @@ jobs: if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' id: hash-frontend run: | - pipenv run python tools/frontend/hash_frontend.py + pipenv run python tools/frontend/hash_source.py echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT rm hash.txt From 141d26d32ca103f29aaa3fbf58980662525da659 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 04:12:20 +0700 Subject: [PATCH 08/13] update cache id fe --- .github/actions/install/action.yml | 16 +++------------- .github/workflows/frontend.yml | 21 +++++++++------------ .github/workflows/partial-tests.yml | 16 +++------------- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index d2bb1d7b4c..7d5af68f84 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -61,13 +61,13 @@ runs: id: hash-frontend run: | pipenv run python tools/frontend/hash_source.py - echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT rm hash.txt shell: bash - name: Restore cached frontend build if: inputs.install-gui - id: cache-fe-build-restore + id: cache-fe-build uses: actions/cache@v4 with: path: | @@ -76,20 +76,10 @@ runs: key: ${{ inputs.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build - if: inputs.install-gui && steps.cache-fe-build-restore.outputs.cache-hit != 'true' + if: inputs.install-gui && steps.cache-fe-build.outputs.cache-hit != 'true' run: pipenv run python tools/frontend/bundle_build.py shell: bash - - name: Save frontend build as cache - if: inputs.install-gui - id: cache-fe-build-save - uses: actions/cache/save@v4 - with: - path: | - taipy/gui/webapp - taipy/gui_core/lib - key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }} - - name: Install Playwright if: inputs.install-gui run: pipenv run playwright install chromium --with-deps diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 3f4c0d7453..5ef93f1b87 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -35,31 +35,28 @@ jobs: cache-dependency-path: '**/package-lock.json' - name: Hash taipy-gui source code - id: hash-taipy-gui-fe + id: hash-gui-fe + working-directory: ./ run: | python tools/frontend/hash_source.py --taipy-gui-only - echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT rm hash.txt - name: Restore cached frontend build - id: cache-taipy-gui-fe-build-restore + id: cache-gui-fe-build uses: actions/cache@v4 with: path: taipy/gui/webapp - key: ${{ matrix.os }}-taipy-gui-build-${{ steps.hash-taipy-gui-fe.outputs.HASH }} + key: ${{ matrix.os }}-taipy-gui-build-${{ steps.hash-gui-fe.outputs.HASH }} - name: Install dom dependencies + if: steps.cache-gui-fe-build.outputs.cache-hit != 'true' working-directory: ./frontend/taipy-gui/dom run: npm ci - name: Install dependencies + if: steps.cache-gui-fe-build.outputs.cache-hit != 'true' run: npm ci --omit=optional - - run: npm run build --if-present - - - name: Save frontend build as cache - id: cache-fe-build-save - uses: actions/cache/save@v4 - with: - path: taipy/gui/webapp - key: ${{ steps.cache-taipy-gui-fe-build-restore.outputs.cache-primary-key }} + - if: steps.cache-gui-fe-build.outputs.cache-hit != 'true' + run: npm run build --if-present - run: npm test diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 47bb7e46c9..878544d635 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -79,12 +79,12 @@ jobs: id: hash-frontend run: | pipenv run python tools/frontend/hash_source.py - echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT + echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT rm hash.txt - name: Restore cached frontend build if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' - id: cache-fe-build-restore + id: cache-fe-build uses: actions/cache@v4 with: path: | @@ -93,19 +93,9 @@ jobs: key: ${{ matrix.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build - if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build-restore.outputs.cache-hit != 'true' + if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build.outputs.cache-hit != 'true' run: pipenv run python tools/frontend/bundle_build.py - - name: Save frontend build as cache - if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' - id: cache-fe-build-save - uses: actions/cache/save@v4 - with: - path: | - taipy/gui/webapp - taipy/gui_core/lib - key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }} - - name: Install Playwright if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' run: pipenv run playwright install chromium --with-deps From afa532265e929f687208ead6eee62ee94e2345b1 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 10:23:30 +0700 Subject: [PATCH 09/13] Change something on the frontend side to trigger the test --- frontend/taipy-gui/base/src/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/taipy-gui/base/src/app.ts b/frontend/taipy-gui/base/src/app.ts index 59c789281d..f6069eab1c 100644 --- a/frontend/taipy-gui/base/src/app.ts +++ b/frontend/taipy-gui/base/src/app.ts @@ -40,6 +40,7 @@ export class TaipyApp { this.appId = ""; this.path = path; this.socket = socket; + // Init socket io connection initSocket(socket, this); } From 7f55a9b78f1c7c352fbc7d126b485b4d9bdf6997 Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 10:35:06 +0700 Subject: [PATCH 10/13] use bash so windows get the correct id hash --- .github/workflows/frontend.yml | 1 + .github/workflows/partial-tests.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 5ef93f1b87..571481ca5f 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -41,6 +41,7 @@ jobs: python tools/frontend/hash_source.py --taipy-gui-only echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT rm hash.txt + shell: bash - name: Restore cached frontend build id: cache-gui-fe-build uses: actions/cache@v4 diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 878544d635..40f5ea07d4 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -81,7 +81,7 @@ jobs: pipenv run python tools/frontend/hash_source.py echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT rm hash.txt - + shell: bash - name: Restore cached frontend build if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true' id: cache-fe-build From 95b77de2f6d3c89ee7b55295a7f9cc5dc261d90f Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 10:39:52 +0700 Subject: [PATCH 11/13] use runner os instead of matrix os --- .github/actions/install/action.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/partial-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 7d5af68f84..053fe23c14 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -73,7 +73,7 @@ runs: path: | taipy/gui/webapp taipy/gui_core/lib - key: ${{ inputs.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + key: ${{ runner.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build if: inputs.install-gui && steps.cache-fe-build.outputs.cache-hit != 'true' diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 571481ca5f..81a4e7973a 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -47,7 +47,7 @@ jobs: uses: actions/cache@v4 with: path: taipy/gui/webapp - key: ${{ matrix.os }}-taipy-gui-build-${{ steps.hash-gui-fe.outputs.HASH }} + key: ${{ runner.os }}-taipy-gui-build-${{ steps.hash-gui-fe.outputs.HASH }} - name: Install dom dependencies if: steps.cache-gui-fe-build.outputs.cache-hit != 'true' diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 40f5ea07d4..528210285f 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -90,7 +90,7 @@ jobs: path: | taipy/gui/webapp taipy/gui_core/lib - key: ${{ matrix.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + key: ${{ runner.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build.outputs.cache-hit != 'true' From 49daf4905f7c6b837c107e7b4cf7dce3600fe5ab Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Fri, 31 May 2024 10:47:49 +0700 Subject: [PATCH 12/13] change naming scheme --- .github/actions/install/action.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/partial-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 053fe23c14..affdfd99aa 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -73,7 +73,7 @@ runs: path: | taipy/gui/webapp taipy/gui_core/lib - key: ${{ runner.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build if: inputs.install-gui && steps.cache-fe-build.outputs.cache-hit != 'true' diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 81a4e7973a..8e4b573baf 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -47,7 +47,7 @@ jobs: uses: actions/cache@v4 with: path: taipy/gui/webapp - key: ${{ runner.os }}-taipy-gui-build-${{ steps.hash-gui-fe.outputs.HASH }} + key: taipy-gui-frontend-build-${{ runner.os }}-${{ steps.hash-gui-fe.outputs.HASH }} - name: Install dom dependencies if: steps.cache-gui-fe-build.outputs.cache-hit != 'true' diff --git a/.github/workflows/partial-tests.yml b/.github/workflows/partial-tests.yml index 528210285f..b8c49a59ec 100644 --- a/.github/workflows/partial-tests.yml +++ b/.github/workflows/partial-tests.yml @@ -90,7 +90,7 @@ jobs: path: | taipy/gui/webapp taipy/gui_core/lib - key: ${{ runner.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }} + key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }} - name: Frontend Bundle Build if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build.outputs.cache-hit != 'true' From 97cad62e70a9e5a6762c41fc9f31d0554d7c186b Mon Sep 17 00:00:00 2001 From: dinhlongviolin1 Date: Tue, 18 Jun 2024 11:01:54 +0700 Subject: [PATCH 13/13] per FV --- tools/frontend/hash_source.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/frontend/hash_source.py b/tools/frontend/hash_source.py index b504883993..f85826f10b 100644 --- a/tools/frontend/hash_source.py +++ b/tools/frontend/hash_source.py @@ -51,12 +51,10 @@ def hash_files_in_frontend_folder(frontend_folder): continue # Sort before looping to ensure consistent cache key for file in sorted(files): - for ignore in ignore_file_name: - if ignore in file: - continue - for ignore in ignore_file_extension: - if file.endswith(ignore): - continue + if any(ignore in file for ignore in ignore_file_name): + continue + if any(file.endswith(ignore) for ignore in ignore_file_extension): + continue file_path = os.path.join(root, file) file_hash = hash_file(file_path) file_hashes[file_path] = file_hash