Skip to content

Commit

Permalink
Cache dependencies on github actions (#1296) (#1337)
Browse files Browse the repository at this point in the history
  • Loading branch information
dinhlongviolin1 authored Jun 21, 2024
1 parent 7e43f3a commit fbadcf3
Show file tree
Hide file tree
Showing 8 changed files with 175 additions and 21 deletions.
45 changes: 33 additions & 12 deletions .github/actions/install/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,28 @@ outputs:
runs:
using: "composite"
steps:
- name: Replace Pipfile if pipfile-version = max
if: inputs.pipfile-version == 'max'
run: |
# Override the current Pipfile with the Pipfile containing the max versions
rm Pipfile
mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile
shell: bash

- uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
cache: 'pipenv'
cache-dependency-path: 'Pipfile'

- name: Install pipenv
run: pip install pipenv --upgrade
shell: bash

- name: Install minimum supported dependencies
if: inputs.pipfile-version == 'min'
- name: Install pipenv dependencies
run: pipenv install --dev --python=${{ inputs.python-version }}
shell: bash

- name: Install maximum version supported dependencies
if: inputs.pipfile-version == 'max'
run: |
# Override the current Pipfile with the Pipfile containing the max versions
rm Pipfile
mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile
pipenv install --dev --python=${{ inputs.python-version }}
shell: bash

- name: Setup LibMagic (MacOS)
if: inputs.os == 'macos-13' && inputs.install-gui
run: brew install libmagic
Expand All @@ -53,9 +53,30 @@ runs:
if: inputs.install-gui
with:
node-version: 20
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Frontend Bundle Build
- name: Hash frontend source code
if: inputs.install-gui
id: hash-frontend
run: |
pipenv run python tools/frontend/hash_source.py
echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
shell: bash

- name: Restore cached frontend build
if: inputs.install-gui
id: cache-fe-build
uses: actions/cache@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }}

- name: Frontend Bundle Build
if: inputs.install-gui && steps.cache-fe-build.outputs.cache-hit != 'true'
run: pipenv run python tools/frontend/bundle_build.py
shell: bash

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-and-release-single-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:
- uses: actions/checkout@v4
with:
ssh-key: ${{secrets.DEPLOY_KEY}}
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: 3.9
- uses: actions/setup-node@v4
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-and-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
- uses: actions/checkout@v4
with:
ssh-key: ${{secrets.DEPLOY_KEY}}
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: 3.9
- uses: actions/setup-node@v4
Expand Down
26 changes: 25 additions & 1 deletion .github/workflows/frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,41 @@ jobs:

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: npm build and test with node ${{ matrix.node-version }} on ${{ matrix.os }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Hash taipy-gui source code
id: hash-gui-fe
working-directory: ./
run: |
python tools/frontend/hash_source.py --taipy-gui-only
echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
shell: bash
- name: Restore cached frontend build
id: cache-gui-fe-build
uses: actions/cache@v4
with:
path: taipy/gui/webapp
key: taipy-gui-frontend-build-${{ runner.os }}-${{ steps.hash-gui-fe.outputs.HASH }}

- name: Install dom dependencies
if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
working-directory: ./frontend/taipy-gui/dom
run: npm ci
- name: Install dependencies
if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
run: npm ci --omit=optional
- run: npm run build --if-present
- if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
run: npm run build --if-present

- run: npm test

- name: Code coverage
Expand Down
41 changes: 37 additions & 4 deletions .github/workflows/partial-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.python-version}}
cache: 'pipenv'
cache-dependency-path: 'Pipfile'
- name: Install Setuptools and wheel
run: pip install --upgrade setuptools wheel

Expand All @@ -64,11 +66,34 @@ jobs:
if: matrix.os == 'macos-13'
run: brew install libmagic

- uses: actions/setup-node@v4
- name: Setup Node
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
uses: actions/setup-node@v4
with:
node-version: 20
- name: Frontend Bundle Build
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Hash frontend source code
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
id: hash-frontend
run: |
pipenv run python tools/frontend/hash_source.py
echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
shell: bash
- name: Restore cached frontend build
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
id: cache-fe-build
uses: actions/cache@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }}

- name: Frontend Bundle Build
if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build.outputs.cache-hit != 'true'
run: pipenv run python tools/frontend/bundle_build.py

- name: Install Playwright
Expand Down Expand Up @@ -128,9 +153,13 @@ jobs:
core:
- 'taipy/core/**'
- uses: actions/setup-python@v5
- name: Setup Python with cache
if: steps.changes.outputs.core == 'true'
uses: actions/setup-python@v5
with:
python-version: ${{matrix.python-version}}
cache: 'pipenv'
cache-dependency-path: 'Pipfile'

- name: Install pipenv
if: steps.changes.outputs.core == 'true'
Expand Down Expand Up @@ -167,9 +196,13 @@ jobs:
core:
- 'taipy/core/**'
- uses: actions/setup-python@v5
- name: Setup Python with cache
if: steps.changes.outputs.core == 'true'
uses: actions/setup-python@v5
with:
python-version: ${{matrix.python-version}}
cache: 'pipenv'
cache-dependency-path: 'Pipfile'

- name: Install pipenv
if: steps.changes.outputs.core == 'true'
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: 3.8

Expand Down Expand Up @@ -109,7 +109,7 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-versions }}

Expand Down
1 change: 1 addition & 0 deletions frontend/taipy-gui/base/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export class TaipyApp {
this.routes = undefined;
this.path = path;
this.socket = socket;
// Init socket io connection
initSocket(socket, this);
}

Expand Down
75 changes: 75 additions & 0 deletions tools/frontend/hash_source.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Copyright 2021-2024 Avaiga Private Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import hashlib
import os
import sys

ignore_folder = ["node_modules", "dist", "coverage", "build", "scripts", "test-config"]
ignore_file_name = [
"DS_Store",
"env.local",
"env.development.local",
"env.test.local",
"env.production.local",
".eslintrc.js",
".gitignore",
"jest.config.js",
"typedoc-mkdocs.json",
".env",
"dev.env",
]
ignore_file_extension = [".md"]


def hash_file(file_path):
hasher = hashlib.sha256()
with open(file_path, "rb") as f:
buf = f.read()
hasher.update(buf)
return hasher.hexdigest()


def hash_files_in_frontend_folder(frontend_folder):
combined_hasher = hashlib.sha256()
file_hashes = {}
lookup_fe_folder = [f"{frontend_folder}{os.sep}taipy", f"{frontend_folder}{os.sep}taipy-gui"]
if len(sys.argv) > 1 and sys.argv[1] == "--taipy-gui-only":
lookup_fe_folder.pop(0)
for root_folder in lookup_fe_folder:
# Sort before looping to ensure consistent cache key
for root, _, files in sorted(os.walk(root_folder)):
if any(ignore in root for ignore in ignore_folder):
continue
# Sort before looping to ensure consistent cache key
for file in sorted(files):
if any(ignore in file for ignore in ignore_file_name):
continue
if any(file.endswith(ignore) for ignore in ignore_file_extension):
continue
file_path = os.path.join(root, file)
file_hash = hash_file(file_path)
file_hashes[file_path] = file_hash
combined_hasher.update(file_hash.encode())

combined_hash = combined_hasher.hexdigest()
return file_hashes, combined_hash


if __name__ == "__main__":
frontend_folder = "frontend"
file_hashes, combined_hash = hash_files_in_frontend_folder(frontend_folder)
# for path, file_hash in sorted(file_hashes.items()):
# print(f"{path}: {file_hash}")
print(f"Taipy Frontend hash {combined_hash}") # noqa: T201
# write combined hash to file
with open("hash.txt", "w") as f:
f.write(combined_hash)

0 comments on commit fbadcf3

Please sign in to comment.