Skip to content

Commit

Permalink
add support for frontend build cache
Browse files Browse the repository at this point in the history
  • Loading branch information
dinhlongviolin1 committed May 30, 2024
1 parent f1c854d commit 494579b
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 2 deletions.
31 changes: 30 additions & 1 deletion .github/actions/install/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,40 @@ runs:
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Frontend Bundle Build
- name: Hash frontend source code
if: inputs.install-gui
id: hash-frontend
run: |
pipenv run python tools/frontend/hash_frontend.py
echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
shell: bash

- name: Restore cached frontend build
if: inputs.install-gui
id: cache-fe-build-restore
uses: actions/cache@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: ${{ inputs.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }}

- name: Frontend Bundle Build
if: inputs.install-gui && steps.cache-fe-build-restore.outputs.cache-hit != 'true'
run: pipenv run python tools/frontend/bundle_build.py
shell: bash

- name: Save frontend build as cache
if: inputs.install-gui
id: cache-fe-build-save
uses: actions/cache/save@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }}

- name: Install Playwright
if: inputs.install-gui
run: pipenv run playwright install chromium --with-deps
Expand Down
24 changes: 24 additions & 0 deletions .github/workflows/frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,43 @@ jobs:

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: npm build and test with node ${{ matrix.node-version }} on ${{ matrix.os }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Hash taipy-gui source code
id: hash-taipy-gui-fe
run: |
python tools/frontend/hash_frontend.py --taipy-gui-only
echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
- name: Restore cached frontend build
id: cache-taipy-gui-fe-build-restore
uses: actions/cache@v4
with:
path: taipy/gui/webapp
key: ${{ matrix.os }}-taipy-gui-build-${{ steps.hash-taipy-gui-fe.outputs.HASH }}

- name: Install dom dependencies
working-directory: ./frontend/taipy-gui/dom
run: npm ci
- name: Install dependencies
run: npm ci --omit=optional
- run: npm run build --if-present

- name: Save frontend build as cache
id: cache-fe-build-save
uses: actions/cache/save@v4
with:
path: taipy/gui/webapp
key: ${{ steps.cache-taipy-gui-fe-build-restore.outputs.cache-primary-key }}

- run: npm test

- name: Code coverage
Expand Down
30 changes: 29 additions & 1 deletion .github/workflows/partial-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,38 @@ jobs:
cache: 'npm'
cache-dependency-path: '**/package-lock.json'

- name: Frontend Bundle Build
- name: Hash frontend source code
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
id: hash-frontend
run: |
pipenv run python tools/frontend/hash_frontend.py
echo "HASH=$(hash.txt)" >> $GITHUB_OUTPUT
rm hash.txt
- name: Restore cached frontend build
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
id: cache-fe-build-restore
uses: actions/cache@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: ${{ matrix.os }}-frontend-build-${{ steps.hash-frontend.outputs.HASH }}

- name: Frontend Bundle Build
if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build-restore.outputs.cache-hit != 'true'
run: pipenv run python tools/frontend/bundle_build.py

- name: Save frontend build as cache
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
id: cache-fe-build-save
uses: actions/cache/save@v4
with:
path: |
taipy/gui/webapp
taipy/gui_core/lib
key: ${{ steps.cache-fe-build-restore.outputs.cache-primary-key }}

- name: Install Playwright
if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
run: pipenv run playwright install chromium --with-deps
Expand Down
77 changes: 77 additions & 0 deletions tools/frontend/hash_source.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Copyright 2021-2024 Avaiga Private Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import hashlib
import os
import sys

ignore_folder = ["node_modules", "dist", "coverage", "build", "scripts", "test-config"]
ignore_file_name = [
"DS_Store",
"env.local",
"env.development.local",
"env.test.local",
"env.production.local",
".eslintrc.js",
".gitignore",
"jest.config.js",
"typedoc-mkdocs.json",
".env",
"dev.env",
]
ignore_file_extension = [".md"]


def hash_file(file_path):
hasher = hashlib.sha256()
with open(file_path, "rb") as f:
buf = f.read()
hasher.update(buf)
return hasher.hexdigest()


def hash_files_in_frontend_folder(frontend_folder):
combined_hasher = hashlib.sha256()
file_hashes = {}
lookup_fe_folder = [f"{frontend_folder}{os.sep}taipy", f"{frontend_folder}{os.sep}taipy-gui"]
if len(sys.argv) > 1 and sys.argv[1] == "--taipy-gui-only":
lookup_fe_folder.pop(0)
for root_folder in lookup_fe_folder:
# Sort before looping to ensure consistent cache key
for root, _, files in sorted(os.walk(root_folder)):
if any(ignore in root for ignore in ignore_folder):
continue
# Sort before looping to ensure consistent cache key
for file in sorted(files):
for ignore in ignore_file_name:
if ignore in file:
continue
for ignore in ignore_file_extension:
if file.endswith(ignore):
continue
file_path = os.path.join(root, file)
file_hash = hash_file(file_path)
file_hashes[file_path] = file_hash
combined_hasher.update(file_hash.encode())

combined_hash = combined_hasher.hexdigest()
return file_hashes, combined_hash


if __name__ == "__main__":
frontend_folder = "frontend"
file_hashes, combined_hash = hash_files_in_frontend_folder(frontend_folder)
# for path, file_hash in sorted(file_hashes.items()):
# print(f"{path}: {file_hash}")
print(f"Taipy Frontend hash {combined_hash}") # noqa: T201
# write combined hash to file
with open("hash.txt", "w") as f:
f.write(combined_hash)

0 comments on commit 494579b

Please sign in to comment.