Skip to content

Commit

Permalink
feat(python): replace zenroom.h in favour of zencode-exec (#725)
Browse files Browse the repository at this point in the history
* feat: use zencode-exec instead of zenroom.h in python bindings

* fix: one more python-sdist dependency

* fix: add extra argument
  • Loading branch information
albertolerda authored Aug 28, 2023
1 parent 2654cad commit 7421f08
Show file tree
Hide file tree
Showing 9 changed files with 106 additions and 307 deletions.
186 changes: 46 additions & 140 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ jobs:
if: runner.os == 'Linux'
- run: brew install zsh
if: runner.os == 'macOS'
- run: |
cd ../..
make linux
cp src/zencode-exec /usr/local/bin/
- run: |
./prepare.sh
pipx install meson
Expand Down Expand Up @@ -350,142 +354,6 @@ jobs:
./bindings/javascript/README.md
./bindings/javascript/dist/
python-build-check:
name: 🐍 Python build & checks
needs: [semantic-release]
runs-on: ${{ matrix.os }}
if: ${{ needs.semantic-release.outputs.release == 'True' }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
python-version: [38, 39, 310, 311]
platform_id: [manylinux_x86_64, manylinux_i686, manylinux_aarch64, macosx_x86_64]
exclude:
- os: macos-latest
platform_id: manylinux_x86_64
- os: macos-latest
platform_id: manylinux_i686
- os: macos-latest
platform_id: manylinux_aarch64
- os: ubuntu-latest
platform_id: macosx_x86_64
steps:
- uses: actions/checkout@v3
with:
ref: master
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: docker/setup-qemu-action@v2
if: runner.os == 'Linux'
with:
platforms: all
- name: Build wheels
env:
CIBW_BUILD: cp${{ matrix.python-version }}-${{ matrix.platform_id }}
CIBW_ARCHS_LINUX: auto aarch64
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_24
CIBW_MANYLINUX_I686_IMAGE: manylinux_2_24
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_24
CIBW_BEFORE_BUILD_LINUX: |
# deb.debian.org -> archive.debian.org
sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
# security.debian.org -> archive.debian.org/debian-security/
sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list
# remove line that contains source stretch-update
sed -i '/stretch-updates/d' /etc/apt/sources.list
# update and install
apt-get update && apt-get install -y cmake zsh vim && pip install meson ninja
CIBW_BEFORE_BUILD_MACOS: brew install cmake zsh && pip install meson ninja
CIBW_TEST_EXTRAS: "test"
CIBW_TEST_COMMAND: "pytest -s {project}/tests"
run: |
# Remove shared library
rm -f bindings/golang/zenroom/lib/libzenroom.so
cd bindings/python3
./prepare.sh
cp -r ../../.git .
pip install cibuildwheel
cibuildwheel --output-dir wheelhouse
- uses: actions/upload-artifact@v3
with:
path: ./bindings/python3/wheelhouse/*.whl

python-build-check-armv7:
name: 🐍 Python build & check on armv7
runs-on: ubuntu-22.04
needs: [semantic-release]
if: ${{ needs.semantic-release.outputs.release == 'True' }}
strategy:
matrix:
python_version: ['3.8', '3.9', '3.10', '3.11']
steps:
- uses: actions/checkout@v3
with:
ref: master
fetch-depth: 0
- uses: uraimo/run-on-arch-action@v2
name: Build wheels
with:
arch: armv7
distro: ubuntu22.04
install: |
apt-get update
apt-get install -y git vim zsh software-properties-common cmake ninja-build
run: |
# preparation
git config --global --add safe.directory /home/runner/work/Zenroom/Zenroom
# default python version is 3.10
if [[ ${{ matrix.python_version }} != '3.10' ]]; then
add-apt-repository -y ppa:deadsnakes/ppa && apt-get update
apt-get install -y python${{ matrix.python_version }} python${{ matrix.python_version }}-distutils
py_3=`which python3`
py_310=`which python3.10`
py=`which python${{ matrix.python_version }}`
update-alternatives --install $py_3 python3 $py_310 1
update-alternatives --install $py_3 python3 $py 2
update-alternatives --set python3 $py
fi
apt-get install -y python3-pip && python3 -m pip install --upgrade pip
pip install meson wheel pytest schema
# build
rm -f bindings/golang/zenroom/lib/libzenroom.so
cd bindings/python3
./prepare.sh
python3 setup.py bdist_wheel
mkdir ../../wheelhouse && cp dist/*.whl ../../wheelhouse
# tests
pip install ./dist/*.whl
pytest -s tests
- uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl

python-sdist:
name: 🐍 Python create source distribution
runs-on: ubuntu-latest
needs: [semantic-release]
if: ${{ needs.semantic-release.outputs.release == 'True' }}
defaults:
run:
working-directory: bindings/python3
steps:
- uses: actions/checkout@v3
with:
ref: master
fetch-depth: 0
- run: |
sudo apt-get install --force-yes cmake vim zsh
pip install meson ninja twine
./prepare.sh
- run: pipx run build --sdist
- run: twine check dist/*.tar.gz
- uses: actions/upload-artifact@v3
with:
path: ./bindings/python3/dist/*.tar.gz

build-release-on-ubuntu:
name: 🐧 Binary builds on Ubuntu
runs-on: ubuntu-latest
Expand Down Expand Up @@ -543,6 +411,44 @@ jobs:
src/libzenroom_dll.lib
src/zenroom.dll
file-changes:
name: "🗃️ Check file changes"
runs-on: ubuntu-latest
needs: [semantic-release]
if: "needs.semantic-release.outputs.release == 'True'"
outputs:
python3: ${{ steps.filter.outputs.python3 }}
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
python3:
- 'bindings/python3/**'
python-sdist:
name: 🐍 Python create source distribution
runs-on: ubuntu-latest
needs: [semantic-release, file-changes]
if: "needs.semantic-release.outputs.release == 'True' && needs.file-changes.outputs.python3 == 'true'"
defaults:
run:
working-directory: bindings/python3
steps:
- uses: actions/checkout@v3
with:
ref: master
fetch-depth: 0
- run: |
pip install ninja twine
./prepare.sh
- run: pipx run build --sdist
- run: twine check dist/*.tar.gz
- uses: actions/upload-artifact@v3
with:
path: ./bindings/python3/dist/*.tar.gz

build-release-on-osx:
name: 🍎 mac osx release build
needs: [semantic-release]
Expand All @@ -565,7 +471,7 @@ jobs:
npm-release:
name: 📦 npm release
runs-on: ubuntu-latest
needs: [js-build-release, python-build-check, python-build-check-armv7, python-sdist, build-release-on-osx, build-release-on-ubuntu]
needs: [js-build-release, build-release-on-osx, build-release-on-ubuntu]
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -588,7 +494,7 @@ jobs:

pypi-release:
name: 📦 PyPI release
needs: [js-build-release, python-build-check, python-build-check-armv7, python-sdist, build-release-on-osx, build-release-on-ubuntu]
needs: [js-build-release, build-release-on-osx, build-release-on-ubuntu]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v3
Expand All @@ -602,7 +508,7 @@ jobs:

draft-binary-release:
name: 📦 Pack release
needs: [semantic-release, js-build-release, python-build-check, python-build-check-armv7, python-sdist, build-release-on-osx, build-release-on-ubuntu]
needs: [semantic-release, js-build-release, build-release-on-osx, build-release-on-ubuntu]
runs-on: ubuntu-latest
steps:
- name: download binary artifacts
Expand All @@ -627,7 +533,7 @@ jobs:

remove-tag-on-fail:
name: 🧻 Remove tag on fail
needs: [semantic-release, js-build-release, python-build-check, python-build-check-armv7, python-sdist, build-release-on-osx, build-release-on-ubuntu]
needs: [semantic-release, js-build-release, build-release-on-osx, build-release-on-ubuntu]
runs-on: ubuntu-latest
if: ${{ (failure() || cancelled()) && needs.semantic-release.outputs.release == 'True' }}
steps:
Expand Down
5 changes: 0 additions & 5 deletions bindings/python3/prepare.sh
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
#!/bin/bash
mkdir -p src/docs/pages
cp ../../docs/pages/python.md src/docs/pages/python.md
cp ../../Makefile src/Makefile
cp -r ../../build src/build
cp -r ../../lib src/lib
cp -r ../../src src/src
cp -r ../../test src/test

py_version=`git describe --tags --abbrev=0 | sed 's/^v//'`
py_version_hash=${py_version}
Expand Down
105 changes: 3 additions & 102 deletions bindings/python3/setup.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,8 @@
import os
import subprocess
import time
from setuptools import Extension, setup

ECP_CURVE = 'BLS381'
ECDH_CURVE = 'SECP256K1'

PYTHON_ROOT = os.getcwd()
ZENROOM_ROOT = os.path.join( os.getcwd(), 'src')

ZENROOM_LIB_ROOT_REL= 'src/src'
LUA_ROOT = os.path.join(ZENROOM_ROOT, 'lib/lua54/src')
MILAGRO_INCLUDE_DIR = os.path.join(ZENROOM_ROOT, 'lib/milagro-crypto-c/include')
QP_ROOT = os.path.join(ZENROOM_ROOT, 'lib/pqclean')
ED25519_INCLUDE_DIR = os.path.join(ZENROOM_ROOT, 'lib/ed25519-donna')
MIMALLOC_INCLUDE_DIR = os.path.join(ZENROOM_ROOT, 'lib/mimalloc/include')
ZENROOM_ROOT = os.path.join(os.getcwd(), 'src')

def get_versions():
with open(os.path.join(ZENROOM_ROOT, 'git_utils')) as f:
Expand All @@ -23,96 +11,10 @@ def get_versions():
return python_version, zenroom_version


ZENROOM_SOURCES = [
'rmd160.c',
'base58.c',
'segwit_addr.c',
'cortex_m.c',
'encoding.c',
'lua_functions.c',
'lualibs_detected.c',
'lua_modules.c',
'lua_shims.c',
'mutt_sprintf.c',
'randombytes.c',
'repl.c',
'zen_aes.c',
'zen_big.c',
'zen_config.c',
'zen_ecdh.c',
'zen_ecp2.c',
'zen_ecp.c',
'zen_error.c',
'zen_fp12.c',
'zen_hash.c',
'zen_io.c',
'zen_memory.c',
'zen_octet.c',
'zen_parse.c',
'zen_qp.c',
'zen_float.c',
'zen_ed.c',
'zen_random.c',
'zenroom.c',
'zen_ecdh_factory.c'
]

# Add meson build variables to the environment
# source_root = os.path.join(ZENROOM_ROOT, 'build')
meson_root = os.path.join(ZENROOM_ROOT, 'meson')
# env = dict(os.environ,
# MESON_SOURCE_ROOT=source_root,
# MESON_BUILD_ROOT=PYTHON_ROOT)

os.chdir(ZENROOM_ROOT)
subprocess.check_call(['make', 'clean'])
subprocess.check_call(['make', 'meson'])
python_version, zenroom_version = get_versions()
os.chdir(PYTHON_ROOT)

zenroom_lib = Extension('zenroom',
sources=[
os.path.join(ZENROOM_LIB_ROOT_REL, src)
for src in ZENROOM_SOURCES
],
# + [zenroom_ecdh_factory] + [
# os.path.join(LUA_ROOT, src)
# for src in LUA_SOURCES
# ],
include_dirs=[
os.getcwd(),
ZENROOM_LIB_ROOT_REL,
LUA_ROOT,
MILAGRO_INCLUDE_DIR,
ED25519_INCLUDE_DIR,
MIMALLOC_INCLUDE_DIR,
os.path.join(meson_root, 'milagro-crypto-c/include'),
os.path.join(meson_root, 'milagro-crypto-c/include'),
# os.path.join(QP_ROOT, 'dilithium2'),
# os.path.join(QP_ROOT, 'kyber512'),
# os.path.join(QP_ROOT, 'sntrup761'),
],

extra_compile_args=[
'-DVERSION="' + zenroom_version + '"',
# '-DLUA_COMPAT_5_3',
# '-DLUA_COMPAT_MODULE',
# '-DLUA_COMPAT_BITLIB'
],
extra_objects=[
os.path.join(meson_root, 'milagro-crypto-c/lib/libamcl_core.a'),
os.path.join(meson_root, 'milagro-crypto-c/lib/libamcl_core.a'),
os.path.join(meson_root, 'milagro-crypto-c/lib/libamcl_curve_' + ECDH_CURVE + '.a'),
os.path.join(meson_root, 'milagro-crypto-c/lib/libamcl_pairing_' + ECP_CURVE + '.a'),
os.path.join(meson_root, 'milagro-crypto-c/lib/libamcl_curve_' + ECP_CURVE + '.a'),
os.path.join(meson_root, 'libqpz.a'),
os.path.join(meson_root, 'liblua.a'),
os.path.join(meson_root, 'libed25519.a'),
os.path.join(meson_root, 'libmimalloc-static.a'),
],
extra_link_args=['-lm']
)


def get_readme():
try:
Expand All @@ -128,8 +30,8 @@ def get_readme():
version=python_version,
long_description=get_readme(),
long_description_content_type='text/markdown',
license = 'AGPLv3',
keywords = 'zenroom crypto-language-processing virtual-machine blockchain crypto ecc dyne ecdh ecdsa zero-knowledge-proofs javascript npm ecp2 miller-loop hamming-distance elgamal aes-gcm aead seccomp goldilocks'.split(),
license='AGPLv3',
keywords='zenroom crypto-language-processing virtual-machine blockchain crypto ecc dyne ecdh ecdsa zero-knowledge-proofs javascript npm ecp2 miller-loop hamming-distance elgamal aes-gcm aead seccomp goldilocks'.split(),
url='https://github.com/dyne/Zenroom',
author='Danilo Spinella, David Dashyan, Puria Nafisi Azizi',
author_email='[email protected], [email protected], [email protected]',
Expand All @@ -154,7 +56,6 @@ def get_readme():
},
packages=['zenroom'],
include_package_data=True,
ext_modules=[zenroom_lib],
python_requires='>=3.8, <4',
extras_require={
'dev': [],
Expand Down
Loading

0 comments on commit 7421f08

Please sign in to comment.