From 748d5adfab3acaac1bd4aca40702d6138e5a1256 Mon Sep 17 00:00:00 2001 From: Chris Hewison Date: Tue, 19 Jan 2021 18:01:44 +0900 Subject: [PATCH] Add wasmer web assembly engine as POC This adds the wasmer-d submodule and an engine which can run in d at runtime. Currently the Engine is only run in the unit tests in `WasmEngine.d` to experiment with running Web Assembly programs. `dtest=WasmEngine dub test` --- .github/workflows/documentation.yml | 5 + .github/workflows/integration.yml | 5 + .github/workflows/main.yml | 5 + .gitmodules | 3 + Dockerfile | 8 ++ README.md | 1 + dub.json | 3 +- dub.selections.json | 3 +- source/agora/node/WasmEngine.d | 155 ++++++++++++++++++++++++++++ submodules/wasmer-d | 1 + 10 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 source/agora/node/WasmEngine.d create mode 160000 submodules/wasmer-d diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f8f83483e7b..6629be7cd8d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -37,6 +37,11 @@ jobs: sudo apt-get install libsodium-dev sudo apt-get install clang + - name: Install Wasmer + run: | + curl https://get.wasmer.io -sSfL | WASMER_DIR=${HOME}/.wasmer sh + echo 'WASMER_DIR=${HOME}/.wasmer' >> $GITHUB_ENV + - name: Build documentation run: | AGORA_VERSION=HEAD dub build -b ddox diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7b3d2aeef6d..6abeaae8437 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -27,6 +27,11 @@ jobs: sudo apt-get update sudo apt-get install libsodium-dev libsqlite3-dev clang + - name: Install Wasmer + run: | + curl https://get.wasmer.io -sSfL | WASMER_DIR=${HOME}/.wasmer sh + echo 'WASMER_DIR=${HOME}/.wasmer' >> $GITHUB_ENV + - name: Checkout faucet uses: actions/checkout@v2 with: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd9700e7b46..2065806edf0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,12 +62,17 @@ jobs: brew install coreutils libsodium pkg-config echo "LIBRARY_PATH=${LD_LIBRARY_PATH-}:/usr/local/lib/" >> $GITHUB_ENV echo "PKG_CONFIG_PATH=/usr/local/opt/sqlite/lib/pkgconfig:/usr/local/opt/openssl@1.1/lib/pkgconfig/" >> $GITHUB_ENV + env | grep HOME + curl https://get.wasmer.io -sSfL | WASMER_DIR=${HOME}/.wasmer sh + echo 'WASMER_DIR=${HOME}/.wasmer' >> $GITHUB_ENV - name: '[Linux] Install dependencies & setup environment' if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install libsodium-dev libsqlite3-dev clang + curl https://get.wasmer.io -sSfL | WASMER_DIR=${HOME}/.wasmer sh + echo 'WASMER_DIR=${HOME}/.wasmer' >> $GITHUB_ENV - name: '[Windows] Install dependencies & setup environment' if: runner.os == 'Windows' diff --git a/.gitmodules b/.gitmodules index 756b4740731..e55a3f83e98 100644 --- a/.gitmodules +++ b/.gitmodules @@ -61,3 +61,6 @@ [submodule "config"] path = submodules/configy url = https://github.com/bosagora/configy.git +[submodule "wasmer-d"] + path = submodules/wasmer-d + url = https://github.com/chances/wasmer-d.git diff --git a/Dockerfile b/Dockerfile index 01bf5e1f517..0b39c6151db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,10 @@ ARG AGORA_VERSION="HEAD" ADD . /root/agora/ WORKDIR /root/agora/talos/ RUN if [ -z ${AGORA_STANDALONE+x} ]; then npm ci && npm run build; else mkdir -p build; fi +WORKDIR / +RUN wget https://github.com/wasmerio/wasmer/releases/download/2.1.1/wasmer-linux-musl-amd64.tar.gz +RUN tar -zxvf wasmer-linux-musl-amd64.tar.gz +RUN wasmer -V WORKDIR /root/agora/ # Build Agora RUN AGORA_VERSION=${AGORA_VERSION} dub build --skip-registry=all --compiler=ldc2 ${DUB_OPTIONS} @@ -20,6 +24,10 @@ RUN if [ -z ${AGORA_STANDALONE+x} ]; then dub build --skip-registry=all --compil # Uses edge as we need the same `ldc-runtime` as the LDC that compiled Agora, # and `bosagora/agora-builder:latest` uses edge. FROM alpine:3.15 +WORKDIR / +RUN wget https://github.com/wasmerio/wasmer/releases/download/2.1.1/wasmer-linux-musl-amd64.tar.gz +RUN tar -zxvf wasmer-linux-musl-amd64.tar.gz +RUN wasmer -V # The following makes debugging Agora much easier on server # Since it's a tiny configuration file read by GDB at init, it won't affect release build COPY devel/dotgdbinit /root/.gdbinit diff --git a/README.md b/README.md index d4262295977..3435d0c6f56 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Additionally, the following are dependencies: - `openssl`: Binary (to detect the version) and development library - `sqlite3`: Development library - `zlib`: Development library +- `wasmer`: Web Assembly Engine - [Install](https://github.com/wasmerio/wasmer-install#readme) Additionally, on OSX, `PKG_CONFIG_PATH` needs to be properly set to pick up `sqlite3` and `openssl`. Provided you installed those packages via `brew`, you can run the following command prior to building: diff --git a/dub.json b/dub.json index e5c5b6e4f0a..73825972977 100644 --- a/dub.json +++ b/dub.json @@ -204,6 +204,7 @@ "localrest": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] }, "ocean": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] }, "serialization": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] }, - "vibe-d": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] } + "vibe-d": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] }, + "wasmer": { "version": "*", "dflags" : [ "-preview=in", "-revert=dtorfields" ] } } } diff --git a/dub.selections.json b/dub.selections.json index d5ef6324278..0e909731675 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -23,6 +23,7 @@ "taggedalgebraic": {"path":"submodules/taggedalgebraic/"}, "tinyendian": {"path":"submodules/tinyendian/"}, "vibe-core": {"path":"submodules/vibe-core/"}, - "vibe-d": {"path":"submodules/vibe.d/"} + "vibe-d": {"path":"submodules/vibe.d/"}, + "wasmer": {"path":"submodules/wasmer-d/"} } } diff --git a/source/agora/node/WasmEngine.d b/source/agora/node/WasmEngine.d new file mode 100644 index 00000000000..8621cdead00 --- /dev/null +++ b/source/agora/node/WasmEngine.d @@ -0,0 +1,155 @@ +/******************************************************************************* + + Contains the wasmer WebAssembly engine to be used for Trust Contracts. + The submodule `wasmer-d` provides a `D` wrapper of the `wamser` runtime. + (https://chances.github.io/wasmer-d/wasmer.html) + + Copyright: + Copyright (c) 2021 BOS Platform Foundation Korea + All rights reserved. + + License: + MIT License. See LICENSE for details. + +*******************************************************************************/ + +module agora.node.WasmEngine; + +import agora.utils.Log; + +mixin AddLogger!(); + +/// Ditto +public class WasmEngine +{ + import wasmer; + + /// The JIT engine for executing WebAssembly Programs (WASM) + Engine engine; + + /// The global state that can be manipulated by WebAssembly programs + /// https://webassembly.github.io/spec/core/exec/runtime.html#syntax-store + Store store; + + public void start () + { + log.info("Starting WasmEngine"); + engine = new Engine(); + store = new Store(engine); + log.info("Started WasmEngine"); + } + + public void stop () + { + log.info("Stopping WasmEngine"); + destroy(store); + destroy(engine); + log.info("Stopped WasmEngine"); + } + + public void exampleTime () + { + log.info("Running {} using WasmEngine", __FUNCTION__); + const wat_callback_module_time = "(module + (type $FUNCSIG$ii (func (param i32) (result i32))) + (import \"env\" \"time\" (func $time (param i32) (result i32))) + (table 0 anyfunc) + (memory $0 1) + (export \"memory\" (memory $0)) + (export \"main\" (func $main)) + (func $main (; 1 ;) (result i32) + (call $time + (i32.const 0) + ) + ) + )"; + log.trace("wat (text representation of wasm)"); + log.trace("=============================="); + log.trace("{}", wat_callback_module_time); + log.trace("=============================="); + auto module_ = Module.from(store, wat_callback_module_time); + assert(module_.valid, "Error compiling module!"); + + auto print = (Module module_, int value) => + { + return value; + }(); + auto imports = [new Function(store, module_, print.toDelegate).asExtern]; + auto instance = module_.instantiate(imports); + assert(instance.valid, "Could not instantiate module!"); + auto runFunc = Function.from(instance.exports[1]); + assert(instance.exports[1].name == "main", "Failed to get the `main` function!"); + assert(runFunc.valid, "`main` function invalid!"); + Value[] results; + assert(runFunc.call(results), "Error calling the `main` function!"); + assert(results.length == 1, "length of results should be 1!"); + // assert(results[0].value.of.i32 > 0, "time should be greater than 0"); + log.info("Used web assembly engine to get time. Result = {}", results[0].value.of.i32); + destroy(instance); + destroy(module_); + } + + + public void exampleAdd () + { + const wat_callback_module_add = "(module" ~ + " (func $print (import \"\" \"print\") (param i32) (result i32))" ~ + " (func (export \"run\") (param $x i32) (param $y i32) (result i32)" ~ + " (call $print (i32.add (local.get $x) (local.get $y)))" ~ + " )" ~ + ")"; + log.info("Running {} using WasmEngine", __FUNCTION__); + log.trace("wat (text representation of wasm)"); + log.trace("=============================="); + log.trace("{}", wat_callback_module_add); + log.trace("=============================="); + + auto module_ = Module.from(store, wat_callback_module_add); + assert(module_.valid, "Error compiling module!"); + + auto print = (Module module_, int value) => + { + return value; + }(); + auto imports = [new Function(store, module_, print.toDelegate).asExtern]; + auto instance = module_.instantiate(imports); + assert(instance.valid, "Could not instantiate module!"); + auto runFunc = Function.from(instance.exports[0]); + assert(instance.exports[0].name == "run" && runFunc.valid, "Failed to get the `run` function!"); + + auto three = new Value(3); + auto four = new Value(4); + Value[] results; + assert(runFunc.call([three, four], results), "Error calling the `run` function!"); + assert(results.length == 1 && results[0].value.of.i32 == 7); + log.info("Used web assembly engine to add 3 and 4. Result = {}", results[0].value.of.i32); + + destroy(three); + destroy(four); + destroy(instance); + destroy(module_); + } +} + +unittest +{ + import std.stdio; + import std.format; + + Log.root.level(LogLevel.Trace, true); + auto output = stdout.lockingTextWriter(); + try + { + auto wasmEngine = new WasmEngine(); + wasmEngine.start(); + wasmEngine.exampleAdd(); + wasmEngine.exampleTime(); + wasmEngine.stop(); + } + catch (Exception e) + { + output.formattedWrite("%s", e); + } + // print logs of the work thread + CircularAppender!()().print(output); +} diff --git a/submodules/wasmer-d b/submodules/wasmer-d new file mode 160000 index 00000000000..c94c72e251c --- /dev/null +++ b/submodules/wasmer-d @@ -0,0 +1 @@ +Subproject commit c94c72e251c4d4fb2c7d8e84f201728f823c619c