Merge branch 'testnet' into accelerator

.github/workflows/build-ton-wasm-emscripten.yml

@@ -19,7 +19,7 @@ jobs:
 
     - name: Build TON WASM artifacts
       run: |
-        cd assembly/wasm
+        cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
         chmod +x fift-func-wasm-build-ubuntu.sh
         ./fift-func-wasm-build-ubuntu.sh -a
 

assembly/cicd/jenkins/test-builds.groovy

@@ -219,7 +219,7 @@ pipeline {
                     steps {
                         timeout(time: 180, unit: 'MINUTES') {
                             sh '''
-                                cd assembly/wasm
+                                cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
                                 chmod +x fift-func-wasm-build-ubuntu.sh
                                 ./fift-func-wasm-build-ubuntu.sh -a
                             '''

assembly/wasm/fift-func-wasm-build-ubuntu.sh

@@ -1,5 +1,3 @@
-# The script builds funcfift compiler to WASM
-
 # Execute these prerequisites first
 # sudo apt update
 # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \
@@ -11,10 +9,12 @@
 # sudo ./llvm.sh 16 all
 
 with_artifacts=false
+scratch_new=false
 
-while getopts 'a' flag; do
+while getopts 'af' flag; do
   case "${flag}" in
     a) with_artifacts=true ;;
+    f) scratch_new=true ;;
     *) break
        ;;
   esac
@@ -24,131 +24,155 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
-cd ../..
-rm -rf openssl zlib emsdk secp256k1 libsodium build
 echo `pwd`
+if [ "$scratch_new" = true ]; then
+  echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton
+  rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build
+fi
 
-git clone https://github.com/openssl/openssl.git
-cd openssl
-git checkout checkout openssl-3.1.4
-./config
-make -j16
-OPENSSL_DIR=`pwd`
-cd ..
-
-git clone https://github.com/madler/zlib.git
-cd zlib
-git checkout v1.3.1
-ZLIB_DIR=`pwd`
-cd ..
-
-git clone https://github.com/lz4/lz4.git
-cd lz4
-git checkout v1.9.4
-LZ4_DIR=`pwd`
-cd ..
-
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-git checkout v0.3.2
-./autogen.sh
-SECP256K1_DIR=`pwd`
-cd ..
-
-git clone https://github.com/jedisct1/libsodium
-cd libsodium
-git checkout 1.0.18-RELEASE
-SODIUM_DIR=`pwd`
-cd ..
-
-mkdir build
-cd build
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_CXX_STANDARD=17 \
--DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
--DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
--DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so \
--DTON_USE_ABSEIL=OFF ..
-
-test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
-
-ninja fift smc-envelope
 
-test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+if [ ! -d "openssl" ]; then
+  git clone https://github.com/openssl/openssl.git
+  cd openssl
+  git checkout openssl-3.1.4
+  ./config
+  make -j16
+  OPENSSL_DIR=`pwd`
+  cd ..
+else
+  OPENSSL_DIR=`pwd`/openssl
+  echo Using compiled openssl at $OPENSSL_DIR
+fi
 
-rm -rf *
+if [ ! -d "build" ]; then
+  mkdir build
+  cd build
+  cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_STANDARD=17 \
+  -DOPENSSL_FOUND=1 \
+  -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
+  -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
+  -DTON_USE_ABSEIL=OFF ..
+
+  test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
+  ninja fift smc-envelope
+  test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+  rm -rf *
+  cd ..
+else
+  echo cleaning build...
+  rm -rf build/*
+fi
 
-cd ..
+if [ ! -d "emsdk" ]; then
+  git clone https://github.com/emscripten-core/emsdk.git
+echo
+  echo Using cloned emsdk
+fi
 
-git clone https://github.com/emscripten-core/emsdk.git
 cd emsdk
 ./emsdk install 3.1.19
 ./emsdk activate 3.1.19
 EMSDK_DIR=`pwd`
-ls $EMSDK_DIR
 
 . $EMSDK_DIR/emsdk_env.sh
 export CC=$(which emcc)
 export CXX=$(which em++)
 export CCACHE_DISABLE=1
 
-cd ../openssl
-
-make clean
-emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
-sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
-sed -i 's/-ldl//g' Makefile
-sed -i 's/-O3/-Os/g' Makefile
-emmake make depend
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
-
-cd ../zlib
-
-emconfigure ./configure --static
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+cd ..
 
-cd ../lz4
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+if [ ! -f "openssl/openssl_em" ]; then
+  cd openssl
+  make clean
+  emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
+  sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
+  sed -i 's/-ldl//g' Makefile
+  sed -i 's/-O3/-Os/g' Makefile
+  emmake make depend
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
+  touch openssl_em
+  cd ..
+else
+  echo Using compiled openssl with emscripten
+fi
 
-cd ../secp256k1
+if [ ! -d "zlib" ]; then
+  git clone https://github.com/madler/zlib.git
+  cd zlib
+  git checkout v1.3.1
+  ZLIB_DIR=`pwd`
+  emconfigure ./configure --static
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+  cd ..
+else
+  ZLIB_DIR=`pwd`/zlib
+  echo Using compiled zlib with emscripten at $ZLIB_DIR
+fi
 
-emconfigure ./configure --enable-module-recovery
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+if [ ! -d "lz4" ]; then
+  git clone https://github.com/lz4/lz4.git
+  cd lz4
+  git checkout v1.9.4
+  LZ4_DIR=`pwd`
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+  cd ..
+else
+  LZ4_DIR=`pwd`/lz4
+  echo Using compiled lz4 with emscripten at $LZ4_DIR
+fi
 
-cd ../libsodium
+if [ ! -d "secp256k1" ]; then
+  git clone https://github.com/bitcoin-core/secp256k1.git
+  cd secp256k1
+  git checkout v0.3.2
+  ./autogen.sh
+  SECP256K1_DIR=`pwd`
+  emconfigure ./configure --enable-module-recovery
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+  cd ..
+else
+  SECP256K1_DIR=`pwd`/secp256k1
+  echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR
+fi
 
-emconfigure ./configure --disable-ssp
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+if [ ! -d "libsodium" ]; then
+  git clone https://github.com/jedisct1/libsodium
+  cd libsodium
+  git checkout 1.0.18-RELEASE
+  SODIUM_DIR=`pwd`
+  emconfigure ./configure --disable-ssp
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+  cd ..
+else
+  SODIUM_DIR=`pwd`/libsodium
+  echo Using compiled libsodium with emscripten at $SODIUM_DIR
+fi
 
-cd ../build
+cd build
 
-emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
+emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 -DZLIB_FOUND=1 \
 -DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
 -DZLIB_INCLUDE_DIR=$ZLIB_DIR \
 -DLZ4_FOUND=1 \
 -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
 -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
 -DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
 -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
 -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a \
 -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
 -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \
--DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \
 -DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \
 -DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \
--DSODIUM_USE_STATIC_LIBS=ON ..
+..
 
 test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; }
 cp -R ../crypto/smartcont ../crypto/fift/lib crypto

crypto/CMakeLists.txt

@@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 
 add_library(fift-lib STATIC ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
+
 if (USE_EMSCRIPTEN)
   target_link_options(fift-lib PRIVATE -fexceptions)
   target_compile_options(fift-lib PRIVATE -fexceptions)

crypto/block/transaction.cpp

@@ -1661,7 +1661,12 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
   std::unique_ptr<StringLoggerTail> logger;
   auto vm_log = vm::VmLog();
   if (cfg.with_vm_log) {
-    size_t log_max_size = cfg.vm_log_verbosity > 0 ? 1024 * 1024 : 256;
+    size_t log_max_size = 256;
+    if (cfg.vm_log_verbosity > 4) {
+      log_max_size = 32 << 20;
+    } else if (cfg.vm_log_verbosity > 0) {
+      log_max_size = 1 << 20;
+    }
     logger = std::make_unique<StringLoggerTail>(log_max_size);
     vm_log.log_interface = logger.get();
     vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
@@ -1673,6 +1678,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
           vm_log.log_mask |= vm::VmLog::DumpStack;
           if (cfg.vm_log_verbosity > 4) {
             vm_log.log_mask |= vm::VmLog::DumpStackVerbose;
+            vm_log.log_mask |= vm::VmLog::DumpC5;
           }
         }
       }

crypto/common/bitstring.cpp

@@ -640,11 +640,11 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con
   return bits;
 }
 
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end) {
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end) {
   const char* ptr = str;
-  while (ptr < str_end && buff_size && (*ptr == '0' || *ptr == '1')) {
+  while (ptr < str_end && buff_size_bits && (*ptr == '0' || *ptr == '1')) {
     *buff++ = (bool)(*ptr++ & 1);
-    --buff_size;
+    --buff_size_bits;
   }
   return td::narrow_cast<long>(ptr == str_end ? ptr - str : str - ptr - 1);
 }

crypto/common/bitstring.h

@@ -58,7 +58,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits);
 long long bits_load_long(ConstBitPtr from, unsigned bits);
 unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits);
 long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, const char* str, const char* str_end);
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end);
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end);
 
 void bits_sha256(BitPtr to, ConstBitPtr from, std::size_t size);
 

crypto/fift/Fift.cpp

@@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
 }
 
 td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
-  ctx.ton_db = &config_.ton_db;
   ctx.source_lookup = &config_.source_lookup;
   ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
   ctx.output_stream = config_.output_stream;

crypto/fift/Fift.h

@@ -19,7 +19,6 @@
 #pragma once
 
 #include "SourceLookup.h"
-#include "vm/db/TonDb.h"
 #include "Dictionary.h"
 
 #include "td/utils/Status.h"
@@ -31,13 +30,11 @@ struct Fift {
  public:
   struct Config {
     fift::SourceLookup source_lookup;
-    vm::TonDb ton_db;
     fift::Dictionary dictionary;
     std::ostream* output_stream{&std::cout};
     std::ostream* error_stream{&std::cerr};
     bool show_backtrace{true};
   };
-  // Fift must own ton_db and dictionary, no concurrent access is allowed
   explicit Fift(Config config);
 
   td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);