WIP: wasi-native

.buildconfig

@@ -1,4 +1,7 @@
-PG_VERSION=16.4
-SDK_VERSION=3.1.72.3bi
-WASI_SDK_VERSION=24.0.4
+PG_VERSION=git
+SDK_VERSION=3.1.74.2bi
+WASI_SDK_VERSION=24.0.5
 SDKROOT=/opt/python-wasm-sdk
+PG_GIT_COMMIT=8e6af359aabdf0b687dfdb8e42b489fb7d27ce0a
+COPTS="-Os -g0"
+

.github/workflows/build_and_test.yml

@@ -207,7 +207,7 @@ jobs:
 
       - name: Build demo site
         run: |
-          NO_SDK_CHECK=true SDKROOT=/opt/python-wasm-sdk bash ./cibuild.sh demo-site
+          bash ./cibuild/demo-site.sh
 
       - name: Build docs
         working-directory: ./docs

.github/workflows/build_wasm_postgres.yml

@@ -19,13 +19,13 @@ jobs:
       SDKROOT: /opt/python-wasm-sdk
       SYS_PYTHON: /usr/bin/python3
       PGROOT: /tmp/pglite
-      TOTAL_MEMORY: 128MB
+      TOTAL_MEMORY: 192MB
       CMA_MB: 16
-      DEBUG: false
       OBJDUMP: true
       contrib: contrib
       extra: extra
       EXTRA_EXT: vector postgis
+      DEBUG: false
 
     steps:
       - uses: actions/checkout@v4
@@ -84,6 +84,11 @@ jobs:
         run: |
           bash ./cibuild.sh linkweb
 
+      - name: Build and Link postgres WASI for wasi/native
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+          bash ./extra/native/wip.sh
+
       - name: Copy release for PGlite build
         run: |
           bash ./cibuild.sh pglite-prep

cibuild.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+#set -x;
+#set -e;
+
 # data transfer zone this is == (wire query size + result size ) + 2
 # expressed in EMSDK MB
 export CMA_MB=${CMA_MB:-64}
@@ -10,7 +13,7 @@ chmod +x ./extra/*.sh cibuild/*.sh
 
 . .buildconfig
 
-export PG_VERSION SDK_VERSION WASI_SDK_VERSION SDKROOT
+export PG_VERSION SDK_VERSION WASI_SDK_VERSION SDKROOT COPTS
 
 export PG_VERSION=${PG_VERSION:-16.4}
 export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)}
@@ -30,10 +33,10 @@ EOE=false
 
 if ./cibuild/sdk.sh
 then
-    echo "sdk check passed (emscripten)"
+    echo "sdk check passed (emscripten+wasi)"
 else
     echo sdk failed
-    exit 44
+    exit 39
 fi
 
 
@@ -231,9 +234,10 @@ END
     fi
 
     mkdir -p ${PGROOT}/include/postgresql/server
-    cp ${PG_DEBUG_HEADER} ${PGROOT}/include/
-    cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql
-    cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql/server
+    for dest in ${PGROOT}/include ${PGROOT}/include/postgresql ${PGROOT}/include/postgresql/server
+    do
+        [ -f $dest/pg_debug.h ] || cp ${PG_DEBUG_HEADER} $dest/
+    done
 
     # store all pg options that have impact on cmd line initdb/boot
     cat > ${PGROOT}/pgopts.sh <<END
@@ -288,7 +292,12 @@ END
 
     # install emsdk-shared along with pg config  tool
     # for building user ext.
-    cp build/postgres/bin/emsdk-shared $PGROOT/bin/
+    if [ -f $PGROOT/bin/emsdk-shared ]
+    then
+        echo emsdk-shared already installed
+    else
+        cp -vf build/postgres/bin/emsdk-shared $PGROOT/bin/
+    fi
 
     export PGLITE=$(pwd)/packages/pglite
 
@@ -318,14 +327,26 @@ export PATH=${WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH
 
 if echo " $*"|grep -q " contrib"
 then
-    # TEMP FIX for SDK
-    SSL_INCDIR=$EMSDK/upstream/emscripten/cache/sysroot/include/openssl
-    [ -f $SSL_INCDIR/evp.h ] || ln -s $PREFIX/include/openssl $SSL_INCDIR
-    SKIP="\
+
+    if $WASI
+    then
+        echo " ========= TODO WASI openssl ============== "
+        SKIP="\
  [\
  sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\
  ltree_plpython sepgsql bool_plperl start-scripts\
+ pgcrypto uuid-ossp xml2\
  ]"
+    else
+        # TEMP FIX for SDK
+        SSL_INCDIR=$EMSDK/upstream/emscripten/cache/sysroot/include/openssl
+        [ -f $SSL_INCDIR/evp.h ] || ln -s $PREFIX/include/openssl $SSL_INCDIR
+        SKIP="\
+ [\
+ sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\
+ ltree_plpython sepgsql bool_plperl start-scripts\
+ ]"
+    fi
 
     for extdir in postgresql/contrib/*
     do
@@ -479,19 +500,6 @@ ________________________________________________________________________________
 
                 mv $packed /tmp/sdk/pg${PG_VERSION}-${packed}
 
-                # for repl demo
-#                mkdir -p /tmp/web/pglite
-
-                #cp -r ${PGLITE}/dist ${WEBROOT}/pglite/
-                #cp -r ${PGLITE}/examples ${WEBROOT}/pglite/
-
-#                for dir in /tmp/web ${WEBROOT}/pglite/examples
-#                do
-#                    pushd "$dir"
-#                    cp ${PGLITE}/dist/postgres.data ./
-#                    popd
-#                done
-
                 echo "<html>
                 <body>
                     <ul>
@@ -518,7 +526,7 @@ ________________________________________________________________________________
                 tar -cpRz ${PGROOT} > /tmp/sdk/pglite-pg${PG_VERSION}.tar.gz
 
                 # build sdk (node)
-                cp /tmp/sdk/postgres-${PG_VERSION}.tar.gz ${WEBROOT}/
+                cp /tmp/sdk/postgres-${PG_VERSION}-*.tar.gz ${WEBROOT}/
 
                 # pglite (web)
                 cp /tmp/sdk/pglite-pg${PG_VERSION}.tar.gz ${WEBROOT}/
@@ -560,18 +568,7 @@ ________________________________________________________________________________
         ;;
 
         demo-site) echo "==================== demo-site =========================="
-            # Move all existing files to a subfolder
-            mkdir -p /tmp/web/x-term-repl
-            mv /tmp/web/* /tmp/web/x-term-repl/
-
-            mkdir -p /tmp/web/dist
-            mkdir -p /tmp/web/examples
-            mkdir -p /tmp/web/benchmark
-
-            PGLITE=$(pwd)/packages/pglite
-            cp -r ${PGLITE}/dist/* /tmp/web/dist/
-            cp -r ${PGLITE}/examples/* /tmp/web/examples/
-            cp -r ${WORKSPACE}/packages/benchmark/dist/* /tmp/web/benchmark/
+            ./cibuild/demo-site.sh
         ;;
     esac
     shift

cibuild/demo-site.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Move all existing files to a subfolder
+mkdir -p /tmp/web/x-term-repl
+mv /tmp/web/* /tmp/web/x-term-repl/
+
+mkdir -p /tmp/web/dist
+mkdir -p /tmp/web/examples
+mkdir -p /tmp/web/benchmark
+
+PGLITE=$(pwd)/packages/pglite
+cp -r ${PGLITE}/dist/* /tmp/web/dist/
+cp -r ${PGLITE}/examples/* /tmp/web/examples/
+cp -r ${WORKSPACE:-$GITHUB_WORKSPACE}/packages/benchmark/dist/* /tmp/web/benchmark/
+

cibuild/linkimports.sh

@@ -8,6 +8,7 @@ echo "============= link imports : begin ==============="
 
 pushd ${WORKSPACE}
     > patches/imports/pgcore
+
     for extra_pg_so in $(find $PGROOT/lib/postgresql/|grep \.so$)
     do
         SOBASE=patches/imports.pgcore/$(basename $extra_pg_so .so)
@@ -50,6 +51,7 @@ matches = list( imports.intersection(exports) )
 
 # ?
 for sym in """
+_ErrorContext
 _check_function_bodies
 _clock_gettime
 _CurrentMemoryContext
@@ -61,9 +63,7 @@ _interactive_read
 _interactive_write
 _lowerstr
 _main
-_main_repl
 _pg_initdb
-_pg_repl_raf
 _pg_shutdown
 _readstoplist
 _searchstoplist
@@ -76,8 +76,6 @@ _TopMemoryContext
     if sym and not sym in matches:
         matches.append(sym)
 
-#__ZNSt13runtime_errorD1Ev
-
 matches.sort()
 
 for sym in matches:

cibuild/linkweb.sh

@@ -35,7 +35,10 @@ emcc $CDEBUG -shared -o ${WEBROOT}/libpgc.so \
 
 # this override completely pg server main loop for web use purpose
 pushd src
-    rm pg_initdb.o backend/main/main.o ./backend/tcop/postgres.o ./backend/utils/init/postinit.o
+    for obj in pg_initdb.o backend/main/main.o ./backend/tcop/postgres.o ./backend/utils/init/postinit.o
+    do
+        [ -f $obj ] && rm $obj
+    done
 
     emcc -DPG_INITDB_MAIN=1 -sFORCE_FILESYSTEM -DPREFIX=${PGROOT} ${CC_PGLITE} \
      -I${PGROOT}/include -I${PGROOT}/include/postgresql/server -I${PGROOT}/include/postgresql/internal \
@@ -94,17 +97,22 @@ pushd src/backend
 # -sSINGLE_FILE  => Uncaught SyntaxError: Cannot use 'import.meta' outside a module (at postgres.html:1:6033)
 # -sENVIRONMENT=web => XHR
 
-    export EMCC_WEB="-sNO_EXIT_RUNTIME=1 -sFORCE_FILESYSTEM=1"
+    export EMCC_WEB="-sENVIRONMENT=node,web" -sNO_EXIT_RUNTIME=1 -sFORCE_FILESYSTEM=1"
 
     if ${PGES6:-true}
     then
         # es6
-        MODULE="$LDEBUG --closure 0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module"
-        export COPTS="-O2 -g0"
+        MODULE="$LDEBUG --closure 1 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module"
+        if $DEBUG
+        then
+            export COPTS=${COPTS:-"-O2 -g3"}
+        else
+            export COPTS=${COPTS:-"-Os -g0"}
+        fi
     else
-        export COPTS="-O0 -g3"
+        export COPTS="-O2 -g3"
         # local debug always fast build
-        MODULE="-sMODULARIZE=0 -sEXPORT_ES6=0"
+        MODULE="--closure 0 -sMODULARIZE=0 -sEXPORT_ES6=0"
     fi
 
     MODULE="$MODULE --shell-file ${WORKSPACE}/tests/repl.html"
@@ -113,8 +121,8 @@ pushd src/backend
     # =======================================================
     # size optimisations
     # =======================================================
-
-    rm ${PGROOT}/lib/lib*.so.? 2>/dev/null
+    touch ${PGROOT}/lib/libany.so.x
+    rm ${PGROOT}/lib/lib*.so.?
 
     echo "#!/bin/true" > placeholder
     chmod +x placeholder
@@ -132,7 +140,8 @@ pushd src/backend
 
     # encodings ?
     # ./lib/postgresql/utf8_and*.so
-    rm ${PGROOT}/lib/postgresql/utf8_and*.so
+    touch ${PGROOT}/lib/postgresql/utf8_andany.so
+    rm -f ${PGROOT}/lib/postgresql/utf8_and*.so
 
 
     # =========================================================
@@ -193,7 +202,7 @@ _________________________________________________________
     # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@exports"
 
 
-    emcc $EMCC_WEB $LINKER $MODULE  \
+    emcc $EMCC_WEB $LINKER $MODULE \
      -sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=4MB -sGLOBAL_BASE=${CMA_MB}MB \
      -fPIC -D__PYDK__=1 -DPREFIX=${PGROOT} \
      -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
@@ -227,7 +236,7 @@ popd
 
 
 echo "
-============= link web : end ===============
+============= link web : end COPTS=$COPTS ===============
 
 
 

cibuild/pack_extension.py

@@ -73,6 +73,9 @@ async def archive(target_folder):
                 if asp.startswith('/base/'):
                     continue
 
+                if asp.startswith('/dump.'):
+                    continue
+
                 fp = PGROOT / asp[1:]
                 if fp.is_symlink():
                     continue
@@ -107,8 +110,8 @@ async def archive(target_folder):
 print("="*80)
 
 if not EXTNAME:
-    print("ERROR: no new installed extension found, is it builtin ?")
-    sys.exit(105)
+    print("MAYBE ERROR: no new installed extension found, is it builtin ?")
+    sys.exit(0)
 
 print(f"""
 PG installed in : {PGROOT=}

cibuild/pg-16.x.sh

@@ -50,7 +50,7 @@ else
     # either a submodule dir or a symlink.
     # release only use symlink
 
-    rm postgresql 2>/dev/null
+    [ -f postgresql/configure ] && rm postgresql 2>/dev/null
     ln -s postgresql-${PG_VERSION} postgresql
 
 fi