Skip to content

Commit

Permalink
Fix pl/pgsql (#125)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pmp-p
pmp-p authored Jul 25, 2024
1 parent 97abee5 commit b1832ae
Show file tree
Hide file tree
Showing 28 changed files with 2,183 additions and 696 deletions.
1 change: 1 addition & 0 deletions .github/workflows/build_pglite.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
SYS_PYTHON: /usr/bin/python3
PGROOT: /tmp/pglite
DEBUG: false
OBJDUMP: false

steps:
- uses: actions/checkout@v4
Expand Down
139 changes: 74 additions & 65 deletions cibuild.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ export CMA_MB=${CMA_MB:-64}

export PGVERSION=${PGVERSION:-16.3}
export CI=${CI:-false}
export GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)}
export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)}
export PGROOT=${PGROOT:-/tmp/pglite}
export WEBROOT=${WEBROOT:-/tmp/web}
export DEBUG=${DEBUG:-false}
export PGDATA=${PGROOT}/base
export PGUSER=postgres
export PGPATCH=${WORKSPACE}/patches


# exit on error
Expand All @@ -22,7 +23,7 @@ if mkdir -p ${PGROOT}/sdk
then
echo "checking for valid prefix ${PGROOT}"
else
sudo mkdir -p ${PGROOT}/sdk
sudo mkdir -p ${PGROOT}/sdk ${PGROOT}/bin
sudo chown $(whoami) -R ${PGROOT}
fi

Expand All @@ -41,7 +42,10 @@ fi

export PGPASS


if which wasm-objdump
then
cp $(which wasm-objdump) $PGROOT/bin/
fi

# default to web/release size optim.
if $DEBUG
Expand All @@ -55,32 +59,36 @@ fi

# setup compiler+node. emsdk provides node (18), recent enough for bun.
# TODO: but may need to adjust $PATH with stock emsdk.

if which emcc
if ${WASI:-false}
then
echo "Using provided emsdk from $(which emcc)"
echo "Wasi build (experimental)"
. /opt/python-wasm-sdk/wasm32-wasi-shell.sh
else
. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
fi
if which emcc
then
echo "Using provided emsdk from $(which emcc)"
else
. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
fi

# custom code for node/web builds that modify pg main/tools behaviour
# this used by both node/linkweb build stages

# custom code for node/web builds that modify pg main/tools behaviour
# this used by both node/linkweb build stages
# pass the "kernel" contiguous memory zone size to the C compiler.
CC_PGLITE="-DCMA_MB=${CMA_MB}"

# pass the "kernel" contiguous memory zone size to the C compiler.
CC_PGLITE="-DCMA_MB=${CMA_MB}"
# these are files that shadow original portion of pg core, with minimal changes
# to original code
# some may be included multiple time
CC_PGLITE="-DPATCH_MAIN=${WORKSPACE}/patches/pg_main.c ${CC_PGLITE}"
CC_PGLITE="-DPATCH_LOOP=${WORKSPACE}/patches/interactive_one.c ${CC_PGLITE}"
CC_PGLITE="-DPATCH_PLUGIN=${WORKSPACE}/patches/pg_plugin.h ${CC_PGLITE}"

# these are files that shadow original portion of pg core, with minimal changes
# to original code
# some may be included multiple time
CC_PGLITE="-DPATCH_MAIN=${GITHUB_WORKSPACE}/patches/pg_main.c ${CC_PGLITE}"
CC_PGLITE="-DPATCH_LOOP=${GITHUB_WORKSPACE}/patches/interactive_one.c ${CC_PGLITE}"
CC_PGLITE="-DPATCH_PLUGIN=${GITHUB_WORKSPACE}/patches/pg_plugin.h ${CC_PGLITE}"
fi

export CC_PGLITE



if [ -f ${WEBROOT}/postgres.js ]
then
echo using current from ${WEBROOT}
Expand Down Expand Up @@ -117,7 +125,8 @@ END

# workaround no "locale -a" for Node.
# this is simply the minimal result a popen call would give.
cat > ${PGROOT}/locale <<END
mkdir -p ${PGROOT}/etc/postgresql
cat > ${PGROOT}/etc/postgresql/locale <<END
C
C.UTF-8
POSIX
Expand All @@ -135,20 +144,21 @@ END
. cibuild/pg-git.sh
fi

# install wasm-shared along with pg config tool
# for building user ext.
cp build/postgres/bin/wasm-shared $PGROOT/bin/

export PGLITE=$(pwd)/packages/pglite

echo "export PGSRC=${PGSRC}" >> ${PGROOT}/pgopts.sh
echo "export PGLITE=${PGLITE}" >> ${PGROOT}/pgopts.sh

if [ -f /opt/sdk/wasisdk/wabt/bin/wasm-objdump ]
then
./cibuild/symtab.sh ${PGROOT}/lib/postgresql/plpgsql.so > ${PGROOT}/symbols
fi

fi

# put wasm-shared the pg extension linker from build dir in the path
# and also pg_config from the install dir.
export PATH=${GITHUB_WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH
export PATH=${WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH



Expand All @@ -167,18 +177,17 @@ export PATH=${GITHUB_WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH

if echo "$*"|grep -q vector
then
echo "================================================="
echo "====================== vector : $(pwd) ================="

pushd build

# [ -d pgvector ] || git clone --no-tags --depth 1 --single-branch --branch master https://github.com/pgvector/pgvector
# git clone --branch v0.7.2 https://github.com/pgvector/pgvector.git

if [ -d pgvector ]
then
echo using local pgvector
else
wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.7.2.tar.gz -Opgvector.tar.gz
wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.7.3.tar.gz -Opgvector.tar.gz
tar xvfz pgvector.tar.gz && rm pgvector.tar.gz
mv pgvector-?.?.? pgvector
fi
Expand All @@ -187,7 +196,7 @@ then
# path for wasm-shared already set to (pwd:pg build dir)/bin
# OPTFLAGS="" turns off arch optim (sse/neon).
PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install
cp sql/vector.sql sql/vector--0.7.2.sql ${PGROOT}/share/postgresql/extension
cp sql/vector.sql sql/vector--0.7.3.sql ${PGROOT}/share/postgresql/extension
rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/vector.sql
popd

Expand All @@ -197,15 +206,23 @@ then

fi

if echo "$*"|grep " postgis"
if echo "$*"|grep " pg_stat_statements"
then
echo "================================================="
PG_LINK=em++ echo "WIP - requires latests python-wasm-sdk, not just emsdk"

pushd build/postgres/contrib/pg_stat_statements
emmake make install
popd
python3 cibuild/pack_extension.py
fi


if echo "$*"|grep " postgis"
then
echo "======================= postgis : $(pwd) ==================="

./cibuild/postgis.sh

python3 cibuild/pack_extension.py
fi

if echo "$*"|grep " quack"
then
Expand Down Expand Up @@ -246,7 +263,7 @@ fi

if echo "$*"|grep "node"
then
echo "================================================="
echo "====================== node : $(pwd) ========================"
mkdir -p /tmp/sdk/

# remove versionned symlinks
Expand All @@ -260,18 +277,16 @@ fi
# run linkweb after node build because it will remove some wasm .so used by node from fs
# they don't need to be in MEMFS as they are fetched.


# include current pglite source for easy local rebuild with just npm run build:js.


if echo "$*"|grep "linkweb"
then

# build web version
echo "========== linkweb : $(pwd) =================="
pushd build/postgres
echo "=================== $(pwd) ========================="

. $GITHUB_WORKSPACE/cibuild/linkweb.sh
. $WORKSPACE/cibuild/linkweb.sh

# upload all to gh pages,
# TODO: include node archive and samples ?
Expand All @@ -290,7 +305,7 @@ fi
while test $# -gt 0
do
case "$1" in
pglite) echo "=================== pglite ======================="
pglite) echo "=================== pglite : $(pwd) ======================="
# TODO: SAMs NOTE - Not using this in GitHub action as it doesnt resolve pnpm correctly
# replaced with pglite-prep and pglite-bundle-sdk

Expand All @@ -303,11 +318,7 @@ do
cp -r $PGLITE ${PGROOT}/sdk/packages/

mkdir /tmp/web/repl/dist-webcomponent -p
cp -r ${GITHUB_WORKSPACE}/packages/repl/dist-webcomponent /tmp/web/repl/

pushd /tmp/web/pglite/examples
ln -s ../dist/postgres.data
popd
cp -r ${WORKSPACE}/packages/repl/dist-webcomponent /tmp/web/repl/

if $CI
then
Expand All @@ -317,44 +328,42 @@ do
du -hs ${WEBROOT}/*
;;

pglite-prep) echo "==================== pglite-prep =========================="
mkdir $PGLITE/release || rm $PGLITE/release/*
pglite-prep) echo "==================== pglite-prep =========================="
mkdir -p $PGLITE/release
rm $PGLITE/release/*

# copy packed extensions
cp ${WEBROOT}/*.tar.gz ${PGLITE}/release/


cp -vf ${WEBROOT}/postgres.{js,data,wasm} $PGLITE/release/
cp -vf ${WEBROOT}/libecpg.so $PGLITE/release/postgres.so

;;
;;

pglite-bundle-interim) echo "================== pglite-bundle-interim ======================"
tar -cpRz ${PGLITE}/release > /tmp/sdk/pglite-interim-${PGVERSION}.tar.gz
;;
;;

demo-site) echo "==================== demo-site =========================="

echo "<html>
<body>
<ul>
<li><a href=./pglite/examples/repl.html>PGlite REPL (in-memory)</a></li>
<li><a href=./pglite/examples/repl-idb.html>PGlite REPL (indexedDB)</a></li>
<li><a href=./pglite/examples/notify.html>list/notify test</a></li>
<li><a href=./pglite/examples/index.html>All PGlite Examples</a></li>
<li><a href=./pglite/benchmark/index.html>Benchmarks</a> / <a href=./pglite/benchmark/rtt.html>RTT Benchmarks</a></li>
<li><a href=./postgres.html>Postgres xterm REPL</a></li>
</ul>
</body>
</html>" > /tmp/web/index.html
<body>
<ul>
<li><a href=./pglite/examples/repl.html>PGlite REPL (in-memory)</a></li>
<li><a href=./pglite/examples/repl-idb.html>PGlite REPL (indexedDB)</a></li>
<li><a href=./pglite/examples/notify.html>list/notify test</a></li>
<li><a href=./pglite/examples/index.html>All PGlite Examples</a></li>
<li><a href=./pglite/benchmark/index.html>Benchmarks</a> / <a href=./pglite/benchmark/rtt.html>RTT Benchmarks</a></li>
<li><a href=./postgres.html>Postgres xterm REPL</a></li>
</ul>
</body>
</html>" > /tmp/web/index.html

mkdir -p /tmp/web/pglite
mkdir -p /tmp/web/repl

PGLITE=$(pwd)/packages/pglite
cp -r ${PGLITE}/dist /tmp/web/pglite/
cp -r ${PGLITE}/examples /tmp/web/pglite/
cp -r ${GITHUB_WORKSPACE}/packages/repl/dist-webcomponent /tmp/web/repl/
cp -r ${GITHUB_WORKSPACE}/packages/benchmark /tmp/web/pglite/
cp -r ${WORKSPACE}/packages/repl/dist-webcomponent /tmp/web/repl/
cp -r ${WORKSPACE}/packages/benchmark /tmp/web/pglite/
;;
esac
shift
Expand Down
Loading

1 comment on commit b1832ae

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚀 Deployed on https://66a29f7c1bacb99489816cae--pglite-dev-demos.netlify.app

Please sign in to comment.