Switch to vanilla emsdk for building pglite (#48)

* first pass

* skip postgresql tests as usual

* preloaded files; pglite.data file autotools handling

* new pglite and install-pglite Makefile targets; workflow succesful;

* add pglite.{data,js,html} to gitignore

* remove the need for emsdk-shared script

* add password and PGPASSFILE static files

* use system zic from /usr/sbin/zic for timezone

* use static files from repo when preloading password and PGPASSFILE

* add empty file

* builder dockerfile; external deps source code

* use -sMAIN_MODULE=2 which reduces size code

* building with -sMAIN_MODULE=2 to reduce size

* use file containing needed EXPORTED_FUNCTIONS - tested on data generated from the *.o files of plpgsql + the already predefined ones;

-sDYLINK_DEBUG=2 flag is very useful in debugging emscripten's dynamic linking;

generating the EXPORTED_FUNCTIONS needs to be automated

* automate creation of imports needed by plpgsql;
automate creation of EXPORTED_FUNCTIONS emscripten parameter;

* infrastructure for contrib extensions

* more excluded imports

* link pglite.wasm against more libs so the exports are visible for emscripten

* more EXPORTED_FUNCTIONS excludes

* more EXPORTED_FUNCTIONS includes

* extended docker image to contain the exports of the prebuilt libraries

* change the exports format of shared libs

* extend backend makefile to better handle EXPORTED_FUNCTIONS depending on shared libs' exports

* git: ignore .map files

* added pg_ivm as submodule; added Makefile to build such submodules

* added pgvector as submodule and updated the Makefile to build it

* use env var to point to llvm-nm for exports and imports defs

* exclude libossp-uuid exports and dont link libossp-uuid to the exe

* git ignore excluded.imports file

* new simple bash script to build everything; allow debug builds

* cleanup

* added bashs script to clean everything

* unset DEBUG var if it is false - there is a bug somewhere and until we fix that, this works;
bug fix in passing PGLITE_CFLAGS

* further simplify build-with-docker2.sh script

* build pglite script remove SOURCE_MAP_PREFIXES

* comments

* rename script

* add 2 more exported functions; sort the always included imports

* separate dockerfiles for different archs - this is needed because emsdk does not publish different archs under the same tag; hopefully this will be fixed in the future

* contrib extensions packaging; integration into the build of the said extensions

* better error message when building/disting contribs

* change custom build root for pglite contrib

* add LDFLAGS_SL="-sSIDE_MODULE=1" when building contrib extensions

* renamed pgvector submodule to vector

* dist extra extensions insted of install (packaging pgvector and pg_ivm into tar.gz)

* use pushed docker image

* build only using native build platform, since wasm is the same on all platforms

* remove unused dockerfiles

* cleanup src/backend/Makefile

* remove wasm-builder folder

* added comment to other extensions Makefile

* comments

* remove binaries used for pglite-builder

* remove config.site since the param that it contains is already passed in the command line

* update clean-pglite.sh script

---------

Co-authored-by: tudor <[email protected]>

Author: tdrz

.gitignore

@@ -44,3 +44,12 @@ lib*.pc
 /tmp_install/
 /portlock/
 /dist
+
+# emscripten build excludes
+*.cjs
+*.wasm
+pglite.data
+pglite.js
+pglite.html
+*.map
+pglite-wasm/excluded.imports

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "pglite/pg_ivm"]
+	path = pglite/pg_ivm
+	url = https://github.com/sraoss/pg_ivm.git
+[submodule "pglite/pgvector"]
+	path = pglite/vector
+	url = https://github.com/pgvector/pgvector.git

build-pglite.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+### NOTES ###
+# $INSTALL_PREFIX is expected to point to the installation folder of various libraries built to wasm (see pglite-builder)
+#############
+
+# final output folder
+INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"}
+
+# build with optimizations by default aka release
+PGLITE_CFLAGS="-O2"
+if [ "$DEBUG" = true ]
+then
+    echo "pglite: building debug version."
+    PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt"
+else
+    echo "pglite: building release version."
+    # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set
+    unset DEBUG
+fi
+
+echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS"
+
+# Step 1: configure the project
+LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+
+# Step 2: make and install all except pglite
+emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; }
+emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; }
+
+# Step 3.1: make all contrib extensions - do not install
+emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
+# Step 3.2: make dist contrib extensions - this will create an archive for each extension
+emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; }
+# the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive
+
+# Step 4: make and dist other extensions
+SAVE_PATH=$PATH
+PATH=$PATH:$INSTALL_FOLDER/bin
+emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; }
+emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; }
+PATH=$SAVE_PATH
+
+# Step 5: make and install pglite
+# Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
+PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }

build-with-docker.sh

@@ -0,0 +1,13 @@
+# although we could use any path inside docker, using the same path as on the host
+# allows the DWARF info (when building in DEBUG) to contain the correct file paths
+DOCKER_WORKSPACE=$(pwd)
+
+docker run $@ \
+  --rm \
+  -e DEBUG=${DEBUG:-false} \
+  --workdir=${DOCKER_WORKSPACE} \
+  -v .:${DOCKER_WORKSPACE}:rw \
+  -v ./dist:/install/pglite:rw \
+  electricsql/pglite-builder:3.1.74_1 \
+  ./build-pglite.sh
+  

clean-pglite.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+emmake make -C src/backend uninstall; emmake make -C src/backend clean;
+emmake make -C pglite/ clean; emmake make -C pglite/ uninstall;
+emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall;
+emmake make clean; emmake make uninstal

contrib/Makefile

@@ -90,6 +90,7 @@ endif
 # Missing:
 #		start-scripts	\ (does not have a makefile)
 
+include ./dist.mk
 
 $(recurse)
 $(recurse_always)

contrib/dist.mk

@@ -0,0 +1,27 @@
+# contrib/dist.mk
+#
+# Package each contrib extension into its own .tar.gz archive
+
+prefix ?= /install/pglite
+CONTRIB_BUILD_ROOT := /tmp/extensions/build
+ARCHIVE_DIR := /install/pglite/extensions
+
+CONTRIBS := $(SUBDIRS)
+
+# Default target: build tarballs for all contribs
+dist: $(addsuffix .tar.gz,$(CONTRIBS))
+
+# Pattern rule: build $(EXT).tar.gz for each contrib
+%.tar.gz:
+	@echo "=== Staging $* ==="
+	rm -rf $(CONTRIB_BUILD_ROOT)/$*
+	bash -c 'mkdir -p $(CONTRIB_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}'
+	$(MAKE) -C $* install DESTDIR=$(CONTRIB_BUILD_ROOT)/$*
+	@echo "=== Packaging $* ==="
+	mkdir -p $(ARCHIVE_DIR)
+	cd $(CONTRIB_BUILD_ROOT)/$*/$(prefix) && \
+	files=$$(find . -type f -o -type l | sed 's|^\./||') && \
+	tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files
+# 	tar -C $(CONTRIB_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz .
+
+.PHONY: dist

other/PGPASSFILE

@@ -0,0 +1,3 @@
+localhost:5432:postgres:password:md532e12f215ba27cb750c9e093ce4b5127
+localhost:5432:postgres:postgres:md53175bce1d3201d16594cebf9d7eb3f9d
+localhost:5432:postgres:login:md5d5745f9425eceb269f9fe01d0bef06ff

other/empty

@@ -0,0 +1 @@
+password