Switch to vanilla emsdk for building pglite

.gitignore

@@ -44,3 +44,12 @@ lib*.pc
 /tmp_install/
 /portlock/
 /dist
+
+# emscripten build excludes
+*.cjs
+*.wasm
+pglite.data
+pglite.js
+pglite.html
+*.map
+pglite-wasm/excluded.imports

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "pglite/pg_ivm"]
+	path = pglite/pg_ivm
+	url = https://github.com/sraoss/pg_ivm.git
+[submodule "pglite/pgvector"]
+	path = pglite/vector
+	url = https://github.com/pgvector/pgvector.git

build-pglite.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+### NOTES ###
+# $INSTALL_PREFIX is expected to point to the installation folder of various libraries built to wasm (see pglite-builder)
+#############
+
+# final output folder
+INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"}
+
+# build with optimizations by default aka release
+PGLITE_CFLAGS="-O2"
+if [ "$DEBUG" = true ]
+then
+    echo "pglite: building debug version."
+    PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt"
+else
+    echo "pglite: building release version."
+    # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set
+    unset DEBUG
+fi
+
+echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS"
+
+# Step 1: configure the project
+LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+
+# Step 2: make and install all except pglite
+emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; }
+emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; }
+
+# Step 3.1: make all contrib extensions - do not install
+emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
+# Step 3.2: make dist contrib extensions - this will create an archive for each extension
+emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; }
+# the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive
+
+# Step 4: make and dist other extensions
+SAVE_PATH=$PATH
+PATH=$PATH:$INSTALL_FOLDER/bin
+emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; }
+emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; }
+PATH=$SAVE_PATH
+
+# Step 5: make and install pglite
+# Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
+PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }

build-with-docker.sh

@@ -0,0 +1,13 @@
+# although we could use any path inside docker, using the same path as on the host
+# allows the DWARF info (when building in DEBUG) to contain the correct file paths
+DOCKER_WORKSPACE=$(pwd)
+
+docker run $@ \
+  --rm \
+  -e DEBUG=${DEBUG:-false} \
+  --workdir=${DOCKER_WORKSPACE} \
+  -v .:${DOCKER_WORKSPACE}:rw \
+  -v ./dist:/install/pglite:rw \
+  electricsql/pglite-builder:3.1.74_1 \
+  ./build-pglite.sh
+

clean-pglite.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+emmake make -C src/backend uninstall; emmake make -C src/backend clean;
+emmake make -C pglite/ clean; emmake make -C pglite/ uninstall;
+emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall;
+emmake make clean; emmake make uninstal

contrib/Makefile

@@ -90,6 +90,7 @@ endif
 # Missing:
 #		start-scripts	\ (does not have a makefile)
 
+include ./dist.mk
 
 $(recurse)
 $(recurse_always)

contrib/dist.mk

@@ -0,0 +1,27 @@
+# contrib/dist.mk
+#
+# Package each contrib extension into its own .tar.gz archive
+
+prefix ?= /install/pglite
+CONTRIB_BUILD_ROOT := /tmp/extensions/build
+ARCHIVE_DIR := /install/pglite/extensions
+
+CONTRIBS := $(SUBDIRS)
+
+# Default target: build tarballs for all contribs
+dist: $(addsuffix .tar.gz,$(CONTRIBS))
+
+# Pattern rule: build $(EXT).tar.gz for each contrib
+%.tar.gz:
+	@echo "=== Staging $* ==="
+	rm -rf $(CONTRIB_BUILD_ROOT)/$*
+	bash -c 'mkdir -p $(CONTRIB_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}'
+	$(MAKE) -C $* install DESTDIR=$(CONTRIB_BUILD_ROOT)/$*
+	@echo "=== Packaging $* ==="
+	mkdir -p $(ARCHIVE_DIR)
+	cd $(CONTRIB_BUILD_ROOT)/$*/$(prefix) && \
+	files=$$(find . -type f -o -type l | sed 's|^\./||') && \
+	tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files
+# 	tar -C $(CONTRIB_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz .
+
+.PHONY: dist

other/PGPASSFILE

@@ -0,0 +1,3 @@
+localhost:5432:postgres:password:md532e12f215ba27cb750c9e093ce4b5127
+localhost:5432:postgres:postgres:md53175bce1d3201d16594cebf9d7eb3f9d
+localhost:5432:postgres:login:md5d5745f9425eceb269f9fe01d0bef06ff

other/password

@@ -0,0 +1 @@
+password

pglite-wasm/builder/Dockerfile

@@ -0,0 +1,89 @@
+ARG EMSDK_VER=3.1.74
+ARG BUILDPLATFORM
+# we only need to build on one platform, since we're interested in the WASM output
+# building on the native (BUILDPLATFORM) is much faster, so use that
+# remove "-arm64" suffix if building on x86_64
+FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER}-arm64 AS builder
+
+ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm
+
+RUN apt update && apt upgrade -y && apt install -y \
+    xz-utils autoconf libtool automake pkgconf bison flex
+
+SHELL ["/bin/bash", "-c"]
+
+RUN mkdir -p /install/libs /install/exports
+
+WORKDIR /install/libs
+
+# zlib CAN be installed by using the emsdk like this
+# RUN embuilder --pic --verbose build zlib
+
+WORKDIR /src
+
+# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn"
+ENV EMCC_COMMON_FLAGS="-O2 -fPIC"
+
+WORKDIR /src
+RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz
+WORKDIR /src/zlib-1.3.1
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --static --prefix=/install/libs
+RUN emmake make -j && emmake make install
+RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libz.exports
+
+WORKDIR /src
+RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz
+WORKDIR /src/libxml2-v2.14.5
+RUN ./autogen.sh --with-python=no
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs
+RUN emmake make -j && emmake make install
+# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
+RUN ${LLVM_NM} /install/libs/lib/libxml2.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxml2.exports
+
+WORKDIR /src
+RUN curl -L https://gitlab.gnome.org/GNOME/libxslt/-/archive/v1.1.43/libxslt-v1.1.43.tar.gz | tar -xz
+WORKDIR /src/libxslt-v1.1.43
+RUN ./autogen.sh --with-python=no
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs --with-libxml-src=/src/libxml2-v2.14.5/ --with-pic=yes
+RUN emmake make -j && emmake make install
+# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
+RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxslt.exports
+
+WORKDIR /src
+RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz
+WORKDIR /src/openssl-3.0.17
+RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs
+RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454
+RUN emmake make -j && emmake make install
+# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
+RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports
+
+WORKDIR /src
+RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz
+WORKDIR /src/uuid-1.6.2
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes
+RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm
+WORKDIR /install/libs/lib
+RUN ln -s libuuid.a libossp-uuid.a # contrib extensions use -lossp-uuid
+RUN ${LLVM_NM} /install/libs/lib/libossp-uuid.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libossp-uuid.exports
+
+ARG TARGETARCH
+FROM emscripten/emsdk:${EMSDK_VER} AS base-amd64
+FROM emscripten/emsdk:${EMSDK_VER}-arm64 AS base-arm64
+FROM base-${TARGETARCH} AS runner
+
+RUN apt update && apt upgrade -y && apt install -y \
+    xz-utils autoconf libtool automake pkgconf bison flex
+
+# this is where the libraries will be installed and subsequently where the LIBS and INCLUDES can be found
+ARG INSTALL_PREFIX=/install/libs
+ENV INSTALL_PREFIX=${INSTALL_PREFIX}
+
+ARG LIB_EXPORTS_DIR=/install/exports
+ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR}
+
+COPY --from=builder /install/libs ${INSTALL_PREFIX}
+COPY --from=builder /install/exports ${LIB_EXPORTS_DIR}
+
+# needed in building pglite.wasm
+ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm

pglite-wasm/excluded.pglite.imports

@@ -0,0 +1,32 @@
+__indirect_function_table
+invoke_di
+invoke_didi
+invoke_dii
+invoke_i
+invoke_ii
+invoke_iii
+invoke_iiii
+invoke_iiiii
+invoke_iiiiii
+invoke_iiiiiii
+invoke_iiiiiiii
+invoke_iij
+invoke_ij
+invoke_ijj
+invoke_j
+invoke_ji
+invoke_jiiiii
+invoke_v
+invoke_vi
+invoke_vii
+invoke_viii
+invoke_viiidi
+invoke_viiii
+invoke_viiiii
+invoke_viiiiii
+invoke_viiiiii
+invoke_viiiiiiii
+invoke_viiiiiiiii
+invoke_viiji
+invoke_vij
+invoke_vijiiidjiiii

pglite-wasm/included.pglite.imports

@@ -0,0 +1,38 @@
+close
+fcntl
+free
+get_buffer_addr
+get_buffer_size
+get_channel
+getpid
+gettimeofday
+gmtime
+interactive_one
+interactive_read
+interactive_write
+ioctl
+isalnum
+isxdigit
+lowerstr
+main
+malloc
+memcmp
+memcpy
+memset
+nanosleep
+open
+pgl_backend
+pgl_initdb
+pgl_shutdown
+rand
+read
+readstoplist
+realloc
+searchstoplist
+socket
+srand
+strcmp
+strftime
+strlen
+strtoul
+use_wire

pglite-wasm/pg_main.c

@@ -26,6 +26,10 @@
 #include <unistd.h>             /* chdir */
 #include <sys/stat.h>           /* mkdir */
 
+#if defined(__EMSCRIPTEN__)
+#include <emscripten.h>
+#endif
+
 // globals
 
 
@@ -482,7 +486,8 @@ __attribute__ ((export_name("pgl_backend")))
 #else
         remove(IDB_PIPE_BOOT);
 #endif
-        stdin = fdopen(saved_stdin, "r");
+        // tdrz: I've commented this out!!!
+        // stdin = fdopen(saved_stdin, "r");
 
         PDEBUG("# 479: initdb faking shutdown to complete WAL/OID states");
         pg_proc_exit(66);

pglite/Makefile

@@ -0,0 +1,35 @@
+# Packages other extensions into individual tar.gz archives
+
+top_builddir = ..
+include $(top_builddir)/src/Makefile.global
+
+SUBDIRS = \
+		pg_ivm \
+		vector
+
+prefix ?= /install/pglite
+EXTENSIONS_BUILD_ROOT := /tmp/extensions/build
+ARCHIVE_DIR := /install/pglite/extensions
+
+EXTENSIONS := $(SUBDIRS)
+
+# Default target: build tarballs for all contribs
+dist: $(addsuffix .tar.gz,$(EXTENSIONS))
+
+# Pattern rule: build $(EXT).tar.gz for each extra extension
+%.tar.gz:
+	@echo "=== Staging $* ==="
+	rm -rf $(EXTENSIONS_BUILD_ROOT)/$*
+	bash -c 'mkdir -p $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}'
+	$(MAKE) -C $* install DESTDIR=$(EXTENSIONS_BUILD_ROOT)/$*
+	@echo "=== Packaging $* ==="
+	mkdir -p $(ARCHIVE_DIR)
+	cd $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) && \
+	files=$$(find . -type f -o -type l | sed 's|^\./||') && \
+	tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files
+# 	tar -C $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz .
+
+.PHONY: dist
+
+
+$(recurse)

src/Makefile.shlib

@@ -225,13 +225,14 @@ ifeq ($(SHLIB_EXPORTS),)
 endif
 
 ifeq ($(PORTNAME), emscripten)
-  LINK.shared       = emsdk-shared
+  LINK.shared       = $(COMPILER) -shared -sSIDE_MODULE=1 -Wno-unused-function
   ifdef soname
     # emscripten uses unversioned shared libraries
     shlib     = $(shlib_bare)
     soname        = $(shlib_bare)
   endif
-  BUILD.exports     = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@
+  BUILD.exports     = ( $(AWK) '/^[^\#]/ {printf "%s\n",$$1}' $< ) | sort -u >$@
+#  BUILD.exports     = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@
   exports_file      = $(SHLIB_EXPORTS:%.txt=%.list)
 #  ifneq (,$(exports_file))
 #    LINK.shared     += -Wl,--version-script=$(exports_file)