Lots of Changes

.github/workflows/push.yml

@@ -13,19 +13,51 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [x64, arm64]
+        platform: [x64]
     steps:
       - uses: actions/checkout@v3
       - name: compile runtime
         run: |
           make ARCH=${{ matrix.platform }} lo
-#          ./lo build binding core
       - name: check
-        if: ${{ matrix.platform == 'x64' }}
+        env:
+          LO_HOME: ${{ github.workspace }}
         run: |
           make check
-          brew install libffi lz4 zstd jsonschema
-          LO_PREFIX=/usr/local/opt ./lo test/build.js
+          brew install lz4 zstd openssl@3
+          ./lo test/build.js
+          ./lo build runtime runtimes/lo
+      - name: upload runtime artifact
+        uses: actions/upload-artifact@v3
+        with:
+          path: ${{ github.workspace }}/lo
+          name: lo-mac-${{ matrix.platform }}
+
+  build-mac-arm:
+#    if: ${{ false }}  # disable
+    name: mac
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [arm64]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '^1.22.1'
+      - name: compile runtime
+        run: |
+          make ARCH=${{ matrix.platform }} lo
+      - name: check
+        env:
+          LO_HOME: ${{ github.workspace }}
+          PREFIX: /opt/homebrew/opt
+        run: |
+          make check
+          brew install lz4 zstd openssl@3
+          ./lo test/build.js
+          ./lo build runtime runtimes/lo
       - name: upload runtime artifact
         uses: actions/upload-artifact@v3
         with:
@@ -49,12 +81,14 @@ jobs:
       - name: compile
         run: |
           make lo
-#          ./lo build binding core
       - name: check
+        env:
+          LO_HOME: ${{ github.workspace }}
         run: |
           make check
           sudo apt-get install -qy libffi-dev liblz4-dev libseccomp-dev libsqlite3-dev libtcc-dev zlib1g-dev tcc
           ./lo test/build.js
+          ./lo build runtime runtimes/lo
       - name: upload artifact
         uses: actions/upload-artifact@v3
         with:
@@ -79,7 +113,7 @@ jobs:
           sudo apt-get install -qy binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
       - name: compile
         run: |
-          make ARCH=arm64 C=aarch64-linux-gnu-gcc CC=aarch64-linux-gnu-g++ lo
+          make ARCH=arm64 CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ lo
       - name: upload artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/release.yml

@@ -17,16 +17,53 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [x64, arm64]
+        platform: [x64]
+    steps:
+      - uses: actions/checkout@v3
+      - name: compile
+        run: |
+          make ARCH=${{ matrix.platform }} lo
+      - name: check
+        env:
+          LO_HOME: ${{ github.workspace }}
+        run: |
+          make check
+          brew install lz4 zstd openssl@3
+          ./lo build runtime runtimes/lo
+      - name: compress runtime
+        run: |
+          gzip -9 -c lo > lo-mac-${{ matrix.platform }}.gz
+      - name: upload compressed artifact
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: lo-mac-${{ matrix.platform }}.gz
+          asset_name: lo-mac-${{ matrix.platform }}.gz
+          asset_content_type: application/gzip
+
+  build-mac-arm:
+#    if: ${{ false }}  # disable
+    name: mac
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [arm64]
     steps:
       - uses: actions/checkout@v3
       - name: compile
         run: |
           make ARCH=${{ matrix.platform }} lo
       - name: check
-        if: ${{ matrix.platform == 'x64' }}      
+        env:
+          LO_HOME: ${{ github.workspace }}
+          PREFIX: /opt/homebrew/opt
         run: |
           make check
+          brew install lz4 zstd openssl@3
+          ./lo build runtime runtimes/lo
       - name: compress runtime
         run: |
           gzip -9 -c lo > lo-mac-${{ matrix.platform }}.gz
@@ -58,8 +95,12 @@ jobs:
         run: |
           make lo
       - name: check
+        env:
+          LO_HOME: ${{ github.workspace }}
         run: |
           make check
+          sudo apt-get install -qy libffi-dev liblz4-dev libseccomp-dev libsqlite3-dev libtcc-dev zlib1g-dev tcc
+          ./lo build runtime runtimes/lo
       - name: compress runtime
         run: |
           gzip -9 -c lo > lo-linux-x64.gz

.gitignore

@@ -18,5 +18,11 @@ lib/inflate/em_inflate.c
 lib/inflate/em_inflate.h
 lib/**/*.cc
 !lib/core/core.cc
+!lib/curl/curl.cc
 !lib/inflate/inflate.cc
 !lib/mach/mach.cc
+.lo_history
+lib/bestlines/bestline.c
+lib/bestlines/bestline.h
+lib/pico/picohttpparser.c
+lib/pico/picohttpparser.h

Makefile

@@ -1,38 +1,44 @@
-C=clang
-CC=clang++
-LARGS=-rdynamic -pthread
+CC=clang
+CXX=clang++
+LINK=clang++
+LARGS=-rdynamic -pthread -static-libstdc++
 CCARGS=-std=c++17 -c -fno-omit-frame-pointer -fno-rtti -fno-exceptions
 CARGS=-c -fno-omit-frame-pointer
 WARN=-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter
 OPT=-O3
-VERSION=0.0.13-pre
-V8_VERSION=1.0.0
+VERSION=0.0.14-pre
+V8_VERSION=12.3
 RUNTIME=lo
 LO_HOME=$(shell pwd)
-BINDINGS=core.o curl.o inflate.a
+BINDINGS=core.o inflate.a curl.o
 ARCH=x64
 os=linux
 TARGET=${RUNTIME}
-LIBS=-lcurl -ldl
-V8_FLAGS=-DV8_COMPRESS_POINTERS -DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=0
+LIBS=-ldl -lcurl -lssl -lz
+V8_FLAGS=-DV8_COMPRESS_POINTERS -DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=0 -DV8_INTL_SUPPORT=1
+LIB_DIRS=
 
 ifeq ($(OS),Windows_NT)
 	os=win
 else
-    UNAME_S := $(shell uname -s)
-    ifeq ($(UNAME_S),Linux)
-			os=linux
-			LARGS+=-s
-    else ifeq ($(UNAME_S),Darwin)
-			os=mac
-			BINDINGS+=mach.o
-			LARGS+=-s -w
-			ifeq ($(ARCH),arm64)
-				LARGS+=-arch arm64
-				CARGS+=-arch arm64
-				CCARGS+=-arch arm64
-			endif
-    endif
+	UNAME_S := $(shell uname -s)
+	ifeq ($(UNAME_S),Linux)
+		os=linux
+		LARGS+=-s -static-libgcc
+		CC=gcc
+		CXX=g++
+		LINK=g++
+	else ifeq ($(UNAME_S),Darwin)
+		os=mac
+		BINDINGS+=mach.o
+		LARGS+=-s -w
+		LIB_DIRS+=-L"/opt/homebrew/lib"
+		ifeq ($(ARCH),arm64)
+			LARGS+=-arch arm64
+			CARGS+=-arch arm64
+			CCARGS+=-arch arm64
+		endif
+	endif
 endif
 
 .PHONY: help clean
@@ -64,42 +70,50 @@ v8/v8_monolith.lib: ## download the v8 static library for windows
 	gzip -d v8/v8_monolith.lib.gz
 
 main.o: ## compile the main.cc object file
-	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' -I./v8 -I./v8/include ${WARN} ${V8_FLAGS} main.cc
+	$(CXX) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' -I./v8 -I./v8/include ${WARN} ${V8_FLAGS} main.cc
 
 builtins.o: ## link all source files and assets into an object file
 ifeq (${os},linux)
-	$(C) ${CARGS} builtins_linux.S -o builtins.o
+	$(CC) ${CARGS} builtins_linux.S -o builtins.o
 else
-	$(C) ${CARGS} builtins.S -o builtins.o
+	$(CC) ${CARGS} builtins.S -o builtins.o
 endif
 
 ${RUNTIME}.o: ## compile runtime into an object file 
-	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' ${V8_FLAGS} -I./v8 -I./v8/include ${WARN} ${RUNTIME}.cc
+	$(CXX) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' ${V8_FLAGS} -I./v8 -I./v8/include ${WARN} ${RUNTIME}.cc
 
 ${RUNTIME}: v8/include v8/libv8_monolith.a main.js ${BINDINGS} builtins.o main.o ${RUNTIME}.o ## link the runtime for linux/macos
 	@echo building ${RUNTIME} for ${os} on ${ARCH}
-	$(CC) $(LARGS) ${OPT} main.o ${RUNTIME}.o builtins.o ${BINDINGS} v8/libv8_monolith.a ${LIBS} -o ${TARGET}
+	$(LINK) $(LARGS) ${OPT} main.o ${RUNTIME}.o builtins.o ${BINDINGS} ${LIBS} -o ${TARGET} -L"./v8" -lv8_monolith ${LIB_DIRS}
 
 ${RUNTIME}.exe: v8/include v8/v8_monolith.lib main.js ## link the runtime for windows
 	cl /EHsc /std:c++17 /DRUNTIME='"${RUNTIME}"' /DVERSION='"${VERSION}"' /I./v8 /I./v8/include /c main.cc
 	cl /EHsc /std:c++17 /DRUNTIME='"${RUNTIME}"' /DVERSION='"${VERSION}"' /I./v8 /I./v8/include /c ${RUNTIME}.cc
 	cl v8/v8_monolith.lib ${RUNTIME}.obj main.obj winmm.lib dbghelp.lib advapi32.lib /link /out:${TARGET}.exe
 
-mach.o: lib/mach/mach.cc ## build the core binding
-	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o mach.o lib/mach/mach.cc
+mach.o: lib/mach/mach.cc ## build the mach binding
+	$(CXX) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o mach.o lib/mach/mach.cc
 
 core.o: lib/core/core.cc ## build the core binding
-	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o core.o lib/core/core.cc
+	$(CXX) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o core.o lib/core/core.cc
 
 curl.o: lib/curl/curl.cc ## build the curl binding
-	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o curl.o lib/curl/curl.cc
+	$(CXX) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o curl.o lib/curl/curl.cc
 
-inflate.a: lib/inflate/inflate.cc ## build the curl binding
+lib/inflate/em_inflate.h:
 	curl -L -o lib/inflate/em_inflate.h https://raw.githubusercontent.com/emmanuel-marty/em_inflate/master/lib/em_inflate.h
+
+lib/inflate/em_inflate.c:
 	curl -L -o lib/inflate/em_inflate.c https://raw.githubusercontent.com/emmanuel-marty/em_inflate/master/lib/em_inflate.c
-	$(C) -fPIC $(CARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate -o em_inflate.o lib/inflate/em_inflate.c
-	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate $(WARN) ${V8_FLAGS} -o inflate.o lib/inflate/inflate.cc
-	ar crsT inflate.a inflate.o em_inflate.o
+
+lib/inflate/em_inflate.o: lib/inflate/em_inflate.h lib/inflate/em_inflate.c ## build the em_inflate object
+	$(CC) -fPIC $(CARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate -o lib/inflate/em_inflate.o lib/inflate/em_inflate.c
+
+inflate.o: lib/inflate/inflate.cc lib/inflate/em_inflate.h ## build the em_inflate object
+	$(CXX) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate $(WARN) ${V8_FLAGS} -o inflate.o lib/inflate/inflate.cc
+
+inflate.a: inflate.o lib/inflate/em_inflate.o ## build the inflate binding
+	ar crsT inflate.a inflate.o lib/inflate/em_inflate.o
 
 check: ## run the runtime sanity tests
 	./${RUNTIME} test/runtime.js
@@ -126,6 +140,7 @@ else
 	rm -f *.o
 	rm -f *.a
 	rm -f lib/**/*.a
+	rm -f lib/**/*.o
 	rm -f lib/**/*.so
 	rm -f ${RUNTIME}
 endif

README.md

@@ -13,19 +13,30 @@
 
 ## linux
 
+the default build of lo runtime links to system libssl and zlib but we build a 
+custom version of libcurl as the default libcurl links to a ton of support 
+libraries we don't need and this slows down startup time. we will likely remove 
+dependency on libcurl at some point (i.e. when we have a robust implementation 
+of fetch in the runtime) but, for now, you will need to install libcurl as 
+follows.
+
+```shell
+sudo apt install -y libcurl4-dev
+```
+
 ### gcc/x64
 ```
-make cleanall lo
+make lo
 ```
 
 ### arm64
 ```
-make ARCH=arm64 cleanall lo
+make ARCH=arm64 lo
 ```
 
 ### clang
 ```
-make C="clang" CC="clang++" clean lo
+make CC="clang" CXX="clang++" lo
 ```
 
 ### ccache and mold for fast rebuilds
@@ -35,6 +46,25 @@ mold -run make C="ccache gcc" CC="ccache g++" lo
 
 ## macos
 
+the default build of lo runtime depends on libcurl, which is available out
+of the box on macos. it also depends on openssl, which does not seem to be 
+available as a static library by default.
+
+as such, you will need to install openssl using homebrew or some other 
+mechanism for the default build to succeed.
+
+```shell
+brew install openssl@3
+```
+
+this will install libssl libs in /opt/homebrew/lib by default. if you need to 
+override this default in the Makefile you can pass the LIB_DIRS argument to make
+as follows
+
+```shell
+make LIB_DIRS=/opt/somewhere_else/lib ARCH=arm64 lo
+```
+
 ### x64
 ```
 make cleanall lo

TODO.md

@@ -73,6 +73,20 @@
 - [ ] **question**: how do we model constants like RTLD_DEFAULT which seem to be pointers to things?
 - [ ] **bug**:  on macos, when i do a ```lo build``` it generates same main.h for main.h main_win.h and main_mac.h
 - [ ] **todo**: we need a way to ignore auto-generated files for git, but also to be able to re-generate them to check in when we need to.
+- [ ] **todo**: figure out a way to create builtins for v8 and have them automatically baked in, e.g.
+    https://github.com/denoland/v8/commit/8feae8c6166c2867554fb3f99fdd2b59a4a83c37
+    https://github.com/denoland/v8/commit/2ca3e486b84398f163775450e7d0cc3f4868c981
+    https://github.com/denoland/v8/commit/ae31efc15ffc948d1616d5e393a8f62fff1bfce8
+- [ ] **todo**: use CC and CXX instead of C and CC
+- [ ] **todo**: when building, do some pre-processing like stripping comments, minification and compression
+- [ ] **bug**: bug in linux event loop where event loop drops out after all tcp server connections go away. but we should still have the server fd and the timer on the loop
+- [ ] **todo**: allow setting arch: ```['arm64', 'x64']``` on bindings definitions so we can exclude for certain arches (e.g. boringssl)
+- [ ] **bug**: cannot compile boringssl with -fPIC
+- [ ] **bug**: SIGWNCH kills the app
+- [ ] **todo**: change console.log and console.error in threads so they don't write to stdout and stderr
+- [ ] **todo**: try/catch in loop callback and call on_error if we get an error
+- [ ] **todo**: make build.js build method async
+- [ ] **todo**: add a method for reading a bindary builtin into a buffer, or even just reading it as a memory stream. - wrapmemory? mmap? memfd_create?
 
 ## features