Demo fixes (#131)

* exportapp: include in filesystem and accept app path

Usage has be changed to:
`GOOS=<os> GOARCH=<arch> exportapp <output-name> <app-path>`

* build: cache pkg in `/sys/build` instead of temp dir

Using the temp dir isn't as practical considering we have to reload
to update the shell binary, causing needless unpacking every time.
In the future this may become a non-issue with process reloading.

That being said, build-pkg data doesn't change very often and it makes sense
to have a semi-permanent cache.

* fs: only copy in kernel binaries if they don't already exist

* fs: always copy terminal app

Fixes 404 error on localhost.

build/build-pkgs/main.go

@@ -150,7 +150,7 @@ func generateLinkConfig(zw *zip.Writer, target string, packages []string) {
 	}
 
 	for _, pkg := range packages {
-		io.WriteString(wr, "packagefile "+pkg+"=/sys/tmp/build/pkg/targets/"+target+"/"+pkg+".a\n")
+		io.WriteString(wr, "packagefile "+pkg+"=/sys/build/pkg/targets/"+target+"/"+pkg+".a\n")
 	}
 }
 

build/main.go

@@ -178,7 +178,7 @@ func main2(flags BuildFlags, args []string) int {
 
 	bw := bufio.NewWriter(importcfg)
 	for i := range imports {
-		fmt.Fprintf(bw, "packagefile %s=/sys/tmp/build/pkg/targets/%s_%s/%[1]s.a\n", strings.Trim(i, "\""), target.os, target.arch)
+		fmt.Fprintf(bw, "packagefile %s=/sys/build/pkg/targets/%s_%s/%[1]s.a\n", strings.Trim(i, "\""), target.os, target.arch)
 	}
 	if err := bw.Flush(); err != nil {
 		fmt.Println("unable to write to importcfg:", err)
@@ -189,8 +189,7 @@ func main2(flags BuildFlags, args []string) int {
 		return 1
 	}
 
-	fmt.Println("Unpacking pkg.zip...")
-	if err := openZipPkg("/sys/tmp/build", target); err != nil {
+	if err := openZipPkg("/sys/build", target); err != nil {
 		fmt.Println("unable to open pkg.zip:", err)
 		return 1
 	}
@@ -212,7 +211,7 @@ func main2(flags BuildFlags, args []string) int {
 
 	// run compile.wasm
 	fmt.Printf("Compiling %s to %s\n", args[0], objPath)
-	exitcode, err := run("/sys/tmp/build/pkg/compile.wasm", compileArgs...)
+	exitcode, err := run("/sys/build/pkg/compile.wasm", compileArgs...)
 	if exitcode != 0 || err != nil {
 		if err != nil {
 			fmt.Println(err)
@@ -223,7 +222,7 @@ func main2(flags BuildFlags, args []string) int {
 		return exitcode
 	}
 
-	linkcfg := fmt.Sprintf("/sys/tmp/build/pkg/importcfg_%s_%s.link", target.os, target.arch)
+	linkcfg := fmt.Sprintf("/sys/build/pkg/importcfg_%s_%s.link", target.os, target.arch)
 
 	output, err := getOutputPath(target.arch, *flags.Output, pkg.Name, absPkgPath)
 	if err != nil {
@@ -234,7 +233,7 @@ func main2(flags BuildFlags, args []string) int {
 	// run link.wasm using importcfg_$GOOS_$GOARCH.link
 	fmt.Println("Linking", objPath)
 	exitcode, err = run(
-		"/sys/tmp/build/pkg/link.wasm",
+		"/sys/build/pkg/link.wasm",
 		"-importcfg", linkcfg,
 		"-buildmode=exe",
 		"-o", output,
@@ -319,11 +318,15 @@ func run(name string, args ...string) (int, error) {
 }
 
 func openZipPkg(dest string, tgt Target) error {
+	fmt.Printf("Unpacking pkg.zip/%s to %s...\n", tgt.print(), dest)
+
 	pkg, err := zip.NewReader(bytes.NewReader(zipEmbed), int64(len(zipEmbed)))
 	if err != nil {
 		return err
 	}
 
+	os.MkdirAll(filepath.Join(dest, "pkg"), 0755)
+
 	dfs := os.DirFS(dest)
 
 	// TODO: Optimize

dev/bootloader.js

@@ -125,13 +125,11 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
     await kernelReady;
     globalThis.sys = new task.Task(initfs);
 
-    // start kernel
     console.log("Starting kernel...")
     await sys.exec("kernel");
 
     // load host API
     await import(URL.createObjectURL(initfs["host.js"].blob));
-
   });
 }
 

dev/initdata.go

@@ -21,6 +21,9 @@ var files = []File{
 	{Name: "host.js", Path: "./kernel/web/lib/host.js"},
 	{Name: "indexedfs.js", Path: "./internal/indexedfs/indexedfs.js"},
 
+	{Name: "export/exportapp.sh", Path: "./internal/export/exportapp.sh"},
+	{Name: "export/main.go", Path: "./internal/export/main.go"},
+
 	// Shell source files
 	{Name: "shell/main.go", Path: "shell/main.go"},
 	{Name: "shell/copy.go", Path: "shell/copy.go"},

internal/export/exportapp.sh

@@ -1,5 +1,5 @@
 rm -r /sys/export/assets
-cp -r /app/$1 /sys/export/assets
+cp -r $2 /sys/export/assets
 mkdir /sys/tmp/export
 build -os $GOOS -arch $GOARCH -output /sys/tmp/export/$1 /sys/export
 dl /sys/tmp/export/$1

kernel/fs/fs.go

@@ -80,6 +80,7 @@ func (s *Service) Initialize(kernelSource embed.FS, p *proc.Service) {
 	fs.MkdirAll(s.fsys, "sys/cmd", 0755)
 	fs.MkdirAll(s.fsys, "sys/dev", 0755)
 	fs.MkdirAll(s.fsys, "sys/tmp", 0755)
+	fs.MkdirAll(s.fsys, "sys/build", 0755)
 
 	devURL := fmt.Sprintf("%ssys/dev", js.Global().Get("hostURL").String())
 	devMode := false
@@ -99,8 +100,8 @@ func (s *Service) Initialize(kernelSource embed.FS, p *proc.Service) {
 	}
 
 	// copy some apps including terminal
+	must(s.copyAllFS(s.fsys, "sys/app/terminal", internal.Dir, "app/terminal"))
 	if !devMode {
-		must(s.copyAllFS(s.fsys, "sys/app/terminal", internal.Dir, "app/terminal"))
 		must(s.copyAllFS(s.fsys, "sys/app/todo", internal.Dir, "app/todo"))
 		must(s.copyAllFS(s.fsys, "sys/app/jazz-todo", internal.Dir, "app/jazz-todo"))
 		must(s.copyAllFS(s.fsys, "sys/app/explorer", internal.Dir, "app/explorer"))
@@ -113,13 +114,24 @@ func (s *Service) Initialize(kernelSource embed.FS, p *proc.Service) {
 		must(s.copyFromInitFS(filepath.Join("sys/cmd", path), path))
 	}
 
-	// copy of kernel source into filesystem.
-	must(s.copyAllFS(s.fsys, "sys/cmd/kernel", kernelSource, "."))
+	// copy all of kernel source except `/bin` into filesystem.
+	must(s.copyKernelSource("sys/cmd/kernel", kernelSource))
 
-	// move builtin kernel exe's into filesystem
-	must(s.fsys.Rename("sys/cmd/kernel/bin/build", "sys/cmd/build.wasm"))
-	must(s.fsys.Rename("sys/cmd/kernel/bin/shell", "sys/bin/shell.wasm"))
-	must(s.fsys.Rename("sys/cmd/kernel/bin/micro", "sys/cmd/micro.wasm"))
+	// copy embedded kernel exe's into filesystem if they don't already exist
+	if exists, _ := fs.Exists(s.fsys, "sys/cmd/build.wasm"); !exists {
+		must(copyFileAcrossFS(s.fsys, "sys/cmd/build.wasm", kernelSource, "bin/build"))
+	}
+	if exists, _ := fs.Exists(s.fsys, "sys/cmd/micro.wasm"); !exists {
+		must(copyFileAcrossFS(s.fsys, "sys/cmd/micro.wasm", kernelSource, "bin/micro"))
+	}
+	if exists, _ := fs.Exists(s.fsys, "sys/bin/shell.wasm"); !exists {
+		must(copyFileAcrossFS(s.fsys, "sys/bin/shell.wasm", kernelSource, "bin/shell"))
+	}
+
+	// setup exportapp
+	fs.MkdirAll(s.fsys, "sys/export", 0755)
+	must(s.copyFromInitFS("sys/export/main.go", "export/main.go"))
+	must(s.copyFromInitFS("sys/cmd/exportapp.sh", "export/exportapp.sh"))
 
 	must(s.fsys.(*mountablefs.FS).Mount(memfs.New(), "/sys/tmp"))
 
@@ -175,6 +187,18 @@ func getPrefixedInitFiles(prefix string) []string {
 	return result
 }
 
+func copyFileAcrossFS(dstFS fs.MutableFS, dstPath string, srcFS fs.FS, srcPath string) error {
+	srcData, err := fs.ReadFile(srcFS, srcPath)
+	if err != nil {
+		return err
+	}
+	err = fs.MkdirAll(dstFS, filepath.Dir(dstPath), 0755)
+	if err != nil {
+		return err
+	}
+	return fs.WriteFile(dstFS, dstPath, srcData, 0644)
+}
+
 func (s *Service) copyAllFS(dstFS fs.MutableFS, dstDir string, srcFS fs.FS, srcDir string) error {
 	if err := fs.MkdirAll(dstFS, dstDir, 0755); err != nil {
 		return err
@@ -184,13 +208,28 @@ func (s *Service) copyAllFS(dstFS fs.MutableFS, dstDir string, srcFS fs.FS, srcD
 			return err
 		}
 		if !d.IsDir() {
-			srcData, err := fs.ReadFile(srcFS, path)
-			if err != nil {
-				return err
-			}
 			dstPath := filepath.Join(dstDir, strings.TrimPrefix(path, srcDir))
-			fs.MkdirAll(dstFS, filepath.Dir(dstPath), 0755)
-			return fs.WriteFile(dstFS, dstPath, srcData, 0644)
+			return copyFileAcrossFS(dstFS, dstPath, srcFS, path)
+		}
+		return nil
+	}))
+}
+
+// Avoids copying the bin directory
+func (s *Service) copyKernelSource(dstDir string, srcFS fs.FS) error {
+	if err := fs.MkdirAll(s.fsys, dstDir, 0755); err != nil {
+		return err
+	}
+	return fs.WalkDir(srcFS, ".", fs.WalkDirFunc(func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() && path == "bin" {
+			return fs.SkipDir
+		}
+		if !d.IsDir() {
+			dstPath := filepath.Join(dstDir, strings.TrimPrefix(path, "."))
+			return copyFileAcrossFS(s.fsys, dstPath, srcFS, path)
 		}
 		return nil
 	}))

kernel/proc/proc.go

@@ -53,7 +53,7 @@ func (s *Service) GetAll() map[int]*Process {
 }
 
 func (s *Service) Spawn(path string, args []string, env map[string]string, dir string) (*Process, error) {
-	// TODO: check path exists, execute bit
+	// TODO: execute bit
 
 	// can't use jsutil.WanixSyscall inside the kernel
 	stat, err := jsutil.AwaitErr(js.Global().Get("api").Get("fs").Call("stat", path))