Skip to content

Commit

Permalink
fsapi: adds Oflag to decouple from syscall package
Browse files Browse the repository at this point in the history 
This adds `fsapi.Oflag` which includes oflag values needed to support
`fsapi.FS`. Notably, this covers more area than Go does. Go defines
portable abstractions for most open flags, but not all. Flags like
`O_DIRECTORY` are needed by wasip1 and not defined in Go as
`os.O_DIRECTORY`. Also, some operating systems don't define these
either. Instead we have a new `fsapi.Oflag` type which includes all
flags needed, even if notably in windows behavior is emulated.

Signed-off-by: Adrian Cole <adrian@tetrate.io>
  • Loading branch information
Adrian Cole committed Jul 19, 2023
1 parent 1e0c73d commit 26c0b44
Show file tree
Hide file tree
Showing 47 changed files with 628 additions and 496 deletions.
2 changes: 1 addition & 1 deletion experimental/sys/errno.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ type Errno uint16
// the Errno it returns, and we export fs.FS. This is not in /internal/sys as
// that would introduce a package cycle.

// This is a subset of errors to reduce implementation burden. `wasip` defines
// This is a subset of errors to reduce implementation burden. `wasip1` defines
// almost all POSIX error numbers, but not all are used in practice. wazero
// will add ones needed in POSIX order, as needed by functions that explicitly
// document returning them.
Expand Down
38 changes: 24 additions & 14 deletions imports/wasi_snapshot_preview1/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1687,53 +1687,63 @@ func preopenPath(fsc *sys.FSContext, fd int32) (string, experimentalsys.Errno) {
}
}

func openFlags(dirflags, oflags, fdflags uint16, rights uint32) (openFlags int) {
func openFlags(dirflags, oflags, fdflags uint16, rights uint32) (openFlags fsapi.Oflag) {
if dirflags&wasip1.LOOKUP_SYMLINK_FOLLOW == 0 {
openFlags |= fsapi.O_NOFOLLOW
}
if oflags&wasip1.O_DIRECTORY != 0 {
openFlags |= fsapi.O_DIRECTORY
return // Early return for directories as the rest of flags doesn't make sense for it.
} else if oflags&wasip1.O_EXCL != 0 {
openFlags |= syscall.O_EXCL
openFlags |= fsapi.O_EXCL
}
// Because we don't implement rights, we paritally rely on the open flags
// Because we don't implement rights, we partially rely on the open flags
// to determine the mode in which the file will be opened. This will create
// divergent behavior compared to WASI runtimes which have a more strict
// interpretation of the WASI capabilities model; for example, a program
// which sets O_CREAT but does not give read or write permissions will
// successfully create a file when running with wazero, but might get a
// permission denied error on other runtimes.
defaultMode := syscall.O_RDONLY
defaultMode := fsapi.O_RDONLY
if oflags&wasip1.O_TRUNC != 0 {
openFlags |= syscall.O_TRUNC
defaultMode = syscall.O_RDWR
openFlags |= fsapi.O_TRUNC
defaultMode = fsapi.O_RDWR
}
if oflags&wasip1.O_CREAT != 0 {
openFlags |= syscall.O_CREAT
defaultMode = syscall.O_RDWR
openFlags |= fsapi.O_CREAT
defaultMode = fsapi.O_RDWR
}
if fdflags&wasip1.FD_NONBLOCK != 0 {
openFlags |= fsapi.O_NONBLOCK
}
if fdflags&wasip1.FD_APPEND != 0 {
openFlags |= syscall.O_APPEND
defaultMode = syscall.O_RDWR
openFlags |= fsapi.O_APPEND
defaultMode = fsapi.O_RDWR
}
if fdflags&wasip1.FD_DSYNC != 0 {
openFlags |= fsapi.O_DSYNC
}
if fdflags&wasip1.FD_RSYNC != 0 {
openFlags |= fsapi.O_RSYNC
}
if fdflags&wasip1.FD_SYNC != 0 {
openFlags |= fsapi.O_SYNC
}

// Since rights were discontinued in wasi, we only interpret RIGHT_FD_WRITE
// because it is the only way to know that we need to set write permissions
// on a file if the application did not pass any of O_CREATE, O_APPEND, nor
// on a file if the application did not pass any of O_CREAT, O_APPEND, nor
// O_TRUNC.
const r = wasip1.RIGHT_FD_READ
const w = wasip1.RIGHT_FD_WRITE
const rw = r | w
switch {
case (rights & rw) == rw:
openFlags |= syscall.O_RDWR
openFlags |= fsapi.O_RDWR
case (rights & w) == w:
openFlags |= syscall.O_WRONLY
openFlags |= fsapi.O_WRONLY
case (rights & r) == r:
openFlags |= syscall.O_RDONLY
openFlags |= fsapi.O_RDONLY
default:
openFlags |= defaultMode
}
Expand Down
53 changes: 26 additions & 27 deletions imports/wasi_snapshot_preview1/fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"path"
"runtime"
"strings"
"syscall"
"testing"
gofstest "testing/fstest"
"time"
Expand Down Expand Up @@ -60,7 +59,7 @@ func Test_fdAllocate(t *testing.T) {
preopen := fsc.RootFS()
defer r.Close(testCtx)

fd, errno := fsc.OpenFile(preopen, fileName, os.O_RDWR, 0)
fd, errno := fsc.OpenFile(preopen, fileName, fsapi.O_RDWR, 0)
require.EqualErrno(t, 0, errno)

f, ok := fsc.LookupFile(fd)
Expand Down Expand Up @@ -138,10 +137,10 @@ func Test_fdClose(t *testing.T) {
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
preopen := fsc.RootFS()

fdToClose, errno := fsc.OpenFile(preopen, path1, os.O_RDONLY, 0)
fdToClose, errno := fsc.OpenFile(preopen, path1, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

fdToKeep, errno := fsc.OpenFile(preopen, path2, os.O_RDONLY, 0)
fdToKeep, errno := fsc.OpenFile(preopen, path2, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

// Close
Expand Down Expand Up @@ -253,10 +252,10 @@ func Test_fdFdstatGet(t *testing.T) {
stdin.File = stdinFile

// Make this file writeable, to ensure flags read-back correctly.
fileFD, errno := fsc.OpenFile(preopen, file, os.O_RDWR, 0)
fileFD, errno := fsc.OpenFile(preopen, file, fsapi.O_RDWR, 0)
require.EqualErrno(t, 0, errno)

dirFD, errno := fsc.OpenFile(preopen, dir, os.O_RDONLY, 0)
dirFD, errno := fsc.OpenFile(preopen, dir, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

tests := []struct {
Expand Down Expand Up @@ -503,12 +502,12 @@ func Test_fdFdstatSetFlags(t *testing.T) {
preopen := fsc.RootFS()
defer r.Close(testCtx)

// First, O_CREATE the file with O_APPEND. We use O_EXCL because that
// triggers an EEXIST error if called a second time with O_CREATE. Our
// logic should clear O_CREATE preventing this.
// First, O_CREAT the file with O_APPEND. We use O_EXCL because that
// triggers an EEXIST error if called a second time with O_CREAT. Our
// logic should clear O_CREAT preventing this.
const fileName = "file.txt"
// Create the target file.
fd, errno := fsc.OpenFile(preopen, fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE|syscall.O_EXCL, 0o600)
fd, errno := fsc.OpenFile(preopen, fileName, fsapi.O_RDWR|fsapi.O_APPEND|fsapi.O_CREAT|fsapi.O_EXCL, 0o600)
require.EqualErrno(t, 0, errno)

// Write the initial text to the file.
Expand Down Expand Up @@ -615,10 +614,10 @@ func Test_fdFilestatGet(t *testing.T) {
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
preopen := fsc.RootFS()

fileFD, errno := fsc.OpenFile(preopen, file, os.O_RDONLY, 0)
fileFD, errno := fsc.OpenFile(preopen, file, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

dirFD, errno := fsc.OpenFile(preopen, dir, os.O_RDONLY, 0)
dirFD, errno := fsc.OpenFile(preopen, dir, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

tests := []struct {
Expand Down Expand Up @@ -2157,7 +2156,7 @@ func Test_fdReaddir(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
defer log.Reset()

fd, errno := fsc.OpenFile(preopen, tc.initialDir, os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(preopen, tc.initialDir, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)
defer fsc.CloseFile(fd) // nolint

Expand Down Expand Up @@ -2199,7 +2198,7 @@ func Test_fdReaddir_Rewind(t *testing.T) {

fsc := mod.(*wasm.ModuleInstance).Sys.FS()

fd, errno := fsc.OpenFile(fsc.RootFS(), ".", os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(fsc.RootFS(), ".", fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

mem := mod.Memory()
Expand Down Expand Up @@ -2248,7 +2247,7 @@ func Test_fdReaddir_Errors(t *testing.T) {
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
preopen := fsc.RootFS()

fileFD, errno := fsc.OpenFile(preopen, "animals.txt", os.O_RDONLY, 0)
fileFD, errno := fsc.OpenFile(preopen, "animals.txt", fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

// Directories are stateful, so we open them during the test.
Expand Down Expand Up @@ -2352,7 +2351,7 @@ func Test_fdReaddir_Errors(t *testing.T) {

// Reset the directory so that tests don't taint each other.
if tc.fd == dirFD {
dirFD, errno = fsc.OpenFile(preopen, "dir", os.O_RDONLY, 0)
dirFD, errno = fsc.OpenFile(preopen, "dir", fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)
defer fsc.CloseFile(dirFD) // nolint
}
Expand Down Expand Up @@ -2465,11 +2464,11 @@ func Test_fdRenumber(t *testing.T) {
preopen := fsc.RootFS()

// Sanity check of the file descriptor assignment.
fileFDAssigned, errno := fsc.OpenFile(preopen, "animals.txt", os.O_RDONLY, 0)
fileFDAssigned, errno := fsc.OpenFile(preopen, "animals.txt", fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)
require.Equal(t, int32(fileFD), fileFDAssigned)

dirFDAssigned, errno := fsc.OpenFile(preopen, "dir", os.O_RDONLY, 0)
dirFDAssigned, errno := fsc.OpenFile(preopen, "dir", fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)
require.Equal(t, int32(dirFD), dirFDAssigned)

Expand Down Expand Up @@ -3605,7 +3604,7 @@ func Test_pathLink(t *testing.T) {
uint64(newFd), uint64(destination), uint64(len(destinationName)))
require.Contains(t, log.String(), wasip1.ErrnoName(wasip1.ErrnoSuccess))

f := openFile(t, destinationRealPath, os.O_RDONLY, 0)
f := openFile(t, destinationRealPath, fsapi.O_RDONLY, 0)
defer f.Close()

st, errno := f.Stat()
Expand Down Expand Up @@ -3927,7 +3926,7 @@ func requireOpenFD(t *testing.T, mod api.Module, path string) int32 {
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
preopen := fsc.RootFS()

fd, errno := fsc.OpenFile(preopen, path, os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(preopen, path, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)
return fd
}
Expand Down Expand Up @@ -4856,14 +4855,14 @@ func Test_pathUnlinkFile_Errors(t *testing.T) {
}

func requireOpenFile(t *testing.T, tmpDir string, pathName string, data []byte, readOnly bool) (api.Module, int32, *bytes.Buffer, api.Closer) {
oflags := os.O_RDWR
oflags := fsapi.O_RDWR
if readOnly {
oflags = os.O_RDONLY
oflags = fsapi.O_RDONLY
}

realPath := joinPath(tmpDir, pathName)
if data == nil {
oflags = os.O_RDONLY
oflags = fsapi.O_RDONLY
require.NoError(t, os.Mkdir(realPath, 0o700))
} else {
require.NoError(t, os.WriteFile(realPath, data, 0o600))
Expand Down Expand Up @@ -4910,7 +4909,7 @@ func Test_fdReaddir_dotEntryHasARealInode(t *testing.T) {
uint64(sys.FdPreopen), uint64(0), uint64(len(readDirTarget)))

// Open the directory, before writing files!
fd, errno := fsc.OpenFile(preopen, readDirTarget, os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(preopen, readDirTarget, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

// get the real inode of the current directory
Expand Down Expand Up @@ -4968,11 +4967,11 @@ func Test_fdReaddir_opened_file_written(t *testing.T) {
uint64(sys.FdPreopen), uint64(0), uint64(len(dirName)))

// Open the directory, before writing files!
dirFD, errno := fsc.OpenFile(preopen, dirName, os.O_RDONLY, 0)
dirFD, errno := fsc.OpenFile(preopen, dirName, fsapi.O_RDONLY, 0)
require.EqualErrno(t, 0, errno)

// Then write a file to the directory.
f := openFile(t, joinPath(dirPath, "file"), os.O_CREATE, 0)
f := openFile(t, joinPath(dirPath, "file"), fsapi.O_CREAT, 0)
defer f.Close()

// get the real inode of the current directory
Expand Down Expand Up @@ -5021,7 +5020,7 @@ func joinPath(dirName, baseName string) string {
return path.Join(dirName, baseName)
}

func openFile(t *testing.T, path string, flag int, perm fs.FileMode) fsapi.File {
func openFile(t *testing.T, path string, flag fsapi.Oflag, perm fs.FileMode) fsapi.File {
f, errno := sysfs.OpenOSFile(path, flag, perm)
require.EqualErrno(t, 0, errno)
return f
Expand Down
25 changes: 12 additions & 13 deletions imports/wasi_snapshot_preview1/fs_unit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package wasi_snapshot_preview1

import (
"os"
"syscall"
"testing"

"github.com/tetratelabs/wazero/internal/fsapi"
Expand Down Expand Up @@ -110,7 +109,7 @@ func Test_maxDirents(t *testing.T) {
var (
testDirents = func() []fsapi.Dirent {
dPath := "dir"
d, errno := sysfs.OpenFSFile(fstest.FS, dPath, syscall.O_RDONLY, 0)
d, errno := sysfs.OpenFSFile(fstest.FS, dPath, fsapi.O_RDONLY, 0)
if errno != 0 {
panic(errno)
}
Expand Down Expand Up @@ -208,16 +207,16 @@ func Test_openFlags(t *testing.T) {
name string
dirflags, oflags, fdflags uint16
rights uint32
expectedOpenFlags int
expectedOpenFlags fsapi.Oflag
}{
{
name: "oflags=0",
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDONLY,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDONLY,
},
{
name: "oflags=O_CREAT",
oflags: wasip1.O_CREAT,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDWR | syscall.O_CREAT,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDWR | fsapi.O_CREAT,
},
{
name: "oflags=O_DIRECTORY",
Expand All @@ -227,42 +226,42 @@ func Test_openFlags(t *testing.T) {
{
name: "oflags=O_EXCL",
oflags: wasip1.O_EXCL,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDONLY | syscall.O_EXCL,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDONLY | fsapi.O_EXCL,
},
{
name: "oflags=O_TRUNC",
oflags: wasip1.O_TRUNC,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDWR | syscall.O_TRUNC,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDWR | fsapi.O_TRUNC,
},
{
name: "fdflags=FD_APPEND",
fdflags: wasip1.FD_APPEND,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDWR | syscall.O_APPEND,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDWR | fsapi.O_APPEND,
},
{
name: "oflags=O_TRUNC|O_CREAT",
oflags: wasip1.O_TRUNC | wasip1.O_CREAT,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDWR | syscall.O_TRUNC | syscall.O_CREAT,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDWR | fsapi.O_TRUNC | fsapi.O_CREAT,
},
{
name: "dirflags=LOOKUP_SYMLINK_FOLLOW",
dirflags: wasip1.LOOKUP_SYMLINK_FOLLOW,
expectedOpenFlags: syscall.O_RDONLY,
expectedOpenFlags: fsapi.O_RDONLY,
},
{
name: "rights=FD_READ",
rights: wasip1.RIGHT_FD_READ,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDONLY,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDONLY,
},
{
name: "rights=FD_WRITE",
rights: wasip1.RIGHT_FD_WRITE,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_WRONLY,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_WRONLY,
},
{
name: "rights=FD_READ|FD_WRITE",
rights: wasip1.RIGHT_FD_READ | wasip1.RIGHT_FD_WRITE,
expectedOpenFlags: fsapi.O_NOFOLLOW | syscall.O_RDWR,
expectedOpenFlags: fsapi.O_NOFOLLOW | fsapi.O_RDWR,
},
}

Expand Down
5 changes: 3 additions & 2 deletions imports/wasi_snapshot_preview1/wasi_bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
"github.com/tetratelabs/wazero/internal/fsapi"
"github.com/tetratelabs/wazero/internal/sys"
"github.com/tetratelabs/wazero/internal/testing/proxy"
"github.com/tetratelabs/wazero/internal/wasip1"
Expand Down Expand Up @@ -198,7 +199,7 @@ func Benchmark_fdReaddir(b *testing.B) {

// Open the root directory as a file-descriptor.
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
fd, errno := fsc.OpenFile(fsc.RootFS(), ".", os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(fsc.RootFS(), ".", fsapi.O_RDONLY, 0)
if errno != 0 {
b.Fatal(errno)
}
Expand Down Expand Up @@ -307,7 +308,7 @@ func Benchmark_pathFilestat(b *testing.B) {
fd := sys.FdPreopen
if bc.fd != sys.FdPreopen {
fsc := mod.(*wasm.ModuleInstance).Sys.FS()
fd, errno := fsc.OpenFile(fsc.RootFS(), "zig", os.O_RDONLY, 0)
fd, errno := fsc.OpenFile(fsc.RootFS(), "zig", fsapi.O_RDONLY, 0)
if errno != 0 {
b.Fatal(errno)
}
Expand Down
Loading

0 comments on commit 26c0b44

Please sign in to comment.