From feec6bdcb4f01a6a958ecfebac598f1f80313ea5 Mon Sep 17 00:00:00 2001 From: uz Date: Tue, 9 Nov 2021 11:42:13 +0900 Subject: [PATCH 1/3] Add memfs package --- memfs/example_test.go | 31 +++ memfs/memfs.go | 347 ++++++++++++++++++++++++++++ memfs/memfs_test.go | 514 ++++++++++++++++++++++++++++++++++++++++++ memfs/store.go | 176 +++++++++++++++ memfs/store_test.go | 225 ++++++++++++++++++ 5 files changed, 1293 insertions(+) create mode 100644 memfs/example_test.go create mode 100644 memfs/memfs.go create mode 100644 memfs/memfs_test.go create mode 100644 memfs/store.go create mode 100644 memfs/store_test.go diff --git a/memfs/example_test.go b/memfs/example_test.go new file mode 100644 index 0000000..d76adcc --- /dev/null +++ b/memfs/example_test.go @@ -0,0 +1,31 @@ +package memfs_test + +import ( + "fmt" + "log" + "io/fs" + + "github.com/jarxorg/io2" + "github.com/jarxorg/io2/memfs" +) + +func ExampleNew() { + name := "path/to/example.txt" + content := []byte(`Hello`) + + fsys := memfs.New() + var err error + _, err = io2.WriteFile(fsys, name, content, fs.ModePerm) + if err != nil { + log.Fatal(err) + } + + wrote, err := fs.ReadFile(fsys, name) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", string(wrote)) + + // Output: Hello +} diff --git a/memfs/memfs.go b/memfs/memfs.go new file mode 100644 index 0000000..0873766 --- /dev/null +++ b/memfs/memfs.go @@ -0,0 +1,347 @@ +// Package memfs provides an in-memory filesystem. +package memfs + +import ( + "bytes" + "io" + "io/fs" + "path/filepath" + "strings" + "sync" + + "github.com/jarxorg/io2" +) + +// MemFS represents an in-memory filesystem. +// MemFS keeps fs.FileMode but that permission is not checked. +type MemFS struct { + mutex sync.Mutex + dir string + store *store +} + +var ( + _ fs.FS = (*MemFS)(nil) + _ fs.GlobFS = (*MemFS)(nil) + _ fs.ReadDirFS = (*MemFS)(nil) + _ fs.ReadFileFS = (*MemFS)(nil) + _ fs.StatFS = (*MemFS)(nil) + _ fs.SubFS = (*MemFS)(nil) + _ io2.WriteFileFS = (*MemFS)(nil) + _ io2.RemoveFileFS = (*MemFS)(nil) +) + +// New returns a new MemFS. +func New() *MemFS { + return &MemFS{ + dir: "/", + store: newStore(), + } +} + +func (fsys *MemFS) key(name string) string { + return filepath.Clean(filepath.Join(fsys.dir, name)) +} + +func (fsys *MemFS) open(name string) (*value, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "Open", Path: name, Err: fs.ErrInvalid} + } + v := fsys.store.get(fsys.key(name)) + if v == nil { + return nil, &fs.PathError{Op: "Open", Path: name, Err: fs.ErrNotExist} + } + return v, nil +} + +func (fsys *MemFS) mkdirAll(dir string, mode fs.FileMode) error { + if !fs.ValidPath(dir) { + return &fs.PathError{Op: "MkdirAll", Path: dir, Err: fs.ErrInvalid} + } + keys := strings.Split(fsys.key(dir), "/") + for i, k := range keys { + key := fsys.key(filepath.Join(keys[0 : i+1]...)) + if v := fsys.store.get(key); v != nil { + if !v.isDir { + return &fs.PathError{Op: "MkdirAll", Path: dir, Err: fs.ErrInvalid} + } + continue + } + if k == "" { + k = "." + } + v := &value{name: k, mode: mode | fs.ModeDir, isDir: true} + fsys.store.put(key, v) + } + return nil +} + +func (fsys *MemFS) create(name string, mode fs.FileMode) (*value, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "Create", Path: name, Err: fs.ErrInvalid} + } + err := fsys.mkdirAll(filepath.Dir(name), mode) + if err != nil { + return nil, err + } + key := fsys.key(name) + v := fsys.store.get(key) + if v == nil { + v = &value{name: key, mode: mode} + fsys.store.put(key, v) + } else if v.isDir { + return nil, &fs.PathError{Op: "Create", Path: name, Err: fs.ErrInvalid} + } + return v, nil +} + +// Open opens the named file. +func (fsys *MemFS) Open(name string) (fs.File, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + v, err := fsys.open(name) + if err != nil { + return nil, err + } + + f := &MemFile{ + fsys: fsys, + name: name, + } + if !v.isDir { + f.buf = bytes.NewBuffer(v.data) + } + return f, nil +} + +var filepathRel = func(basepath, targpath string) (string, error) { + return filepath.Rel(basepath, targpath) +} + +// Glob returns the names of all files matching pattern, providing an implementation +// of the top-level Glob function. +func (fsys *MemFS) Glob(pattern string) ([]string, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + keys, err := fsys.store.prefixGlobKeys(fsys.dir, pattern) + if err != nil { + return nil, err + } + + var names []string + for _, key := range keys { + name, err := filepathRel(fsys.dir, key) + if err != nil { + return nil, err + } + names = append(names, name) + } + return names, nil +} + +// ReadDir reads the named directory and returns a list of directory entries sorted +// by filename. +func (fsys *MemFS) ReadDir(dir string) ([]fs.DirEntry, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + if !fs.ValidPath(dir) { + return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrInvalid} + } + prefix := fsys.key(dir) + keys := fsys.store.prefixKeys(prefix) + var dirEntries []fs.DirEntry + for _, key := range keys { + dirEntries = append(dirEntries, fsys.store.get(key)) + } + return dirEntries, nil +} + +// ReadFile reads the named file and returns its contents. +func (fsys *MemFS) ReadFile(name string) ([]byte, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + v, err := fsys.open(name) + if err != nil { + return nil, err + } + if v.isDir { + return nil, &fs.PathError{Op: "ReadFile", Path: name, Err: fs.ErrInvalid} + } + return v.data, nil +} + +// Stat returns a FileInfo describing the file. If there is an error, it should be +// of type *PathError. +func (fsys *MemFS) Stat(name string) (fs.FileInfo, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + return fsys.open(name) +} + +// Sub returns an FS corresponding to the subtree rooted at dir. +func (fsys *MemFS) Sub(dir string) (fs.FS, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + if !fs.ValidPath(dir) { + return nil, &fs.PathError{Op: "Sub", Path: dir, Err: fs.ErrInvalid} + } + info, err := fsys.open(dir) + if err != nil { + return nil, err + } + if !info.isDir { + return nil, &fs.PathError{Op: "Sub", Path: dir, Err: fs.ErrInvalid} + } + return &MemFS{ + dir: filepath.Join(fsys.dir, dir), + store: fsys.store, + }, nil +} + +// MkdirAll creates the named directory. +func (fsys *MemFS) MkdirAll(dir string, mode fs.FileMode) error { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + return fsys.mkdirAll(dir, mode) +} + +// CreateFile creates the named file. +func (fsys *MemFS) CreateFile(name string, mode fs.FileMode) (io2.WriterFile, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + v, err := fsys.create(name, mode) + if err != nil { + return nil, err + } + return &MemFile{ + fsys: fsys, + name: name, + buf: bytes.NewBuffer(v.data), + mode: mode, + }, nil +} + +// WriteFile writes the specified bytes to the named file. +func (fsys *MemFS) WriteFile(name string, p []byte, mode fs.FileMode) (int, error) { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + v, err := fsys.create(name, mode) + if err != nil { + return 0, err + } + v.data = p[:] + return len(p), nil +} + +// RemoveFile removes the specified named file. +func (fsys *MemFS) RemoveFile(name string) error { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + if !fs.ValidPath(name) { + return &fs.PathError{Op: "RemoveFile", Path: name, Err: fs.ErrInvalid} + } + + fsys.store.remove(fsys.key(name)) + return nil +} + +// RemoveAll removes path and any children it contains. +func (fsys *MemFS) RemoveAll(path string) error { + fsys.mutex.Lock() + defer fsys.mutex.Unlock() + + if !fs.ValidPath(path) { + return &fs.PathError{Op: "RemoveAll", Path: path, Err: fs.ErrInvalid} + } + + fsys.store.removeAll(fsys.key(path)) + return nil +} + +// MemFile represents an in-memory file. +// MemFile implements fs.File, fs.ReadDirFile and io2.WriterFile. +type MemFile struct { + fsys *MemFS + name string + buf *bytes.Buffer + mode fs.FileMode + dirRead bool + dirEntries []fs.DirEntry + dirIndex int + wrote bool +} + +var ( + _ fs.File = (*MemFile)(nil) + _ fs.ReadDirFile = (*MemFile)(nil) + _ io2.WriterFile = (*MemFile)(nil) +) + +// Read reads bytes from this file. +func (f *MemFile) Read(p []byte) (int, error) { + return f.buf.Read(p) +} + +// Stat returns the fs.FileInfo of this file. +func (f *MemFile) Stat() (fs.FileInfo, error) { + return f.fsys.Stat(f.name) +} + +// Close closes streams. +func (f *MemFile) Close() error { + if f.wrote { + var err error + _, err = f.fsys.WriteFile(f.name, f.buf.Bytes(), f.mode) + return err + } + f.dirEntries = nil + return nil +} + +// ReadDir reads sub directories. +func (f *MemFile) ReadDir(n int) ([]fs.DirEntry, error) { + if !f.dirRead { + f.dirRead = true + var err error + f.dirEntries, err = f.fsys.ReadDir(f.name) + if err != nil { + return nil, err + } + } + max := len(f.dirEntries) + if f.dirIndex >= max { + if n == -1 { + return nil, nil + } + return nil, io.EOF + } + if n == 0 { + return nil, nil + } + if n == -1 { + n = max - f.dirIndex + } + end := f.dirIndex + n + if end > max { + end = max + } + defer func() { f.dirIndex = end }() + + return f.dirEntries[f.dirIndex:end], nil +} + +// Write writes the specified bytes to this file. +func (f *MemFile) Write(p []byte) (int, error) { + f.wrote = true + return f.buf.Write(p) +} diff --git a/memfs/memfs_test.go b/memfs/memfs_test.go new file mode 100644 index 0000000..f55516a --- /dev/null +++ b/memfs/memfs_test.go @@ -0,0 +1,514 @@ +package memfs + +import ( + "errors" + "io" + "io/fs" + "os" + "reflect" + "strings" + "testing" + "testing/fstest" + + "github.com/jarxorg/io2" +) + +func newMemFSTest(t *testing.T) *MemFS { + fsys := New() + err := io2.CopyFS(fsys, os.DirFS("../osfs/testdata"), ".") + if err != nil { + t.Fatal(err) + } + return fsys +} + +func TestFS(t *testing.T) { + fsys := newMemFSTest(t) + + if err := fstest.TestFS(fsys, "dir0"); err != nil { + t.Errorf(`Error testing/fstest: %+v`, err) + } + if err := fstest.TestFS(fsys, "dir0/file01.txt"); err != nil { + t.Errorf(`Error testing/fstest: %+v`, err) + } +} + +func TestCreateFile(t *testing.T) { + testCases := []struct { + name string + errStr string + }{ + { + name: "file.txt", + }, { + name: "newDir/file.txt", + }, { + name: "newDir", + errStr: "Create newDir: invalid argument", + }, { + name: "newDir/file.txt/invalid", + errStr: "MkdirAll newDir/file.txt: invalid argument", + }, { + name: "../invalid", + errStr: "Create ../invalid: invalid argument", + }, { + name: "dir0/file01.txt", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + _, err := fsys.CreateFile(tc.name, fs.ModePerm) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error Create("%s") error got "%s"; want "%s"`, tc.name, errStr, tc.errStr) + } + if err != nil { + continue + } + info, err := fsys.Stat(tc.name) + if err != nil { + t.Fatal(err) + } + if info.IsDir() { + t.Errorf(`Error %s IsDir() returns true; want false`, tc.name) + } + } +} + +func TestMkdirAll(t *testing.T) { + testCases := []struct { + dir string + errStr string + }{ + { + dir: "test0", + }, { + dir: "test0/test1", + }, { + dir: "test2/test3", + }, { + dir: "../invalid", + errStr: "MkdirAll ../invalid: invalid argument", + }, { + dir: "dir0/file01.txt", + errStr: "MkdirAll dir0/file01.txt: invalid argument", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + err := fsys.MkdirAll(tc.dir, fs.ModePerm) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error MkdirAll("%s") error got "%s"; want "%s"`, tc.dir, errStr, tc.errStr) + } + if err != nil { + continue + } + info, err := fsys.Stat(tc.dir) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() { + t.Errorf(`Error %s IsDir() returns false; want true`, tc.dir) + } + } +} + +func TestGlob(t *testing.T) { + testCases := []struct { + want []string + pattern string + errStr string + }{ + { + want: []string{ + "dir0/file01.txt", + }, + pattern: "*/*1.txt", + }, { + want: []string{ + "dir0/file01.txt", + "dir0/file02.txt", + }, + pattern: "dir0/*.txt", + }, { + pattern: "no-match", + }, { + pattern: "[[", + errStr: "syntax error in pattern", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + got, err := fsys.Glob(tc.pattern) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error Glob("%s") error got "%s"; want "%s"`, tc.pattern, errStr, tc.errStr) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf(`Error Glob("%s") got %v; want %v`, tc.pattern, got, tc.want) + } + } +} + +func TestGlob_filepathRelError(t *testing.T) { + orgFilepathRel := filepathRel + defer func() { filepathRel = orgFilepathRel }() + + wantErr := errors.New("test") + filepathRel = func(basepath, targpath string) (string, error) { + return "", wantErr + } + + fsys := newMemFSTest(t) + var gotErr error + _, gotErr = fsys.Glob("*") + if gotErr != wantErr { + t.Errorf(`Error Glob error got %v; want %v`, gotErr, wantErr) + } +} + +func TestReadDir(t *testing.T) { + testCases := []struct { + want []string + dir string + errStr string + }{ + { + want: []string{ + "dir0", + }, + dir: ".", + }, { + want: []string{ + "file01.txt", + "file02.txt", + }, + dir: "dir0", + }, { + dir: "not-found", + }, { + dir: "../invalid", + errStr: "ReadDir ../invalid: invalid argument", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + entries, err := fsys.ReadDir(tc.dir) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error ReadDir("%s") error got "%s"; want "%s"`, tc.dir, errStr, tc.errStr) + } + var got []string + for _, entry := range entries { + got = append(got, entry.Name()) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf(`Error ReadDir("%s") got %v; want %v`, tc.dir, got, tc.want) + } + } +} + +func TestReadFile(t *testing.T) { + testCases := []struct { + want []byte + name string + errStr string + }{ + { + want: []byte("content01\n"), + name: "dir0/file01.txt", + }, { + name: "not-found", + errStr: "Open not-found: file does not exist", + }, { + name: "dir0", + errStr: "ReadFile dir0: invalid argument", + }, { + name: "../invalid.txt", + errStr: "Open ../invalid.txt: invalid argument", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + got, err := fsys.ReadFile(tc.name) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error ReadFile("%s") error got "%s"; want "%s"`, tc.name, errStr, tc.errStr) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf(`Error ReadFile("%s") got "%s"; want "%s"`, tc.name, got, tc.want) + } + } +} + +func TestSub(t *testing.T) { + fsys := newMemFSTest(t) + dir0, err := fsys.Sub("dir0") + if err != nil { + t.Fatal(err) + } + memfsDir0 := dir0.(*MemFS) + + // NOTE: Write to sub filesystem. + name := "test.txt" + want := []byte(`test`) + _, err = memfsDir0.WriteFile(name, want, fs.ModePerm) + if err != nil { + t.Fatal(err) + } + + // NOTE: Read from parent filesystem. + got, err := fsys.ReadFile("dir0/" + name) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf(`Error ReadFile("%s") got "%s"; want "%s"`, name, got, want) + } +} + +func TestSub_Errors(t *testing.T) { + testCases := []struct { + dir string + errStr string + }{ + { + dir: "../invalid", + errStr: "Sub ../invalid: invalid argument", + }, { + dir: "not-found", + errStr: "Open not-found: file does not exist", + }, { + dir: "dir0/file01.txt", + errStr: "Sub dir0/file01.txt: invalid argument", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + var err error + _, err = fsys.Sub(tc.dir) + if err == nil { + t.Fatalf(`Fatal Sub("%s") return no error`, tc.dir) + } + if err.Error() != tc.errStr { + t.Errorf(`Error Sub("%s") error got "%v"; want "%s"`, tc.dir, err, tc.errStr) + } + } +} + +func TestWriteFile(t *testing.T) { + data := []byte(`testdata`) + testCases := []struct { + name string + errStr string + }{ + { + name: "new.txt", + }, { + name: "dir0/file01.txt", + }, { + name: "dir0", + errStr: "Create dir0: invalid argument", + }, { + name: "../invalid.txt", + errStr: "Create ../invalid.txt: invalid argument", + }, + } + + fsys := newMemFSTest(t) + for _, tc := range testCases { + n, err := fsys.WriteFile(tc.name, data, fs.ModePerm) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error WriteFile("%s") error got "%s"; want "%s"`, tc.name, errStr, tc.errStr) + } + if errStr == "" && n != len(data) { + t.Errorf(`Error WriteFile("%s") returns %d; want %d`, tc.name, n, len(data)) + } + } +} + +func TestRemoveFile(t *testing.T) { + fsys := newMemFSTest(t) + name := "dir0/file01.txt" + + // NOTE: Check exists. + var err error + _, err = fsys.Stat(name) + if err != nil { + t.Fatal(err) + } + + err = fsys.RemoveFile(name) + if err != nil { + t.Fatal(err) + } + + info, err := fsys.Stat(name) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf(`Error RemoveFile("%s") after Stat returns %v`, name, info) + } +} + +func TestRemoveFile_Errors(t *testing.T) { + fsys := newMemFSTest(t) + name := "../invalid" + + want := &fs.PathError{Op: "RemoveFile", Path: name, Err: fs.ErrInvalid} + got := fsys.RemoveFile(name) + + if !reflect.DeepEqual(got, want) { + t.Errorf(`Error RemoveFile("%s") returns %v; want %v`, name, got, want) + } +} + +func TestRemoveAll(t *testing.T) { + fsys := newMemFSTest(t) + dir := "dir0" + + var want []string + for _, k := range fsys.store.keys { + if !strings.HasPrefix(k, "/" + dir) { + want = append(want, k) + } + } + + err := fsys.RemoveAll("dir0") + if err != nil { + t.Fatal(err) + } + + got := fsys.store.keys[:] + if !reflect.DeepEqual(got, want) { + t.Errorf(`Error RemoveAll("%s") after keys %v; want %v`, dir, got, want) + } +} + +func TestRemoveAll_Errors(t *testing.T) { + fsys := newMemFSTest(t) + name := "../invalid" + + want := &fs.PathError{Op: "RemoveAll", Path: name, Err: fs.ErrInvalid} + got := fsys.RemoveAll(name) + + if !reflect.DeepEqual(got, want) { + t.Errorf(`Error RemoveAll("%s") returns %v; want %v`, name, got, want) + } +} + +func TestMemFile_ReadDir(t *testing.T) { + fsys := newMemFSTest(t) + dir := "dir0" + + f, err := fsys.Open(dir) + if err != nil { + t.Fatal(err) + } + + memf, ok := f.(*MemFile) + if !ok { + t.Fatalf(`Fatal not MemFile: %#v`, f) + } + + testCases := []struct { + name string + err error + }{ + { + name: "file01.txt", + }, { + name: "file02.txt", + }, { + err: io.EOF, + }, + } + + for _, tc := range testCases { + entries, err := memf.ReadDir(1) + if tc.err != nil { + if !errors.Is(err, tc.err) { + t.Errorf(`Error ReadDir(1) error %v; want %v`, err, tc.err) + } + continue + } + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf(`Error ReadDir(1) returns %d entries; want 1`, len(entries)) + } + if entries[0].Name() != tc.name { + t.Errorf(`Error ReadDir(1) returns unknown entries %v`, entries) + } + } +} + +func TestMemFile_ReadDir_Errors(t *testing.T) { + fsys := newMemFSTest(t) + dir := "dir0" + + f, err := fsys.Open(dir) + if err != nil { + t.Fatal(err) + } + + memf, ok := f.(*MemFile) + if !ok { + t.Fatalf(`Fatal not MemFile: %#v`, f) + } + + memf.name = "../invalid" + _, err = memf.ReadDir(1) + if err == nil { + t.Fatalf(`Fatal ReadDir(1) returns no error`) + } +} + +func TestMemFile_ReadDir0(t *testing.T) { + fsys := newMemFSTest(t) + dir := "dir0" + + f, err := fsys.Open(dir) + if err != nil { + t.Fatal(err) + } + + memf, ok := f.(*MemFile) + if !ok { + t.Fatalf(`Fatal not MemFile: %#v`, f) + } + + entries, err := memf.ReadDir(0) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf(`Error ReadDir(0) returns unknown entries %v`, entries) + } +} diff --git a/memfs/store.go b/memfs/store.go new file mode 100644 index 0000000..96f9c7c --- /dev/null +++ b/memfs/store.go @@ -0,0 +1,176 @@ +package memfs + +import ( + "io/fs" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +// Value works as fs.DirEntry or fs.FileInfo. +type value struct { + name string + data []byte + mode fs.FileMode + modTime time.Time + isDir bool +} + +var ( + _ fs.DirEntry = (*value)(nil) + _ fs.FileInfo = (*value)(nil) +) + +func (e *value) Name() string { + return filepath.Base(e.name) +} + +func (e *value) Size() int64 { + if e.isDir { + return 0 + } + return int64(len(e.data)) +} + +func (e *value) Mode() fs.FileMode { + return e.mode +} + +func (e *value) ModTime() time.Time { + return e.modTime +} + +func (e *value) IsDir() bool { + return e.isDir +} + +func (e *value) Sys() interface{} { + return nil +} + +func (e *value) Type() fs.FileMode { + return e.mode +} + +func (e *value) Info() (fs.FileInfo, error) { + return e, nil +} + +// Store represents an in-memory key value store. +// store.keys is always sorted. +// All functions of the store are not thread safety. +type store struct { + keys []string + values map[string]*value +} + +func newStore() *store { + return &store{ + values: map[string]*value{}, + } +} + +func (s *store) get(k string) *value { + return s.values[k] +} + +func (s *store) put(k string, v *value) *value { + if _, ok := s.values[k]; !ok { + s.keys = append(s.keys, k) + sort.Strings(s.keys) + } + + s.values[k] = v + return v +} + +func (s *store) remove(key string) *value { + i := s.keyIndex(key) + if i == -1 { + return nil + } + v := s.values[key] + s.keys = append(s.keys[0:i], s.keys[i+1:]...) + delete(s.values, key) + return v +} + +func (s *store) removeAll(prefix string) { + from := s.keyIndex(prefix) + if from == -1 { + return + } + + max := len(s.keys) + to := -1 + for i := from; i < max; i++ { + key := s.keys[i] + if !strings.HasPrefix(key, prefix) { + break + } + delete(s.values, key) + to = i + } + s.keys = append(s.keys[0:from], s.keys[to+1:]...) +} + +func (s *store) keyIndex(key string) int { + i := sort.SearchStrings(s.keys, key) + if i < len(s.keys) && s.keys[i] == key { + return i + } + return -1 +} + +func (s *store) prefixKeys(prefix string) []string { + i := s.keyIndex(prefix) + if i == -1 { + return nil + } + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + var keys []string + max := len(s.keys) + for i++; i < max; i++ { + key := s.keys[i] + if !strings.HasPrefix(key, prefix) { + break + } + if strings.Contains(key[len(prefix):], "/") { + break + } + keys = append(keys, key) + } + return keys +} + +func (s *store) prefixGlobKeys(prefix, pattern string) ([]string, error) { + i := s.keyIndex(prefix) + if i == -1 { + return nil, nil + } + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + var keys []string + max := len(s.keys) + for i++; i < max; i++ { + key := s.keys[i] + if !strings.HasPrefix(key, prefix) { + break + } + ok, err := path.Match(pattern, key[len(prefix):]) + if err != nil { + return nil, err + } + if ok { + keys = append(keys, key) + } + } + return keys, nil +} diff --git a/memfs/store_test.go b/memfs/store_test.go new file mode 100644 index 0000000..4103c4f --- /dev/null +++ b/memfs/store_test.go @@ -0,0 +1,225 @@ +package memfs + +import ( + "io/fs" + "reflect" + "sort" + "strings" + "testing" + "time" +) + +func TestValue(t *testing.T) { + v := &value{ + name: "path/to/name", + data: []byte(`content`), + mode: fs.ModePerm, + modTime: time.Now(), + } + if name := v.Name(); name != "name" { + t.Errorf(`Name returns %s; want name`, name) + } + if size := v.Size(); size != int64(len(v.data)) { + t.Errorf(`Size returns %d; want %d`, size, len(v.data)) + } + v.isDir = true + if size := v.Size(); size != 0 { + t.Errorf(`dir Size returns %d; want 0`, size) + } + if mode := v.Mode(); mode != v.mode { + t.Errorf(`Mode returns %v; want %v`, mode, v.mode) + } + if modTime := v.ModTime(); modTime != v.modTime { + t.Errorf(`ModTime returns %v; want %v`, modTime, v.modTime) + } + if isDir := v.IsDir(); isDir != v.isDir { + t.Errorf(`IsDir returns %v; want %v`, isDir, v.isDir) + } + if sys := v.Sys(); sys != nil { + t.Errorf(`Sys returns %v; want nil`, sys) + } + if typ := v.Type(); typ != v.mode { + t.Errorf(`Type returns %v; want %v`, typ, v.mode) + } + info, err := v.Info() + if err != nil { + t.Fatal(err) + } + if info != v { + t.Errorf(`Info returns %v; want %v`, info, v) + } +} + +var testStoreSrc = map[string]*value{ + "/": &value{name: ".", mode: fs.ModePerm, isDir: true}, + "/dir0": &value{name: "dir0", mode: fs.ModePerm, isDir: true}, + "/dir0/file01.txt": &value{name: "dir0/file01.txt", mode: fs.ModePerm, isDir: false}, + "/dir0/file02.txt": &value{name: "dir0/file02.txt", mode: fs.ModePerm, isDir: false}, + "/dir1": &value{name: "dir0", mode: fs.ModePerm, isDir: true}, + "/dir1/file11.txt": &value{name: "dir1/file11.txt", mode: fs.ModePerm, isDir: false}, + "/dir1/file12.txt": &value{name: "dir1/file12.txt", mode: fs.ModePerm, isDir: false}, +} + +func newStoreTest() *store { + s := newStore() + for k, v := range testStoreSrc { + s.put(k, v) + } + return s +} + +func TestStore(t *testing.T) { + s := newStoreTest() + + var wantKeys []string + for k := range testStoreSrc { + wantKeys = append(wantKeys, k) + } + sort.Strings(wantKeys) + + if !reflect.DeepEqual(s.keys, wantKeys) { + t.Errorf(`Error store.keys is %v; want %v`, s.keys, wantKeys) + } + + key := "/dir0/file02.txt" + wantValue := testStoreSrc[key] + gotValue := s.get(key) + if !reflect.DeepEqual(gotValue, wantValue) { + t.Errorf(`Error store.get("%s") returns %v; want %v`, key, gotValue, wantValue) + } +} + +func TestStore_remove(t *testing.T) { + s := newStoreTest() + + key := "/dir1/file11.txt" + v := s.get(key) + if v == nil { + t.Errorf(`Error not found %s`, key) + } + + s.remove(key) + v = s.get(key) + if v != nil { + t.Errorf(`Error found %s: %v`, key, v) + } + + for _, k := range s.keys { + if k == key { + t.Errorf(`Error found %s`, key) + } + } + + v = s.remove(key) + if v != nil { + t.Errorf(`Error found %s: %v`, key, v) + } +} + +func TestStore_removeAll(t *testing.T) { + s := newStoreTest() + + prefix := "/dir0" + s.removeAll(prefix) + + for _, key := range s.keys { + if strings.HasPrefix(key, prefix) { + t.Errorf(`Error found %s`, key) + } + v := s.get(key) + if v == nil { + t.Errorf(`Error not found %s: %v`, key, v) + } + } + + want := len(s.keys) + s.removeAll(prefix) + got := len(s.keys) + + if got != want { + t.Errorf(`Error keys length %d; want %d`, got, want) + } +} + +func TestStore_prefixKeys(t *testing.T) { + testCases := []struct { + want []string + prefix string + }{ + { + want: []string{ + "/dir0", + }, + prefix: "/", + }, { + want: []string{ + "/dir0/file01.txt", + "/dir0/file02.txt", + }, + prefix: "/dir0", + }, { + prefix: "/not-found", + }, + } + + s := newStoreTest() + for _, tc := range testCases { + got := s.prefixKeys(tc.prefix) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf(`Error prefixKeys("%s") got %v; want %v`, tc.prefix, got, tc.want) + } + } +} + +func TestStore_prefixGlobKeys(t *testing.T) { + testCases := []struct { + want []string + prefix string + pattern string + errStr string + }{ + { + want: []string{ + "/dir0/file01.txt", + "/dir1/file11.txt", + }, + prefix: "/", + pattern: "*/*1.txt", + }, { + want: []string{ + "/dir0/file01.txt", + "/dir0/file02.txt", + }, + prefix: "/dir0", + pattern: "*.txt", + }, { + prefix: "/not-found", + pattern: "*.*", + }, { + prefix: "/", + pattern: "*.go", + }, { + prefix: "/", + pattern: "[[", + errStr: "syntax error in pattern", + }, + } + + s := newStoreTest() + for _, tc := range testCases { + got, err := s.prefixGlobKeys(tc.prefix, tc.pattern) + errStr := "" + if err != nil { + errStr = err.Error() + } + if errStr != tc.errStr { + t.Errorf(`Error prefixGlobKeys("%s", "%s") error got "%s"; want "%s"`, + tc.prefix, tc.pattern, errStr, tc.errStr) + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf(`Error prefixGlobKeys("%s", "%s") got %v; want %v`, + tc.prefix, tc.pattern, got, tc.want) + } + } +} From d7f397a030e58aa5c386eec144bd0400f8f66e50 Mon Sep 17 00:00:00 2001 From: uz Date: Tue, 9 Nov 2021 11:42:24 +0900 Subject: [PATCH 2/3] Update README.md --- README.md | 44 +++----------------------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 5376279..ded550c 100644 --- a/README.md +++ b/README.md @@ -6,48 +6,10 @@ Go "io" and "io/fs" package utilities. -## Writable io/fs implementations for the OS +## Writable io/fs.FS implementations -```go -package main - -import ( - "fmt" - "io/fs" - "io/ioutil" - "log" - "os" - - "github.com/jarxorg/io2" - "github.com/jarxorg/io2/osfs" -) - -func func main() { - tmpDir, err := ioutil.TempDir("", "example") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - name := "example.txt" - content := []byte(`Hello`) - - fsys := osfs.DirFS(tmpDir) - _, err = io2.WriteFile(fsys, name, content, fs.ModePerm) - if err != nil { - log.Fatal(err) - } - - wrote, err := ioutil.ReadFile(tmpDir + "/" + name) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("%s\n", string(wrote)) - - // Output: Hello -} -``` +- [osfs](https://github.com/jarxorg/io2/tree/main/osfs) +- [memfs](https://github.com/jarxorg/io2/tree/main/memfs) ## Delegator From 3fc373b0409e958f08845a2f98732c931a3ae8ae Mon Sep 17 00:00:00 2001 From: uz Date: Tue, 9 Nov 2021 11:42:32 +0900 Subject: [PATCH 3/3] Improve tests --- osfs/osfs_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/osfs/osfs_test.go b/osfs/osfs_test.go index bf0af12..1b59034 100644 --- a/osfs/osfs_test.go +++ b/osfs/osfs_test.go @@ -12,7 +12,7 @@ import ( "github.com/jarxorg/io2" ) -func TestDirFS_TestFS(t *testing.T) { +func TestFS(t *testing.T) { if err := fstest.TestFS(DirFS("testdata"), "dir0"); err != nil { t.Errorf("Error testing/fstest: %+v", err) } @@ -21,6 +21,25 @@ func TestDirFS_TestFS(t *testing.T) { } } +func TestMkdirAll(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + fsys := NewOSFS(tmpDir) + err = fsys.MkdirAll("dir", fs.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = fsys.MkdirAll("../invalid", fs.ModePerm) + if err == nil { + t.Fatal(err) + } +} + func TestCreateFile(t *testing.T) { tmpDir, err := ioutil.TempDir("", "test") if err != nil {