From ec2214e8bf1d7615271d0ef9eb0809fe4913e940 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Fri, 4 Feb 2022 08:14:25 +0000 Subject: [PATCH 1/4] add zip file support --- internal/fs/fs_real.go | 13 +- internal/fs/fs_zip.go | 292 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 internal/fs/fs_zip.go diff --git a/internal/fs/fs_real.go b/internal/fs/fs_real.go index d0f294b1652..7a4cf4e8a42 100644 --- a/internal/fs/fs_real.go +++ b/internal/fs/fs_real.go @@ -110,12 +110,21 @@ func RealFS(options RealFSOptions) (FS, error) { watchData = make(map[string]privateWatchData) } - return &realFS{ + var result FS = &realFS{ entries: make(map[string]entriesOrErr), fp: fp, watchData: watchData, doNotCacheEntries: options.DoNotCache, - }, nil + } + + // Add a wrapper that lets us traverse into ".zip" files. This is what yarn + // uses as a package format when in yarn is in its "PnP" mode. + result = &zipFS{ + inner: result, + zipFiles: make(map[string]*zipFile), + } + + return result, nil } func (fs *realFS) ReadDirectory(dir string) (entries DirEntries, canonicalError error, originalError error) { diff --git a/internal/fs/fs_zip.go b/internal/fs/fs_zip.go new file mode 100644 index 00000000000..ad3f84958a7 --- /dev/null +++ b/internal/fs/fs_zip.go @@ -0,0 +1,292 @@ +package fs + +import ( + "archive/zip" + "io/ioutil" + "strings" + "sync" + "syscall" +) + +type zipFS struct { + inner FS + + zipFilesMutex sync.Mutex + zipFiles map[string]*zipFile +} + +type zipFile struct { + reader *zip.ReadCloser + err error + + dirs map[string]*compressedDir + files map[string]*compressedFile + wait sync.WaitGroup +} + +type compressedDir struct { + entries map[string]EntryKind + path string + + // Compatible entries are decoded lazily + mutex sync.Mutex + dirEntries DirEntries +} + +type compressedFile struct { + compressed *zip.File + + // The file is decompressed lazily + mutex sync.Mutex + contents string + err error + wasRead bool +} + +func (fs *zipFS) checkForZip(path string, kind EntryKind) (*zipFile, string) { + var zipPath string + var pathTail string + + // Do a quick check for a ".zip" in the path at all + path = strings.ReplaceAll(path, "\\", "/") + if i := strings.Index(path, ".zip/"); i != -1 { + zipPath = path[:i+len(".zip")] + pathTail = path[i+len(".zip/"):] + } else if kind == DirEntry && strings.HasSuffix(path, ".zip") { + zipPath = path + } else { + return nil, "" + } + + // If there is one, then check whether it's a file on the file system or not + fs.zipFilesMutex.Lock() + archive := fs.zipFiles[zipPath] + if archive != nil { + fs.zipFilesMutex.Unlock() + archive.wait.Wait() + } else { + archive = &zipFile{} + archive.wait.Add(1) + fs.zipFiles[zipPath] = archive + fs.zipFilesMutex.Unlock() + defer archive.wait.Done() + + // Try reading the zip archive if it's not in the cache + tryToReadZipArchive(zipPath, archive) + } + + if archive.err != nil { + return nil, "" + } + return archive, pathTail +} + +func tryToReadZipArchive(zipPath string, archive *zipFile) { + reader, err := zip.OpenReader(zipPath) + if err != nil { + archive.err = err + return + } + + dirs := make(map[string]*compressedDir) + files := make(map[string]*compressedFile) + + // Build an index of all files in the archive + for _, file := range reader.File { + baseName := file.Name + if strings.HasSuffix(baseName, "/") { + baseName = baseName[:len(baseName)-1] + } + dirPath := "" + if slash := strings.LastIndexByte(baseName, '/'); slash != -1 { + dirPath = baseName[:slash] + baseName = baseName[slash+1:] + } + if file.FileInfo().IsDir() { + // Handle a directory + lowerDir := strings.ToLower(dirPath) + if _, ok := dirs[lowerDir]; !ok { + dirs[lowerDir] = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + } + } else { + // Handle a file + files[strings.ToLower(file.Name)] = &compressedFile{compressed: file} + lowerDir := strings.ToLower(dirPath) + dir, ok := dirs[lowerDir] + if !ok { + dir = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + dirs[lowerDir] = dir + } + dir.entries[baseName] = FileEntry + } + } + + // Populate child directories + seeds := make([]string, 0, len(dirs)) + for dir := range dirs { + seeds = append(seeds, dir) + } + for _, baseName := range seeds { + for baseName != "" { + dirPath := "" + if slash := strings.LastIndexByte(baseName, '/'); slash != -1 { + dirPath = baseName[:slash] + baseName = baseName[slash+1:] + } + lowerDir := strings.ToLower(dirPath) + dir, ok := dirs[lowerDir] + if !ok { + dir = &compressedDir{ + path: dirPath, + entries: make(map[string]EntryKind), + } + dirs[lowerDir] = dir + } + dir.entries[baseName] = DirEntry + baseName = dirPath + } + } + + archive.dirs = dirs + archive.files = files + archive.reader = reader +} + +func (fs *zipFS) ReadDirectory(path string) (entries DirEntries, canonicalError error, originalError error) { + entries, canonicalError, originalError = fs.inner.ReadDirectory(path) + if canonicalError != syscall.ENOENT && canonicalError != syscall.ENOTDIR { + return + } + + // If the directory doesn't exist, try reading from an enclosing zip archive + zip, pathTail := fs.checkForZip(path, DirEntry) + if zip == nil { + return + } + + // Does the zip archive have this directory? + dir, ok := zip.dirs[strings.ToLower(pathTail)] + if !ok { + return DirEntries{}, syscall.ENOENT, syscall.ENOENT + } + + // Check whether it has already been converted + dir.mutex.Lock() + defer dir.mutex.Unlock() + if dir.dirEntries.data != nil { + return dir.dirEntries, nil, nil + } + + // Otherwise, fill in the entries + dir.dirEntries = DirEntries{dir: path, data: make(map[string]*Entry, len(dir.entries))} + for name, kind := range dir.entries { + dir.dirEntries.data[strings.ToLower(name)] = &Entry{ + dir: path, + base: name, + kind: kind, + } + } + + return dir.dirEntries, nil, nil +} + +func (fs *zipFS) ReadFile(path string) (contents string, canonicalError error, originalError error) { + contents, canonicalError, originalError = fs.inner.ReadFile(path) + if canonicalError != syscall.ENOENT { + return + } + + // If the file doesn't exist, try reading from an enclosing zip archive + zip, pathTail := fs.checkForZip(path, FileEntry) + if zip == nil { + return + } + + // Does the zip archive have this file? + file, ok := zip.files[strings.ToLower(pathTail)] + if !ok { + return "", syscall.ENOENT, syscall.ENOENT + } + + // Check whether it has already been read + file.mutex.Lock() + defer file.mutex.Unlock() + if file.wasRead { + return file.contents, file.err, file.err + } + file.wasRead = true + + // If not, try to open it + reader, err := file.compressed.Open() + if err != nil { + file.err = err + return "", err, err + } + defer reader.Close() + + // Then try to read it + bytes, err := ioutil.ReadAll(reader) + if err != nil { + file.err = err + return "", err, err + } + + file.contents = string(bytes) + return file.contents, nil, nil +} + +func (fs *zipFS) OpenFile(path string) (result OpenedFile, canonicalError error, originalError error) { + result, canonicalError, originalError = fs.inner.OpenFile(path) + return +} + +func (fs *zipFS) ModKey(path string) (modKey ModKey, err error) { + modKey, err = fs.inner.ModKey(path) + return +} + +func (fs *zipFS) IsAbs(path string) bool { + return fs.inner.IsAbs(path) +} + +func (fs *zipFS) Abs(path string) (string, bool) { + return fs.inner.Abs(path) +} + +func (fs *zipFS) Dir(path string) string { + return fs.inner.Dir(path) +} + +func (fs *zipFS) Base(path string) string { + return fs.inner.Base(path) +} + +func (fs *zipFS) Ext(path string) string { + return fs.inner.Ext(path) +} + +func (fs *zipFS) Join(parts ...string) string { + return fs.inner.Join(parts...) +} + +func (fs *zipFS) Cwd() string { + return fs.inner.Cwd() +} + +func (fs *zipFS) Rel(base string, target string) (string, bool) { + return fs.inner.Rel(base, target) +} + +func (fs *zipFS) kind(dir string, base string) (symlink string, kind EntryKind) { + return fs.inner.kind(dir, base) +} + +func (fs *zipFS) WatchData() WatchData { + return fs.inner.WatchData() +} From 12aca158d9fb93bb7fd25d17e2eec73d1a54a01d Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Wed, 27 Jul 2022 20:52:01 +0000 Subject: [PATCH 2/4] implement yarn pnp `__virtual__` path mangling --- internal/fs/fs_zip.go | 77 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/internal/fs/fs_zip.go b/internal/fs/fs_zip.go index ad3f84958a7..5cb981c87a2 100644 --- a/internal/fs/fs_zip.go +++ b/internal/fs/fs_zip.go @@ -3,6 +3,7 @@ package fs import ( "archive/zip" "io/ioutil" + "strconv" "strings" "sync" "syscall" @@ -159,6 +160,8 @@ func tryToReadZipArchive(zipPath string, archive *zipFile) { } func (fs *zipFS) ReadDirectory(path string) (entries DirEntries, canonicalError error, originalError error) { + path = mangleYarnPnPVirtualPath(path) + entries, canonicalError, originalError = fs.inner.ReadDirectory(path) if canonicalError != syscall.ENOENT && canonicalError != syscall.ENOTDIR { return @@ -197,6 +200,8 @@ func (fs *zipFS) ReadDirectory(path string) (entries DirEntries, canonicalError } func (fs *zipFS) ReadFile(path string) (contents string, canonicalError error, originalError error) { + path = mangleYarnPnPVirtualPath(path) + contents, canonicalError, originalError = fs.inner.ReadFile(path) if canonicalError != syscall.ENOENT { return @@ -242,11 +247,15 @@ func (fs *zipFS) ReadFile(path string) (contents string, canonicalError error, o } func (fs *zipFS) OpenFile(path string) (result OpenedFile, canonicalError error, originalError error) { + path = mangleYarnPnPVirtualPath(path) + result, canonicalError, originalError = fs.inner.OpenFile(path) return } func (fs *zipFS) ModKey(path string) (modKey ModKey, err error) { + path = mangleYarnPnPVirtualPath(path) + modKey, err = fs.inner.ModKey(path) return } @@ -260,6 +269,9 @@ func (fs *zipFS) Abs(path string) (string, bool) { } func (fs *zipFS) Dir(path string) string { + if prefix, suffix, ok := parseYarnPnPVirtualPath(path); ok && suffix == "" { + return prefix + } return fs.inner.Dir(path) } @@ -290,3 +302,68 @@ func (fs *zipFS) kind(dir string, base string) (symlink string, kind EntryKind) func (fs *zipFS) WatchData() WatchData { return fs.inner.WatchData() } + +func parseYarnPnPVirtualPath(path string) (string, string, bool) { + i := 0 + + for { + start := i + slash := strings.IndexAny(path[i:], "/\\") + if slash == -1 { + break + } + i += slash + 1 + + // Replace the segments "__virtual__//" with N times the ".." operation + if path[start:i-1] == "__virtual__" { + if slash := strings.IndexAny(path[i:], "/\\"); slash != -1 { + var count string + var suffix string + j := i + slash + 1 + + // Find the range of the count + if slash := strings.IndexAny(path[j:], "/\\"); slash != -1 { + count = path[j : j+slash] + suffix = path[j+slash:] + } else { + count = path[j:] + } + + // Parse the count + if n, err := strconv.ParseInt(count, 10, 64); err == nil { + prefix := path[:start] + + // Apply N times the ".." operator + for n > 0 && (strings.HasSuffix(prefix, "/") || strings.HasSuffix(prefix, "\\")) { + slash := strings.LastIndexAny(prefix[:len(prefix)-1], "/\\") + if slash == -1 { + break + } + prefix = prefix[:slash+1] + n-- + } + + // Make sure the prefix and suffix work well when joined together + if suffix == "" && strings.IndexAny(prefix, "/\\") != strings.LastIndexAny(prefix, "/\\") { + prefix = prefix[:len(prefix)-1] + } else if prefix == "" { + prefix = "." + } else if strings.HasPrefix(suffix, "/") || strings.HasPrefix(suffix, "\\") { + suffix = suffix[1:] + } + + return prefix, suffix, true + } + } + } + } + + return "", "", false +} + +func mangleYarnPnPVirtualPath(path string) string { + if prefix, suffix, ok := parseYarnPnPVirtualPath(path); ok { + return prefix + suffix + } + return path +} From 053616779a2cda7abc4b5d3c7b61defb77fe0020 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Mon, 8 Aug 2022 13:47:53 -0400 Subject: [PATCH 3/4] add simple test coverage for zip and `__virtual__` --- scripts/js-api-tests.js | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/scripts/js-api-tests.js b/scripts/js-api-tests.js index 097d432c05b..3f8be7b568a 100644 --- a/scripts/js-api-tests.js +++ b/scripts/js-api-tests.js @@ -2644,6 +2644,75 @@ require("/assets/file.png"); assert.strictEqual(await tryTargetESM('node14.18'), `// \nvar import_node_fs = __toESM(require("node:fs"));\nimport("node:fs");\n(0, import_node_fs.default)();\n`) assert.strictEqual(await tryTargetESM('node14.17'), `// \nvar import_node_fs = __toESM(require("fs"));\nimport("fs");\n(0, import_node_fs.default)();\n`) }, + + async zipFile({ esbuild, testDir }) { + const entry = path.join(testDir, 'entry.js') + const zip = path.join(testDir, 'test.zip') + + await writeFileAsync(entry, ` + import foo from './test.zip/foo.js' + import bar from './test.zip/bar/bar.js' + + import virtual1 from './test.zip/__virtual__/ignored/0/foo.js' + import virtual2 from './test.zip/ignored/__virtual__/ignored/1/foo.js' + import virtual3 from './test.zip/__virtual__/ignored/1/test.zip/foo.js' + + console.log({ + foo, + bar, + + virtual1, + virtual2, + virtual3, + }) + `) + + // This uses the real file system instead of the mock file system so that + // we can check that everything works as expected on Windows, which is not + // a POSIX environment. + await writeFileAsync(zip, Buffer.from( + `UEsDBAoAAgAAAG1qCFUSAXosFQAAABUAAAAGABwAZm9vLmpzVVQJAAOeRfFioEXxYnV4C` + + `wABBPUBAAAEFAAAAGV4cG9ydCBkZWZhdWx0ICdmb28nClBLAwQKAAIAAABzaghVwuDbLR` + + `UAAAAVAAAACgAcAGJhci9iYXIuanNVVAkAA6lF8WKrRfFidXgLAAEE9QEAAAQUAAAAZXh` + + `wb3J0IGRlZmF1bHQgJ2JhcicKUEsBAh4DCgACAAAAbWoIVRIBeiwVAAAAFQAAAAYAGAAA` + + `AAAAAQAAAKSBAAAAAGZvby5qc1VUBQADnkXxYnV4CwABBPUBAAAEFAAAAFBLAQIeAwoAA` + + `gAAAHNqCFXC4NstFQAAABUAAAAKABgAAAAAAAEAAACkgVUAAABiYXIvYmFyLmpzVVQFAA` + + `OpRfFidXgLAAEE9QEAAAQUAAAAUEsFBgAAAAACAAIAnAAAAK4AAAAAAA==`, 'base64')) + + const value = await esbuild.build({ + entryPoints: [entry], + bundle: true, + write: false, + }) + + assert.strictEqual(value.outputFiles.length, 1) + assert.strictEqual(value.outputFiles[0].text, `(() => { + // scripts/.js-api-tests/zipFile/test.zip/foo.js + var foo_default = "foo"; + + // scripts/.js-api-tests/zipFile/test.zip/bar/bar.js + var bar_default = "bar"; + + // scripts/.js-api-tests/zipFile/test.zip/__virtual__/ignored/0/foo.js + var foo_default2 = "foo"; + + // scripts/.js-api-tests/zipFile/test.zip/ignored/__virtual__/ignored/1/foo.js + var foo_default3 = "foo"; + + // scripts/.js-api-tests/zipFile/test.zip/__virtual__/ignored/1/test.zip/foo.js + var foo_default4 = "foo"; + + // scripts/.js-api-tests/zipFile/entry.js + console.log({ + foo: foo_default, + bar: bar_default, + virtual1: foo_default2, + virtual2: foo_default3, + virtual3: foo_default4 + }); +})(); +`) + }, } function fetch(host, port, path, headers) { From 905a486f6c698a7d1756e33683b7be25bd093268 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Tue, 9 Aug 2022 13:28:17 -0400 Subject: [PATCH 4/4] zip: fix bug with readdir and trailing slashes --- internal/fs/fs_zip.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/fs/fs_zip.go b/internal/fs/fs_zip.go index 5cb981c87a2..316abf92388 100644 --- a/internal/fs/fs_zip.go +++ b/internal/fs/fs_zip.go @@ -107,10 +107,14 @@ func tryToReadZipArchive(zipPath string, archive *zipFile) { // Handle a directory lowerDir := strings.ToLower(dirPath) if _, ok := dirs[lowerDir]; !ok { - dirs[lowerDir] = &compressedDir{ + dir := &compressedDir{ path: dirPath, entries: make(map[string]EntryKind), } + + // List the same directory both with and without the slash + dirs[lowerDir] = dir + dirs[lowerDir+"/"] = dir } } else { // Handle a file @@ -122,7 +126,10 @@ func tryToReadZipArchive(zipPath string, archive *zipFile) { path: dirPath, entries: make(map[string]EntryKind), } + + // List the same directory both with and without the slash dirs[lowerDir] = dir + dirs[lowerDir+"/"] = dir } dir.entries[baseName] = FileEntry } @@ -147,7 +154,10 @@ func tryToReadZipArchive(zipPath string, archive *zipFile) { path: dirPath, entries: make(map[string]EntryKind), } + + // List the same directory both with and without the slash dirs[lowerDir] = dir + dirs[lowerDir+"/"] = dir } dir.entries[baseName] = DirEntry baseName = dirPath