fix #211: add support for nested source maps

CHANGELOG.md

@@ -6,6 +6,10 @@
 
     The wrapper for immediately-invoked function expressions is hard-coded to an arrow function and was not updated when the ES5 target was added. This meant that bundling ES5 code would generate a bundle what wasn't ES5-compatible. Doing this now uses a function expression instead.
 
+* Add support for nested source maps ([#211](https://github.com/evanw/esbuild/issues/211))
+
+    Source map comments of the form `//# sourceMappingURL=...` inside input files are now respected. This means you can bundle files with source maps and esbuild will generate a source map that maps all the way back to the original files instead of to the intermediate file with the source map.
+
 ## 0.6.4
 
 * Allow extending `tsconfig.json` paths inside packages ([#269](https://github.com/evanw/esbuild/issues/269))

internal/ast/ast.go

@@ -3,6 +3,8 @@ package ast
 import (
 	"strings"
 
+	"github.com/evanw/esbuild/internal/sourcemap"
+
 	"github.com/evanw/esbuild/internal/compat"
 )
 
@@ -248,6 +250,11 @@ type LocRef struct {
 	Ref Ref
 }
 
+type Span struct {
+	Text  string
+	Range Range
+}
+
 // This is used to represent both file system paths (IsAbsolute == true) and
 // abstract module paths (IsAbsolute == false). Abstract module paths represent
 // "virtual modules" when used for an input file and "package paths" when used
@@ -1273,6 +1280,9 @@ type AST struct {
 	NamedExports            map[string]Ref
 	TopLevelSymbolToParts   map[Ref][]uint32
 	ExportStarImportRecords []uint32
+
+	SourceMapComment Span
+	SourceMap        *sourcemap.SourceMap
 }
 
 func (ast *AST) HasCommonJSFeatures() bool {

internal/bundler/bundler.go

@@ -236,9 +236,15 @@ func parseFile(args parseArgs) {
 			fmt.Sprintf("File extension not supported: %s", args.prettyPath))
 	}
 
+	// Stop now if parsing failed
+	if !result.ok {
+		args.results <- result
+		return
+	}
+
 	// Run the resolver on the parse thread so it's not run on the main thread.
 	// That way the main thread isn't blocked if the resolver takes a while.
-	if result.ok && args.options.IsBundling {
+	if args.options.IsBundling {
 		result.resolveResults = make([]*resolver.ResolveResult, len(result.file.ast.ImportRecords))
 
 		if len(result.file.ast.ImportRecords) > 0 {
@@ -291,9 +297,57 @@ func parseFile(args parseArgs) {
 		}
 	}
 
+	// Attempt to parse the source map if present
+	if args.options.SourceMap != config.SourceMapNone && result.file.ast.SourceMapComment.Text != "" {
+		if path, contents := extractSourceMapFromComment(args.log, args.fs, &source, result.file.ast.SourceMapComment); contents != nil {
+			prettyPath := path.Text
+			if path.IsAbsolute {
+				prettyPath = args.res.PrettyPath(prettyPath)
+			}
+			result.file.ast.SourceMap = parser.ParseSourceMap(args.log, logging.Source{
+				KeyPath:    path,
+				PrettyPath: prettyPath,
+				Contents:   *contents,
+			})
+		}
+	}
+
 	args.results <- result
 }
 
+func extractSourceMapFromComment(log logging.Log, fs fs.FS, source *logging.Source, comment ast.Span) (ast.Path, *string) {
+	// Data URL
+	if strings.HasPrefix(comment.Text, "data:") {
+		if strings.HasPrefix(comment.Text, "data:application/json;base64,") {
+			n := int32(len("data:application/json;base64,"))
+			encoded := comment.Text[n:]
+			decoded, err := base64.StdEncoding.DecodeString(encoded)
+			if err != nil {
+				r := ast.Range{Loc: ast.Loc{Start: comment.Range.Loc.Start + n}, Len: comment.Range.Len - n}
+				log.AddRangeWarning(source, r, "Invalid base64 data in source map")
+				return ast.Path{}, nil
+			}
+			contents := string(decoded)
+			return ast.Path{Text: "sourceMappingURL in " + source.PrettyPath}, &contents
+		}
+	}
+
+	// Absolute path
+	if source.KeyPath.IsAbsolute {
+		absPath := fs.Join(fs.Dir(source.KeyPath.Text), comment.Text)
+		contents, ok := fs.ReadFile(absPath)
+		if !ok {
+			log.AddRangeWarning(source, comment.Range, fmt.Sprintf("Could not find source map file: %s", absPath))
+			return ast.Path{}, nil
+		}
+		return ast.Path{IsAbsolute: true, Text: absPath}, &contents
+	}
+
+	// Anything else is unsupported
+	log.AddRangeWarning(source, comment.Range, "Unsupported source map comment")
+	return ast.Path{}, nil
+}
+
 func loaderFromFileExtension(extensionToLoader map[string]config.Loader, base string) config.Loader {
 	// Pick the loader with the longest matching extension. So if there's an
 	// extension for ".css" and for ".module.css", we want to match the one for
@@ -576,11 +630,6 @@ type compileResult struct {
 	// This is the line and column offset since the previous JavaScript string
 	// or the start of the file if this is the first JavaScript string.
 	generatedOffset lineColumnOffset
-
-	// The source map contains the original source code, which is quoted in
-	// parallel for speed. This is only filled in if the SourceMap option is
-	// enabled.
-	quotedSource string
 }
 
 func (b *Bundle) Compile(log logging.Log, options config.Options) []OutputFile {

internal/bundler/linker.go

@@ -2631,9 +2631,9 @@ func (c *linkerContext) generateCodeForFileInChunk(
 	}
 
 	// Only generate a source map if needed
-	sourceMapContents := &c.sources[sourceIndex].Contents
+	sourceForSourceMap := &c.sources[sourceIndex]
 	if c.options.SourceMap == config.SourceMapNone {
-		sourceMapContents = nil
+		sourceForSourceMap = nil
 	}
 
 	// Indent the file if everything is wrapped in an IIFE
@@ -2648,9 +2648,10 @@ func (c *linkerContext) generateCodeForFileInChunk(
 		OutputFormat:        c.options.OutputFormat,
 		RemoveWhitespace:    c.options.RemoveWhitespace,
 		ToModuleRef:         toModuleRef,
-		SourceMapContents:   sourceMapContents,
 		ExtractComments:     c.options.IsBundling && c.options.RemoveWhitespace,
 		UnsupportedFeatures: c.options.UnsupportedFeatures,
+		SourceForSourceMap:  sourceForSourceMap,
+		InputSourceMap:      file.ast.SourceMap,
 	}
 	tree := file.ast
 	tree.Parts = []ast.Part{{Stmts: stmts}}
@@ -2669,11 +2670,6 @@ func (c *linkerContext) generateCodeForFileInChunk(
 		result.entryPointTail = &entryPointTail
 	}
 
-	// Also quote the source for the source map while we're running in parallel
-	if c.options.SourceMap != config.SourceMapNone {
-		result.quotedSource = printer.QuoteForJSON(c.sources[sourceIndex].Contents)
-	}
-
 	waitGroup.Done()
 }
 
@@ -2757,15 +2753,15 @@ func (c *linkerContext) generateChunk(chunk chunkMeta) (results []OutputFile) {
 		// Start with the hashbang if there is one
 		if file.ast.Hashbang != "" {
 			hashbang := file.ast.Hashbang + "\n"
-			prevOffset.advance(hashbang)
+			prevOffset.advanceString(hashbang)
 			j.AddString(hashbang)
 			newlineBeforeComment = true
 		}
 
 		// Add the top-level directive if present
 		if file.ast.Directive != "" {
 			quoted := printer.Quote(file.ast.Directive) + ";" + newline
-			prevOffset.advance(quoted)
+			prevOffset.advanceString(quoted)
 			j.AddString(quoted)
 			newlineBeforeComment = true
 		}
@@ -2783,7 +2779,7 @@ func (c *linkerContext) generateChunk(chunk chunkMeta) (results []OutputFile) {
 		if c.options.ModuleName != "" {
 			text = "var " + c.options.ModuleName + space + "=" + space + text
 		}
-		prevOffset.advance(text)
+		prevOffset.advanceString(text)
 		j.AddString(text)
 		newlineBeforeComment = false
 	}
@@ -2840,12 +2836,12 @@ func (c *linkerContext) generateChunk(chunk chunkMeta) (results []OutputFile) {
 		// Don't add a file name comment for the runtime
 		if c.options.IsBundling && !c.options.RemoveWhitespace && !isRuntime {
 			if newlineBeforeComment {
-				prevOffset.advance("\n")
+				prevOffset.advanceString("\n")
 				j.AddString("\n")
 			}
 
 			text := fmt.Sprintf("%s// %s\n", indent, c.sources[compileResult.sourceIndex].PrettyPath)
-			prevOffset.advance(text)
+			prevOffset.advanceString(text)
 			j.AddString(text)
 		}
 
@@ -2856,13 +2852,19 @@ func (c *linkerContext) generateChunk(chunk chunkMeta) (results []OutputFile) {
 
 		// Don't include the runtime in source maps
 		if isRuntime {
-			prevOffset.advance(string(compileResult.JS))
+			prevOffset.advanceString(string(compileResult.JS))
 			j.AddBytes(compileResult.JS)
 		} else {
 			// Save the offset to the start of the stored JavaScript
 			compileResult.generatedOffset = prevOffset
 			j.AddBytes(compileResult.JS)
-			prevOffset = lineColumnOffset{}
+
+			// Ignore empty source map chunks
+			if compileResult.SourceMapChunk.ShouldIgnore {
+				prevOffset.advanceBytes(compileResult.JS)
+			} else {
+				prevOffset = lineColumnOffset{}
+			}
 
 			// Include this file in the source map
 			if c.options.SourceMap != config.SourceMapNone {
@@ -2973,8 +2975,19 @@ func (c *linkerContext) generateChunk(chunk chunkMeta) (results []OutputFile) {
 	return
 }
 
-func (offset *lineColumnOffset) advance(text string) {
-	for i := 0; i < len(text); i++ {
+func (offset *lineColumnOffset) advanceBytes(bytes []byte) {
+	for i, n := 0, len(bytes); i < n; i++ {
+		if bytes[i] == '\n' {
+			offset.lines++
+			offset.columns = 0
+		} else {
+			offset.columns++
+		}
+	}
+}
+
+func (offset *lineColumnOffset) advanceString(text string) {
+	for i, n := 0, len(text); i < n; i++ {
 		if text[i] == '\n' {
 			offset.lines++
 			offset.columns = 0
@@ -3146,22 +3159,31 @@ func (c *linkerContext) generateSourceMapForChunk(results []compileResult) []byt
 
 	// Write the sources
 	j.AddString(",\n  \"sources\": [")
-	for i, result := range results {
-		sourceFile := c.sources[result.sourceIndex].PrettyPath
-		if i > 0 {
-			j.AddString(", ")
+	needComma := false
+	for _, result := range results {
+		for _, source := range result.SourceMapChunk.QuotedSources {
+			if needComma {
+				j.AddString(", ")
+			} else {
+				needComma = true
+			}
+			j.AddString(source.QuotedPath)
 		}
-		j.AddString(printer.QuoteForJSON(sourceFile))
 	}
 	j.AddString("]")
 
 	// Write the sourcesContent
 	j.AddString(",\n  \"sourcesContent\": [")
-	for i, result := range results {
-		if i > 0 {
-			j.AddString(", ")
+	needComma = false
+	for _, result := range results {
+		for _, source := range result.SourceMapChunk.QuotedSources {
+			if needComma {
+				j.AddString(", ")
+			} else {
+				needComma = true
+			}
+			j.AddString(source.QuotedContents)
 		}
-		j.AddString(result.quotedSource)
 	}
 	j.AddString("]")
 
@@ -3174,20 +3196,24 @@ func (c *linkerContext) generateSourceMapForChunk(results []compileResult) []byt
 		chunk := result.SourceMapChunk
 		offset := result.generatedOffset
 
+		// Ignore empty source map chunks
+		if chunk.ShouldIgnore {
+			continue
+		}
+
 		// Because each file for the bundle is converted to a source map once,
 		// the source maps are shared between all entry points in the bundle.
 		// The easiest way of getting this to work is to have all source maps
 		// generate as if their source index is 0. We then adjust the source
 		// index per entry point by modifying the first source mapping. This
 		// is done by AppendSourceMapChunk() using the source index passed
 		// here.
-		startState := printer.SourceMapState{SourceIndex: sourceMapIndex}
-
-		// Advance the state by the line/column offset from the previous chunk
-		startState.GeneratedColumn += offset.columns
-		if offset.lines > 0 {
-			j.AddBytes(bytes.Repeat([]byte{';'}, offset.lines))
-		} else {
+		startState := printer.SourceMapState{
+			SourceIndex:     sourceMapIndex,
+			GeneratedLine:   offset.lines,
+			GeneratedColumn: offset.columns,
+		}
+		if offset.lines == 0 {
 			startState.GeneratedColumn += prevColumnOffset
 		}
 
@@ -3196,7 +3222,7 @@ func (c *linkerContext) generateSourceMapForChunk(results []compileResult) []byt
 
 		// Generate the relative offset to start from next time
 		prevEndState = chunk.EndState
-		prevEndState.SourceIndex = sourceMapIndex
+		prevEndState.SourceIndex += sourceMapIndex
 		prevColumnOffset = chunk.FinalGeneratedColumn
 
 		// If this was all one line, include the column offset from the start
@@ -3205,7 +3231,7 @@ func (c *linkerContext) generateSourceMapForChunk(results []compileResult) []byt
 			prevColumnOffset += startState.GeneratedColumn
 		}
 
-		sourceMapIndex++
+		sourceMapIndex += len(result.SourceMapChunk.QuotedSources)
 	}
 	j.AddString("\"")