Move the skip table from lexer-related data to parser-related data

driver/parser.go

@@ -35,6 +35,9 @@ type Grammar interface {
 	// TerminalCount returns a terminal symbol count of grammar.
 	TerminalCount() int
 
+	// SkipTerminal returns true when a terminal symbol must be skipped on syntax analysis.
+	SkipTerminal(terminal int) bool
+
 	// EOF returns the EOF symbol.
 	EOF() int
 
@@ -63,9 +66,6 @@ type VToken interface {
 
 	// Position returns (row, column) pair.
 	Position() (int, int)
-
-	// Skip returns true when a token must be skipped on syntax analysis.
-	Skip() bool
 }
 
 type TokenStream interface {
@@ -275,7 +275,7 @@ func (p *Parser) nextToken() (VToken, error) {
 			return nil, err
 		}
 
-		if tok.Skip() {
+		if p.gram.SkipTerminal(tok.TerminalID()) {
 			continue
 		}
 

driver/spec.go

@@ -40,6 +40,10 @@ func (g *grammarImpl) TerminalCount() int {
 	return g.g.ParsingTable.TerminalCount
 }
 
+func (g *grammarImpl) SkipTerminal(terminal int) bool {
+	return g.g.ParsingTable.TerminalSkip[terminal] == 1
+}
+
 func (g *grammarImpl) ErrorTrapperState(state int) bool {
 	return g.g.ParsingTable.ErrorTrapperStates[state] != 0
 }

driver/template.go

@@ -147,6 +147,7 @@ type grammarImpl struct {
 	nonTerminals            []string
 	lhsSymbols              []int
 	terminals               []string
+	terminalSkip            []int
 	astActions              [][]int
 }
 
@@ -160,6 +161,7 @@ func NewGrammar() *grammarImpl {
 		nonTerminals:            {{ genNonTerminals }},
 		lhsSymbols:              {{ genLHSSymbols }},
 		terminals:               {{ genTerminals }},
+		terminalSkip:            {{ genTerminalSkip }},
 		astActions:              {{ genASTActions }},
 	}
 }
@@ -192,6 +194,10 @@ func (g *grammarImpl) TerminalCount() int {
 	return {{ .terminalCount }}
 }
 
+func (g *grammarImpl) SkipTerminal(terminal int) bool {
+	return g.terminalSkip[terminal] == 1
+}
+
 func (g *grammarImpl) ErrorTrapperState(state int) bool {
 	return g.errorTrapperStates[state] != 0
 }
@@ -355,6 +361,25 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap {
 			fmt.Fprintf(&b, "}")
 			return b.String()
 		},
+		"genTerminalSkip": func() string {
+			var b strings.Builder
+			fmt.Fprintf(&b, "[]int{\n")
+			c := 1
+			for _, v := range cgram.ParsingTable.TerminalSkip {
+				fmt.Fprintf(&b, "%v, ", v)
+				if c == 20 {
+					fmt.Fprintf(&b, "\n")
+					c = 1
+				} else {
+					c++
+				}
+			}
+			if c > 1 {
+				fmt.Fprintf(&b, "\n")
+			}
+			fmt.Fprintf(&b, "}")
+			return b.String()
+		},
 		"genASTActions": func() string {
 			var b strings.Builder
 			fmt.Fprintf(&b, "[][]int{\n")
@@ -389,7 +414,6 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap {
 const lexerSrcTmplate = `
 type vToken struct {
 	terminalID int
-	skip       bool
 	tok        *Token
 }
 
@@ -409,22 +433,15 @@ func (t *vToken) Invalid() bool {
 	return t.tok.Invalid
 }
 
-func (t *vToken) Skip() bool {
-	return t.skip
-}
-
 func (t *vToken) Position() (int, int) {
 	return t.tok.Row, t.tok.Col
 }
 
 var kindToTerminal = {{ genKindToTerminal }}
 
-var skip = {{ genSkip }}
-
 type tokenStream struct {
 	lex            *Lexer
 	kindToTerminal []int
-	skip           []int
 }
 
 func NewTokenStream(src io.Reader) (*tokenStream, error) {
@@ -445,7 +462,6 @@ func (t *tokenStream) Next() (VToken, error) {
 	}
 	return &vToken{
 		terminalID: kindToTerminal[tok.KindID],
-		skip:       skip[tok.KindID] > 0,
 		tok:        tok,
 	}, nil
 }
@@ -472,25 +488,6 @@ func genLexerTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap {
 			fmt.Fprintf(&b, "}")
 			return b.String()
 		},
-		"genSkip": func() string {
-			var b strings.Builder
-			fmt.Fprintf(&b, "[]int{\n")
-			c := 1
-			for _, v := range cgram.LexicalSpecification.Maleeni.Skip {
-				fmt.Fprintf(&b, "%v, ", v)
-				if c == 20 {
-					fmt.Fprintf(&b, "\n")
-					c = 1
-				} else {
-					c++
-				}
-			}
-			if c > 1 {
-				fmt.Fprintf(&b, "\n")
-			}
-			fmt.Fprintf(&b, "}")
-			return b.String()
-		},
 	}
 }
 

driver/token_stream.go

@@ -9,7 +9,6 @@ import (
 
 type vToken struct {
 	terminalID int
-	skip       bool
 	tok        *mldriver.Token
 }
 
@@ -29,18 +28,13 @@ func (t *vToken) Invalid() bool {
 	return t.tok.Invalid
 }
 
-func (t *vToken) Skip() bool {
-	return t.skip
-}
-
 func (t *vToken) Position() (int, int) {
 	return t.tok.Row, t.tok.Col
 }
 
 type tokenStream struct {
 	lex            *mldriver.Lexer
 	kindToTerminal []int
-	skip           []int
 }
 
 func NewTokenStream(g *spec.CompiledGrammar, src io.Reader) (TokenStream, error) {
@@ -52,7 +46,6 @@ func NewTokenStream(g *spec.CompiledGrammar, src io.Reader) (TokenStream, error)
 	return &tokenStream{
 		lex:            lex,
 		kindToTerminal: g.LexicalSpecification.Maleeni.KindToTerminal,
-		skip:           g.LexicalSpecification.Maleeni.Skip,
 	}, nil
 }
 
@@ -63,7 +56,6 @@ func (l *tokenStream) Next() (VToken, error) {
 	}
 	return &vToken{
 		terminalID: l.kindToTerminal[tok.KindID],
-		skip:       l.skip[tok.KindID] > 0,
 		tok:        tok,
 	}, nil
 }

grammar/grammar.go

@@ -166,20 +166,20 @@ func (b *GrammarBuilder) Build() (*Grammar, error) {
 
 	// When a terminal symbol that cannot be reached from the start symbol has the skip directive,
 	// the compiler treats its terminal as a used symbol, not unused.
-	for _, sym := range symTabAndLexSpec.skipSyms {
-		if _, ok := syms.unusedTerminals[sym]; !ok {
-			prod := syms.usedTerminals[sym]
-
+	for _, sym := range symTabAndLexSpec.skip {
+		s := sym.String()
+		if _, ok := syms.unusedTerminals[s]; !ok {
+			prod := syms.usedTerminals[s]
 			b.errs = append(b.errs, &verr.SpecError{
 				Cause:  semErrTermCannotBeSkipped,
-				Detail: sym,
+				Detail: s,
 				Row:    prod.Pos.Row,
 				Col:    prod.Pos.Col,
 			})
 			continue
 		}
 
-		delete(syms.unusedTerminals, sym)
+		delete(syms.unusedTerminals, s)
 	}
 
 	for sym, prod := range syms.unusedProductions {
@@ -381,11 +381,10 @@ func collectUserDefinedIDsFromDirective(dir *spec.DirectiveNode) []string {
 }
 
 type symbolTableAndLexSpec struct {
-	symTab   *symbolTable
-	lexSpec  *mlspec.LexSpec
-	errSym   symbol
-	skip     []mlspec.LexKindName
-	skipSyms []string
+	symTab  *symbolTable
+	lexSpec *mlspec.LexSpec
+	errSym  symbol
+	skip    []mlspec.LexKindName
 }
 
 func (b *GrammarBuilder) genSymbolTableAndLexSpec(root *spec.RootNode) (*symbolTableAndLexSpec, error) {
@@ -403,7 +402,6 @@ func (b *GrammarBuilder) genSymbolTableAndLexSpec(root *spec.RootNode) (*symbolT
 	}
 
 	skipKinds := []mlspec.LexKindName{}
-	skipSyms := []string{}
 	for _, prod := range root.LexProductions {
 		if sym, exist := symTab.toSymbol(prod.LHS); exist {
 			if sym == errSym {
@@ -439,7 +437,6 @@ func (b *GrammarBuilder) genSymbolTableAndLexSpec(root *spec.RootNode) (*symbolT
 		}
 		if skip {
 			skipKinds = append(skipKinds, mlspec.LexKindName(prod.LHS))
-			skipSyms = append(skipSyms, prod.LHS)
 		}
 		entries = append(entries, entry)
 	}
@@ -469,9 +466,8 @@ func (b *GrammarBuilder) genSymbolTableAndLexSpec(root *spec.RootNode) (*symbolT
 		lexSpec: &mlspec.LexSpec{
 			Entries: entries,
 		},
-		errSym:   errSym,
-		skip:     skipKinds,
-		skipSyms: skipSyms,
+		errSym: errSym,
+		skip:   skipKinds,
 	}, nil
 }
 
@@ -1218,7 +1214,6 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec
 	}
 
 	kind2Term := make([]int, len(lexSpec.KindNames))
-	skip := make([]int, len(lexSpec.KindNames))
 	for i, k := range lexSpec.KindNames {
 		if k == mlspec.LexKindNameNil {
 			kind2Term[mlspec.LexKindIDNil] = symbolNil.num().Int()
@@ -1230,21 +1225,24 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec
 			return nil, nil, fmt.Errorf("terminal symbol '%v' was not found in a symbol table", k)
 		}
 		kind2Term[i] = sym.num().Int()
+	}
+
+	termTexts, err := gram.symbolTable.terminalTexts()
+	if err != nil {
+		return nil, nil, err
+	}
 
+	termSkip := make([]int, len(termTexts))
+	for i, k := range lexSpec.KindNames {
 		for _, sk := range gram.skipLexKinds {
 			if k != sk {
 				continue
 			}
-			skip[i] = 1
+			termSkip[kind2Term[i]] = 1
 			break
 		}
 	}
 
-	termTexts, err := gram.symbolTable.terminalTexts()
-	if err != nil {
-		return nil, nil, err
-	}
-
 	nonTerms, err := gram.symbolTable.nonTerminalTexts()
 	if err != nil {
 		return nil, nil, err
@@ -1332,7 +1330,6 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec
 			Maleeni: &spec.Maleeni{
 				Spec:           lexSpec,
 				KindToTerminal: kind2Term,
-				Skip:           skip,
 			},
 		},
 		ParsingTable: &spec.ParsingTable{
@@ -1345,6 +1342,7 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec
 			AlternativeSymbolCounts: altSymCounts,
 			Terminals:               termTexts,
 			TerminalCount:           tab.terminalCount,
+			TerminalSkip:            termSkip,
 			NonTerminals:            nonTerms,
 			NonTerminalCount:        tab.nonTerminalCount,
 			EOFSymbol:               symbolEOF.num().Int(),

spec/grammar/grammar.go

@@ -17,7 +17,6 @@ type LexicalSpecification struct {
 type Maleeni struct {
 	Spec           *mlspec.CompiledLexSpec `json:"spec"`
 	KindToTerminal []int                   `json:"kind_to_terminal"`
-	Skip           []int                   `json:"skip"`
 }
 
 type ParsingTable struct {
@@ -30,6 +29,7 @@ type ParsingTable struct {
 	AlternativeSymbolCounts []int    `json:"alternative_symbol_counts"`
 	Terminals               []string `json:"terminals"`
 	TerminalCount           int      `json:"terminal_count"`
+	TerminalSkip            []int    `json:"terminal_skip"`
 	NonTerminals            []string `json:"non_terminals"`
 	NonTerminalCount        int      `json:"non_terminal_count"`
 	EOFSymbol               int      `json:"eof_symbol"`