update token tests (#5270)

src/ext/modelist.js

@@ -77,6 +77,7 @@ var supportedModes = {
     D:           ["d|di"],
     Dart:        ["dart"],
     Diff:        ["diff|patch"],
+    Django:      ["html"],
     Dockerfile:  ["^Dockerfile"],
     Dot:         ["dot"],
     Drools:      ["drl"],
@@ -225,9 +226,7 @@ var supportedModes = {
     XML:         ["xml|rdf|rss|wsdl|xslt|atom|mathml|mml|xul|xbl|xaml"],
     XQuery:      ["xq"],
     YAML:        ["yaml|yml"],
-    Zeek:        ["zeek|bro"],
-    // Add the missing mode "Django" to ext-modelist
-    Django:      ["html"]
+    Zeek:        ["zeek|bro"]
 };
 
 var nameOverrides = {

src/mode/_test/Readme.md

@@ -1,5 +1,5 @@
 `tokens_<modeName>.json` files keep information about correct tokens and tokenizer states for all modes supported by ace.
-They are generated from `text_<modeName>.txt` or `demo/kitchen-sink/doc/*` with
+They are generated from `text_<modeName>.txt` or `demo/kitchen-sink/docs/*` with
 
 ```sh
 node highlight_rules_test.js -gen

src/mode/_test/tokens_aql.json

@@ -0,0 +1,28 @@
+[[
+   "start",
+  ["keyword","FOR"],
+  ["text"," "],
+  ["identifier","user"],
+  ["text"," "],
+  ["keyword","IN"],
+  ["text"," "],
+  ["identifier","users"]
+],[
+   "start",
+  ["text","  "],
+  ["keyword","FILTER"],
+  ["text"," "],
+  ["identifier","user"],
+  ["text","."],
+  ["identifier","username"],
+  ["text"," "],
+  ["keyword.operator","=="],
+  ["text"," "],
+  ["string","\"olivier\""]
+],[
+   "start",
+  ["text","  "],
+  ["keyword","RETURN"],
+  ["text"," "],
+  ["identifier","user"]
+]]

src/mode/_test/tokens_bibtex.json

@@ -0,0 +1,206 @@
+[[
+   "bibtexComment",
+  ["comment","@Comment{"]
+],[
+   ["bibtexComment","bibtexComment"],
+  ["comment","  @Book{tes03,"]
+],[
+   ["bibtexComment","bibtexComment"],
+  ["comment","    author =\t {John Smith},"]
+],[
+   ["bibtexComment","bibtexComment"],
+  ["comment","    title =\t {Cooking},"]
+],[
+   ["bibtexComment","bibtexComment"],
+  ["comment","    publisher =\t {Culinary Expert},"]
+],[
+   ["bibtexComment","bibtexComment"],
+  ["comment","    year =\t 1890"]
+],[
+   "bibtexComment",
+  ["comment","  }"]
+],[
+   "start",
+  ["comment","}"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword","@String"],
+  ["paren.lparen","("],
+  ["variable","mar"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"march\""],
+  ["paren.rparen",")"]
+],[
+   "start"
+],[
+   "keyword3",
+  ["keyword","@Book"],
+  ["paren.lparen","{"],
+  ["support.class","sweig-42"],
+  ["punctuation",","]
+],[
+   "keyword3",
+  ["text","  "],
+  ["variable","Author"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text","\t "],
+  ["paren.lparen","{"],
+  ["string"," Stefan Sweig "],
+  ["paren.rparen","}"],
+  ["punctuation",","]
+],[
+   "keyword3",
+  ["text","  "],
+  ["variable","title"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text","\t "],
+  ["paren.lparen","{"],
+  ["string"," The impossible book "],
+  ["paren.rparen","}"],
+  ["punctuation",","]
+],[
+   "keyword3",
+  ["text","  "],
+  ["variable","publisher"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text","\t "],
+  ["paren.lparen","{"],
+  ["string"," Dead Poet Society"],
+  ["paren.rparen","}"],
+  ["punctuation",","]
+],[
+   "keyword3",
+  ["text","  "],
+  ["variable","year"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text","\t "],
+  ["constant.numeric.bibtex","1942"],
+  ["punctuation",","]
+],[
+   ["variable","keyword3"],
+  ["text","  "],
+  ["variable","month"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text","        mar"]
+],[
+   "start",
+  ["paren.rparen","}"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword","@String"],
+  ["text"," "],
+  ["paren.lparen","{"],
+  ["variable","firstname"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"John\""],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["keyword","@String"],
+  ["text"," "],
+  ["paren.lparen","{"],
+  ["variable","lastname"],
+  ["text","  "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"Smith\""],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["keyword","@String"],
+  ["text"," "],
+  ["paren.lparen","{"],
+  ["variable","email"],
+  ["text","      "],
+  ["keyword.operator","="],
+  ["text"," firstname "],
+  ["keyword.operator","#"],
+  ["text"," "],
+  ["string","\".\""],
+  ["text"," "],
+  ["keyword.operator","#"],
+  ["text"," lastname "],
+  ["keyword.operator","#"],
+  ["text"," "],
+  ["string","\"@imag.en\""],
+  ["paren.rparen","}"]
+],[
+   "start"
+],[
+   "keyword3",
+  ["keyword","@Article"],
+  ["paren.lparen","{"],
+  ["support.class","key01"],
+  ["punctuation",","]
+],[
+   ["variable","keyword3"],
+  ["text","  "],
+  ["variable","title"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["paren.lparen","{"],
+  ["string"," The history of "],
+  ["invalid.illegal","@"],
+  ["string"," sign "],
+  ["paren.rparen","}"]
+],[
+   "start",
+  ["paren.rparen","}"]
+],[
+   "start"
+],[
+   "start",
+  ["comment","Everything \" \" outside {entries} is treated as comment in BibTeX."]
+],[
+   "start"
+],[
+   "keyword3",
+  ["keyword","@Article"],
+  ["paren.lparen","{"],
+  ["support.class","key03"],
+  ["punctuation",","]
+],[
+   ["variable","keyword3"],
+  ["text","  "],
+  ["variable","title"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["string","\"A "],
+  ["paren.lparen","{"],
+  ["string","bunch "],
+  ["paren.lparen","{"],
+  ["string","of"],
+  ["paren.rparen","}"],
+  ["string"," braces "],
+  ["paren.lparen","{"],
+  ["string","in"],
+  ["paren.rparen","}}"],
+  ["string"," title\""]
+],[
+   "start",
+  ["paren.rparen","}"]
+],[
+   "start"
+],[
+   "start",
+  ["keyword","@preamble"],
+  ["text"," "],
+  ["paren.lparen","{"],
+  ["string","\"This bibliography was generated on \\today\""],
+  ["paren.rparen","}"]
+]]