diff --git a/README.md b/README.md index 5c494d05..853d5423 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ To start using it in your project, see the [Getting Started](https://dlang-commu - Supports all YAML 1.1 constructs. All examples from the YAML 1.1 specification are parsed correctly. - Reads from and writes from/to YAML files or in-memory buffers. - - UTF-8, UTF-16 and UTF-32 encodings are supported, both big and - little endian (plain ASCII also works as it is a subset of UTF-8). + - UTF-8 is the only supported encoding. - Support for both block (Python-like, based on indentation) and flow (JSON-like, based on bracing) constructs. - Support for YAML anchors and aliases. diff --git a/dub.json b/dub.json index 601f9027..143269a3 100644 --- a/dub.json +++ b/dub.json @@ -6,9 +6,6 @@ "Cameron \"Herringway\" Ross" ], "license": "BSL-1.0", - "dependencies": { - "tinyendian" : "~>0.2.0" - }, "homepage": "https://github.com/dlang-community/D-YAML", "copyright": "Copyright © 2011-2018, Ferdinand Majerech", "configurations": [ @@ -16,10 +13,7 @@ { "name": "unittest" }, { "name": "unittest-dip1000", - "dflags": [ "-preview=dip1000" ], - "dependencies": { - "tinyendian": { "version": "*", "dflags" : [ "-preview=dip1000" ] }, - } + "dflags": [ "-preview=dip1000" ] } ], "subPackages": [ diff --git a/examples/tokens/source/app.d b/examples/tokens/source/app.d index d87c69db..875dbeec 100644 --- a/examples/tokens/source/app.d +++ b/examples/tokens/source/app.d @@ -22,7 +22,7 @@ import std.uni; void dumpEventString(string str) @safe { - auto events = new Parser(Scanner(Reader(cast(ubyte[])str.dup))); + auto events = new Parser(Scanner(Reader(str.dup))); foreach (event; events) { writeln(event); @@ -30,7 +30,7 @@ void dumpEventString(string str) @safe } void dumpTokens(string str) @safe { - writefln("%(%s\n%)", new Parser(Scanner(Reader(cast(ubyte[])str.dup)))); + writefln("%(%s\n%)", new Parser(Scanner(Reader(str.dup)))); } diff --git a/examples/yaml_bench/yaml_bench.d b/examples/yaml_bench/yaml_bench.d index 061c6a77..ffc29532 100644 --- a/examples/yaml_bench/yaml_bench.d +++ b/examples/yaml_bench/yaml_bench.d @@ -94,9 +94,9 @@ void main(string[] args) //@safe string file = args[1]; auto stopWatch = StopWatch(AutoStart.yes); - void[] fileInMemory; - if(!reload) { fileInMemory = std.file.read(file); } - void[] fileWorkingCopy = fileInMemory.dup; + char[] fileInMemory; + if(!reload) { fileInMemory = cast(char[])std.file.read(file); } + char[] fileWorkingCopy = fileInMemory.dup; auto loadTime = stopWatch.peek(); stopWatch.reset(); try @@ -113,13 +113,13 @@ void main(string[] args) //@safe { // Loading the file rewrites the loaded buffer, so if we don't reload from // disk, we need to use a copy of the originally loaded file. - if(reload) { fileInMemory = std.file.read(file); } + if(reload) { fileInMemory = cast(char[])std.file.read(file); } else { fileWorkingCopy[] = fileInMemory[]; } - void[] fileToLoad = reload ? fileInMemory : fileWorkingCopy; + char[] fileToLoad = reload ? fileInMemory : fileWorkingCopy; if(scanOnly) { - auto reader = Reader(cast(ubyte[])fileToLoad, "benchmark"); + auto reader = Reader(fileToLoad, "benchmark"); auto scanner = Scanner(reader); while(!scanner.empty) { @@ -128,7 +128,7 @@ void main(string[] args) //@safe } else { - auto loader = Loader.fromBuffer(fileToLoad); + auto loader = Loader.fromString(fileToLoad); loader.resolver = resolver; nodes = loader.array; } diff --git a/examples/yaml_gen/yaml_gen.d b/examples/yaml_gen/yaml_gen.d index b970c315..38ea039a 100644 --- a/examples/yaml_gen/yaml_gen.d +++ b/examples/yaml_gen/yaml_gen.d @@ -298,16 +298,10 @@ void main(string[] args) Node[] generated = generate(configFile); auto dumper = dumper(); - auto encoding = config["encoding"]; - dumper.indent = config["indent"].as!uint; dumper.textWidth = config["text-width"].as!uint; - switch(encoding.as!string) - { - case "utf-16": dumper.dump!wchar(File(args[1], "w").lockingTextWriter, generated); break; - case "utf-32": dumper.dump!dchar(File(args[1], "w").lockingTextWriter, generated); break; - default: dumper.dump!char(File(args[1], "w").lockingTextWriter, generated); break; - } + + dumper.dump(File(args[1], "w").lockingTextWriter, generated); } catch(YAMLException e) { diff --git a/meson.build b/meson.build index 43c00f39..8c74b1a0 100644 --- a/meson.build +++ b/meson.build @@ -14,7 +14,6 @@ dyaml_src = [ 'source/dyaml/constructor.d', 'source/dyaml/dumper.d', 'source/dyaml/emitter.d', - 'source/dyaml/encoding.d', 'source/dyaml/escapes.d', 'source/dyaml/event.d', 'source/dyaml/exception.d', @@ -39,12 +38,9 @@ dyaml_src = [ ] install_subdir('source/dyaml', install_dir: 'include/d/yaml/') -tinyendian_dep = dependency('tinyendian', version: '>=0.2.0', fallback: ['tinyendian', 'tinyendian_dep']) - dyaml_lib = library('dyaml', [dyaml_src], include_directories: [src_dir], - dependencies: [tinyendian_dep], install: true, version: meson.project_version(), soversion: project_soversion @@ -59,6 +55,5 @@ pkgc.generate(name: 'dyaml', # Make D-YAML easy to use as subproject dyaml_dep = declare_dependency( link_with: dyaml_lib, - include_directories: [src_dir], - dependencies: [tinyendian_dep] + include_directories: [src_dir] ) diff --git a/source/dyaml/dumper.d b/source/dyaml/dumper.d index 03d3620f..f2dcd43a 100644 --- a/source/dyaml/dumper.d +++ b/source/dyaml/dumper.d @@ -143,13 +143,12 @@ struct Dumper * Throws: YAMLException on error (e.g. invalid nodes, * unable to write to file/stream). */ - void dump(CharacterType = char, Range)(Range range, Node[] documents ...) - if (isOutputRange!(Range, CharacterType) && - isOutputRange!(Range, char) || isOutputRange!(Range, wchar) || isOutputRange!(Range, dchar)) + void dump(Range)(Range range, Node[] documents ...) + if (isOutputRange!(Range, char)) { try { - auto emitter = new Emitter!(Range, CharacterType)(range, canonical, indent_, textWidth, lineBreak); + auto emitter = new Emitter!Range(range, canonical, indent_, textWidth, lineBreak); auto serializer = Serializer(resolver, explicitStart ? Yes.explicitStart : No.explicitStart, explicitEnd ? Yes.explicitEnd : No.explicitEnd, YAMLVersion, tags_); serializer.startStream(emitter); diff --git a/source/dyaml/emitter.d b/source/dyaml/emitter.d index 29ebf2ea..e0bdf2ef 100644 --- a/source/dyaml/emitter.d +++ b/source/dyaml/emitter.d @@ -14,7 +14,6 @@ import std.algorithm; import std.array; import std.ascii; import std.conv; -import std.encoding; import std.exception; import std.format; import std.range; @@ -23,7 +22,6 @@ import std.system; import std.typecons; import std.utf; -import dyaml.encoding; import dyaml.escapes; import dyaml.event; import dyaml.exception; @@ -67,7 +65,7 @@ private alias isFlowIndicator = among!(',', '?', '[', ']', '{', '}'); private alias isSpace = among!('\0', '\n', '\r', '\u0085', '\u2028', '\u2029', ' ', '\t'); //Emits YAML events into a file/stream. -struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) +struct Emitter(Range) if (isOutputRange!(Range, char)) { private: ///Default tag handle shortcuts and replacements. @@ -222,20 +220,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) ///Write a string to the file/stream. void writeString(const scope char[] str) @safe { - static if(is(CharType == char)) - { - copy(str, stream_); - } - static if(is(CharType == wchar)) - { - const buffer = to!wstring(str); - copy(buffer, stream_); - } - static if(is(CharType == dchar)) - { - const buffer = to!dstring(str); - copy(buffer, stream_); - } + copy(str, stream_); } ///In some cases, we wait for a few next events before emitting. @@ -731,7 +716,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) //{ // writeIndent(); //} - auto writer = ScalarWriter!(Range, CharType)(&this, analysis_.scalar, + auto writer = ScalarWriter!Range(&this, analysis_.scalar, context_ != Context.mappingSimpleKey); final switch(style_) { @@ -1172,14 +1157,9 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) //Writers. - ///Start the YAML stream (write the unicode byte order mark). + ///Start the YAML stream (do nothing). void writeStreamStart() @safe { - //Write BOM (except for UTF-8) - static if(is(CharType == wchar) || is(CharType == dchar)) - { - stream_.put(cast(CharType)'\uFEFF'); - } } ///End the YAML stream. @@ -1274,7 +1254,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType)) private: ///RAII struct used to write out scalar values. -struct ScalarWriter(Range, CharType) +struct ScalarWriter(Range) { invariant() { @@ -1283,14 +1263,14 @@ struct ScalarWriter(Range, CharType) } private: - @disable int opCmp(ref Emitter!(Range, CharType)); - @disable bool opEquals(ref Emitter!(Range, CharType)); + @disable int opCmp(ref Emitter!(Range)); + @disable bool opEquals(ref Emitter!(Range)); ///Used as "null" UTF-32 character. static immutable dcharNone = dchar.max; ///Emitter used to emit the scalar. - Emitter!(Range, CharType)* emitter_; + Emitter!Range* emitter_; ///UTF-8 encoded text of the scalar to write. string text_; @@ -1311,7 +1291,7 @@ struct ScalarWriter(Range, CharType) public: ///Construct a ScalarWriter using emitter to output text. - this(Emitter!(Range, CharType)* emitter, string text, const bool split = true) @safe nothrow + this(Emitter!Range* emitter, string text, const bool split = true) @safe nothrow { emitter_ = emitter; text_ = text; @@ -1506,7 +1486,7 @@ struct ScalarWriter(Range, CharType) ///Write text as plain scalar. void writePlain() @safe { - if(emitter_.context_ == Emitter!(Range, CharType).Context.root){emitter_.openEnded_ = true;} + if(emitter_.context_ == Emitter!Range.Context.root){emitter_.openEnded_ = true;} if(text_ == ""){return;} if(!emitter_.whitespace_) { diff --git a/source/dyaml/encoding.d b/source/dyaml/encoding.d deleted file mode 100644 index 50c10b9e..00000000 --- a/source/dyaml/encoding.d +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Ferdinand Majerech 2014. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module dyaml.encoding; - - -import tinyendian; - -alias Encoding = tinyendian.UTFEncoding; diff --git a/source/dyaml/loader.d b/source/dyaml/loader.d index 6b79ad5a..3be141de 100644 --- a/source/dyaml/loader.d +++ b/source/dyaml/loader.d @@ -57,7 +57,7 @@ struct Loader { try { - auto loader = Loader(std.file.read(filename), filename); + auto loader = Loader(cast(char[])std.file.read(filename), filename); return loader; } catch(FileException e) @@ -69,7 +69,7 @@ struct Loader /// ditto static Loader fromFile(File file) @system { - auto loader = Loader(file.byChunk(4096).join, file.name); + auto loader = Loader(cast(char[])file.byChunk(4096).join, file.name); return loader; } @@ -88,7 +88,7 @@ struct Loader */ static Loader fromString(char[] data, string filename = "") @safe { - return Loader(cast(ubyte[])data, filename); + return Loader(data, filename); } /// Ditto static Loader fromString(string data, string filename = "") @safe @@ -105,40 +105,9 @@ struct Loader { assert(Loader.fromString("42").load().as!int == 42); } - - /** Construct a Loader to load YAML from a buffer. - * - * Params: yamlData = Buffer with YAML data to load. This may be e.g. a file - * loaded to memory or a string with YAML data. Note that - * buffer $(B will) be overwritten, as D:YAML minimizes - * memory allocations by reusing the input _buffer. - * $(B Must not be deleted or modified by the user as long - * as nodes loaded by this Loader are in use!) - Nodes may - * refer to data in this buffer. - * - * Note that D:YAML looks for byte-order-marks YAML files encoded in - * UTF-16/UTF-32 (and sometimes UTF-8) use to specify the encoding and - * endianness, so it should be enough to load an entire file to a buffer and - * pass it to D:YAML, regardless of Unicode encoding. - * - * Throws: YAMLException if yamlData contains data illegal in YAML. - */ - static Loader fromBuffer(ubyte[] yamlData) @safe - { - return Loader(yamlData); - } - /// Ditto - static Loader fromBuffer(void[] yamlData) @system - { - return Loader(yamlData); - } - /// Ditto - private this(void[] yamlData, string name = "") @system - { - this(cast(ubyte[])yamlData, name); - } + deprecated("Use Loader.fromString instead") alias fromBuffer = fromString; /// Ditto - private this(ubyte[] yamlData, string name = "") @safe + private this(char[] yamlData, string name = "") @safe { try { @@ -430,36 +399,3 @@ EOS"; { assertThrown(Loader.fromFile("test/data/odd-utf16.stream-error").load()); } - -// UTF-16 and 32 test -@safe unittest -{ - import std.conv : to; - import std.range : only; - enum string yaml = `ABCDØ`; - enum bom = '\uFEFF'; - foreach (doc; only( - cast(ubyte[])(bom~yaml.to!(wchar[])), - cast(ubyte[])(bom~yaml.to!(dchar[])), - )) - { - assert(Loader.fromBuffer(doc).load().as!string == yaml); - } -} -// Invalid unicode test -@safe unittest -{ - import std.conv : to; - import std.range : only; - enum string yaml = `ABCDØ`; - enum badBOM = '\uFFFE'; - foreach (doc; only( - cast(ubyte[])yaml.to!(wchar[]), - cast(ubyte[])yaml.to!(dchar[]), - cast(ubyte[])(badBOM~yaml.to!(wchar[])), - cast(ubyte[])(badBOM~yaml.to!(dchar[])), - )) - { - assertThrown(Loader.fromBuffer(doc).load()); - } -} diff --git a/source/dyaml/package.d b/source/dyaml/package.d index e61b716a..6d811c8b 100644 --- a/source/dyaml/package.d +++ b/source/dyaml/package.d @@ -6,7 +6,6 @@ module dyaml; public import dyaml.dumper; -public import dyaml.encoding; public import dyaml.exception; public import dyaml.linebreak; public import dyaml.loader; diff --git a/source/dyaml/reader.d b/source/dyaml/reader.d index 824c1d1c..856da3a4 100644 --- a/source/dyaml/reader.d +++ b/source/dyaml/reader.d @@ -21,9 +21,6 @@ import std.system; import std.typecons; import std.utf; -import tinyendian; - -import dyaml.encoding; import dyaml.exception; alias isBreak = among!('\n', '\u0085', '\u2028', '\u2029'); @@ -53,15 +50,6 @@ struct Reader // Current column in file. uint column_; - // Original Unicode encoding of the data. - Encoding encoding_; - - version(unittest) - { - // Endianness of the input before it was converted (for testing) - Endian endian_; - } - // The number of consecutive ASCII characters starting at bufferOffset_. // // Used to minimize UTF-8 decoding. @@ -85,31 +73,12 @@ struct Reader /// /// Throws: ReaderException on a UTF decoding error or if there are /// nonprintable Unicode characters illegal in YAML. - this(ubyte[] buffer, string name = "") @safe pure + this(char[] buffer, string name = "") @safe pure { name_ = name; - auto endianResult = fixUTFByteOrder(buffer); - if(endianResult.bytesStripped > 0) - { - // TODO: add line and column - throw new ReaderException("Size of UTF-16 or UTF-32 input not aligned " ~ - "to 2 or 4 bytes, respectively", Mark(name, 0, 0)); - } + buffer_ = buffer; - version(unittest) { endian_ = endianResult.endian; } - encoding_ = endianResult.encoding; - - auto utf8Result = toUTF8(endianResult.array, endianResult.encoding); - const msg = utf8Result.errorMessage; - if(msg !is null) - { - // TODO: add line and column - throw new ReaderException("Error when converting to UTF-8: " ~ msg, Mark(name, 0, 0)); - } - - buffer_ = utf8Result.utf8; - - characterCount_ = utf8Result.characterCount; + characterCount_ = buffer.walkLength; // Check that all characters in buffer are printable. // TODO: add line and column enforce(isPrintableValidUTF8(buffer_), @@ -423,9 +392,6 @@ struct Reader /// Get index of the current character in the buffer. size_t charIndex() const @safe pure nothrow @nogc { return charIndex_; } - /// Get encoding of the input buffer. - Encoding encoding() const @safe pure nothrow @nogc { return encoding_; } - private: // Update upcomingASCII_ (should be called forward()ing over a UTF-8 sequence) void checkASCII() @safe pure nothrow @nogc @@ -456,103 +422,6 @@ private: private: -// Convert a UTF-8/16/32 buffer to UTF-8, in-place if possible. -// -// Params: -// -// input = Buffer with UTF-8/16/32 data to decode. May be overwritten by the -// conversion, in which case the result will be a slice of this buffer. -// encoding = Encoding of input. -// -// Returns: -// -// A struct with the following members: -// -// $(D string errorMessage) In case of an error, the error message is stored here. If -// there was no error, errorMessage is NULL. Always check -// this first. -// $(D char[] utf8) input converted to UTF-8. May be a slice of input. -// $(D size_t characterCount) Number of characters (code points) in input. -public auto toUTF8(ubyte[] input, const UTFEncoding encoding) @safe pure nothrow -{ - // Documented in function ddoc. - struct Result - { - string errorMessage; - char[] utf8; - size_t characterCount; - } - - Result result; - - // Encode input_ into UTF-8 if it's encoded as UTF-16 or UTF-32. - // - // Params: - // - // buffer = The input buffer to encode. - // result = A Result struct to put encoded result and any error messages to. - // - // On error, result.errorMessage will be set. - static void encode(C)(C[] input, ref Result result) @safe pure - { - // We can do UTF-32->UTF-8 in place because all UTF-8 sequences are 4 or - // less bytes. - static if(is(C == dchar)) - { - char[4] encodeBuf; - auto utf8 = cast(char[])input; - auto length = 0; - foreach(dchar c; input) - { - ++result.characterCount; - // ASCII - if(c < 0x80) - { - utf8[length++] = cast(char)c; - continue; - } - - std.utf.encode(encodeBuf, c); - const bytes = codeLength!char(c); - utf8[length .. length + bytes] = encodeBuf[0 .. bytes]; - length += bytes; - } - result.utf8 = utf8[0 .. length]; - } - // Unfortunately we can't do UTF-16 in place so we just use std.conv.to - else - { - result.characterCount = std.utf.count(input); - result.utf8 = input.to!(char[]); - } - } - - try final switch(encoding) - { - case UTFEncoding.UTF_8: - result.utf8 = cast(char[])input; - result.utf8.validate(); - result.characterCount = std.utf.count(result.utf8); - break; - case UTFEncoding.UTF_16: - assert(input.length % 2 == 0, "UTF-16 buffer size must be even"); - encode(cast(wchar[])input, result); - break; - case UTFEncoding.UTF_32: - assert(input.length % 4 == 0, "UTF-32 buffer size must be a multiple of 4"); - encode(cast(dchar[])input, result); - break; - } - catch(ConvException e) { result.errorMessage = e.msg; } - catch(UTFException e) { result.errorMessage = e.msg; } - catch(Exception e) - { - assert(false, "Unexpected exception in encode(): " ~ e.msg); - } - - return result; -} - /// Determine if all characters (code points, not bytes) in a string are printable. bool isPrintableValidUTF8(const char[] chars) @safe pure { @@ -576,73 +445,6 @@ size_t countASCII(const(char)[] buffer) @safe pure nothrow @nogc } // Unittests. -void testEndian(R)() -{ - void endian_test(ubyte[] data, Encoding encoding_expected, Endian endian_expected) - { - auto reader = new R(data); - assert(reader.encoding == encoding_expected); - assert(reader.endian_ == endian_expected); - } - ubyte[] little_endian_utf_16 = [0xFF, 0xFE, 0x7A, 0x00]; - ubyte[] big_endian_utf_16 = [0xFE, 0xFF, 0x00, 0x7A]; - endian_test(little_endian_utf_16, Encoding.UTF_16, Endian.littleEndian); - endian_test(big_endian_utf_16, Encoding.UTF_16, Endian.bigEndian); -} - -void testPeekPrefixForward(R)() -{ - import std.encoding; - ubyte[] data = bomTable[BOM.utf8].sequence ~ cast(ubyte[])"data"; - auto reader = new R(data); - assert(reader.peek() == 'd'); - assert(reader.peek(1) == 'a'); - assert(reader.peek(2) == 't'); - assert(reader.peek(3) == 'a'); - assert(reader.peek(4) == '\0'); - assert(reader.prefix(4) == "data"); - // assert(reader.prefix(6) == "data\0"); - reader.forward(2); - assert(reader.peek(1) == 'a'); - // assert(collectException(reader.peek(3))); -} - -void testUTF(R)() -{ - import std.encoding; - dchar[] data = cast(dchar[])"data"; - void utf_test(T)(T[] data, BOM bom) - { - ubyte[] bytes = bomTable[bom].sequence ~ - (cast(ubyte[])data)[0 .. data.length * T.sizeof]; - auto reader = new R(bytes); - assert(reader.peek() == 'd'); - assert(reader.peek(1) == 'a'); - assert(reader.peek(2) == 't'); - assert(reader.peek(3) == 'a'); - } - utf_test!char(to!(char[])(data), BOM.utf8); - utf_test!wchar(to!(wchar[])(data), endian == Endian.bigEndian ? BOM.utf16be : BOM.utf16le); - utf_test(data, endian == Endian.bigEndian ? BOM.utf32be : BOM.utf32le); -} - -void test1Byte(R)() -{ - ubyte[] data = [97]; - - auto reader = new R(data); - assert(reader.peek() == 'a'); - assert(reader.peek(1) == '\0'); - // assert(collectException(reader.peek(2))); -} - -@system unittest -{ - testEndian!Reader(); - testPeekPrefixForward!Reader(); - testUTF!Reader(); - test1Byte!Reader(); -} //Issue 257 - https://github.com/dlang-community/D-YAML/issues/257 @safe unittest { diff --git a/source/dyaml/scanner.d b/source/dyaml/scanner.d index 633bec11..74a5ef4d 100644 --- a/source/dyaml/scanner.d +++ b/source/dyaml/scanner.d @@ -384,7 +384,7 @@ struct Scanner /// Add STREAM-START token. void fetchStreamStart() @safe nothrow { - tokens_.push(streamStartToken(reader_.mark, reader_.mark, reader_.encoding)); + tokens_.push(streamStartToken(reader_.mark, reader_.mark)); } ///Add STREAM-END token. diff --git a/source/dyaml/test/representer.d b/source/dyaml/test/representer.d index eeac1571..a6e2fcbc 100644 --- a/source/dyaml/test/representer.d +++ b/source/dyaml/test/representer.d @@ -28,11 +28,10 @@ module dyaml.test.representer; assert((baseName in expected) !is null, "Unimplemented representer test: " ~ baseName); Node[] expectedNodes = expected[baseName]; - foreach (encoding; AliasSeq!(char, wchar, dchar)) { - auto emitStream = new Appender!(immutable(encoding)[]); + auto emitStream = new Appender!string; auto dumper = dumper(); - dumper.dump!encoding(emitStream, expectedNodes); + dumper.dump(emitStream, expectedNodes); immutable output = emitStream.data; diff --git a/source/dyaml/test/suitehelpers.d b/source/dyaml/test/suitehelpers.d index 30168436..c62ca010 100644 --- a/source/dyaml/test/suitehelpers.d +++ b/source/dyaml/test/suitehelpers.d @@ -32,7 +32,7 @@ enum dchar[] bom32 = ['\uFEFF', '\uFFFE']; Parser parseData(string data, string name = "TEST") @safe { - auto reader = Reader(cast(ubyte[])data.dup, name); + auto reader = Reader(data.dup, name); auto scanner = Scanner(reader); return new Parser(scanner); } @@ -46,7 +46,7 @@ Params: */ void testScanner(string name, string data) @safe { - ubyte[] yamlData = cast(ubyte[])data.dup; + char[] yamlData = data.dup; string[] tokens; foreach (token; Scanner(Reader(yamlData, name))) { @@ -88,7 +88,7 @@ bool testImplicitResolver(string name, string data, string detectData, out strin Event[] emitTestCommon(string name, Event[] events, bool canonical) @safe { auto emitStream = new Appender!string(); - auto emitter = Emitter!(typeof(emitStream), char)(emitStream, canonical, 2, 80, LineBreak.unix); + auto emitter = Emitter!(typeof(emitStream))(emitStream, canonical, 2, 80, LineBreak.unix); foreach (event; events) { emitter.emit(event); diff --git a/source/dyaml/token.d b/source/dyaml/token.d index 5400a3f9..55c67a46 100644 --- a/source/dyaml/token.d +++ b/source/dyaml/token.d @@ -11,7 +11,6 @@ module dyaml.token; import std.conv; -import dyaml.encoding; import dyaml.exception; import dyaml.reader; import dyaml.style; @@ -83,9 +82,6 @@ struct Token /// Style of scalar token, if this is a scalar token. ScalarStyle style; // 1B - /// Encoding, if this is a stream start token. - Encoding encoding; - // 1B /// Type of directive for directiveToken. DirectiveType directive; // 4B @@ -106,8 +102,7 @@ struct Token Token directiveToken(const Mark start, const Mark end, char[] value, DirectiveType directive, const uint nameEnd) @safe pure nothrow @nogc { - return Token(value, start, end, TokenID.directive, ScalarStyle.init, Encoding.init, - directive, nameEnd); + return Token(value, start, end, TokenID.directive, ScalarStyle.init, directive, nameEnd); } /// Construct a simple (no value) token with specified type. @@ -120,17 +115,8 @@ Token simpleToken(TokenID id)(const Mark start, const Mark end) return Token(null, start, end, id); } -/// Construct a stream start token. -/// -/// Params: start = Start position of the token. -/// end = End position of the token. -/// encoding = Encoding of the stream. -Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) @safe pure nothrow @nogc -{ - return Token(null, start, end, TokenID.streamStart, ScalarStyle.invalid, encoding); -} - /// Aliases for construction of simple token types. +alias streamStartToken = simpleToken!(TokenID.streamStart); alias streamEndToken = simpleToken!(TokenID.streamEnd); alias blockSequenceStartToken = simpleToken!(TokenID.blockSequenceStart); alias blockMappingStartToken = simpleToken!(TokenID.blockMappingStart); @@ -151,8 +137,7 @@ alias flowEntryToken = simpleToken!(TokenID.flowEntry); Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value, const uint valueDivider = uint.max) { - return Token(value, start, end, id, ScalarStyle.invalid, Encoding.init, - DirectiveType.init, valueDivider); + return Token(value, start, end, id, ScalarStyle.invalid, DirectiveType.init, valueDivider); } /// Alias for construction of tag token. diff --git a/testsuite/source/app.d b/testsuite/source/app.d index daf34aaf..6ef0393f 100644 --- a/testsuite/source/app.d +++ b/testsuite/source/app.d @@ -24,7 +24,7 @@ auto dumpEventString(string str) @safe string[] output; try { - auto events = new Parser(Scanner(Reader(cast(ubyte[])str.dup))); + auto events = new Parser(Scanner(Reader(str.dup))); foreach (event; events) { output ~= event.text;