remove support for UTF-16/32

dlang-community · Jan 15, 2025 · f69ae09 · f69ae09
1 parent 3a5a2c5
commit f69ae09
Show file tree

Hide file tree

Showing 17 changed files with 45 additions and 374 deletions.
diff --git a/README.md b/README.md
@@ -20,8 +20,7 @@ To start using it in your project, see the [Getting Started](https://dlang-commu
   - Supports all YAML 1.1 constructs. All examples from the YAML 1.1
     specification are parsed correctly.
   - Reads from and writes from/to YAML files or in-memory buffers.
-  - UTF-8, UTF-16 and UTF-32 encodings are supported, both big and
-    little endian (plain ASCII also works as it is a subset of UTF-8).
+  - UTF-8 is the only supported encoding.
   - Support for both block (Python-like, based on indentation) and flow
     (JSON-like, based on bracing) constructs.
   - Support for YAML anchors and aliases.

diff --git a/dub.json b/dub.json
@@ -6,20 +6,14 @@
         "Cameron \"Herringway\" Ross"
     ],
     "license": "BSL-1.0",
-    "dependencies": {
-        "tinyendian" :  "~>0.2.0"
-    },
     "homepage": "https://github.com/dlang-community/D-YAML",
     "copyright": "Copyright © 2011-2018, Ferdinand Majerech",
     "configurations": [
         { "name": "library"  },
         { "name": "unittest" },
         {
             "name": "unittest-dip1000",
-            "dflags": [ "-preview=dip1000" ],
-            "dependencies": {
-                "tinyendian":          { "version": "*", "dflags" : [ "-preview=dip1000" ] },
-            }
+            "dflags": [ "-preview=dip1000" ]
         }
     ],
     "subPackages": [

diff --git a/examples/tokens/source/app.d b/examples/tokens/source/app.d
@@ -22,15 +22,15 @@ import std.uni;
 
 void dumpEventString(string str) @safe
 {
-    auto events = new Parser(Scanner(Reader(cast(ubyte[])str.dup)));
+    auto events = new Parser(Scanner(Reader(str.dup)));
     foreach (event; events)
     {
         writeln(event);
     }
 }
 void dumpTokens(string str) @safe
 {
-    writefln("%(%s\n%)", new Parser(Scanner(Reader(cast(ubyte[])str.dup))));
+    writefln("%(%s\n%)", new Parser(Scanner(Reader(str.dup))));
 }
 
 

diff --git a/examples/yaml_bench/yaml_bench.d b/examples/yaml_bench/yaml_bench.d
@@ -94,9 +94,9 @@ void main(string[] args) //@safe
     string file = args[1];
 
     auto stopWatch = StopWatch(AutoStart.yes);
-    void[] fileInMemory;
-    if(!reload) { fileInMemory = std.file.read(file); }
-    void[] fileWorkingCopy = fileInMemory.dup;
+    char[] fileInMemory;
+    if(!reload) { fileInMemory = cast(char[])std.file.read(file); }
+    char[] fileWorkingCopy = fileInMemory.dup;
     auto loadTime = stopWatch.peek();
     stopWatch.reset();
     try
@@ -113,13 +113,13 @@ void main(string[] args) //@safe
         {
             // Loading the file rewrites the loaded buffer, so if we don't reload from
             // disk, we need to use a copy of the originally loaded file.
-            if(reload) { fileInMemory = std.file.read(file); }
+            if(reload) { fileInMemory = cast(char[])std.file.read(file); }
             else       { fileWorkingCopy[] = fileInMemory[]; }
-            void[] fileToLoad = reload ? fileInMemory : fileWorkingCopy;
+            char[] fileToLoad = reload ? fileInMemory : fileWorkingCopy;
 
             if(scanOnly)
             {
-                auto reader = Reader(cast(ubyte[])fileToLoad, "benchmark");
+                auto reader = Reader(fileToLoad, "benchmark");
                 auto scanner = Scanner(reader);
                 while(!scanner.empty)
                 {
@@ -128,7 +128,7 @@ void main(string[] args) //@safe
             }
             else
             {
-                auto loader = Loader.fromBuffer(fileToLoad);
+                auto loader = Loader.fromString(fileToLoad);
                 loader.resolver = resolver;
                 nodes = loader.array;
             }

diff --git a/examples/yaml_gen/yaml_gen.d b/examples/yaml_gen/yaml_gen.d
@@ -298,16 +298,10 @@ void main(string[] args)
         Node[] generated = generate(configFile);
 
         auto dumper     = dumper();
-        auto encoding   = config["encoding"];
-
         dumper.indent = config["indent"].as!uint;
         dumper.textWidth = config["text-width"].as!uint;
-        switch(encoding.as!string)
-        {
-            case "utf-16": dumper.dump!wchar(File(args[1], "w").lockingTextWriter, generated); break;
-            case "utf-32": dumper.dump!dchar(File(args[1], "w").lockingTextWriter, generated); break;
-            default: dumper.dump!char(File(args[1], "w").lockingTextWriter, generated); break;
-        }
+
+        dumper.dump(File(args[1], "w").lockingTextWriter, generated);
     }
     catch(YAMLException e)
     {

diff --git a/meson.build b/meson.build
@@ -14,7 +14,6 @@ dyaml_src = [
     'source/dyaml/constructor.d',
     'source/dyaml/dumper.d',
     'source/dyaml/emitter.d',
-    'source/dyaml/encoding.d',
     'source/dyaml/escapes.d',
     'source/dyaml/event.d',
     'source/dyaml/exception.d',
@@ -39,12 +38,9 @@ dyaml_src = [
 ]
 install_subdir('source/dyaml', install_dir: 'include/d/yaml/')
 
-tinyendian_dep = dependency('tinyendian', version: '>=0.2.0', fallback: ['tinyendian', 'tinyendian_dep'])
-
 dyaml_lib = library('dyaml',
         [dyaml_src],
         include_directories: [src_dir],
-        dependencies: [tinyendian_dep],
         install: true,
         version: meson.project_version(),
         soversion: project_soversion
@@ -59,6 +55,5 @@ pkgc.generate(name: 'dyaml',
 # Make D-YAML easy to use as subproject
 dyaml_dep = declare_dependency(
     link_with: dyaml_lib,
-    include_directories: [src_dir],
-    dependencies: [tinyendian_dep]
+    include_directories: [src_dir]
 )
diff --git a/source/dyaml/dumper.d b/source/dyaml/dumper.d
@@ -143,13 +143,12 @@ struct Dumper
          * Throws:  YAMLException on error (e.g. invalid nodes,
          *          unable to write to file/stream).
          */
-        void dump(CharacterType = char, Range)(Range range, Node[] documents ...)
-            if (isOutputRange!(Range, CharacterType) &&
-                isOutputRange!(Range, char) || isOutputRange!(Range, wchar) || isOutputRange!(Range, dchar))
+        void dump(Range)(Range range, Node[] documents ...)
+            if (isOutputRange!(Range, char))
         {
             try
             {
-                auto emitter = new Emitter!(Range, CharacterType)(range, canonical, indent_, textWidth, lineBreak);
+                auto emitter = new Emitter!Range(range, canonical, indent_, textWidth, lineBreak);
                 auto serializer = Serializer(resolver, explicitStart ? Yes.explicitStart : No.explicitStart,
                                              explicitEnd ? Yes.explicitEnd : No.explicitEnd, YAMLVersion, tags_);
                 serializer.startStream(emitter);

diff --git a/source/dyaml/emitter.d b/source/dyaml/emitter.d
@@ -14,7 +14,6 @@ import std.algorithm;
 import std.array;
 import std.ascii;
 import std.conv;
-import std.encoding;
 import std.exception;
 import std.format;
 import std.range;
@@ -23,7 +22,6 @@ import std.system;
 import std.typecons;
 import std.utf;
 
-import dyaml.encoding;
 import dyaml.escapes;
 import dyaml.event;
 import dyaml.exception;
@@ -67,7 +65,7 @@ private alias isFlowIndicator = among!(',', '?', '[', ']', '{', '}');
 private alias isSpace = among!('\0', '\n', '\r', '\u0085', '\u2028', '\u2029', ' ', '\t');
 
 //Emits YAML events into a file/stream.
-struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType))
+struct Emitter(Range) if (isOutputRange!(Range, char))
 {
     private:
         ///Default tag handle shortcuts and replacements.
@@ -222,20 +220,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType))
         ///Write a string to the file/stream.
         void writeString(const scope char[] str) @safe
         {
-            static if(is(CharType == char))
-            {
-                copy(str, stream_);
-            }
-            static if(is(CharType == wchar))
-            {
-                const buffer = to!wstring(str);
-                copy(buffer, stream_);
-            }
-            static if(is(CharType == dchar))
-            {
-                const buffer = to!dstring(str);
-                copy(buffer, stream_);
-            }
+            copy(str, stream_);
         }
 
         ///In some cases, we wait for a few next events before emitting.
@@ -731,7 +716,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType))
             //{
             //    writeIndent();
             //}
-            auto writer = ScalarWriter!(Range, CharType)(&this, analysis_.scalar,
+            auto writer = ScalarWriter!Range(&this, analysis_.scalar,
                                        context_ != Context.mappingSimpleKey);
             final switch(style_)
             {
@@ -1172,14 +1157,9 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType))
 
         //Writers.
 
-        ///Start the YAML stream (write the unicode byte order mark).
+        ///Start the YAML stream (do nothing).
         void writeStreamStart() @safe
         {
-            //Write BOM (except for UTF-8)
-            static if(is(CharType == wchar) || is(CharType == dchar))
-            {
-                stream_.put(cast(CharType)'\uFEFF');
-            }
         }
 
         ///End the YAML stream.
@@ -1274,7 +1254,7 @@ struct Emitter(Range, CharType) if (isOutputRange!(Range, CharType))
 private:
 
 ///RAII struct used to write out scalar values.
-struct ScalarWriter(Range, CharType)
+struct ScalarWriter(Range)
 {
     invariant()
     {
@@ -1283,14 +1263,14 @@ struct ScalarWriter(Range, CharType)
     }
 
     private:
-        @disable int opCmp(ref Emitter!(Range, CharType));
-        @disable bool opEquals(ref Emitter!(Range, CharType));
+        @disable int opCmp(ref Emitter!(Range));
+        @disable bool opEquals(ref Emitter!(Range));
 
         ///Used as "null" UTF-32 character.
         static immutable dcharNone = dchar.max;
 
         ///Emitter used to emit the scalar.
-        Emitter!(Range, CharType)* emitter_;
+        Emitter!Range* emitter_;
 
         ///UTF-8 encoded text of the scalar to write.
         string text_;
@@ -1311,7 +1291,7 @@ struct ScalarWriter(Range, CharType)
 
     public:
         ///Construct a ScalarWriter using emitter to output text.
-        this(Emitter!(Range, CharType)* emitter, string text, const bool split = true) @safe nothrow
+        this(Emitter!Range* emitter, string text, const bool split = true) @safe nothrow
         {
             emitter_ = emitter;
             text_ = text;
@@ -1506,7 +1486,7 @@ struct ScalarWriter(Range, CharType)
         ///Write text as plain scalar.
         void writePlain() @safe
         {
-            if(emitter_.context_ == Emitter!(Range, CharType).Context.root){emitter_.openEnded_ = true;}
+            if(emitter_.context_ == Emitter!Range.Context.root){emitter_.openEnded_ = true;}
             if(text_ == ""){return;}
             if(!emitter_.whitespace_)
             {

diff --git a/source/dyaml/encoding.d b/source/dyaml/encoding.d
diff --git a/source/dyaml/loader.d b/source/dyaml/loader.d
@@ -57,7 +57,7 @@ struct Loader
          {
             try
             {
-                auto loader = Loader(std.file.read(filename), filename);
+                auto loader = Loader(cast(char[])std.file.read(filename), filename);
                 return loader;
             }
             catch(FileException e)
@@ -69,7 +69,7 @@ struct Loader
          /// ditto
          static Loader fromFile(File file) @system
          {
-            auto loader = Loader(file.byChunk(4096).join, file.name);
+            auto loader = Loader(cast(char[])file.byChunk(4096).join, file.name);
             return loader;
          }
 
@@ -88,7 +88,7 @@ struct Loader
          */
         static Loader fromString(char[] data, string filename = "<unknown>") @safe
         {
-            return Loader(cast(ubyte[])data, filename);
+            return Loader(data, filename);
         }
         /// Ditto
         static Loader fromString(string data, string filename = "<unknown>") @safe
@@ -105,40 +105,9 @@ struct Loader
         {
             assert(Loader.fromString("42").load().as!int == 42);
         }
-
-        /** Construct a Loader to load YAML from a buffer.
-         *
-         * Params: yamlData = Buffer with YAML data to load. This may be e.g. a file
-         *                    loaded to memory or a string with YAML data. Note that
-         *                    buffer $(B will) be overwritten, as D:YAML minimizes
-         *                    memory allocations by reusing the input _buffer.
-         *                    $(B Must not be deleted or modified by the user  as long
-         *                    as nodes loaded by this Loader are in use!) - Nodes may
-         *                    refer to data in this buffer.
-         *
-         * Note that D:YAML looks for byte-order-marks YAML files encoded in
-         * UTF-16/UTF-32 (and sometimes UTF-8) use to specify the encoding and
-         * endianness, so it should be enough to load an entire file to a buffer and
-         * pass it to D:YAML, regardless of Unicode encoding.
-         *
-         * Throws:  YAMLException if yamlData contains data illegal in YAML.
-         */
-        static Loader fromBuffer(ubyte[] yamlData) @safe
-        {
-            return Loader(yamlData);
-        }
-        /// Ditto
-        static Loader fromBuffer(void[] yamlData) @system
-        {
-            return Loader(yamlData);
-        }
-        /// Ditto
-        private this(void[] yamlData, string name = "<unknown>") @system
-        {
-            this(cast(ubyte[])yamlData, name);
-        }
+        deprecated("Use Loader.fromString instead") alias fromBuffer = fromString;
         /// Ditto
-        private this(ubyte[] yamlData, string name = "<unknown>") @safe
+        private this(char[] yamlData, string name = "<unknown>") @safe
         {
             try
             {
@@ -430,36 +399,3 @@ EOS";
 {
     assertThrown(Loader.fromFile("test/data/odd-utf16.stream-error").load());
 }
-
-// UTF-16 and 32 test
-@safe unittest
-{
-    import std.conv : to;
-    import std.range : only;
-    enum string yaml = `ABCDØ`;
-    enum bom = '\uFEFF';
-    foreach (doc; only(
-        cast(ubyte[])(bom~yaml.to!(wchar[])),
-        cast(ubyte[])(bom~yaml.to!(dchar[])),
-    ))
-    {
-        assert(Loader.fromBuffer(doc).load().as!string == yaml);
-    }
-}
-// Invalid unicode test
-@safe unittest
-{
-    import std.conv : to;
-    import std.range : only;
-    enum string yaml = `ABCDØ`;
-    enum badBOM = '\uFFFE';
-    foreach (doc; only(
-        cast(ubyte[])yaml.to!(wchar[]),
-        cast(ubyte[])yaml.to!(dchar[]),
-        cast(ubyte[])(badBOM~yaml.to!(wchar[])),
-        cast(ubyte[])(badBOM~yaml.to!(dchar[])),
-    ))
-    {
-        assertThrown(Loader.fromBuffer(doc).load());
-    }
-}
diff --git a/source/dyaml/package.d b/source/dyaml/package.d
@@ -6,7 +6,6 @@
 module dyaml;
 
 public import dyaml.dumper;
-public import dyaml.encoding;
 public import dyaml.exception;
 public import dyaml.linebreak;
 public import dyaml.loader;