Unicode sys tests and fixes

.gitignore

@@ -82,6 +82,7 @@ tests/unit/unit.py.res1.txt
 tests/unit/unit.py.res2.bin
 tests/sys/bin/
 /tests/sys/dump/
+/tests/sys/test-res/
 tests/optimization/dump/
 tests/misc/projects/*/*.n
 tests/misc/*/*/*.lua

appveyor.yml

@@ -83,6 +83,8 @@ build_script:
     - haxelib setup "%HAXELIB_ROOT%"
 
 test_script:
+    # change codepage to UTF-8
+    - cmd: chcp 65001
     - cd %APPVEYOR_BUILD_FOLDER%/tests/
     - haxe -version
     - haxe RunCi.hxml

extra/CHANGES.txt

@@ -44,6 +44,7 @@
 	all : fixed `@:allow(package)` allowing too much (#8306)
 	all : fixed various issues with startIndex handling on String.indexOf and String.lastIndexOf
 	all : fixed infinite recursion related to printing of objects with circular references (#8113)
+	sys : fixed various Unicode issues (#8135)
 	macro : fixed Array.pop handling (#8075)
 	macro : fixed assertion failure when throwing exception (#8039)
 	macro : fixed various uncatchable exceptions being thrown

src/compiler/main.ml

@@ -467,6 +467,8 @@ and init ctx =
 	let com = ctx.com in
 	let classes = ref [([],"Std")] in
 try
+	set_binary_mode_out stdout true;
+	set_binary_mode_out stderr true;
 	let xml_out = ref None in
 	let json_out = ref None in
 	let swf_header = ref None in

src/generators/genjava.ml

@@ -2149,6 +2149,8 @@ let generate con =
 								| _ -> ()
 				with | Not_found -> ()
 				);
+				write w "haxe.java.Init.init();";
+				newline w;
 				(match gen.gcon.main with
 					| Some(expr) ->
 						expr_s w (mk_block expr)

src/generators/genjvm.ml

@@ -2495,6 +2495,7 @@ class tclass_to_jvm gctx c = object(self)
 				load();
 				jm#putstatic ([],"Sys") "_args" (TArray(string_sig,None))
 			end;
+			jm#invokestatic (["haxe"; "java"], "Init") "init" (method_sig [] None);
 			jm#invokestatic jc#get_this_path "main" (method_sig [] None);
 			jm#return
 

src/generators/genpy.ml

@@ -2495,7 +2495,10 @@ module Generator = struct
 		let ctx = mk_context com in
 		Codegen.map_source_header com (fun s -> print ctx "# %s\n# coding: utf-8\n" s);
 		if has_feature ctx "closure_Array" || has_feature ctx "closure_String" then
-			spr ctx "from functools import partial as _hx_partial";
+			spr ctx "from functools import partial as _hx_partial\n";
+		spr ctx "import sys\n";
+		spr ctx "if sys.stdout.encoding != 'utf-8':\n    sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)\n";
+		spr ctx "if sys.stderr.encoding != 'utf-8':\n    sys.stderr = open(sys.stderr.fileno(), mode='w', encoding='utf8', buffering=1)\n\n";
 		gen_imports ctx;
 		gen_resources ctx;
 		gen_types ctx;

src/macro/eval/evalStdLib.ml

@@ -1428,6 +1428,10 @@ module StdLock = struct
 	)
 end
 
+let lineEnd = match Sys.os_type with
+	| "Win32" | "Cygwin" -> "\r\n"
+	| _ -> "\n"
+
 module StdLog = struct
 	let key_fileName = hash "fileName"
 	let key_lineNumber = hash "lineNumber"
@@ -1436,15 +1440,15 @@ module StdLog = struct
 	let trace = vfun2 (fun v infos ->
 		let s = value_string v in
 		let s = match infos with
-			| VNull -> Printf.sprintf "%s\n" s
+			| VNull -> (Printf.sprintf "%s" s) ^ lineEnd
 			| _ ->  let infos = decode_object infos in
 				let file_name = decode_string (object_field infos key_fileName) in
 				let line_number = decode_int (object_field infos key_lineNumber) in
 				let l = match object_field infos key_customParams with
 					| VArray va -> s :: (List.map value_string (EvalArray.to_list va))
 					| _ -> [s]
 				in
-				 (Printf.sprintf "%s:%i: %s\n" file_name line_number (String.concat "," l)) in
+				(Printf.sprintf "%s:%i: %s" file_name line_number (String.concat "," l)) ^ lineEnd in
 		((get_ctx()).curapi.MacroApi.get_com()).Common.print s;
 		vnull
 	)
@@ -2548,7 +2552,7 @@ module StdSys = struct
 	let println = vfun1 (fun v ->
 		let ctx = get_ctx() in
 		let com = ctx.curapi.get_com() in
-		com.print (value_string v ^ "\n");
+		com.print (value_string v ^ lineEnd);
 		vnull
 	)
 

std/cs/Boot.hx

@@ -42,6 +42,7 @@ class Boot
 
 	@:keep public static function init():Void
 	{
+		cs.system.Console.OutputEncoding = new cs.system.text.UTF8Encoding();
 		cs.Lib.applyCultureChanges();
 	}
 

std/haxe/io/Path.hx

@@ -230,8 +230,13 @@ class Path {
 		var acc = new StringBuf();
 		var colon = false;
 		var slashes = false;
+		#if utf16
+		for (c in haxe.iterators.StringIteratorUnicode.unicodeIterator(tmp)) {
+			switch (c) {
+		#else
 		for (i in 0...tmp.length) {
 			switch (StringTools.fastCodeAt(tmp, i)) {
+		#end
 				case ":".code:
 					acc.add(":");
 					colon = true;

std/java/Boot.hx

@@ -25,6 +25,7 @@ import java.internal.Function;
 import java.internal.HxObject;
 import java.internal.Runtime;
 import java.Lib;
+import java.Init;
 //import java.internal.StringExt;
 import java.StdTypes;
 import Reflect;

std/java/Init.hx

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C)2005-2019 Haxe Foundation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+package java;
+
+@:native("haxe.java.Init") @:keep class Init {
+	public static function init():Void {
+		try {
+			java.lang.System.setOut(new java.io.PrintStream(java.lang.System.out, true, "utf-8"));
+			java.lang.System.setErr(new java.io.PrintStream(java.lang.System.err, true, "utf-8"));
+		} catch (e:java.io.UnsupportedEncodingException) {}
+	}
+}

std/jvm/Jvm.hx

@@ -12,6 +12,7 @@ import jvm.annotation.EnumReflectionInformation;
 import jvm.annotation.EnumValueReflectionInformation;
 import java.lang.invoke.*;
 import java.NativeArray;
+import java.Init;
 import haxe.ds.Vector;
 import haxe.ds.Option;
 

std/php/_std/Sys.hx

@@ -26,6 +26,8 @@ import sys.io.FileInput;
 import haxe.SysTools;
 
 @:coreApi class Sys {
+	static var lineEnd:String = Sys.systemName() == "Windows" ? "\r\n" : "\n";
+
 	/** Environment variables set by `Sys.putEnv()` */
 	static var customEnvVars = new NativeAssocArray<String>();
 
@@ -34,7 +36,7 @@ import haxe.SysTools;
 	}
 
 	public static inline function println( v : Dynamic ) : Void {
-		Global.echo(Std.string(v) + "\n");
+		Global.echo(Std.string(v) + lineEnd);
 	}
 
 	public static function args() : Array<String> {

std/python/Lib.hx

@@ -34,6 +34,7 @@ typedef PySys = python.lib.Sys;
 **/
 class Lib {
 
+	static var lineEnd:String = Sys.systemName() == "Windows" ? "\r\n" : "\n";
 	static public var __name__(get, never):String;
 	static inline function get___name__():String return python.Syntax.code('__name__');
 
@@ -54,7 +55,7 @@ class Lib {
 	**/
 	public static inline function println(v:Dynamic):Void {
 		var str = Std.string(v);
-		printString('$str\n');
+		printString('$str$lineEnd');
 	}
 
 	/**

tests/runci/targets/Hl.hx

@@ -6,12 +6,28 @@ import runci.System.*;
 import runci.Config.*;
 
 class Hl {
+    static var hlSrc = switch [ci, systemName] {
+      case [AppVeyor, "Windows"]: "C:\\hashlink";
+      case _: Path.join([Sys.getEnv("HOME"), "hashlink"]);
+    };
+    static var hlBuild = switch [ci, systemName] {
+      case [AppVeyor, "Windows"]: "C:\\hashlink_build";
+      case _: Path.join([Sys.getEnv("HOME"), "hashlink_build"]);
+    };
+    static var hlBinDir = switch [ci, systemName] {
+      case [AppVeyor, "Windows"]: "C:\\hashlink_build\\bin";
+      case _: Path.join([Sys.getEnv("HOME"), "hashlink_build", "bin"]);
+    };
+    static var hlBinary = switch [ci, systemName] {
+      case [AppVeyor, "Windows"]: "C:\\hashlink_build\\bin\\hl";
+      case _: Path.join([Sys.getEnv("HOME"), "hashlink_build", "bin", "hl"]);
+    };
+
     static public function getHlDependencies() {
         if (commandSucceed("hl", ["--version"])) {
             infoMsg('hl has already been installed.');
             return;
         }
-        var hlSrc = Path.join([Sys.getEnv("HOME"), "hashlink"]);
         runCommand("git", ["clone", "https://github.com/HaxeFoundation/hashlink.git", hlSrc]);
 
         switch (systemName) {
@@ -23,7 +39,6 @@ class Hl {
                 //pass
         }
 
-        var hlBuild = Path.join([Sys.getEnv("HOME"), "hashlink_build"]);
         FileSystem.createDirectory(hlBuild);
         var generator = systemName == "Windows" ? [] : ["-GNinja"];
         runCommand("cmake", generator.concat([
@@ -45,19 +60,21 @@ class Hl {
             "--build", hlBuild
         ]);
 
-        addToPATH(Path.join([hlBuild, "bin"]));
-        runCommand("hl", ["--version"]);
+        runCommand(hlBinary, ["--version"]);
+        addToPATH(hlBinDir);
     }
 
     static public function run(args:Array<String>) {
         getHlDependencies();
         runCommand("haxe", ["compile-hl.hxml"].concat(args));
-        runCommand("hl", ["bin/unit.hl"]);
+        runCommand(hlBinary, ["bin/unit.hl"]);
 
 		// changeDirectory(threadsDir);
 		// runCommand("haxe", ["build.hxml", "-hl", "export/threads.hl"]);
 		// runCommand("hl", ["export/threads.hl"]);
 
-        // TODO sys test
+        changeDirectory(sysDir);
+        runCommand("haxe", ["compile-hl.hxml"]);
+        runCommand(hlBinary, ["bin/hl/sys.hl"]);
     }
 }

tests/sys/compile-cpp.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
+-cpp bin/cpp
+
+--next
+compile-each.hxml
+--main UtilityProcess
 -cpp bin/cpp

tests/sys/compile-cs.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
+-cs bin/cs
+
+--next
+compile-each.hxml
+--main UtilityProcess
 -cs bin/cs

tests/sys/compile-hl.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
--hl bin/hl/ExitCode.hl
+-hl bin/hl/ExitCode.hl
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-hl bin/hl/UtilityProcess.hl

tests/sys/compile-java.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
+-java bin/java
+
+--next
+compile-each.hxml
+--main UtilityProcess
 -java bin/java

tests/sys/compile-jvm.hxml

@@ -13,4 +13,10 @@ compile-each.hxml
 compile-each.hxml
 --main ExitCode
 -java bin/jvm
+-D jvm
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-java bin/jvm
 -D jvm

tests/sys/compile-lua.hxml

@@ -12,3 +12,8 @@ compile-each.hxml
 compile-each.hxml
 --main ExitCode
 -lua bin/lua/ExitCode.lua
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-lua bin/lua/UtilityProcess.lua

tests/sys/compile-neko.hxml

@@ -11,3 +11,8 @@ compile-each.hxml
 # compile-each.hxml
 # -main ExitCode
 # -neko bin/neko/ExitCode.n
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-neko bin/neko/UtilityProcess.n

tests/sys/compile-php.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
--php bin/php/ExitCode
+-php bin/php/ExitCode
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-php bin/php/UtilityProcess

tests/sys/compile-python.hxml

@@ -10,4 +10,9 @@ compile-each.hxml
 --next
 compile-each.hxml
 --main ExitCode
--python bin/python/ExitCode.py
+-python bin/python/ExitCode.py
+
+--next
+compile-each.hxml
+--main UtilityProcess
+-python bin/python/UtilityProcess.py