diff --git a/build.sc b/build.sc index facf7b890..15f79569e 100644 --- a/build.sc +++ b/build.sc @@ -26,7 +26,7 @@ val sourcecode = "0.3.0" val dottyCustomVersion = Option(sys.props("dottyVersion")) val scala2JVMVersions = Seq(scala212, scala213) -val scalaVersions = scala2JVMVersions ++ Seq(scala3) ++ dottyCustomVersion +val scalaVersions = scala2JVMVersions// ++ Seq(scala3) ++ dottyCustomVersion trait CommonPlatformModule extends ScalaModule with PlatformScalaModule{ diff --git a/ujson/src/ujson/AstTransformer.scala b/ujson/src/ujson/AstTransformer.scala index 336287f86..b69361018 100644 --- a/ujson/src/ujson/AstTransformer.scala +++ b/ujson/src/ujson/AstTransformer.scala @@ -5,7 +5,7 @@ import upickle.core.compat._ import scala.language.higherKinds -trait AstTransformer[I] extends Transformer[I] with JsVisitor[I, I]{ +trait AstTransformer[I] extends ujson.Transformer[I] with JsVisitor[I, I]{ def apply(t: Readable): I = t.transform(this) def transformArray[T](f: Visitor[_, T], items: Iterable[I]) = { diff --git a/ujson/src/ujson/IndexedValue.scala b/ujson/src/ujson/IndexedValue.scala index a50590ce6..2dd43c0b7 100644 --- a/ujson/src/ujson/IndexedValue.scala +++ b/ujson/src/ujson/IndexedValue.scala @@ -10,10 +10,12 @@ import upickle.core.{Visitor, ObjVisitor, ArrVisitor, Abort, AbortException} * want to work with an AST but still provide source-index error positions if * something goes wrong */ +@deprecated("Left for binary compatibility, use upickle.core.BufferedValue instead", "upickle after 3.1.4") sealed trait IndexedValue { def index: Int } +@deprecated("Left for binary compatibility, use upickle.core.BufferedValue instead", "upickle after 3.1.4") object IndexedValue extends Transformer[IndexedValue]{ case class Str(index: Int, value0: java.lang.CharSequence) extends IndexedValue diff --git a/ujson/src/ujson/Readable.scala b/ujson/src/ujson/Readable.scala index cbe090e5d..9fb073014 100644 --- a/ujson/src/ujson/Readable.scala +++ b/ujson/src/ujson/Readable.scala @@ -10,7 +10,10 @@ trait Readable { def transform[T](f: Visitor[_, T]): T } -object Readable extends ReadableLowPri{ +object Readable extends ReadableLowPri with Transformer[Readable]{ + def transform[T](j: ujson.Readable, f: upickle.core.Visitor[_, T]): T = { + j.transform(f) + } case class fromTransformer[T](t: T, w: Transformer[T]) extends Readable{ def transform[T](f: Visitor[_, T]): T = { w.transform(t, f) diff --git a/ujson/src/ujson/Transformer.scala b/ujson/src/ujson/Transformer.scala index a35dd6596..df3453bbf 100644 --- a/ujson/src/ujson/Transformer.scala +++ b/ujson/src/ujson/Transformer.scala @@ -1,7 +1,6 @@ package ujson import upickle.core.Visitor -trait Transformer[I] { - def transform[T](j: I, f: Visitor[_, T]): T +trait Transformer[I] extends upickle.core.Transformer[I]{ def transformable[T](j: I) = Readable.fromTransformer(j, this) } diff --git a/ujson/src/ujson/package.scala b/ujson/src/ujson/package.scala index 56f3ac655..cf1972ac0 100644 --- a/ujson/src/ujson/package.scala +++ b/ujson/src/ujson/package.scala @@ -1,8 +1,16 @@ import upickle.core.NoOpVisitor +import upickle.core.BufferedValue package object ujson{ - def transform[T](t: Readable, v: upickle.core.Visitor[_, T]) = t.transform(v) + def transform[T](t: Readable, + v: upickle.core.Visitor[_, T], + sortKeys: Boolean = false): T = { + BufferedValue.maybeSortKeysTransform(Readable, t, sortKeys, v) + } +// @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def transform[T](t: Readable, + v: upickle.core.Visitor[_, T]): T = transform(t, v, sortKeys = false) /** * Read the given JSON input as a JSON struct */ @@ -16,36 +24,71 @@ package object ujson{ */ def write(t: Value.Value, indent: Int = -1, - escapeUnicode: Boolean = false): String = { + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): String = { val writer = new java.io.StringWriter - writeTo(t, writer, indent, escapeUnicode) + writeTo(t, writer, indent, escapeUnicode, sortKeys) writer.toString } + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def write(t: Value.Value, + indent: Int, + escapeUnicode: Boolean): String = { + write(t, indent, escapeUnicode, sortKeys = false) + } + /** * Write the given JSON struct as a JSON String to the given Writer */ def writeTo(t: Value.Value, out: java.io.Writer, indent: Int = -1, - escapeUnicode: Boolean = false): Unit = { - transform(t, Renderer(out, indent, escapeUnicode)) + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + transform(t, Renderer(out, indent, escapeUnicode), sortKeys) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeTo(t: Value.Value, + out: java.io.Writer, + indent: Int, + escapeUnicode: Boolean): Unit = { + writeTo(t, out, indent, escapeUnicode, sortKeys = false) } + def writeToOutputStream(t: Value.Value, out: java.io.OutputStream, indent: Int = -1, - escapeUnicode: Boolean = false): Unit = { - transform(t, new BaseByteRenderer(out, indent, escapeUnicode)) + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + transform(t, new BaseByteRenderer(out, indent, escapeUnicode), sortKeys) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeToOutputStream(t: Value.Value, + out: java.io.OutputStream, + indent: Int, + escapeUnicode: Boolean): Unit = { + writeToOutputStream(t, out, indent, escapeUnicode, sortKeys = false) } def writeToByteArray(t: Value.Value, indent: Int = -1, - escapeUnicode: Boolean = false) = { + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Array[Byte] = { val baos = new java.io.ByteArrayOutputStream - writeToOutputStream(t, baos, indent, escapeUnicode) + writeToOutputStream(t, baos, indent, escapeUnicode, sortKeys) baos.toByteArray } + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeToByteArray(t: Value.Value, + indent: Int, + escapeUnicode: Boolean): Array[Byte] = { + writeToByteArray(t, indent, escapeUnicode, sortKeys = false) + } + /** * Parse the given JSON input, failing if it is invalid */ @@ -54,17 +97,39 @@ package object ujson{ * Parse the given JSON input and write it to a string with * the configured formatting */ - def reformat(s: Readable, indent: Int = -1, escapeUnicode: Boolean = false): String = { + def reformat(s: Readable, + indent: Int = -1, + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): String = { val writer = new java.io.StringWriter() - reformatTo(s, writer, indent, escapeUnicode) + reformatTo(s, writer, indent, escapeUnicode, sortKeys) writer.toString } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def reformat(s: Readable, + indent: Int, + escapeUnicode: Boolean): String = { + reformat(s, indent, escapeUnicode, sortKeys = false) + } /** * Parse the given JSON input and write it to a string with * the configured formatting to the given Writer */ - def reformatTo(s: Readable, out: java.io.Writer, indent: Int = -1, escapeUnicode: Boolean = false): Unit = { - transform(s, Renderer(out, indent, escapeUnicode)) + def reformatTo(s: Readable, + out: java.io.Writer, + indent: Int = -1, + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + transform(s, Renderer(out, indent, escapeUnicode), sortKeys) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def reformatTo(s: Readable, + out: java.io.Writer, + indent: Int, + escapeUnicode: Boolean): Unit = { + reformatTo(s, out, indent, escapeUnicode, sortKeys = false) } /** * Parse the given JSON input and write it to a string with @@ -73,16 +138,34 @@ package object ujson{ def reformatToOutputStream(s: Readable, out: java.io.OutputStream, indent: Int = -1, - escapeUnicode: Boolean = false): Unit = { - transform(s, new BaseByteRenderer(out, indent, escapeUnicode)) + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + transform(s, new BaseByteRenderer(out, indent, escapeUnicode), sortKeys) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def reformatToOutputStream(s: Readable, + out: java.io.OutputStream, + indent: Int, + escapeUnicode: Boolean): Unit = { + reformatToOutputStream(s, out, indent, escapeUnicode, sortKeys = false) } + def reformatToByteArray(s: Readable, indent: Int = -1, - escapeUnicode: Boolean = false) = { + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Array[Byte] = { val baos = new java.io.ByteArrayOutputStream - reformatToOutputStream(s, baos, indent, escapeUnicode) + reformatToOutputStream(s, baos, indent, escapeUnicode, sortKeys) baos.toByteArray } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def reformatToByteArray(s: Readable, + indent: Int, + escapeUnicode: Boolean): Array[Byte] = { + reformatToByteArray(s, indent, escapeUnicode, sortKeys = false) + } // End ujson @deprecated("use ujson.Value") type Js = Value diff --git a/ujson/test/src/ujson/JsonTests.scala b/ujson/test/src/ujson/JsonTests.scala index 827fe0122..f028e158b 100644 --- a/ujson/test/src/ujson/JsonTests.scala +++ b/ujson/test/src/ujson/JsonTests.scala @@ -114,5 +114,32 @@ object JsonTests extends TestSuite { reader.getBufferCopyCount() ==> 0 reader.getBufferLength() ==> bytes.length } + test("sortKeys"){ + val raw = """{"d": [{"c": 0, "b": 1}], "a": 2}""" + val sorted = """{ + | "a": 2, + | "d": [ + | { + | "b": 1, + | "c": 0 + | } + | ] + |}""".stripMargin + ujson.reformat(raw, indent = 4, sortKeys = true) ==> sorted + + ujson.write(ujson.read(raw), indent = 4, sortKeys = true) ==> sorted + + val baos = new java.io.ByteArrayOutputStream + ujson.writeToOutputStream(ujson.read(raw), baos, indent = 4, sortKeys = true) + baos.toString ==> sorted + + val writer = new java.io.StringWriter + ujson.writeTo(ujson.read(raw), writer, indent = 4, sortKeys = true) + writer.toString ==> sorted + + new String(ujson.writeToByteArray(ujson.read(raw), indent = 4, sortKeys = true)) ==> sorted + + new String(ujson.reformatToByteArray(raw, indent = 4, sortKeys = true)) ==> sorted + } } } diff --git a/upickle/core/src-2.12/upickle/core/compat/SortInPlace.scala b/upickle/core/src-2.12/upickle/core/compat/SortInPlace.scala new file mode 100644 index 000000000..51fc750ac --- /dev/null +++ b/upickle/core/src-2.12/upickle/core/compat/SortInPlace.scala @@ -0,0 +1,9 @@ +package upickle.core.compat + +object SortInPlace { + def apply[T, B: Ordering](t: collection.mutable.ArrayBuffer[T])(f: T => B): Unit = { + val sorted = t.sortBy(f) + t.clear() + t.appendAll(sorted) + } +} diff --git a/upickle/core/src-2.13+/upickle/core/LinkedHashMapCompat.scala b/upickle/core/src-2.13+/upickle/core/compat/LinkedHashMapCompat.scala similarity index 100% rename from upickle/core/src-2.13+/upickle/core/LinkedHashMapCompat.scala rename to upickle/core/src-2.13+/upickle/core/compat/LinkedHashMapCompat.scala index ef751efc2..e40de32c3 100644 --- a/upickle/core/src-2.13+/upickle/core/LinkedHashMapCompat.scala +++ b/upickle/core/src-2.13+/upickle/core/compat/LinkedHashMapCompat.scala @@ -1,7 +1,7 @@ package upickle.core.compat -import upickle.core.compat.Factory import upickle.core.LinkedHashMap +import upickle.core.compat.Factory import scala.collection.mutable diff --git a/upickle/core/src-2.13+/upickle/core/compat/SortInPlace.scala b/upickle/core/src-2.13+/upickle/core/compat/SortInPlace.scala new file mode 100644 index 000000000..b7767a902 --- /dev/null +++ b/upickle/core/src-2.13+/upickle/core/compat/SortInPlace.scala @@ -0,0 +1,7 @@ +package upickle.core.compat + +object SortInPlace { + def apply[T, B: scala.Ordering](t: collection.mutable.ArrayBuffer[T])(f: T => B): Unit = { + t.sortInPlaceBy(f) + } +} diff --git a/upickle/core/src/upickle/core/BufferedValue.scala b/upickle/core/src/upickle/core/BufferedValue.scala new file mode 100644 index 000000000..d57ae9f06 --- /dev/null +++ b/upickle/core/src/upickle/core/BufferedValue.scala @@ -0,0 +1,160 @@ +package upickle.core + +import upickle.core.ParseUtils.reject +import scala.collection.mutable + +/** + * A reified version of [[Visitor]], allowing visitor method calls to be buffered up, + * stored somewhere, and replayed later. + */ +sealed trait BufferedValue { + def index: Int +} + +object BufferedValue extends Transformer[BufferedValue]{ + def valueToSortKey(b: BufferedValue): String = b match{ + case BufferedValue.Null(i) => "00" + case BufferedValue.True(i) => "01" + "true" + case BufferedValue.False(i) => "02" + "false" + case BufferedValue.Str(s, i) => "03" + s.toString + case BufferedValue.Num(s, _, _, i) => "04" + s.toString + case BufferedValue.Char(c, i) => "05" + c.toString + case BufferedValue.Binary(bytes, o, l, _) => "06" + new String(bytes, o, l) + case BufferedValue.Ext(tag, bytes, o, l, i) => "07" + tag.toString + new String(bytes, o, l) + case BufferedValue.Float32(f, i) => "08" + f.toString + case BufferedValue.Float64String(s, i) => "09" + s + case BufferedValue.Int32(n, i) => "10" + n.toString + case BufferedValue.Int64(n, i) => "11" + n.toString + case BufferedValue.NumRaw(d, i) => "12" + d.toString + case BufferedValue.UInt64(n, i) => "13" + n.toString + case BufferedValue.Arr(vs, i) => "14" + vs.map(valueToSortKey).mkString + case BufferedValue.Obj(kvs, _, i) => "15" + kvs.map{case (k, v) => valueToSortKey(k) + valueToSortKey(v)}.mkString + } + + def maybeSortKeysTransform[T, V](tr: Transformer[T], + t: T, + sortKeys: Boolean, + f: Visitor[_, V]): V = { + def rec(x: BufferedValue): Unit = { + x match { + case BufferedValue.Arr(items, i) => items.map(rec) + case BufferedValue.Obj(items, jsonableKeys, i) => + upickle.core.compat.SortInPlace[(BufferedValue, BufferedValue), String](items) { + case (k, v) => valueToSortKey(k) + } + items.foreach { case (c, v) => (c, rec(v)) } + case v => + } + } + + if (sortKeys) { + val buffered = tr.transform(t, BufferedValue.Builder) + rec(buffered) + BufferedValue.transform(buffered, f) + } else { + tr.transform(t, f) + } + } + + case class Str(value0: java.lang.CharSequence, index: Int) extends BufferedValue + case class Obj(value0: mutable.ArrayBuffer[(BufferedValue, BufferedValue)], jsonableKeys: Boolean, index: Int) extends BufferedValue + case class Arr(value: mutable.ArrayBuffer[BufferedValue], index: Int) extends BufferedValue + case class Num(s: CharSequence, decIndex: Int, expIndex: Int, index: Int) extends BufferedValue + case class NumRaw(d: Double, index: Int) extends BufferedValue + case class False(index: Int) extends BufferedValue{ + def value = false + } + case class True(index: Int) extends BufferedValue{ + def value = true + } + case class Null(index: Int) extends BufferedValue{ + def value = null + } + + case class Binary(bytes: Array[Byte], offset: Int, len: Int, index: Int) extends BufferedValue + case class Char(s: scala.Char, index: Int) extends BufferedValue + case class Ext(tag: Byte, bytes: Array[Byte], offset: Int, len: Int, index: Int) extends BufferedValue + case class Float32(d: Float, index: Int) extends BufferedValue + case class Float64String(s: String, index: Int) extends BufferedValue + case class Int32(i: Int, index: Int) extends BufferedValue + case class Int64(i: Long, index: Int) extends BufferedValue + case class UInt64(i: Long, index: Int) extends BufferedValue + + def transform[T](j: BufferedValue, f: Visitor[_, T]): T = try{ + j match{ + case BufferedValue.Null(i) => f.visitNull(i) + case BufferedValue.True(i) => f.visitTrue(i) + case BufferedValue.False(i) => f.visitFalse(i) + case BufferedValue.Str(s, i) => f.visitString(s, i) + case BufferedValue.Num(s, d, e, i) => f.visitFloat64StringParts(s, d, e, i) + case BufferedValue.NumRaw(d, i) => f.visitFloat64(d, i) + case BufferedValue.Arr(items, i) => + val ctx = f.visitArray(items.length, i).narrow + for(item <- items) try ctx.visitValue(transform(item, ctx.subVisitor), item.index) catch reject(item.index) + ctx.visitEnd(i) + + case BufferedValue.Obj(items, jsonableKeys, i) => + val ctx = f.visitObject(items.length, jsonableKeys, i).narrow + for((k, item) <- items) { + val keyVisitor = try ctx.visitKey(i) catch reject(i) + val key = transform(k, keyVisitor) + ctx.visitKeyValue(key) + try ctx.visitValue(transform(item, ctx.subVisitor), item.index) + catch reject(item.index) + } + ctx.visitEnd(i) + case BufferedValue.Binary(bytes, offset, len, index) => f.visitBinary(bytes, offset, len, index) + case BufferedValue.Char(s, index) => f.visitChar(s, index) + case BufferedValue.Ext(tag, bytes, offset, len, index) => f.visitExt(tag, bytes, offset, len, index) + case BufferedValue.Float32(d, index) => f.visitFloat32(d, index) + case BufferedValue.Float64String(s, index) => f.visitFloat64String(s, index) + case BufferedValue.Int32(i, index) => f.visitInt32(i, index) + case BufferedValue.Int64(i, index) => f.visitInt64(i, index) + case BufferedValue.UInt64(i, index) => f.visitUInt64(i, index) + } + } catch reject(j.index) + + + object Builder extends Visitor[BufferedValue, BufferedValue]{ + def visitArray(length: Int, i: Int) = new ArrVisitor[BufferedValue, BufferedValue.Arr] { + val out = mutable.ArrayBuffer.empty[BufferedValue] + def subVisitor = Builder + def visitValue(v: BufferedValue, index: Int): Unit = { + out.append(v) + } + def visitEnd(index: Int): BufferedValue.Arr = BufferedValue.Arr(out, i) + } + + def visitObject(length: Int, jsonableKeys: Boolean, i: Int) = new ObjVisitor[BufferedValue, BufferedValue.Obj] { + val out = mutable.ArrayBuffer.empty[(BufferedValue, BufferedValue)] + var currentKey: BufferedValue = _ + def subVisitor = Builder + def visitKey(index: Int) = BufferedValue.Builder + def visitKeyValue(s: Any): Unit = currentKey = s.asInstanceOf[BufferedValue] + def visitValue(v: BufferedValue, index: Int): Unit = { + out.append((currentKey, v)) + } + def visitEnd(index: Int): BufferedValue.Obj = BufferedValue.Obj(out, jsonableKeys, i) + } + + def visitNull(i: Int) = BufferedValue.Null(i) + + def visitFalse(i: Int) = BufferedValue.False(i) + + def visitTrue(i: Int) = BufferedValue.True(i) + + def visitFloat64StringParts(s: CharSequence, decIndex: Int, expIndex: Int, i: Int) = BufferedValue.Num(s, decIndex, expIndex, i) + override def visitFloat64(d: Double, i: Int) = BufferedValue.NumRaw(d, i) + + def visitString(s: CharSequence, i: Int) = BufferedValue.Str(s, i) + + def visitBinary(bytes: Array[Byte], offset: Int, len: Int, index: Int) = BufferedValue. Binary(bytes, offset, len, index) + def visitChar(s: scala.Char, index: Int) = BufferedValue. Char(s, index) + def visitExt(tag: Byte, bytes: Array[Byte], offset: Int, len: Int, index: Int) = BufferedValue. Ext(tag, bytes, offset, len, index) + def visitFloat32(d: Float, index: Int) = BufferedValue. Float32(d, index) + def visitFloat64String(s: String, index: Int) = BufferedValue. Float64String(s, index) + def visitInt32(i: Int, index: Int) = BufferedValue. Int32(i, index) + def visitInt64(i: Long, index: Int) = BufferedValue. Int64(i, index) + def visitUInt64(i: Long, index: Int) = BufferedValue. UInt64(i, index) + } +} diff --git a/upickle/core/src/upickle/core/Transformer.scala b/upickle/core/src/upickle/core/Transformer.scala new file mode 100644 index 000000000..1163e90a7 --- /dev/null +++ b/upickle/core/src/upickle/core/Transformer.scala @@ -0,0 +1,5 @@ +package upickle.core + +trait Transformer[I] { + def transform[T](j: I, f: Visitor[_, T]): T +} diff --git a/upickle/core/src/upickle/core/Types.scala b/upickle/core/src/upickle/core/Types.scala index 3598d307b..0da1b7ab9 100644 --- a/upickle/core/src/upickle/core/Types.scala +++ b/upickle/core/src/upickle/core/Types.scala @@ -115,7 +115,7 @@ trait Types{ types => * Generally nothing more than a way of applying the [[T]] to * a [[Visitor]], along with some utility methods */ - trait Writer[T] { + trait Writer[T] extends Transformer[T]{ /** * Whether or not the type being written can be used as a key in a JSON dictionary. * Opt-in, and only applicable to writers that write primitive types like diff --git a/upickle/src/upickle/Api.scala b/upickle/src/upickle/Api.scala index 9c2efc73d..98773c643 100644 --- a/upickle/src/upickle/Api.scala +++ b/upickle/src/upickle/Api.scala @@ -6,7 +6,6 @@ import language.experimental.macros import language.higherKinds import upickle.core._ import scala.reflect.ClassTag -import ujson.IndexedValue /** * An instance of the upickle API. There's a default instance at @@ -24,6 +23,11 @@ trait Api with MsgReadWriters with Annotator{ + private def maybeSortKeysTransform[T: Writer, V](t: T, + sortKeys: Boolean, + f: Visitor[_, V]): V = { + BufferedValue.maybeSortKeysTransform(implicitly[Writer[T]], t, sortKeys, f) + } /** * Reads the given MessagePack input into a Scala value */ @@ -45,16 +49,29 @@ trait Api */ def write[T: Writer](t: T, indent: Int = -1, - escapeUnicode: Boolean = false): String = { - transform(t).to(ujson.StringRenderer(indent, escapeUnicode)).toString + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): String = { + maybeSortKeysTransform(t, sortKeys, ujson.StringRenderer(indent, escapeUnicode)).toString + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def write[T: Writer](t: T, + indent: Int, + escapeUnicode: Boolean): String = { + write(t, indent, escapeUnicode, sortKeys = false) } + /** * Write the given Scala value as a MessagePack binary */ - def writeBinary[T: Writer](t: T): Array[Byte] = { - transform(t).to(new upack.MsgPackWriter(new ByteArrayOutputStream())).toByteArray + def writeBinary[T: Writer](t: T, + sortKeys: Boolean = false): Array[Byte] = { + maybeSortKeysTransform(t, sortKeys, new upack.MsgPackWriter(new ByteArrayOutputStream())).toByteArray } + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeBinary[T: Writer](t: T): Array[Byte] = writeBinary(t, sortKeys = false) + /** * Write the given Scala value as a JSON struct */ @@ -71,50 +88,105 @@ trait Api def writeTo[T: Writer](t: T, out: java.io.Writer, indent: Int = -1, - escapeUnicode: Boolean = false): Unit = { - transform(t).to(new ujson.Renderer(out, indent = indent, escapeUnicode)) + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + maybeSortKeysTransform(t, sortKeys, new ujson.Renderer(out, indent = indent, escapeUnicode)) } + + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeTo[T: Writer](t: T, + out: java.io.Writer, + indent: Int, + escapeUnicode: Boolean): Unit = writeTo(t, out, indent, escapeUnicode, sortKeys = false) + def writeToOutputStream[T: Writer](t: T, out: java.io.OutputStream, indent: Int = -1, - escapeUnicode: Boolean = false): Unit = { - transform(t).to(new ujson.BaseByteRenderer(out, indent = indent, escapeUnicode)) + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Unit = { + maybeSortKeysTransform(t, sortKeys, new ujson.BaseByteRenderer(out, indent = indent, escapeUnicode)) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeToOutputStream[T: Writer](t: T, + out: java.io.OutputStream, + indent: Int, + escapeUnicode: Boolean): Unit = { + writeToOutputStream(t, out, indent, escapeUnicode, sortKeys = false) } + def writeToByteArray[T: Writer](t: T, indent: Int = -1, - escapeUnicode: Boolean = false) = { + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): Array[Byte] = { val out = new java.io.ByteArrayOutputStream() - writeToOutputStream(t, out, indent, escapeUnicode) + writeToOutputStream(t, out, indent, escapeUnicode, sortKeys) out.toByteArray } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeToByteArray[T: Writer](t: T, + indent: Int, + escapeUnicode: Boolean): Array[Byte] = { + writeToByteArray[T](t, indent, escapeUnicode, sortKeys = false) + } /** * Write the given Scala value as a JSON string via a `geny.Writable` */ def stream[T: Writer](t: T, indent: Int = -1, - escapeUnicode: Boolean = false): geny.Writable = new geny.Writable{ + escapeUnicode: Boolean = false, + sortKeys: Boolean = false): geny.Writable = new geny.Writable{ override def httpContentType = Some("application/json") def writeBytesTo(out: java.io.OutputStream) = { - transform(t).to(new ujson.BaseByteRenderer(out, indent = indent, escapeUnicode)) + maybeSortKeysTransform(t, sortKeys, new ujson.BaseByteRenderer(out, indent = indent, escapeUnicode)) } } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def stream[T: Writer](t: T, + indent: Int, + escapeUnicode: Boolean): geny.Writable = { + stream(t, indent, escapeUnicode, sortKeys = false) + } /** * Write the given Scala value as a MessagePack binary to the given OutputStream */ - def writeBinaryTo[T: Writer](t: T, out: java.io.OutputStream): Unit = { - streamBinary[T](t).writeBytesTo(out) + def writeBinaryTo[T: Writer](t: T, + out: java.io.OutputStream, + sortKeys: Boolean = false): Unit = { + streamBinary[T](t, sortKeys = sortKeys).writeBytesTo(out) } - def writeBinaryToByteArray[T: Writer](t: T) = { + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeBinaryTo[T: Writer](t: T, + out: java.io.OutputStream): Unit = { + writeBinaryTo(t, out, sortKeys = false) + } + + def writeBinaryToByteArray[T: Writer](t: T, + sortKeys: Boolean = false): Array[Byte] = { val out = new java.io.ByteArrayOutputStream() - streamBinary[T](t).writeBytesTo(out) + streamBinary[T](t, sortKeys = sortKeys).writeBytesTo(out) out.toByteArray } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def writeBinaryToByteArray[T: Writer](t: T): Array[Byte] = { + writeBinaryToByteArray(t, sortKeys = false) + } /** * Write the given Scala value as a MessagePack binary via a `geny.Writable` */ - def streamBinary[T: Writer](t: T): geny.Writable = new geny.Writable{ + def streamBinary[T: Writer](t: T, sortKeys: Boolean = false): geny.Writable = new geny.Writable{ override def httpContentType = Some("application/octet-stream") - def writeBytesTo(out: java.io.OutputStream) = transform(t).to(new upack.MsgPackWriter(out)) + def writeBytesTo(out: java.io.OutputStream) = maybeSortKeysTransform(t, sortKeys, new upack.MsgPackWriter(out)) + } + + // @deprecated("Binary Compatibility Stub", "upickle after 3.1.4") + def streamBinary[T: Writer](t: T): geny.Writable = { + streamBinary(t, sortKeys = false) } def writer[T: Writer] = implicitly[Writer[T]] @@ -241,6 +313,11 @@ trait AttributeTagged extends Api with Annotator{ } def taggedExpectedMsg = "expected dictionary" + private def isTagName(i: Any) = i match{ + case s: BufferedValue.Str => s.value0.toString == tagName + case s: CharSequence => s.toString == tagName + case _ => false + } override def taggedObjectContext[T](taggedReader: TaggedReader[T], index: Int) = { new ObjVisitor[Any, T]{ private[this] var fastPath = false @@ -256,10 +333,10 @@ trait AttributeTagged extends Api with Annotator{ def visitKeyValue(s: Any): Unit = { if (context != null) context.visitKeyValue(s) else { - if (s.toString == tagName) () //do nothing + if (isTagName(s)) () //do nothing else { // otherwise, go slow path - val slowCtx = IndexedValue.Builder.visitObject(-1, true, index).narrow + val slowCtx = BufferedValue.Builder.visitObject(-1, true, index).narrow val keyVisitor = slowCtx.visitKey(index) val xxx = keyVisitor.visitString(s.toString, index) slowCtx.visitKeyValue(xxx) @@ -286,11 +363,11 @@ trait AttributeTagged extends Api with Annotator{ if (context == null) throw new Abort(missingKeyMsg) else if (fastPath) context.visitEnd(index).asInstanceOf[T] else{ - val x = context.visitEnd(index).asInstanceOf[IndexedValue.Obj] - val keyAttr = x.value0.find(_._1.toString == tagName) + val x = context.visitEnd(index).asInstanceOf[BufferedValue.Obj] + val keyAttr = x.value0.find(t => isTagName(t._1)) .getOrElse(throw new Abort(missingKeyMsg)) ._2 - val key = keyAttr.asInstanceOf[IndexedValue.Str].value0.toString + val key = keyAttr.asInstanceOf[BufferedValue.Str].value0.toString val delegate = taggedReader.findReader(key) if (delegate == null){ throw new AbortException("invalid tag for tagged object: " + key, keyAttr.index, -1, -1, null) @@ -298,12 +375,12 @@ trait AttributeTagged extends Api with Annotator{ val ctx2 = delegate.visitObject(-1, true, -1) for (p <- x.value0) { val (k0, v) = p - val k = k0.toString - if (k != tagName){ + val k = k0 + if (!isTagName(k)){ val keyVisitor = ctx2.visitKey(-1) - ctx2.visitKeyValue(keyVisitor.visitString(k, -1)) - ctx2.visitValue(IndexedValue.transform(v, ctx2.subVisitor), -1) + ctx2.visitKeyValue(BufferedValue.transform(k, keyVisitor)) + ctx2.visitValue(BufferedValue.transform(v, ctx2.subVisitor), -1) } } ctx2.visitEnd(index) diff --git a/upickle/test/src/upickle/StructTests.scala b/upickle/test/src/upickle/StructTests.scala index 24e7e4d26..645644feb 100644 --- a/upickle/test/src/upickle/StructTests.scala +++ b/upickle/test/src/upickle/StructTests.scala @@ -609,6 +609,36 @@ object StructTests extends TestSuite { test("false") - rw(ujson.Bool(false), """false""") test("null") - rw(ujson.Null, """null""") } + test("sortKeys") { + val raw = """{"d": [{"c": 0, "b": 1}], "a": []}""" + val sorted = + """{ + | "a": [], + | "d": [ + | { + | "b": 1, + | "c": 0 + | } + | ] + |}""".stripMargin + val struct = upickle.default.read[Map[String, Seq[Map[String, Int]]]](raw) + + upickle.default.write(struct, indent = 4, sortKeys = true) ==> sorted + + val baos = new java.io.ByteArrayOutputStream + upickle.default.writeToOutputStream(struct, baos, indent = 4, sortKeys = true) + baos.toString ==> sorted + + val writer = new java.io.StringWriter + upickle.default.writeTo(struct, writer, indent = 4, sortKeys = true) + writer.toString ==> sorted + + new String(upickle.default.writeToByteArray(struct, indent = 4, sortKeys = true)) ==> sorted + + val baos2 = new java.io.ByteArrayOutputStream + upickle.default.stream(struct, indent = 4, sortKeys = true).writeBytesTo(baos2) + baos2.toString() ==> sorted + } } } diff --git a/upickle/test/src/upickle/TestUtil.scala b/upickle/test/src/upickle/TestUtil.scala index 359d50fb1..60f6e80a5 100644 --- a/upickle/test/src/upickle/TestUtil.scala +++ b/upickle/test/src/upickle/TestUtil.scala @@ -63,9 +63,17 @@ class TestUtil[Api <: upickle.Api](val api: Api){ checkBinaryJson: Boolean = true) (implicit r: api.Reader[T], w: api.Writer[T])= { val writtenT = api.write(t) + val writtenTSorted = api.write(t, sortKeys = true) val writtenJsT = api.writeJs(t) val writtenTMsg = api.writeMsg(t) val writtenBytesT = api.writeToByteArray(t) + val writtenBytesTMsg = api.writeBinaryToByteArray(t) + val writtenBytesTMsgSorted = api.writeBinaryToByteArray(t, sortKeys = true) + val writeToDest = new java.io.StringWriter + val writeBinaryToDest = new java.io.ByteArrayOutputStream + + api.writeTo(t, writeToDest) + api.writeBinaryTo(t, writeBinaryToDest) // Test JSON round tripping val strings = values.collect{case s: TestValue.Json => s.value.trim} @@ -85,8 +93,10 @@ class TestUtil[Api <: upickle.Api](val api: Api){ utest.assert(normalizedReadString == normalizedValue) } - val normalizedReadWrittenT = normalize(api.read[T](writtenT)) + val normalizedReadWrittenToT = normalize(api.read[T](writeToDest.toString)) + val normalizedReadWrittenTSorted = normalize(api.read[T](writtenTSorted)) + val normalizedReadWrittenJsT = normalize(api.read[T](writtenJsT)) val normalizedReadByteArrayWrittenT = normalize( api.read[T](writtenT.getBytes(StandardCharsets.UTF_8)) @@ -119,8 +129,12 @@ class TestUtil[Api <: upickle.Api](val api: Api){ utest.assert(ujson.reformat(strings.head) == writtenT) utest.assert(ujson.reformat(strings.head) == writtenJsT.render()) } - if (msgs.nonEmpty) utest.assert(msgs.head == writtenTMsg) + if (msgs.nonEmpty) { + utest.assert(msgs.head == writtenTMsg) + } utest.assert(normalizedReadWrittenT == normalizedT) + utest.assert(normalizedReadWrittenToT == normalizedT) + utest.assert(normalizedReadWrittenT == normalizedReadWrittenTSorted) utest.assert(normalizedReadWrittenT == normalizedReadWrittenJsT) utest.assert(normalizedReadByteArrayWrittenT == normalizedT) utest.assert(normalizedReadStreamWrittenT == normalizedT) @@ -133,20 +147,29 @@ class TestUtil[Api <: upickle.Api](val api: Api){ // Test MessagePack round tripping val writtenBinary = api.writeBinary(t) // println(upickle.core.Util.bytesToString(writtenBinary)) - val roundTrippedBinary = api.readBinary[T](writtenBinary) - (roundTrippedBinary, t) match{ - case (lhs: Array[_], rhs: Array[_]) => assert(lhs.toSeq == rhs.toSeq) - case _ => utest.assert(roundTrippedBinary == t) + val cases = Seq( + writtenBinary -> false, + writtenBytesTMsg -> false, + writtenBytesTMsgSorted -> true, + writeBinaryToDest.toByteArray -> false + ) + for((b, isSorted) <- cases){ + val roundTrippedBinary = api.readBinary[T](b) + (roundTrippedBinary, t) match{ + case (lhs: Array[_], rhs: Array[_]) => assert(lhs.toSeq == rhs.toSeq) + case _ => utest.assert(roundTrippedBinary == t) + } + + // Test binary-JSON equivalence + if (!isSorted && checkBinaryJson){ + val rewrittenBinary = api.writeBinary(roundTrippedBinary) + + val writtenBinaryStr = upickle.core.ParseUtils.bytesToString(writtenBinary) + val rewrittenBinaryStr = upickle.core.ParseUtils.bytesToString(rewrittenBinary) + utest.assert(writtenBinaryStr == rewrittenBinaryStr) + } } - // Test binary-JSON equivalence - if (checkBinaryJson){ - val rewrittenBinary = api.writeBinary(roundTrippedBinary) - - val writtenBinaryStr = upickle.core.ParseUtils.bytesToString(writtenBinary) - val rewrittenBinaryStr = upickle.core.ParseUtils.bytesToString(rewrittenBinary) - utest.assert(writtenBinaryStr == rewrittenBinaryStr) - } } def rwNum[T: Numeric: api.Reader: api.Writer](t: T, values: TestValue*) = { diff --git a/upickle/test/src/upickle/example/ExampleTests.scala b/upickle/test/src/upickle/example/ExampleTests.scala index ce2157157..0ed02796d 100644 --- a/upickle/test/src/upickle/example/ExampleTests.scala +++ b/upickle/test/src/upickle/example/ExampleTests.scala @@ -218,6 +218,18 @@ object ExampleTests extends TestSuite { | "myFieldB": "" | } |}""".stripMargin + + write(Big(1, true, "lol", 'Z', Thing(7, "")), indent = 4, sortKeys = true) ==> + """{ + | "b": true, + | "c": "Z", + | "i": 1, + | "str": "lol", + | "t": { + | "myFieldA": 7, + | "myFieldB": "" + | } + |}""".stripMargin }