From c294e7d4d7cac68371deb009012033d8d9ecc508 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sun, 29 May 2022 20:52:11 -0600 Subject: [PATCH 1/7] fix compression and tests therefore --- requests/src/requests/Requester.scala | 76 +++---- .../src-2/requests/Scala2RequestTests.scala | 24 ++- requests/test/src/requests/RequestTests.scala | 199 ++++++++++++++---- 3 files changed, 204 insertions(+), 95 deletions(-) diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index 54f345a..0283cf6 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -6,10 +6,10 @@ import java.util.zip.{GZIPInputStream, InflaterInputStream} import javax.net.ssl._ import collection.JavaConverters._ import scala.collection.mutable +import java.net.URLConnection trait BaseSession{ def headers: Map[String, String] - def cookies: mutable.Map[String, HttpCookie] def readTimeout: Int def connectTimeout: Int @@ -54,7 +54,7 @@ object Requester{ } case class Requester(verb: String, sess: BaseSession){ - + val CMD = verb.toUpperCase // allow submitting as lower case ... /** * Makes a single HTTP request, and returns a [[Response]] object. Requires @@ -102,17 +102,13 @@ case class Requester(verb: String, check: Boolean = sess.check, chunkedUpload: Boolean = sess.chunkedUpload): Response = { val out = new ByteArrayOutputStream() - var streamHeaders: StreamHeaders = null val w = stream( url, auth, params, data.headers, headers, data, readTimeout, connectTimeout, proxy, cert, sslContext, cookies, cookieValues, maxRedirects, verifySslCerts, autoDecompress, compress, keepAlive, check, chunkedUpload, - onHeadersReceived = sh => streamHeaders = sh - ) - + onHeadersReceived = sh => streamHeaders = sh) w.writeBytesTo(out) - Response( streamHeaders.url, streamHeaders.statusCode, @@ -176,9 +172,9 @@ case class Requester(verb: String, var connection: HttpURLConnection = null try { - - val conn = - if (proxy == null) url1.openConnection + val conn: URLConnection = + if (proxy == null) + url1.openConnection else { val (ip, port) = proxy val p = new java.net.Proxy( @@ -187,7 +183,7 @@ case class Requester(verb: String, url1.openConnection(p) } - connection = conn match{ + connection = conn match { case c: HttpsURLConnection => if (cert != null) { c.setSSLSocketFactory(Util.clientCertSocketFactory(cert, verifySslCerts)) @@ -204,9 +200,8 @@ case class Requester(verb: String, } connection.setInstanceFollowRedirects(false) - val upperCaseVerb = verb.toUpperCase - if (Requester.officialHttpMethods.contains(upperCaseVerb)) { - connection.setRequestMethod(upperCaseVerb) + if (Requester.officialHttpMethods.contains(CMD)) { + connection.setRequestMethod(CMD) } else { // HttpURLConnection enforces a list of official http METHODs, but not everyone abides by the spec // this hack allows us set an unofficial http method @@ -214,33 +209,28 @@ case class Requester(verb: String, case cs: HttpsURLConnection => cs.getClass.getDeclaredFields.find(_.getName == "delegate").foreach{ del => del.setAccessible(true) - Requester.methodField.set(del.get(cs), upperCaseVerb) + Requester.methodField.set(del.get(cs), CMD) } case c => - Requester.methodField.set(c, upperCaseVerb) - } + Requester.methodField.set(c, CMD) + } } - for((k, v) <- blobHeaders) connection.setRequestProperty(k, v) - - for((k, v) <- sess.headers) connection.setRequestProperty(k, v) - - for((k, v) <- headers) connection.setRequestProperty(k, v) - - for((k, v) <- compress.headers) connection.setRequestProperty(k, v) + Seq(blobHeaders, sess.headers, headers, compress.headers). + foreach(h => for((k, v) <- h) connection.setRequestProperty(k, v)) connection.setReadTimeout(readTimeout) auth.header.foreach(connection.setRequestProperty("Authorization", _)) connection.setConnectTimeout(connectTimeout) connection.setUseCaches(false) - connection.setDoOutput(true) + connection.setDoOutput(true) val sessionCookieValues = for{ c <- (sess.cookies ++ cookies).valuesIterator if !c.hasExpired if c.getDomain == null || c.getDomain == url1.getHost if c.getPath == null || url1.getPath.startsWith(c.getPath) - } yield (c.getName, c.getValue) + } yield (c.getName, c.getValue) val allCookies = sessionCookieValues ++ cookieValues if (allCookies.nonEmpty){ @@ -250,19 +240,20 @@ case class Requester(verb: String, .map{case (k, v) => s"""$k="$v""""} .mkString("; ") ) - } - if (verb.toUpperCase == "POST" || verb.toUpperCase == "PUT" || verb.toUpperCase == "PATCH" || verb.toUpperCase == "DELETE") { + } + + if (CMD == "POST" || CMD == "PUT" || CMD == "PATCH" || CMD == "DELETE") { if (!chunkedUpload) { - val bytes = new ByteArrayOutputStream() - data.write(compress.wrap(bytes)) - val byteArray = bytes.toByteArray - connection.setFixedLengthStreamingMode(byteArray.length) - if (byteArray.nonEmpty) connection.getOutputStream.write(byteArray) - } else { - connection.setChunkedStreamingMode(0) - data.write(compress.wrap(connection.getOutputStream)) + val bytes = new ByteArrayOutputStream() + withOs(compress.wrap(bytes)) { os => data.write(os) } + val byteArray = bytes.toByteArray + connection.setFixedLengthStreamingMode(byteArray.length) + withOs(connection.getOutputStream) { os => os.write(byteArray) } + } else { + connection.setChunkedStreamingMode(0) + withOs(compress.wrap(connection.getOutputStream)) { os => data.write(os) } + } } - } val (responseCode, responseMsg, headerFields) = try {( connection.getResponseCode, @@ -333,7 +324,7 @@ case class Requester(verb: String, // The HEAD method is identical to GET except that the server // MUST NOT return a message-body in the response. // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 - if (verb == "HEAD") f(new ByteArrayInputStream(Array())) + if (CMD == "HEAD") f(new ByteArrayInputStream(Array())) else if (stream != null) { try f( if (deGzip) new GZIPInputStream(stream) @@ -367,6 +358,15 @@ case class Requester(verb: String, } } + /** + * Do something with an OutputStream and close it + * @param os OutputStream + * @param fn + */ + private def withOs[T](os: OutputStream)(fn: OutputStream => T) : Unit = + try fn(os) finally os.close() + + /** * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration */ diff --git a/requests/test/src-2/requests/Scala2RequestTests.scala b/requests/test/src-2/requests/Scala2RequestTests.scala index 63d73de..08066ac 100644 --- a/requests/test/src-2/requests/Scala2RequestTests.scala +++ b/requests/test/src-2/requests/Scala2RequestTests.scala @@ -14,31 +14,33 @@ object Scala2RequestTests extends TestSuite{ "https://httpbin.org/post", data = Map("hello" -> "world", "foo" -> "baz"), chunkedUpload = chunkedUpload - ).text() + ).text assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } + test("put") { for (chunkedUpload <- Seq(true, false)) { val res1 = requests.put( "https://httpbin.org/put", data = Map("hello" -> "world", "foo" -> "baz"), chunkedUpload = chunkedUpload - ).text() + ).text assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } - test("send"){ - requests.send("get")("https://httpbin.org/get?hello=world&foo=baz") - val res1 = requests.send("put")( - "https://httpbin.org/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = true - ).text + test("send"){ + requests.send("get")("https://httpbin.org/get?hello=world&foo=baz") - assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) - } + val res1 = requests.send("put")( + "https://httpbin.org/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = true + ).text + + assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) + } } } } diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index 98b94b4..03a8f0c 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -1,10 +1,21 @@ package requests +import java.net.InetSocketAddress + +import com.sun.net.httpserver.HttpExchange +import com.sun.net.httpserver.HttpHandler +import com.sun.net.httpserver.HttpServer +import java.io._ +import java.util.zip._ +import scala.collection.mutable.StringBuilder import utest._ import ujson._ +import requests.Compress +import requests.Compress._ +import scala.annotation.tailrec object RequestTests extends TestSuite{ - val tests = Tests{ + val tests = Tests{ test("matchingMethodWorks"){ val requesters = Seq( requests.delete, @@ -29,7 +40,8 @@ object RequestTests extends TestSuite{ } } } - } + } + test("params"){ test("get"){ // All in URL @@ -58,6 +70,7 @@ object RequestTests extends TestSuite{ assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world")) } } + test("multipart"){ for(chunkedUpload <- Seq(true, false)) { val response = requests.post( @@ -73,6 +86,7 @@ object RequestTests extends TestSuite{ assert(read(response).obj("form") == Obj("file2" -> "Goodbye!")) } } + test("cookies"){ test("session"){ @@ -99,35 +113,38 @@ object RequestTests extends TestSuite{ assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world"))) } } - // Tests fail with 'Request to https://httpbin.org/absolute-redirect/4 failed with status code 404' - // test("redirects"){ - // test("max"){ - // val res1 = requests.get("https://httpbin.org/absolute-redirect/4") - // assert(res1.statusCode == 200) - // val res2 = requests.get("https://httpbin.org/absolute-redirect/5") - // assert(res2.statusCode == 200) - // val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false) - // assert(res3.statusCode == 302) - // val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10) - // assert(res4.statusCode == 200) - // } - // test("maxRelative"){ - // val res1 = requests.get("https://httpbin.org/relative-redirect/4") - // assert(res1.statusCode == 200) - // val res2 = requests.get("https://httpbin.org/relative-redirect/5") - // assert(res2.statusCode == 200) - // val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false) - // assert(res3.statusCode == 302) - // val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10) - // assert(res4.statusCode == 200) - // } - // } + + //Tests fail with 'Request to https://httpbin.org/absolute-redirect/4 failed with status code 404' + test("redirects"){ + test("max"){ + val res1 = requests.get("https://httpbin.org/absolute-redirect/4") + assert(res1.statusCode == 200) + val res2 = requests.get("https://httpbin.org/absolute-redirect/5") + assert(res2.statusCode == 200) + val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false) + assert(res3.statusCode == 302) + val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10) + assert(res4.statusCode == 200) + } + test("maxRelative"){ + val res1 = requests.get("https://httpbin.org/relative-redirect/4") + assert(res1.statusCode == 200) + val res2 = requests.get("https://httpbin.org/relative-redirect/5") + assert(res2.statusCode == 200) + val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false) + assert(res3.statusCode == 302) + val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10) + assert(res4.statusCode == 200) + } + } + test("streaming"){ val res1 = requests.get("http://httpbin.org/stream/5").text() assert(res1.linesIterator.length == 5) val res2 = requests.get("http://httpbin.org/stream/52").text() assert(res2.linesIterator.length == 52) } + test("timeouts"){ test("read"){ intercept[TimeoutException] { @@ -144,6 +161,7 @@ object RequestTests extends TestSuite{ } } } + test("failures"){ intercept[UnknownHostException]{ requests.get("https://doesnt-exist-at-all.com/") @@ -156,6 +174,7 @@ object RequestTests extends TestSuite{ requests.get("://doesnt-exist.com/") } } + test("decompress"){ val res1 = requests.get("https://httpbin.org/gzip") assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org") @@ -171,29 +190,33 @@ object RequestTests extends TestSuite{ (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length) } - test("compression"){ - val res1 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.None, - data = new RequestBlob.ByteSourceRequestBlob("Hello World") - ) - assert(res1.text().contains(""""Hello World"""")) - val res2 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.Gzip, - data = new RequestBlob.ByteSourceRequestBlob("I am cow") - ) - assert(res2.text().contains("data:application/octet-stream;base64,H4sIAAAAAAAAAA==")) + test("compression"){ + val res1 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.None, + data = new RequestBlob.ByteSourceRequestBlob("Hello World") + ) + assert(res1.text.contains(""""Hello World"""")) + + val res2 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.Gzip, + data = new RequestBlob.ByteSourceRequestBlob("I am cow") + ) + assert(read(new String(res2.bytes))("data").toString == + """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") + + val res3 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.Deflate, + data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") + ) + assert(read(new String(res2.bytes))("data").toString == + """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") + + } - val res3 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.Deflate, - data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") - ) - assert(res3.text().contains("data:application/octet-stream;base64,eJw=")) - res3.text() - } test("headers"){ test("default"){ val res = requests.get("https://httpbin.org/headers").text() @@ -207,6 +230,7 @@ object RequestTests extends TestSuite{ } } } + test("clientCertificate"){ val base = "./requests/test/resources" val url = "https://client.badssl.com" @@ -253,6 +277,7 @@ object RequestTests extends TestSuite{ assert(res.statusCode == 400) } } + test("selfSignedCertificate"){ val res = requests.get( "https://self-signed.badssl.com", @@ -260,6 +285,7 @@ object RequestTests extends TestSuite{ ) assert(res.statusCode == 200) } + test("gzipError"){ val response = requests.head("https://api.github.com/users/lihaoyi") assert(response.statusCode == 200) @@ -268,5 +294,86 @@ object RequestTests extends TestSuite{ assert(response.headers.keySet.map(_.toLowerCase).contains("content-length")) assert(response.headers.keySet.map(_.toLowerCase).contains("content-type")) } + + /** + * Compress with each compression mode and call server. Server expands + * and passes it back so we can compare + */ + test("compressionData") { + import Compress._ + val str = "I am deflater mouse" + Seq(None, Gzip, Deflate).foreach(c => + Server.use { + assert(str == requests.post( + "http://localhost:58080/test", + compress = c, + data = str + ).data.toString) + }) + } } } + +/** A server we start for above test or two. Assumes port 58080 is available */ +class Server extends HttpHandler { + val server: HttpServer = HttpServer.create(new InetSocketAddress(58080), 0) + server.createContext("/test", this) + server.setExecutor(null); // default executor + server.start(); + + def stop(): Unit = server.stop(0) + + override def handle(t: HttpExchange) { + val h: java.util.List[String] = t.getRequestHeaders.get("Content-encoding") + val c: Compress = if (h == null) None + else if (h.contains("gzip")) Gzip + else if (h.contains("deflate")) Deflate + else None + val msg = new Plumper(c).decompress(t.getRequestBody) + t.sendResponseHeaders(200, msg.length) + t.getResponseBody.write(msg.getBytes()) + } +} + +object Server { + def use(doIt : => Unit) : Unit = { + val server = new Server + try doIt + finally server.stop() + } +} + +/** + * Stream uncompresser + * @param c Compression mode + */ +class Plumper(c: Compress) { + + private def wrap(is: InputStream) : InputStream = + c match { + case None => is + case Gzip => new GZIPInputStream(is) + case Deflate => new InflaterInputStream(is) + } + + def decompress(compressed: InputStream): String = { + val gis = wrap(compressed) + val br = new BufferedReader(new InputStreamReader(gis, "UTF-8")) + val sb = new StringBuilder() + + @tailrec + def read(): Unit = { + val line = br.readLine + if (line != null) { + sb.append(line) + read + } + } + + read() + br.close() + gis.close() + compressed.close() + sb.toString() + } +} \ No newline at end of file From c457ee31fa1ce74c15f38ac859287edafba7a421 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sun, 5 Jun 2022 13:11:25 -0600 Subject: [PATCH 2/7] Comply with recommended changes --- requests/src/requests/Requester.scala | 55 +++++---- requests/test/src/requests/RequestTests.scala | 111 ++++++++---------- 2 files changed, 79 insertions(+), 87 deletions(-) diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index 0283cf6..86737dd 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -53,8 +53,8 @@ object Requester{ } } case class Requester(verb: String, - sess: BaseSession){ - val CMD = verb.toUpperCase // allow submitting as lower case ... + sess: BaseSession) { + val cmd = verb.toUpperCase // allow submitting as lower case ... /** * Makes a single HTTP request, and returns a [[Response]] object. Requires @@ -200,8 +200,8 @@ case class Requester(verb: String, } connection.setInstanceFollowRedirects(false) - if (Requester.officialHttpMethods.contains(CMD)) { - connection.setRequestMethod(CMD) + if (Requester.officialHttpMethods.contains(cmd)) { + connection.setRequestMethod(cmd) } else { // HttpURLConnection enforces a list of official http METHODs, but not everyone abides by the spec // this hack allows us set an unofficial http method @@ -209,15 +209,20 @@ case class Requester(verb: String, case cs: HttpsURLConnection => cs.getClass.getDeclaredFields.find(_.getName == "delegate").foreach{ del => del.setAccessible(true) - Requester.methodField.set(del.get(cs), CMD) + Requester.methodField.set(del.get(cs), cmd) } case c => - Requester.methodField.set(c, CMD) + Requester.methodField.set(c, cmd) } } - Seq(blobHeaders, sess.headers, headers, compress.headers). - foreach(h => for((k, v) <- h) connection.setRequestProperty(k, v)) + for((k, v) <- blobHeaders) connection.setRequestProperty(k, v) + + for((k, v) <- sess.headers) connection.setRequestProperty(k, v) + + for((k, v) <- headers) connection.setRequestProperty(k, v) + + for((k, v) <- compress.headers) connection.setRequestProperty(k, v) connection.setReadTimeout(readTimeout) auth.header.foreach(connection.setRequestProperty("Authorization", _)) @@ -242,18 +247,18 @@ case class Requester(verb: String, ) } - if (CMD == "POST" || CMD == "PUT" || CMD == "PATCH" || CMD == "DELETE") { + if (cmd == "POST" || cmd == "PUT" || cmd == "PATCH" || cmd == "DELETE") { if (!chunkedUpload) { - val bytes = new ByteArrayOutputStream() - withOs(compress.wrap(bytes)) { os => data.write(os) } - val byteArray = bytes.toByteArray - connection.setFixedLengthStreamingMode(byteArray.length) - withOs(connection.getOutputStream) { os => os.write(byteArray) } - } else { - connection.setChunkedStreamingMode(0) - withOs(compress.wrap(connection.getOutputStream)) { os => data.write(os) } - } + val bytes = new ByteArrayOutputStream() + usingOutputStream(compress.wrap(bytes)) { os => data.write(os) } + val byteArray = bytes.toByteArray + connection.setFixedLengthStreamingMode(byteArray.length) + usingOutputStream(connection.getOutputStream) { os => os.write(byteArray) } + } else { + connection.setChunkedStreamingMode(0) + usingOutputStream(compress.wrap(connection.getOutputStream)) { os => data.write(os) } } + } val (responseCode, responseMsg, headerFields) = try {( connection.getResponseCode, @@ -305,7 +310,7 @@ case class Requester(verb: String, compress, keepAlive, check, chunkedUpload, Some(current), onHeadersReceived ).readBytesThrough(f) - }else{ + } else{ persistCookies() val streamHeaders = StreamHeaders( url, @@ -324,7 +329,7 @@ case class Requester(verb: String, // The HEAD method is identical to GET except that the server // MUST NOT return a message-body in the response. // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 - if (CMD == "HEAD") f(new ByteArrayInputStream(Array())) + if (cmd == "HEAD") f(new ByteArrayInputStream(Array())) else if (stream != null) { try f( if (deGzip) new GZIPInputStream(stream) @@ -358,15 +363,9 @@ case class Requester(verb: String, } } - /** - * Do something with an OutputStream and close it - * @param os OutputStream - * @param fn - */ - private def withOs[T](os: OutputStream)(fn: OutputStream => T) : Unit = - try fn(os) finally os.close() + private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T) : Unit = + try fn(os) finally os.close() - /** * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration */ diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index 03a8f0c..2e80039 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -1,21 +1,17 @@ package requests -import java.net.InetSocketAddress - -import com.sun.net.httpserver.HttpExchange -import com.sun.net.httpserver.HttpHandler -import com.sun.net.httpserver.HttpServer import java.io._ -import java.util.zip._ +import java.net.InetSocketAddress +import java.util.zip.{GZIPInputStream, InflaterInputStream} +import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer} +import requests.Compress._ +import scala.annotation.tailrec import scala.collection.mutable.StringBuilder import utest._ import ujson._ -import requests.Compress -import requests.Compress._ -import scala.annotation.tailrec object RequestTests extends TestSuite{ - val tests = Tests{ + val tests = Tests{ test("matchingMethodWorks"){ val requesters = Seq( requests.delete, @@ -40,7 +36,7 @@ object RequestTests extends TestSuite{ } } } - } + } test("params"){ test("get"){ @@ -88,7 +84,6 @@ object RequestTests extends TestSuite{ } test("cookies"){ - test("session"){ val s = requests.Session(cookieValues = Map("hello" -> "world")) val res1 = s.get("https://httpbin.org/cookies").text().trim @@ -114,7 +109,6 @@ object RequestTests extends TestSuite{ } } - //Tests fail with 'Request to https://httpbin.org/absolute-redirect/4 failed with status code 404' test("redirects"){ test("max"){ val res1 = requests.get("https://httpbin.org/absolute-redirect/4") @@ -191,30 +185,29 @@ object RequestTests extends TestSuite{ (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length) } - test("compression"){ - val res1 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.None, - data = new RequestBlob.ByteSourceRequestBlob("Hello World") - ) - assert(res1.text.contains(""""Hello World"""")) - - val res2 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.Gzip, - data = new RequestBlob.ByteSourceRequestBlob("I am cow") - ) - assert(read(new String(res2.bytes))("data").toString == - """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") - - val res3 = requests.post( - "https://httpbin.org/post", - compress = requests.Compress.Deflate, - data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") - ) - assert(read(new String(res2.bytes))("data").toString == - """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") + test("compression"){ + val res1 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.None, + data = new RequestBlob.ByteSourceRequestBlob("Hello World") + ) + assert(res1.text().contains(""""Hello World"""")) + val res2 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.Gzip, + data = new RequestBlob.ByteSourceRequestBlob("I am cow") + ) + assert(read(new String(res2.bytes))("data").toString == + """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") + + val res3 = requests.post( + "https://httpbin.org/post", + compress = requests.Compress.Deflate, + data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") + ) + assert(read(new String(res2.bytes))("data").toString == + """"data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""") } test("headers"){ @@ -299,18 +292,18 @@ object RequestTests extends TestSuite{ * Compress with each compression mode and call server. Server expands * and passes it back so we can compare */ - test("compressionData") { - import Compress._ - val str = "I am deflater mouse" - Seq(None, Gzip, Deflate).foreach(c => - Server.use { + test("compressionData") { + val str = "I am deflater mouse" + Seq(None, Gzip, Deflate).foreach(c => + Server.use { assert(str == requests.post( - "http://localhost:58080/test", - compress = c, - data = str - ).data.toString) - }) - } + "http://localhost:58080/test", + compress = c, + data = new RequestBlob.ByteSourceRequestBlob(str) + ).data.toString) + } + ) + } } } @@ -323,7 +316,7 @@ class Server extends HttpHandler { def stop(): Unit = server.stop(0) - override def handle(t: HttpExchange) { + override def handle(t: HttpExchange): Unit = { val h: java.util.List[String] = t.getRequestHeaders.get("Content-encoding") val c: Compress = if (h == null) None else if (h.contains("gzip")) Gzip @@ -336,11 +329,11 @@ class Server extends HttpHandler { } object Server { - def use(doIt : => Unit) : Unit = { - val server = new Server - try doIt - finally server.stop() - } + def use(doIt : => Unit) : Unit = { + val server = new Server + try doIt + finally server.stop() + } } /** @@ -363,12 +356,12 @@ class Plumper(c: Compress) { @tailrec def read(): Unit = { - val line = br.readLine - if (line != null) { - sb.append(line) - read - } - } + val line = br.readLine + if (line != null) { + sb.append(line) + read() + } + } read() br.close() From 4fb52799cb14017e10eaef18e6bc8a67d3077290 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Wed, 8 Jun 2022 11:25:30 +0200 Subject: [PATCH 3/7] Revert some formatting changes and rename variable --- requests/src/requests/Requester.scala | 50 ++++++++++--------- .../src-2/requests/Scala2RequestTests.scala | 22 ++++---- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index 86737dd..f36b767 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -53,8 +53,8 @@ object Requester{ } } case class Requester(verb: String, - sess: BaseSession) { - val cmd = verb.toUpperCase // allow submitting as lower case ... + sess: BaseSession){ + private val upperCaseVerb = verb.toUpperCase /** * Makes a single HTTP request, and returns a [[Response]] object. Requires @@ -102,13 +102,17 @@ case class Requester(verb: String, check: Boolean = sess.check, chunkedUpload: Boolean = sess.chunkedUpload): Response = { val out = new ByteArrayOutputStream() + var streamHeaders: StreamHeaders = null val w = stream( url, auth, params, data.headers, headers, data, readTimeout, connectTimeout, proxy, cert, sslContext, cookies, cookieValues, maxRedirects, verifySslCerts, autoDecompress, compress, keepAlive, check, chunkedUpload, - onHeadersReceived = sh => streamHeaders = sh) + onHeadersReceived = sh => streamHeaders = sh + ) + w.writeBytesTo(out) + Response( streamHeaders.url, streamHeaders.statusCode, @@ -172,9 +176,9 @@ case class Requester(verb: String, var connection: HttpURLConnection = null try { - val conn: URLConnection = - if (proxy == null) - url1.openConnection + + val conn = + if (proxy == null) url1.openConnection else { val (ip, port) = proxy val p = new java.net.Proxy( @@ -183,7 +187,7 @@ case class Requester(verb: String, url1.openConnection(p) } - connection = conn match { + connection = conn match{ case c: HttpsURLConnection => if (cert != null) { c.setSSLSocketFactory(Util.clientCertSocketFactory(cert, verifySslCerts)) @@ -200,8 +204,8 @@ case class Requester(verb: String, } connection.setInstanceFollowRedirects(false) - if (Requester.officialHttpMethods.contains(cmd)) { - connection.setRequestMethod(cmd) + if (Requester.officialHttpMethods.contains(upperCaseVerb)) { + connection.setRequestMethod(upperCaseVerb) } else { // HttpURLConnection enforces a list of official http METHODs, but not everyone abides by the spec // this hack allows us set an unofficial http method @@ -209,33 +213,33 @@ case class Requester(verb: String, case cs: HttpsURLConnection => cs.getClass.getDeclaredFields.find(_.getName == "delegate").foreach{ del => del.setAccessible(true) - Requester.methodField.set(del.get(cs), cmd) + Requester.methodField.set(del.get(cs), upperCaseVerb) } case c => - Requester.methodField.set(c, cmd) - } + Requester.methodField.set(c, upperCaseVerb) + } } for((k, v) <- blobHeaders) connection.setRequestProperty(k, v) - + for((k, v) <- sess.headers) connection.setRequestProperty(k, v) - + for((k, v) <- headers) connection.setRequestProperty(k, v) - + for((k, v) <- compress.headers) connection.setRequestProperty(k, v) connection.setReadTimeout(readTimeout) auth.header.foreach(connection.setRequestProperty("Authorization", _)) connection.setConnectTimeout(connectTimeout) connection.setUseCaches(false) - connection.setDoOutput(true) + connection.setDoOutput(true) val sessionCookieValues = for{ c <- (sess.cookies ++ cookies).valuesIterator if !c.hasExpired if c.getDomain == null || c.getDomain == url1.getHost if c.getPath == null || url1.getPath.startsWith(c.getPath) - } yield (c.getName, c.getValue) + } yield (c.getName, c.getValue) val allCookies = sessionCookieValues ++ cookieValues if (allCookies.nonEmpty){ @@ -247,7 +251,7 @@ case class Requester(verb: String, ) } - if (cmd == "POST" || cmd == "PUT" || cmd == "PATCH" || cmd == "DELETE") { + if (upperCaseVerb == "POST" || upperCaseVerb == "PUT" || upperCaseVerb == "PATCH" || upperCaseVerb == "DELETE") { if (!chunkedUpload) { val bytes = new ByteArrayOutputStream() usingOutputStream(compress.wrap(bytes)) { os => data.write(os) } @@ -310,7 +314,7 @@ case class Requester(verb: String, compress, keepAlive, check, chunkedUpload, Some(current), onHeadersReceived ).readBytesThrough(f) - } else{ + }else{ persistCookies() val streamHeaders = StreamHeaders( url, @@ -329,7 +333,7 @@ case class Requester(verb: String, // The HEAD method is identical to GET except that the server // MUST NOT return a message-body in the response. // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 - if (cmd == "HEAD") f(new ByteArrayInputStream(Array())) + if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array())) else if (stream != null) { try f( if (deGzip) new GZIPInputStream(stream) @@ -362,10 +366,10 @@ case class Requester(verb: String, } } } + + private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit = + try fn(os) finally os.close() - private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T) : Unit = - try fn(os) finally os.close() - /** * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration */ diff --git a/requests/test/src-2/requests/Scala2RequestTests.scala b/requests/test/src-2/requests/Scala2RequestTests.scala index 08066ac..9a9d787 100644 --- a/requests/test/src-2/requests/Scala2RequestTests.scala +++ b/requests/test/src-2/requests/Scala2RequestTests.scala @@ -14,7 +14,7 @@ object Scala2RequestTests extends TestSuite{ "https://httpbin.org/post", data = Map("hello" -> "world", "foo" -> "baz"), chunkedUpload = chunkedUpload - ).text + ).text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } @@ -25,22 +25,22 @@ object Scala2RequestTests extends TestSuite{ "https://httpbin.org/put", data = Map("hello" -> "world", "foo" -> "baz"), chunkedUpload = chunkedUpload - ).text + ).text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } - test("send"){ - requests.send("get")("https://httpbin.org/get?hello=world&foo=baz") + test("send"){ + requests.send("get")("https://httpbin.org/get?hello=world&foo=baz") - val res1 = requests.send("put")( - "https://httpbin.org/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = true - ).text + val res1 = requests.send("put")( + "https://httpbin.org/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = true + ).text - assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) - } + assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) + } } } } From 4b4c65ecdf82ca0ba78ecb017b554a1c5fdc8e12 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Wed, 8 Jun 2022 12:00:30 +0200 Subject: [PATCH 4/7] Move echo server to a separate file --- requests/test/src/requests/RequestTests.scala | 76 +----------------- requests/test/src/requests/ServerUtils.scala | 77 +++++++++++++++++++ 2 files changed, 80 insertions(+), 73 deletions(-) create mode 100644 requests/test/src/requests/ServerUtils.scala diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index 2e80039..6f63f75 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -1,12 +1,5 @@ package requests -import java.io._ -import java.net.InetSocketAddress -import java.util.zip.{GZIPInputStream, InflaterInputStream} -import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer} -import requests.Compress._ -import scala.annotation.tailrec -import scala.collection.mutable.StringBuilder import utest._ import ujson._ @@ -293,11 +286,12 @@ object RequestTests extends TestSuite{ * and passes it back so we can compare */ test("compressionData") { + import requests.Compress._ val str = "I am deflater mouse" Seq(None, Gzip, Deflate).foreach(c => - Server.use { + ServerUtils.usingEchoServer { port => assert(str == requests.post( - "http://localhost:58080/test", + s"http://localhost:$port/echo", compress = c, data = new RequestBlob.ByteSourceRequestBlob(str) ).data.toString) @@ -306,67 +300,3 @@ object RequestTests extends TestSuite{ } } } - -/** A server we start for above test or two. Assumes port 58080 is available */ -class Server extends HttpHandler { - val server: HttpServer = HttpServer.create(new InetSocketAddress(58080), 0) - server.createContext("/test", this) - server.setExecutor(null); // default executor - server.start(); - - def stop(): Unit = server.stop(0) - - override def handle(t: HttpExchange): Unit = { - val h: java.util.List[String] = t.getRequestHeaders.get("Content-encoding") - val c: Compress = if (h == null) None - else if (h.contains("gzip")) Gzip - else if (h.contains("deflate")) Deflate - else None - val msg = new Plumper(c).decompress(t.getRequestBody) - t.sendResponseHeaders(200, msg.length) - t.getResponseBody.write(msg.getBytes()) - } -} - -object Server { - def use(doIt : => Unit) : Unit = { - val server = new Server - try doIt - finally server.stop() - } -} - -/** - * Stream uncompresser - * @param c Compression mode - */ -class Plumper(c: Compress) { - - private def wrap(is: InputStream) : InputStream = - c match { - case None => is - case Gzip => new GZIPInputStream(is) - case Deflate => new InflaterInputStream(is) - } - - def decompress(compressed: InputStream): String = { - val gis = wrap(compressed) - val br = new BufferedReader(new InputStreamReader(gis, "UTF-8")) - val sb = new StringBuilder() - - @tailrec - def read(): Unit = { - val line = br.readLine - if (line != null) { - sb.append(line) - read() - } - } - - read() - br.close() - gis.close() - compressed.close() - sb.toString() - } -} \ No newline at end of file diff --git a/requests/test/src/requests/ServerUtils.scala b/requests/test/src/requests/ServerUtils.scala new file mode 100644 index 0000000..5eb888b --- /dev/null +++ b/requests/test/src/requests/ServerUtils.scala @@ -0,0 +1,77 @@ +package requests + +import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer} +import java.io._ +import java.net.InetSocketAddress +import java.util.zip.{GZIPInputStream, InflaterInputStream} +import requests.Compress._ +import scala.annotation.tailrec +import scala.collection.mutable.StringBuilder + +object ServerUtils { + def usingEchoServer(f: Int => Unit): Unit = { + val server = new EchoServer + try f(server.getPort) + finally server.stop() + } + + private class EchoServer extends HttpHandler { + private val server: HttpServer = HttpServer.create(new InetSocketAddress(0), 0) + server.createContext("/echo", this) + server.setExecutor(null); // default executor + server.start() + + def getPort(): Int = server.getAddress.getPort + + def stop(): Unit = server.stop(0) + + override def handle(t: HttpExchange): Unit = { + val h: java.util.List[String] = + t.getRequestHeaders.get("Content-encoding") + val c: Compress = + if (h == null) None + else if (h.contains("gzip")) Gzip + else if (h.contains("deflate")) Deflate + else None + val msg = new Plumper(c).decompress(t.getRequestBody) + t.sendResponseHeaders(200, msg.length) + t.getResponseBody.write(msg.getBytes()) + } + } + + /** Stream uncompresser + * @param c + * Compression mode + */ + private class Plumper(c: Compress) { + + private def wrap(is: InputStream): InputStream = + c match { + case None => is + case Gzip => new GZIPInputStream(is) + case Deflate => new InflaterInputStream(is) + } + + def decompress(compressed: InputStream): String = { + val gis = wrap(compressed) + val br = new BufferedReader(new InputStreamReader(gis, "UTF-8")) + val sb = new StringBuilder() + + @tailrec + def read(): Unit = { + val line = br.readLine + if (line != null) { + sb.append(line) + read() + } + } + + read() + br.close() + gis.close() + compressed.close() + sb.toString() + } + } + +} From c8ce3273c268ed703aea6fef77f0215b55180ddc Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Wed, 8 Jun 2022 12:03:02 +0200 Subject: [PATCH 5/7] Remove unused import --- requests/src/requests/Requester.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index f36b767..0534768 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -6,7 +6,6 @@ import java.util.zip.{GZIPInputStream, InflaterInputStream} import javax.net.ssl._ import collection.JavaConverters._ import scala.collection.mutable -import java.net.URLConnection trait BaseSession{ def headers: Map[String, String] From ca21277de8c27e463f5e1145ba801a7a608eb8a7 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Wed, 8 Jun 2022 18:58:57 +0200 Subject: [PATCH 6/7] Fix Mima --- build.sc | 9 ++++++--- mill | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/build.sc b/build.sc index b084e44..96a8241 100644 --- a/build.sc +++ b/build.sc @@ -1,9 +1,9 @@ import mill._ import mill.scalalib.publish.{Developer, License, PomSettings, VersionControl} import scalalib._ -import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version_mill0.9:0.1.1` +import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.1.4` import de.tobiasroeser.mill.vcs.version.VcsVersion -import $ivy.`com.github.lolgab::mill-mima_mill0.9:0.0.4` +import $ivy.`com.github.lolgab::mill-mima::0.0.10` import com.github.lolgab.mill.mima._ val dottyVersion = Option(sys.props("dottyVersion")) @@ -11,7 +11,10 @@ val dottyVersion = Option(sys.props("dottyVersion")) object requests extends Cross[RequestsModule]((List("2.12.13", "2.13.5", "2.11.12", "3.0.0") ++ dottyVersion): _*) class RequestsModule(val crossScalaVersion: String) extends CrossScalaModule with PublishModule with Mima { def publishVersion = VcsVersion.vcsState().format() - def mimaPreviousVersions = VcsVersion.vcsState().lastTag.toSeq + def mimaPreviousVersions = Seq("0.7.0") ++ VcsVersion.vcsState().lastTag.toSeq + override def mimaBinaryIssueFilters = Seq( + ProblemFilter.exclude[ReversedMissingMethodProblem]("requests.BaseSession.send") + ) def artifactName = "requests" def pomSettings = PomSettings( description = "Scala port of the popular Python Requests HTTP client", diff --git a/mill b/mill index d66be9e..c379d48 100755 --- a/mill +++ b/mill @@ -3,7 +3,7 @@ # This is a wrapper script, that automatically download mill from GitHub release pages # You can give the required mill version with MILL_VERSION env variable # If no version is given, it falls back to the value of DEFAULT_MILL_VERSION -DEFAULT_MILL_VERSION=0.9.7 +DEFAULT_MILL_VERSION=0.9.12 set -e From 6884c30035c1e8f5b79243a48c3220aaff002e61 Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Wed, 8 Jun 2022 19:00:09 +0200 Subject: [PATCH 7/7] Fix compilation error in Scala 3 --- requests/test/src/requests/ServerUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/test/src/requests/ServerUtils.scala b/requests/test/src/requests/ServerUtils.scala index 5eb888b..d9ceebd 100644 --- a/requests/test/src/requests/ServerUtils.scala +++ b/requests/test/src/requests/ServerUtils.scala @@ -11,7 +11,7 @@ import scala.collection.mutable.StringBuilder object ServerUtils { def usingEchoServer(f: Int => Unit): Unit = { val server = new EchoServer - try f(server.getPort) + try f(server.getPort()) finally server.stop() }