8000 fix compression and tests therefore (#108) · com-lihaoyi/requests-scala@7e18f64 · GitHub
[go: up one dir, main page]
Skip to content

Commit 7e18f64

Browse files
tballardTom Ballardlolgab
authored
fix compression and tests therefore (#108)
* fix compression and tests therefore * Comply with recommended changes * Revert some formatting changes and rename variable * Move echo server to a separate file * Remove unused import * Fix Mima * Fix compilation error in Scala 3 Co-authored-by: Tom Ballard <tomballard@redangus.org> Co-authored-by: Lorenzo Gabriele <lorenzolespaul@gmail.com>
1 parent 11a9787 commit 7e18f64

File tree

6 files changed

+155
-41
lines changed

6 files changed

+155
-41
lines changed

‎build.sc

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
import mill._
22
import mill.scalalib.publish.{Developer, License, PomSettings, VersionControl}
33
import scalalib._
4-
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version_mill0.9:0.1.1`
4+
import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.1.4`
55
import de.tobiasroeser.mill.vcs.version.VcsVersion
6-
import $ivy.`com.github.lolgab::mill-mima_mill0.9:0.0.4`
6+
import $ivy.`com.github.lolgab::mill-mima::0.0.10`
77
import com.github.lolgab.mill.mima._
88

99
val dottyVersion = Option(sys.props("dottyVersion"))
1010

1111
object requests extends Cross[RequestsModule]((List("2.12.13", "2.13.5", "2.11.12", "3.0.0") ++ dottyVersion): _*)
1212
class RequestsModule(val crossScalaVersion: String) extends CrossScalaModule with PublishModule with Mima {
1313
def publishVersion = VcsVersion.vcsState().format()
14-
def mimaPreviousVersions = VcsVersion.vcsState().lastTag.toSeq
14+
def mimaPreviousVersions = Seq("0.7.0") ++ VcsVersion.vcsState().lastTag.toSeq
15+
override def mimaBinaryIssueFilters = Seq(
16+
ProblemFilter.exclude[ReversedMissingMethodProblem]("requests.BaseSession.send")
17+
)
1518
def artifactName = "requests"
1619
def pomSettings = PomSettings(
1720
description = "Scala port of the popular Python Requests HTTP client",

‎mill

Lines changed: 1 addition & 1 deletion
6D40
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# This is a wrapper script, that automatically download mill from GitHub release pages
44
# You can give the required mill version with MILL_VERSION env variable
55
# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION
6-
DEFAULT_MILL_VERSION=0.9.7
6+
DEFAULT_MILL_VERSION=0.9.12
77

88
set -e
99

‎requests/src/requests/Requester.scala

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import scala.collection.mutable
99

1010
trait BaseSession{
1111
def headers: Map[String, String]
12-
1312
def cookies: mutable.Map[String, HttpCookie]
1413
def readTimeout: Int
1514
def connectTimeout: Int
@@ -54,7 +53,7 @@ object Requester{
5453
}
5554
case class Requester(verb: String,
5655
sess: BaseSession){
57-
56+
private val upperCaseVerb = verb.toUpperCase
5857

5958
/**
6059
* Makes a single HTTP request, and returns a [[Response]] object. Requires
@@ -204,7 +203,6 @@ case class Requester(verb: String,
204203
}
205204

206205
connection.setInstanceFollowRedirects(false)
207-
val upperCaseVerb = verb.toUpperCase
208206
if (Requester.officialHttpMethods.contains(upperCaseVerb)) {
209207
connection.setRequestMethod(upperCaseVerb)
210208
} else {
@@ -250,17 +248,18 @@ case class Requester(verb: String,
250248
.map{case (k, v) => s"""$k="$v""""}
251249
.mkString("; ")
252250
)
253-
}
254-
if (verb.toUpperCase == "POST" || verb.toUpperCase == "PUT" || verb.toUpperCase == "PATCH" || verb.toUpperCase == "DELETE") {
251+
}
252+
253+
if (upperCaseVerb == "POST" || upperCaseVerb == "PUT" || upperCaseVerb == "PATCH" || upperCaseVerb == "DELETE") {
255254
if (!chunkedUpload) {
256255
val bytes = new ByteArrayOutputStream()
257-
data.write(compress.wrap(bytes))
256+
usingOutputStream(compress.wrap(bytes)) { os => data.write(os) }
258257
val byteArray = bytes.toByteArray
259258
connection.setFixedLengthStreamingMode(byteArray.length)
260-
if (byteArray.nonEmpty) connection.getOutputStream.write(byteArray)
259+
usingOutputStream(connection.getOutputStream) { os => os.write(byteArray) }
261260
} else {
262261
connection.setChunkedStreamingMode(0)
263-
data.write(compress.wrap(connection.getOutputStream))
262+
usingOutputStream(compress.wrap(connection.getOutputStream)) { os => data.write(os) }
264263
}
265264
}
266265

@@ -333,7 +332,7 @@ case class Requester(verb: String,
333332
// The HEAD method is identical to GET except that the server
334333
// MUST NOT return a message-body in the response.
335334
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4
336-
if (verb == "HEAD") f(new ByteArrayInputStream(Array()))
335+
if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array()))
337336
else if (stream != null) {
338337
try f(
339338
if (deGzip) new GZIPInputStream(stream)
@@ -366,6 +365,9 @@ case class Requester(verb: String,
366365
}
367366
}
368367
}
368+
369+
private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit =
370+
try fn(os) finally os.close()
369371

370372
/**
371373
* Overload of [[Requester.apply]] that takes a [[Request]] object as configuration

‎requests/test/src-2/requests/Scala2RequestTests.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ object Scala2RequestTests extends TestSuite{
1818
assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world"))
1919
}
2020
}
21+
2122
test("put") {
2223
for (chunkedUpload <- Seq(true, false)) {
2324
val res1 = requests.put(
@@ -28,6 +29,7 @@ object Scala2RequestTests extends TestSuite{
2829
assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world"))
2930
}
3031
}
32+
3133
test("send"){
3234
requests.send("get")("https://httpbin.org/get?hello=world&foo=baz")
3335

‎requests/test/src/requests/RequestTests.scala

Lines changed: 58 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ object RequestTests extends TestSuite{
3030
}
3131
}
3232
}
33+
3334
test("params"){
3435
test("get"){
3536
// All in URL
@@ -58,6 +59,7 @@ object RequestTests extends TestSuite{
5859
assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world"))
5960
}
6061
}
62+
6163
test("multipart"){
6264
for(chunkedUpload <- Seq(true, false)) {
6365
val response = requests.post(
@@ -73,8 +75,8 @@ object RequestTests extends TestSuite{
7375
assert(read(response).obj("form") == Obj("file2" -> "Goodbye!"))
7476
}
7577
}
76-
test("cookies"){
7778

79+
test("cookies"){
7880
test("session"){
7981
val s = requests.Session(cookieValues = Map("hello" -> "world"))
8082
val res1 = s.get("https://httpbin.org/cookies").text().trim
@@ -99,35 +101,37 @@ object RequestTests extends TestSuite{
99101
assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")))
100102
}
101103
}
102-
// Tests fail with 'Request to https://httpbin.org/absolute-redirect/4 failed with status code 404'
103-
// test("redirects"){
104-
// test("max"){
105-
// val res1 = requests.get("https://httpbin.org/absolute-redirect/4")
106-
// assert(res1.statusCode == 200)
107-
// val res2 = requests.get("https://httpbin.org/absolute-redirect/5")
108-
// assert(res2.statusCode == 200)
109-
// val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false)
110-
// assert(res3.statusCode == 302)
111-
// val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10)
112-
// assert(res4.statusCode == 200)
113-
// }
114-
// test("maxRelative"){
115-
// val res1 = requests.get("https://httpbin.org/relative-redirect/4")
116-
// assert(res1.statusCode == 200)
117-
// val res2 = requests.get("https://httpbin.org/relative-redirect/5")
118-
// assert(res2.statusCode == 200)
119-
// val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false)
120-
// assert(res3.statusCode == 302)
121-
// val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10)
122-
// assert(res4.statusCode == 200)
123-
// }
124-
// }
104+
105+
test("redirects"){
106+
test("max"){
107+
val res1 = requests.get("https://httpbin.org/absolute-redirect/4")
108+
assert(res1.statusCode == 200)
109+
val res2 = requests.get("https://httpbin.org/absolute-redirect/5")
110+
assert(res2.statusCode == 200)
111+
val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false)
112+
assert(res3.statusCode == 302)
113+
val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10)
114+
assert(res4.statusCode == 200)
115+
}
116+
test("maxRelative"){
117+
val res1 = requests.get("https://httpbin.org/relative-redirect/4")
118+
assert(res1.statusCode == 200)
119+
val res2 = requests.get("https://httpbin.org/relative-redirect/5")
120+
assert(res2.statusCode == 200)
121+
val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false)
122+
assert(res3.statusCode == 302)
123+
val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10)
124+
assert(res4.statusCode == 200)
125+
}
126+
}
127+
125128
test("streaming"){
126129
val res1 = requests.get("http://httpbin.org/stream/5").text()
127130
assert(res1.linesIterator.length == 5)
128131
val res2 = requests.get("http://httpbin.org/stream/52").text()
129132
assert(res2.linesIterator.length == 52)
130133
}
134+
131135
test("timeouts"){
132136
test("read"){
133137
intercept[TimeoutException] {
@@ -144,6 +148,7 @@ object RequestTests extends TestSuite{
144148
}
145149
}
146150
}
151+
147152
test("failures"){
148153
intercept[UnknownHostException]{
149154
requests.get("https://doesnt-exist-at-all.com/")
@@ -156,6 +161,7 @@ object RequestTests extends TestSuite{
156161
requests.get("://doesnt-exist.com/")
157162
}
158163
}
164+
159165
test("decompress"){
160166
val res1 = requests.get("https://httpbin.org/gzip")
161167
assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org")
@@ -171,6 +177,7 @@ object RequestTests extends TestSuite{
171177

172178
(res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length)
173179
}
180+
174181
test("compression"){
175182
val res1 = requests.post(
176183
"https://httpbin.org/post",
@@ -184,16 +191,18 @@ object RequestTests extends TestSuite{
184191
compress = requests.Compress.Gzip,
185192
data = new RequestBlob.ByteSourceRequestBlob("I am cow")
186193
)
187-
assert(res2.text().contains("data:application/octet-stream;base64,H4sIAAAAAAAAAA=="))
194+
assert(read(new String(res2.bytes))("data").toString ==
195+
""""data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""")
188196

189197
val res3 = requests.post(
190198
"https://httpbin.org/post",
191199
compress = requests.Compress.Deflate,
192200
data = new RequestBlob.ByteSourceRequestBlob("Hear me moo")
193201
)
194-
assert(res3.text().contains("data:application/octet-stream;base64,eJw="))
195-
res3.text()
196-
}
202+
assert(read(new String(res2.bytes))("data").toString ==
203+
""""data:application/octet-stream;base64,H4sIAAAAAAAAAPNUSMxVSM4vBwCAGeD4CAAAAA=="""")
204+
}
205+
197206
test("headers"){
198207
test("default"){
199208
val res = requests.get("https://httpbin.org/headers").text()
@@ -207,6 +216,7 @@ object RequestTests extends TestSuite{
207216
}
208217
}
209218
}
219+
210220
test("clientCertificate"){
211221
val base = "./requests/test/resources"
212222
val url = "https://client.badssl.com"
@@ -253,13 +263,15 @@ object RequestTests extends TestSuite{
253263
assert(res.statusCode == 400)
254264
}
255265
}
266+
256267
test("selfSignedCertificate"){
257268
val res = requests.get(
258269
"https://self-signed.badssl.com",
259270
verifySslCerts = false
260271
)
261272
assert(res.statusCode == 200)
262273
}
274+
263275
test("gzipError"){
264276
val response = requests.head("https://api.github.com/users/lihaoyi")
265277
assert(response.statusCode == 200)
@@ -268,5 +280,23 @@ object RequestTests extends TestSuite{
268280
assert(response.headers.keySet.map(_.toLowerCase).contains("content-length"))
269281
assert(resp 10000 onse.headers.keySet.map(_.toLowerCase).contains("content-type"))
270282
}
283+
284+
/**
285+
* Compress with each compression mode and call server. Server expands
286+
* and passes it back so we can compare
287+
*/
288+
test("compressionData") {
289+
import requests.Compress._
290+
val str = "I am deflater mouse"
291+
Seq(None, Gzip, Deflate).foreach(c =>
292+
ServerUtils.usingEchoServer { port =>
293+
assert(str == requests.post(
294+
s"http://localhost:$port/echo",
295+
compress = c,
296+
data = new RequestBlob.ByteSourceRequestBlob(str)
297+
).data.toString)
298+
}
299+
)
300+
}
271301
}
272302
}

‎requests/test/src/requests/ServerUtils.scala

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package requests
2+
3+
import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer}
4+
import java.io._
5+
import java.net.InetSocketAddress
6+
import java.util.zip.{GZIPInputStream, InflaterInputStream}
7+
import requests.Compress._
8+
import scala.annotation.tailrec
9+
import scala.collection.mutable.StringBuilder
10+
11+
object ServerUtils {
12+
def usingEchoServer(f: Int => Unit): Unit = {
13+
val server = new EchoServer
14+
try f(server.getPort())
15+
finally server.stop()
16+
}
17+
18+
private class EchoServer extends HttpHandler {
19+
private val server: HttpServer = HttpServer.create(new InetSocketAddress(0), 0)
20+
server.createContext("/echo", this)
21+
server.setExecutor(null); // default executor
22+
server.start()
23+
24+
def getPort(): Int = server.getAddress.getPort
25+
26+
def stop(): Unit = server.stop(0)
27+
28+
override def handle(t: HttpExchange): Unit = {
29+
val h: java.util.List[String] =
30+
t.getRequestHeaders.get("Content-encoding")
31+
val c: Compress =
32+
if (h == null) None
33+
else if (h.contains("gzip")) Gzip
34+
else if (h.contains("deflate")) Deflate
35+
else None
36+
val msg = new Plumper(c).decompress(t.getRequestBody)
37+
t.sendResponseHeaders(200, msg.length)
38+
t.getResponseBody.write(msg.getBytes())
39+
}
40+
}
41+
42+
/** Stream uncompresser
43+
* @param c
44+
* Compression mode
45+
*/
46+
private class Plumper(c: Compress) {
47+
48+
private def wrap(is: InputStream): InputStream =
49+
c match {
50+
case None => is
51+
case Gzip => new GZIPInputStream(is)
52+
case Deflate => new InflaterInputStream(is)
53+
}
54+
55+
def decompress(compressed: InputStream): String = {
56+
val gis = wrap(compressed)
57+
val br = new BufferedReader(new InputStreamReader(gis, "UTF-8"))
58+
val sb = new StringBuilder()
59+
60+
@tailrec
61+
def read(): Unit = {
62+
val line = br.readLine
63+
if (line != null) {
64+
sb.append(line)
65+
read()
66+
}
67+
}
68+
69+
read()
70+
br.close()
71+
gis.close()
72+
compressed.close()
73+
sb.toString()
74+
}
75+
}
76+
77+
}

0 commit comments

Comments
 (0)
0