Add support for IDNA 2008 (RFC 5891) when encoding URL (kittinunf#819)

buildSrc/src/main/kotlin/Constants.kt

@@ -238,3 +238,8 @@ object Stetho {
         const val dependency = "com.facebook.stetho:stetho-urlconnection:$version"
     }
 }
+
+object ICU {
+    const val version = "70.1"
+    const val dependency = "com.ibm.icu:icu4j:$version"
+}

fuel/build.gradle.kts

@@ -1,5 +1,6 @@
 dependencies {
     api(Result.dependency)
+    implementation(ICU.dependency)
 
     testImplementation(project(Fuel.Test.name))
     testImplementation(Json.dependency)

fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt

@@ -1,9 +1,9 @@
 package com.github.kittinunf.fuel.core
 
 import com.github.kittinunf.fuel.core.requests.DefaultRequest
+import com.ibm.icu.text.IDNA
 import java.net.MalformedURLException
 import java.net.URI
-import java.net.URISyntaxException
 import java.net.URL
 
 class Encoding(
@@ -13,6 +13,8 @@ class Encoding(
     val parameters: Parameters? = null
 ) : RequestFactory.RequestConvertible {
 
+    private val idna: IDNA = IDNA.getUTS46Instance(flags)
+
     private val encoder: (Method, String, Parameters?) -> Request = { method, path, parameters ->
         DefaultRequest(
             method = method,
@@ -36,13 +38,34 @@ class Encoding(
             }
             URL(base + if (path.startsWith('/') or path.isEmpty()) path else "/$path")
         }
-        val uri = try {
-            url.toURI()
-        } catch (e: URISyntaxException) {
-            URI(url.protocol, url.userInfo, url.host, url.port, url.path, url.query, url.ref)
-        }
+
+        val uri = URI(
+            url.protocol,
+            url.userInfo,
+            domainToAscii(url.host),
+            url.port,
+            url.path,
+            url.query,
+            url.ref
+        )
+
         return URL(uri.toASCIIString())
     }
 
+    private fun domainToAscii(domain: String): String {
+        val info = IDNA.Info()
+        val sb = StringBuilder()
+        val domainAscii = idna.nameToASCII(domain, sb, info).toString()
+        if (info.hasErrors()) {
+            throw MalformedURLException(info.errors.toString())
+        }
+        return domainAscii
+    }
+
     private val defaultHeaders = Headers.from()
+
+    companion object {
+        private const val flags =
+            IDNA.CHECK_BIDI or IDNA.CHECK_CONTEXTJ or IDNA.CHECK_CONTEXTO or IDNA.NONTRANSITIONAL_TO_ASCII or IDNA.USE_STD3_RULES
+    }
 }

fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt

@@ -0,0 +1,38 @@
+package com.github.kittinunf.fuel.core
+
+import org.hamcrest.CoreMatchers.equalTo
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@RunWith(Parameterized::class)
+internal class EncodingTest(private val inputUrl: String, private val encodedUrl: String) {
+
+    companion object {
+        @JvmStatic
+        @Parameterized.Parameters(name = "URLs")
+        fun data() = listOf(
+            arrayOf("https://github.com/kittinunf/fuel/", "https://github.com/kittinunf/fuel/"),
+            arrayOf(
+                "https://xn----f38am99bqvcd5liy1cxsg.test",
+                "https://xn----f38am99bqvcd5liy1cxsg.test"
+            ),
+            arrayOf("https://test.xn--rhqv96g", "https://test.xn--rhqv96g"),
+            arrayOf("https://test.شبك", "https://test.xn--ngbx0c"),
+            arrayOf("https://普遍接受-测试.top", "https://xn----f38am99bqvcd5liy1cxsg.top"),
+            arrayOf("https://मेल.डाटामेल.भारत", "https://xn--r2bi6d.xn--c2bd4bq1db8d.xn--h2brj9c"),
+            arrayOf("http://fußball.de", "http://xn--fuball-cta.de"),
+            arrayOf("http://fußball.de", "http://xn--fuball-cta.de"),
+        )
+    }
+
+    @Test
+    fun testRequestURLIDNAEncoding() {
+        val encoding = Encoding(
+            httpMethod = Method.GET,
+            urlString = inputUrl,
+        )
+        assertThat(encoding.request.url.toString(), equalTo(encodedUrl))
+    }
+}