fix #36 imgur image embedding

add basic request headers to satisfy imgur
Aivean · Apr 22, 2024 · a689c74 · a689c74
1 parent 8820896
commit a689c74
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 24 deletions.
diff --git a/src/main/scala/com/aivean/royalroad/Utils.scala b/src/main/scala/com/aivean/royalroad/Utils.scala
@@ -119,32 +119,39 @@ object Utils {
     }
   }
 
-  def getDataURIForURL(url: URL): URI = withResource(url.openStream()) { is =>
-    val bis = new BufferedInputStream(is)
-    val contentType = URLConnection.guessContentTypeFromStream(bis) match {
-      case null => // try to guess from url
-        val ext = url.toString.split('.').lastOption
-        ext match {
-          case Some("jpg") => "image/jpeg"
-          case Some("png") => "image/png"
-          case Some("gif") => "image/gif"
-          case _ => null
-        }
-      case x => x
-    }
+  def getDataURIForURL(url: URL): URI = {
+    val connection = url.openConnection().asInstanceOf[HttpURLConnection]
+    connection.setRequestProperty("User-Agent", "curl/8.4.0")
+    connection.setRequestProperty("Host", url.getHost)
+    connection.setRequestProperty("Accept", "*/*")
+
+    withResource(connection.getInputStream) { is =>
+      val bis = new BufferedInputStream(is)
+      val contentType = URLConnection.guessContentTypeFromStream(bis) match {
+        case null => // try to guess from url
+          val ext = url.toString.split('.').lastOption
+          ext match {
+            case Some("jpg") => "image/jpeg"
+            case Some("png") => "image/png"
+            case Some("gif") => "image/gif"
+            case _ => null
+          }
+        case x => x
+      }
 
-    if (contentType != null) {
-      withResource(new ByteArrayOutputStream()) { os =>
-        val chunk = new Array[Byte](4096)
-        Stream.continually(bis.read(chunk))
-          .takeWhile(_ > 0)
-          .foreach(readBytes => os.write(chunk, 0, readBytes))
-        os.flush()
-        new URI("data:" + contentType + ";base64," +
-          Base64.getEncoder.encodeToString(os.toByteArray))
+      if (contentType != null) {
+        withResource(new ByteArrayOutputStream()) { os =>
+          val chunk = new Array[Byte](4096)
+          Stream.continually(bis.read(chunk))
+            .takeWhile(_ > 0)
+            .foreach(readBytes => os.write(chunk, 0, readBytes))
+          os.flush()
+          new URI("data:" + contentType + ";base64," +
+            Base64.getEncoder.encodeToString(os.toByteArray))
+        }
+      } else {
+        throw new IOException("could not get content type from " + url.toExternalForm)
       }
-    } else {
-      throw new IOException("could not get content type from " + url.toExternalForm)
     }
   }
 

diff --git a/src/test/scala/com/aivean/royalroad/UtilsTest.scala b/src/test/scala/com/aivean/royalroad/UtilsTest.scala
@@ -15,6 +15,12 @@ class UtilsTest extends FunSuite {
       new URI("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAIAAAACDbGyAAAAEUlEQVR4nGP8z4ACmBgo4wMAUJEBCfBOwRcAAAAASUVORK5CYII="))
   }
 
+  // test url to data uri conversion
+  test("urlToDataUriImgur") {
+    val url = new URL("https://i.imgur.com/kp112Ad.png")
+    val dataUri = Utils.getDataURIForURL(url)
+    assert(dataUri.toString.startsWith("data:image/png;base64,"))
+  }
 
   val reportWarnings = List(
     "A case of theft: this story is not rightfully on Amazon; if you spot it, report the violation.",
@@ -49,6 +55,13 @@ class UtilsTest extends FunSuite {
     }
   }
 
+  test("amazonRegex2") {
+    reportWarnings.foreach { w =>
+      println(w +
+      s" score: ${Utils.WarningFuzzyMatcher.scoreString(w)} > ${Utils.WarningFuzzyMatcher.threshold}")
+    }
+  }
+
 
   test("renderTemplate") {
     assert(Utils.renderTemplate(