fix(video): now sending with audio

Now we're downloading the video, the audio, merging them and uploading to telegram
Massolari · Jan 30, 2024 · d4b30ef · d4b30ef
1 parent 7e6db54
commit d4b30ef
Show file tree

Hide file tree

Showing 8 changed files with 287 additions and 49 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM alpine:edge
 RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
-    build-base erlang gleam rebar3 sqlite
+    build-base erlang gleam rebar3 sqlite ffmpeg
 RUN mkdir /app
 COPY src /app/src
 COPY gleam.toml /app/gleam.toml

diff --git a/devbox.json b/devbox.json
@@ -1,5 +1,6 @@
 {
   "packages": [
+        "github:NixOS/nixpkgs/nixos-unstable#ffmpeg",
         "github:NixOS/nixpkgs/nixos-unstable#gleam",
         "github:NixOS/nixpkgs/nixos-unstable#rebar3",
         "github:NixOS/nixpkgs/nixos-unstable#erlang",

diff --git a/gleam.toml b/gleam.toml
@@ -18,6 +18,7 @@ gleam_erlang = "~> 0.24"
 gleam_json = "~> 0.7"
 dot_env = "~> 0.2"
 sqlight = "~> 0.9"
+shellout = "~> 1.5"
 
 [dev-dependencies]
 gleeunit = "~> 1.0"
diff --git a/manifest.toml b/manifest.toml
@@ -6,16 +6,17 @@ packages = [
   { name = "dot_env", version = "0.2.3", build_tools = ["gleam"], requirements = ["gleam_stdlib", "simplifile"], otp_app = "dot_env", source = "hex", outer_checksum = "E4FEF0AEE27AB4017BC95DFDC403FB284185C181FA076389F3CD64841EF7D849" },
   { name = "esqlite", version = "0.8.6", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "607E45F4DA42601D8F530979417F57A4CD629AB49085891849302057E68EA188" },
   { name = "gleam_erlang", version = "0.24.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "26BDB52E61889F56A291CB34167315780EE4AA20961917314446542C90D1C1A0" },
-  { name = "gleam_hackney", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "gleam_http", "hackney"], otp_app = "gleam_hackney", source = "hex", outer_checksum = "066B1A55D37DBD61CC72A1C4EDE43C6015B1797FAF3818C16FE476534C7B6505" },
+  { name = "gleam_hackney", version = "1.2.0", build_tools = ["gleam"], requirements = ["hackney", "gleam_stdlib", "gleam_http"], otp_app = "gleam_hackney", source = "hex", outer_checksum = "066B1A55D37DBD61CC72A1C4EDE43C6015B1797FAF3818C16FE476534C7B6505" },
   { name = "gleam_http", version = "3.5.3", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "C2FC3322203B16F897C1818D9810F5DEFCE347F0751F3B44421E1261277A7373" },
   { name = "gleam_json", version = "0.7.0", build_tools = ["gleam"], requirements = ["thoas", "gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "CB405BD93A8828BCD870463DE29375E7B2D252D9D124C109E5B618AAC00B86FC" },
   { name = "gleam_stdlib", version = "0.34.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1FB8454D2991E9B4C0C804544D8A9AD0F6184725E20D63C3155F0AEB4230B016" },
   { name = "gleeunit", version = "1.0.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "D364C87AFEB26BDB4FB8A5ABDE67D635DC9FA52D6AB68416044C35B096C6882D" },
-  { name = "hackney", version = "1.20.1", build_tools = ["rebar3"], requirements = ["unicode_util_compat", "ssl_verify_fun", "parse_trans", "certifi", "idna", "metrics", "mimerl"], otp_app = "hackney", source = "hex", outer_checksum = "FE9094E5F1A2A2C0A7D10918FEE36BFEC0EC2A979994CFF8CFE8058CD9AF38E3" },
+  { name = "hackney", version = "1.20.1", build_tools = ["rebar3"], requirements = ["metrics", "ssl_verify_fun", "idna", "certifi", "parse_trans", "mimerl", "unicode_util_compat"], otp_app = "hackney", source = "hex", outer_checksum = "FE9094E5F1A2A2C0A7D10918FEE36BFEC0EC2A979994CFF8CFE8058CD9AF38E3" },
   { name = "idna", version = "6.1.1", build_tools = ["rebar3"], requirements = ["unicode_util_compat"], otp_app = "idna", source = "hex", outer_checksum = "92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA" },
   { name = "metrics", version = "1.0.1", build_tools = ["rebar3"], requirements = [], otp_app = "metrics", source = "hex", outer_checksum = "69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16" },
   { name = "mimerl", version = "1.2.0", build_tools = ["rebar3"], requirements = [], otp_app = "mimerl", source = "hex", outer_checksum = "F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323" },
   { name = "parse_trans", version = "3.4.1", build_tools = ["rebar3"], requirements = [], otp_app = "parse_trans", source = "hex", outer_checksum = "620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A" },
+  { name = "shellout", version = "1.5.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "shellout", source = "hex", outer_checksum = "7B5DE499DBB3DDC25051FC1BB3770DD5466938B6A2AFA91A6FB4A4D49F4CB0D4" },
   { name = "simplifile", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "88D931B120D27A4C91EF5D6CE18E9C8FFD635D0953DA29EEDCCE93432975D2EC" },
   { name = "sqlight", version = "0.9.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "esqlite"], otp_app = "sqlight", source = "hex", outer_checksum = "2D9C9BA420A5E7DCE7DB2DAAE4CAB0BE6218BEB48FD1531C583550B3D1316E94" },
   { name = "ssl_verify_fun", version = "1.1.7", build_tools = ["mix", "rebar3", "make"], requirements = [], otp_app = "ssl_verify_fun", source = "hex", outer_checksum = "FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8" },
@@ -31,5 +32,6 @@ gleam_http = { version = "~> 3.5" }
 gleam_json = { version = "~> 0.7" }
 gleam_stdlib = { version = "~> 0.32" }
 gleeunit = { version = "~> 1.0" }
+shellout = { version = "~> 1.5" }
 simplifile = { version = "~> 1.1" }
 sqlight = { version = "~> 0.9" }
diff --git a/src/form_data.gleam b/src/form_data.gleam
@@ -0,0 +1,39 @@
+import gleam/list
+import gleam/dynamic
+import gleam/bytes_builder.{type BytesBuilder}
+
+@external(erlang, "hackney_multipart", "encode_form")
+fn encode_form(
+  parts: List(dynamic.Dynamic),
+  boundary: String,
+) -> #(BytesBuilder, Int)
+
+@external(erlang, "hackney_multipart", "boundary")
+fn generate_boundary() -> String
+
+pub type Entry {
+  File(path: String, name: String, extra_headers: List(#(String, String)))
+  Text(name: String, value: String)
+}
+
+pub type FormData {
+  FormData(body: BytesBuilder, length: Int, boundary: String)
+}
+
+pub fn new(parts: List(Entry)) -> FormData {
+  let boundary = generate_boundary()
+
+  let #(body, length) =
+    parts
+    |> list.map(fn(part) {
+      case part {
+        File(path, name, extra_headers) ->
+          dynamic.from(File(path: path, name: name, extra_headers: extra_headers,
+          ))
+        Text(name, value) -> dynamic.from(#(name, value))
+      }
+    })
+    |> encode_form(boundary)
+
+  FormData(body: body, length: length, boundary: boundary)
+}
diff --git a/src/reddit.gleam b/src/reddit.gleam
@@ -1,14 +1,18 @@
 import app_data.{type AppData}
 import gleam/bit_array
+import gleam/bytes_builder
 import gleam/bool
 import gleam/dynamic
 import gleam/hackney
 import gleam/http
 import gleam/http/request.{type Request}
 import gleam/io
 import gleam/json
+import gleam/list
 import gleam/result
 import gleam/string
+import shellout
+import simplifile
 
 pub type Post {
   Post(
@@ -156,22 +160,22 @@ fn post_decoder() -> dynamic.Decoder(Post) {
 }
 
 fn media_decoder() -> dynamic.Decoder(Result(Media, Nil)) {
-  fn(json) {
-    dynamic.any([
-      fn(dynamic) {
-        is_video_decoder(json)(dynamic)
-        |> result.map(Ok)
-      },
-      fn(dynamic) {
-        url_decoder()(dynamic)
-        |> result.map(Ok)
-      },
-      fn(_) { Ok(Error(Nil)) },
-    ])(json)
-  }
+  dynamic.any([
+    fn(dynamic) {
+      is_video_decoder(dynamic)
+      |> result.map(Ok)
+    },
+    fn(dynamic) {
+      url_decoder()(dynamic)
+      |> result.map(Ok)
+    },
+    fn(_) { Ok(Error(Nil)) },
+  ])
 }
 
-fn is_video_decoder(json: dynamic.Dynamic) -> dynamic.Decoder(Media) {
+fn is_video_decoder(
+  json: dynamic.Dynamic,
+) -> Result(Media, List(dynamic.DecodeError)) {
   dynamic.field(named: "is_video", of: fn(dynamic_is_video) {
     dynamic.bool(dynamic_is_video)
     |> result.try(fn(is_video) {
@@ -183,12 +187,10 @@ fn is_video_decoder(json: dynamic.Dynamic) -> dynamic.Decoder(Media) {
               named: "reddit_video",
               of: dynamic.decode2(
                 fn(url: String, is_gif: Bool) {
-                  let media = case is_gif {
+                  case is_gif {
                     True -> Media(url, Gif)
                     False -> Media(url, Video)
                   }
-
-                  media
                 },
                 dynamic.field(named: "fallback_url", of: dynamic.string),
                 dynamic.field(named: "is_gif", of: dynamic.bool),
@@ -199,7 +201,7 @@ fn is_video_decoder(json: dynamic.Dynamic) -> dynamic.Decoder(Media) {
         False -> Error([])
       }
     })
-  })
+  })(json)
 }
 
 fn url_decoder() -> dynamic.Decoder(Media) {
@@ -301,3 +303,106 @@ fn external_url_decoder() -> dynamic.Decoder(Result(String, Nil)) {
     })(json)
   }
 }
+
+pub fn get_video(url: String) -> Result(String, String) {
+  let video_filename = "video.mp4"
+  let audio_filename = "audio.mp4"
+
+  use audio_url <- result.try(
+    url
+    |> string.split("DASH")
+    |> list.first
+    |> result.map(fn(url) { url <> "DASH_AUDIO_128.mp4" })
+    |> result.map_error(fn(_) { "Error getting audio url" }),
+  )
+
+  use request <- result.try(
+    request.to(url)
+    |> result.map_error(fn(_) { "Error creating video request" }),
+  )
+
+  io.println("Downloading video...")
+
+  use video_response <- result.try(
+    request
+    |> request.set_body(bytes_builder.new())
+    |> hackney.send_bits
+    |> result.map_error(fn(e) {
+      io.debug(e)
+      "Error getting video"
+    }),
+  )
+  io.println("Downloaded video")
+
+  io.println("Writing video file...")
+  use _ <- result.try(
+    simplifile.write_bits(video_filename, video_response.body)
+    |> result.map_error(fn(_) { "Error writing video file" }),
+  )
+
+  use request <- result.try(
+    request.to(audio_url)
+    |> result.map_error(fn(_) { "Error creating audio request" }),
+  )
+
+  io.println("Downloading audio...")
+  use audio_response <- result.try(
+    request
+    |> request.set_body(bytes_builder.new())
+    |> hackney.send_bits
+    |> result.map_error(fn(_) { "Error getting audio" }),
+  )
+  io.println("Downloaded audio")
+
+  io.println("Writing audio file...")
+  use _ <- result.try(
+    simplifile.write_bits(audio_filename, audio_response.body)
+    |> result.map_error(fn(_) { "Error writing audio file" }),
+  )
+
+  let filename = "video_with_audio.mp4"
+
+  io.println("Merging video and audio...")
+  use _ <- result.try(
+    shellout.command(
+      run: "ffmpeg",
+      with: [
+        "-i",
+        video_filename,
+        "-i",
+        audio_filename,
+        "-c:v",
+        "copy",
+        "-c:a",
+        "aac",
+        "-strict",
+        "experimental",
+        filename,
+        "-hide_banner",
+        "-loglevel",
+        "panic",
+        "-y",
+      ],
+      in: ".",
+      opt: [],
+    )
+    |> result.map_error(fn(error) { error.1 }),
+  )
+  io.println("Merged video and audio")
+
+  io.println("Removing video and audio files...")
+  let rm_result =
+    shellout.command(
+      run: "rm",
+      with: [video_filename, audio_filename],
+      in: ".",
+      opt: [],
+    )
+
+  case rm_result {
+    Ok(_) -> io.println("Removed video and audio files")
+    Error(_) -> io.println("Error removing video and audio files")
+  }
+
+  Ok(filename)
+}
diff --git a/src/reddit_to_telegram.gleam b/src/reddit_to_telegram.gleam
@@ -2,6 +2,8 @@ import gleam/io
 import gleam/list
 import gleam/result
 import gleam/function
+import gleam/pair
+import gleam/string
 import gleam/int
 import reddit
 import telegram
@@ -58,10 +60,16 @@ fn start(
         <> bridge.telegram_channel
         <> "...",
       )
-      let inserted =
+      let #(inserted, errors) =
         filtered_posts
         |> telegram.send_messages(data, bridge.telegram_channel)
-        |> list.filter_map(function.identity)
+        |> list.partition(with: result.is_ok)
+        |> pair.map_first(list.filter_map(_, function.identity))
+        |> pair.map_second(fn(error) {
+          error
+          |> list.map(result.unwrap_error(_, ""))
+          |> string.join("\n")
+        })
 
       io.println(
         inserted