diff --git a/README.md b/README.md index 753d71e..b45e815 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ https://t.me/save_it_playground - [Elixir](https://elixir-lang.org/) - [ex_gram](https://github.com/rockneurotiko/ex_gram) -- [cobalt api](https://github.com/imputnet/cobalt/blob/current/docs/api.md) +- [cobalt api](https://github.com/imputnet/cobalt) - [Typesense](https://typesense.org/) ## Development @@ -61,6 +61,7 @@ docker compose up ```sh # Run +export COBALT_API_URL= export TELEGRAM_BOT_TOKEN= iex -S mix run --no-halt diff --git a/config/runtime.exs b/config/runtime.exs index faaef97..f44b78a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -3,7 +3,9 @@ import Config config :save_it, :telegram_bot_token, System.get_env("TELEGRAM_BOT_TOKEN") config :ex_gram, token: System.get_env("TELEGRAM_BOT_TOKEN") -config :save_it, :typesense_url, System.get_env("TYPESENSE_URL", "http://localhost:8100") +config :save_it, :cobalt_api_url, System.get_env("COBALT_API_URL", "http://localhost:9001") + +config :save_it, :typesense_url, System.get_env("TYPESENSE_URL", "http://localhost:8101") config :save_it, :typesense_api_key, System.get_env("TYPESENSE_API_KEY", "xyz") # optional diff --git a/docker-compose.yml b/docker-compose.yml index eff3ed0..5f227bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,28 @@ services: + cobalt-api: + image: ghcr.io/imputnet/cobalt:10 + + init: true + read_only: true + restart: unless-stopped + + ports: + - 9001:9000/tcp + environment: + API_URL: "http://localhost:9001/" + typesense: image: typesense/typesense:27.1 + restart: on-failure - hostname: typesense + ports: - - "8100:8108" + - "8101:8108" volumes: - ./data/typesense-data:/data command: "--data-dir /data --api-key=xyz --enable-cors" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8100/health"] + test: ["CMD", "curl", "-f", "http://localhost:8101/health"] interval: 30s timeout: 10s retries: 5 diff --git a/docs/assets/save_it_demo.gif b/docs/assets/save_it_demo.gif new file mode 100644 index 0000000..6871499 Binary files /dev/null and b/docs/assets/save_it_demo.gif differ diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index 801f8d8..c60b801 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -1,6 +1,6 @@ defmodule SaveIt.Bot do require Logger - alias SaveIt.CobaltClient + alias SaveIt.FileHelper alias SaveIt.GoogleDrive alias SaveIt.GoogleOAuth2DeviceFlow @@ -8,6 +8,7 @@ defmodule SaveIt.Bot do alias SaveIt.PhotoService alias SmallSdk.Telegram + alias SmallSdk.Cobalt @bot :save_it_bot @@ -225,8 +226,8 @@ defmodule SaveIt.Bot do {:ok, progress_message} = send_message(chat.id, Enum.at(@progress, 0)) url = List.first(urls) - case CobaltClient.get_download_url(url) do - {:ok, url, download_urls} -> + case Cobalt.get_download_url(url) do + {:ok, purge_url, download_urls} -> case FileHelper.get_downloaded_files(download_urls) do nil -> update_message(chat.id, progress_message.message_id, Enum.slice(@progress, 0..1)) @@ -244,7 +245,7 @@ defmodule SaveIt.Bot do bot_send_files(chat.id, files) delete_messages(chat.id, [message_id, progress_message.message_id]) - FileHelper.write_folder(url, files) + FileHelper.write_folder(purge_url, files) # TODO: 给图片添加 emoji GoogleDrive.upload_files(chat.id, files) diff --git a/lib/save_it/cobalt_client.ex b/lib/save_it/cobalt_client.ex deleted file mode 100644 index 6b3eae9..0000000 --- a/lib/save_it/cobalt_client.ex +++ /dev/null @@ -1,73 +0,0 @@ -defmodule SaveIt.CobaltClient do - require Logger - - use Tesla - - plug(Tesla.Middleware.BaseUrl, "https://api.cobalt.tools") - - plug(Tesla.Middleware.Headers, [ - {"Accept", "application/json"}, - {"Content-Type", "application/json"} - ]) - - plug(Tesla.Middleware.JSON) - - @doc """ - - ## Examples - get_download_url("https://www.instagram.com/p/C9pr7NDPAyd/?igsh=azBiNHJ0ZXd3bTFh") #=> - - """ - # def get_download_url("https://www.instagram.com/" <> _ = text) do - def get_download_url(text) do - # https://www.instagram.com/p/C9pr7NDPAyd/?igsh=azBiNHJ0ZXd3bTFh => https://www.instagram.com/p/C9pr7NDPAyd/ - url = String.split(text, "?") |> hd() - - case post("api/json", %{url: url}) do - {:ok, response} -> - case response.body do - %{"url" => url} -> - # memo: ins single video, response.body is %{"url" => url} - # %{ - # "status" => "redirect", - # "url" => "https://scontent.cdninstagram.com/..." - # } - {:ok, url} - - %{"status" => "picker", "picker" => picker_items} -> - # [%{"url" => url}] = picker_items - # error: you attempted to apply a function named :first on [], If you are using Kernel.apply/3, make sure the module is an atom. If you are using the dot syntax, such as module.function(), make sure the left-hand side of the dot is an atom representing a module - {:ok, url, Enum.map(picker_items, &Map.get(&1, "url"))} - - %{"status" => "error", "text" => msg} -> - Logger.warning("response.body is status error, text: #{msg}") - {:error, msg} - - _ -> - Logger.warning("response.body: #{inspect(response.body)}") - {:error, "inner service error"} - end - - {:error, error} -> - {:error, error} - end - end - - # TODO: get_download_urls for multiple urls - # def get_download_url(url) do - # case post("api/json", %{url: url}) do - # {:ok, response} -> - # # %{ - # # "status" => "redirect", - # # "url" => "https://video.twimg.com/amplify_video/1814202798097268736/vid/avc1/720x1192/HAD9zyJn1xoP4oRN.mp4?tag=16" - # # } - # %{"url" => url} = response.body - # url - - # # TODO send photo to telegram - - # {:error, error} -> - # error - # end - # end -end diff --git a/lib/save_it/migration/typesense.ex b/lib/save_it/migration/typesense.ex index 04232fb..9ad8a9d 100644 --- a/lib/save_it/migration/typesense.ex +++ b/lib/save_it/migration/typesense.ex @@ -1,7 +1,7 @@ defmodule SaveIt.Migration.Typesense do alias SmallSdk.Typesense - import Tj.UrlHelper, only: [validate_url!: 1] + import SaveIt.SmallHelper.UrlHelper, only: [validate_url!: 1] def create_collection!(schema) do req = build_request("/collections") diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index 42ea6c0..8331aca 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -2,7 +2,7 @@ defmodule SaveIt.PhotoService do require Logger alias SmallSdk.Typesense - import Tj.UrlHelper, only: [validate_url!: 1] + import SaveIt.SmallHelper.UrlHelper, only: [validate_url!: 1] def create_photo!( %{ diff --git a/lib/tj/url_helper.ex b/lib/save_it/small_helper/url_helper.ex similarity index 82% rename from lib/tj/url_helper.ex rename to lib/save_it/small_helper/url_helper.ex index 3fab418..d7578e9 100644 --- a/lib/tj/url_helper.ex +++ b/lib/save_it/small_helper/url_helper.ex @@ -1,4 +1,4 @@ -defmodule Tj.UrlHelper do +defmodule SaveIt.SmallHelper.UrlHelper do def validate_url!(url) do uri = URI.parse(url) diff --git a/lib/small_sdk/cobalt.ex b/lib/small_sdk/cobalt.ex new file mode 100644 index 0000000..8f4f29c --- /dev/null +++ b/lib/small_sdk/cobalt.ex @@ -0,0 +1,98 @@ +defmodule SmallSdk.Cobalt do + require Logger + + import SaveIt.SmallHelper.UrlHelper, only: [validate_url!: 1] + + def get_download_url(text) do + url = String.split(text, "?") |> hd() + + req = build_request("/") + res = Req.post(req, json: %{url: url}) + + body = handle_response(res) + + case body do + %{"url" => download_url} -> + {:ok, download_url} + + %{"status" => "picker", "picker" => picker_items} -> + {:ok, url, Enum.map(picker_items, &Map.get(&1, "url"))} + + %{"status" => "error", "text" => msg} -> + Logger.warning("response.body is status error, text: #{msg}") + {:error, msg} + + _ -> + Logger.warning("response.body: #{inspect(body)}") + {:error, "inner service error"} + end + end + + defp get_env() do + api_url = Application.fetch_env!(:save_it, :cobalt_api_url) |> validate_url!() + + {api_url} + end + + defp build_request(path) do + {api_url} = get_env() + + Req.new( + base_url: api_url, + url: path, + headers: [ + {"Accept", "application/json"}, + {"Content-Type", "application/json"} + ] + ) + end + + @doc """ + Handle response from Cobalt API return body if status is 200..209 + """ + def handle_response({:ok, %{status: status, body: body}}) do + case status do + status when status in 200..209 -> + body + + 400 -> + Logger.warning("Bad Request: #{inspect(body)}") + raise "Bad Request" + + 401 -> + raise "Unauthorized" + + 404 -> + nil + + 409 -> + raise "Conflict" + + 422 -> + raise "Unprocessable Entity" + + 503 -> + raise "Service Unavailable" + + _ -> + Logger.error("Unhandled status code #{status}: #{inspect(body)}") + raise "Unknown error: #{status}" + end + end + + def handle_response({:error, reason}) do + Logger.error("Request failed: #{inspect(reason)}") + raise "Request failed" + end + + def handle_response!(%{status: status, body: body}) do + case status do + status when status in 200..209 -> + body + + status -> + Logger.warning("Request failed with status #{status}: #{inspect(body)}") + raise "Request failed with status #{status}" + end + end +end diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 004a24f..3bf0ca6 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -1,7 +1,7 @@ defmodule SmallSdk.Typesense do require Logger - import Tj.UrlHelper, only: [validate_url!: 1] + import SaveIt.SmallHelper.UrlHelper, only: [validate_url!: 1] def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") diff --git a/zeabur/readme.md b/zeabur/readme.md index 671d84b..fa97f0e 100644 --- a/zeabur/readme.md +++ b/zeabur/readme.md @@ -1,11 +1,14 @@ -# Zeabur +# SaveIt template on zeabur -## Typesense Template +- zeabur template: https://zeabur.com/templates/FTAONK +- github: https://github.com/ThaddeusJiang/save_it/ + +## Develop ```sh -npx zeabur template update -c FTAONK -f typesense-template.yaml +npx zeabur template update -c FTAONK -f template.yaml ``` -docs: +## Docs -- https://zeabur.com/docs/template/template-in-code +- [zeabur docs | template ](https://zeabur.com/docs/template/template-in-code) diff --git a/zeabur/template.yaml b/zeabur/template.yaml index d0c3d30..a2c327e 100644 --- a/zeabur/template.yaml +++ b/zeabur/template.yaml @@ -5,13 +5,13 @@ metadata: name: SaveIt spec: description: SaveIt is a telegram bot that helps you save photos. - icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + icon: https://raw.githubusercontent.com/ThaddeusJiang/save_it/refs/heads/main/docs/assets/savt_it_bot_logo.jpg tags: - Bot - Telegram - Photos Storage - Search Engine - coverImage: + coverImage: https://raw.githubusercontent.com/ThaddeusJiang/save_it/refs/heads/main/docs/assets/savt_it_demo.gif readme: |- # SaveIt @@ -22,14 +22,11 @@ spec: - Search photos using semantic search - Find similar photos by photo - ## Learn more - https://github.com/ThaddeusJiang/save_it - - + [more](https://github.com/ThaddeusJiang/save_it) services: - name: save_it - icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + icon: https://raw.githubusercontent.com/ThaddeusJiang/save_it/refs/heads/main/docs/assets/savt_it_bot_logo.jpg template: GIT spec: source: @@ -41,6 +38,9 @@ spec: TELEGRAM_BOT_TOKEN: default: "" expose: false + COBALT_API_URL: + default: http://cobalt-api.zeabur.internal:9000 + expose: false TYPESENSE_URL: default: http://typesense.zeabur.internal:8108 expose: false @@ -81,3 +81,19 @@ spec: expose: false configs: [] + - name: cobalt-api + icon: https://github.com/imputnet/cobalt/raw/main/web/static/favicon.png + template: PREBUILT_V2 + spec: + source: + image: ghcr.io/imputnet/cobalt:10 + ports: + - id: api + port: 9000 + type: TCP + env: + API_URL: + default: undefined + expose: false + + configs: []