From 9627c72f1018042c90e74e7ceab3b38130f86b22 Mon Sep 17 00:00:00 2001 From: TJ Date: Mon, 28 Oct 2024 14:02:06 +0900 Subject: [PATCH 01/10] first commit --- .gitignore | 14 +-- README.md | 44 +++++++-- config/runtime.exs | 2 - docs/{worklogs => dev-logs}/2024-10-21.md | 0 docs/dev-logs/2024-10-25.md | 9 ++ docs/dev/readme.md | 0 docs/dev/typesense.md | 14 +++ lib/migration/typesense.ex | 42 +++++++++ lib/migration/typesense/photo.ex | 35 ++++++++ lib/save_it/bot.ex | 90 +++++++------------ .../{typesense_photo.ex => photo_service.ex} | 31 +++++-- lib/small_sdk/typesense.ex | 62 ++++++++++++- lib/small_sdk/typesense_admin.ex | 66 -------------- .../2024-10-24_create_photos_collection.exs | 1 + priv/typesense/reset.exs | 2 +- zeabur/template.yaml | 83 +++++++++++++++++ 16 files changed, 345 insertions(+), 150 deletions(-) rename docs/{worklogs => dev-logs}/2024-10-21.md (100%) create mode 100644 docs/dev-logs/2024-10-25.md create mode 100644 docs/dev/readme.md create mode 100644 docs/dev/typesense.md create mode 100644 lib/migration/typesense.ex create mode 100644 lib/migration/typesense/photo.ex rename lib/save_it/{typesense_photo.ex => photo_service.ex} (74%) delete mode 100644 lib/small_sdk/typesense_admin.ex create mode 100644 priv/typesense/2024-10-24_create_photos_collection.exs create mode 100644 zeabur/template.yaml diff --git a/.gitignore b/.gitignore index c81d2b0..1c7fa49 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,14 @@ save_it-*.tar # Temporary files, for example, from tests. /tmp/ .DS_Store -data -dev.sh -start.sh -run.sh -nohup.out +# data _local +data + +# scripts +_local* +_dev* +_stag* +_prod* +nohup.out diff --git a/README.md b/README.md index fdf3744..72bd025 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A telegram bot can Save photos and Search photos +## Features + - [x] Save photos via a link - [x] Search photos using semantic search - [x] Find similar photos by photo @@ -27,13 +29,8 @@ messages: ``` /search cat - /search dog - /search girl - -/similar photo - /similar photo ``` @@ -58,10 +55,45 @@ mix deps.get ``` ```sh -# run +# start typesense +docker compose up +``` + +```sh +# modify env export TELEGRAM_BOT_TOKEN= export TYPESENSE_URL= export TYPESENSE_API_KEY= iex -S mix run --no-halt ``` + +Pro Tips: create shell script for fast run app + +1. touch start.sh + +```sh +#!/bin/sh + +export TELEGRAM_BOT_TOKEN= + +export TYPESENSE_URL= +export TYPESENSE_API_KEY= + +export GOOGLE_OAUTH_CLIENT_ID= +export GOOGLE_OAUTH_CLIENT_SECRET= + +iex -S mix run --no-halt +``` + +2. execute permission + +```sh +chmod +x start.sh +``` + +3. run + +```sh +./start.sh +``` diff --git a/config/runtime.exs b/config/runtime.exs index 047283f..faaef97 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -9,5 +9,3 @@ config :save_it, :typesense_api_key, System.get_env("TYPESENSE_API_KEY", "xyz") # optional config :save_it, :google_oauth_client_id, System.get_env("GOOGLE_OAUTH_CLIENT_ID") config :save_it, :google_oauth_client_secret, System.get_env("GOOGLE_OAUTH_CLIENT_SECRET") - -config :save_it, :web_url, System.get_env("WEB_URL", "http://localhost:4000") diff --git a/docs/worklogs/2024-10-21.md b/docs/dev-logs/2024-10-21.md similarity index 100% rename from docs/worklogs/2024-10-21.md rename to docs/dev-logs/2024-10-21.md diff --git a/docs/dev-logs/2024-10-25.md b/docs/dev-logs/2024-10-25.md new file mode 100644 index 0000000..ddc6e58 --- /dev/null +++ b/docs/dev-logs/2024-10-25.md @@ -0,0 +1,9 @@ +# 2024-10-25 + +## Req call typesense API alway :timeout, but typesense was updated. + +```elixir +** (MatchError) no match of right hand side value: {:error, %Req.TransportError{reason: :timeout}} + (save_it 0.2.0-rc.1) lib/migration/typesense.ex:11: Migration.Typesense.create_collection!/1 + priv/typesense/reset.exs:3: (file) +``` diff --git a/docs/dev/readme.md b/docs/dev/readme.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/dev/typesense.md b/docs/dev/typesense.md new file mode 100644 index 0000000..c564cf3 --- /dev/null +++ b/docs/dev/typesense.md @@ -0,0 +1,14 @@ +# Typesense + +## Typesense API Errors + +``` +# 400 Bad Request - The request could not be understood due to malformed syntax. +# 401 Unauthorized - Your API key is wrong. +# 404 Not Found - The requested resource is not found. +# 409 Conflict - When a resource already exists. +# 422 Unprocessable Entity - Request is well-formed, but cannot be processed. +# 503 Service Unavailable - We’re temporarily offline. Please try again later. +``` + +docs: https://typesense.org/docs/27.1/api/api-errors.html#api-errors diff --git a/lib/migration/typesense.ex b/lib/migration/typesense.ex new file mode 100644 index 0000000..8bea87e --- /dev/null +++ b/lib/migration/typesense.ex @@ -0,0 +1,42 @@ +defmodule Migration.Typesense do + def create_collection!(schema) do + req = build_request("/collections") + {:ok, res} = Req.post(req, json: schema) + + res.body + end + + def delete_collection!(collection_name) do + req = build_request("/collections/#{collection_name}") + {:ok, res} = Req.delete(req) + + res.body + end + + def list_collections() do + req = build_request("/collections") + {:ok, res} = Req.get(req) + + res.body + end + + defp get_env() do + url = Application.fetch_env!(:save_it, :typesense_url) + api_key = Application.fetch_env!(:save_it, :typesense_api_key) + + {url, api_key} + end + + defp build_request(path) do + {url, api_key} = get_env() + + Req.new( + base_url: url, + url: path, + headers: [ + {"Content-Type", "application/json"}, + {"X-TYPESENSE-API-KEY", api_key} + ] + ) + end +end diff --git a/lib/migration/typesense/photo.ex b/lib/migration/typesense/photo.ex new file mode 100644 index 0000000..ebe686b --- /dev/null +++ b/lib/migration/typesense/photo.ex @@ -0,0 +1,35 @@ +defmodule Migration.Typesense.Photo do + alias Migration.Typesense + + @photos_schema %{ + "name" => "photos", + "fields" => [ + # image: base64 encoded string + %{"name" => "image", "type" => "image", "store" => false}, + %{ + "name" => "image_embedding", + "type" => "float[]", + "embed" => %{ + "from" => ["image"], + "model_config" => %{ + "model_name" => "ts/clip-vit-b-p32" + } + } + }, + %{"name" => "caption", "type" => "string", "optional" => true}, + %{"name" => "file_id", "type" => "string"}, + %{"name" => "belongs_to_id", "type" => "string"}, + %{"name" => "inserted_at", "type" => "int64"} + ], + "default_sorting_field" => "inserted_at" + } + + def create_collection!() do + Typesense.create_collection!(@photos_schema) + end + + def reset!() do + Typesense.delete_collection!(@photos_schema["name"]) + Typesense.create_collection!(@photos_schema) + end +end diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index 3390101..3afba0f 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -5,7 +5,7 @@ defmodule SaveIt.Bot do alias SaveIt.GoogleDrive alias SaveIt.GoogleOAuth2DeviceFlow - alias SaveIt.TypesensePhoto + alias SaveIt.PhotoService alias SmallSdk.Telegram @@ -115,6 +115,10 @@ defmodule SaveIt.Bot do end end + def handle({:command, :search, %{chat: chat, text: nil}}, _context) do + send_message(chat.id, "What do you want to search? animal, food, etc.") + end + def handle({:command, :search, %{chat: chat, text: text}}, _context) when is_binary(text) do q = String.trim(text) @@ -124,7 +128,7 @@ defmodule SaveIt.Bot do send_message(chat.id, "What do you want to search? animal, food, etc.") _ -> - photos = TypesensePhoto.search_photos!(q, belongs_to_id: chat.id) + photos = PhotoService.search_photos!(q, belongs_to_id: chat.id) answer_photos(chat.id, photos) end @@ -134,30 +138,25 @@ defmodule SaveIt.Bot do send_message(chat.id, "Upload a photo to find similar photos.") end - # dev-notes: never reach here, it will be handled by handle({:message, %{chat: chat, caption: nil, photo: photos}}, ctx) - # def handle({:command, :similar, %{chat: _chat, photo: photo}}, _context) do - # end - # caption: nil -> find same photos - def handle({:message, %{chat: chat, caption: nil, photo: photos}}, ctx) do + def handle({:message, %{chat: chat, caption: nil, photo: photos}}, _ctx) do photo = List.last(photos) file = ExGram.get_file!(photo.file_id) photo_file_content = Telegram.download_file_content!(file.file_path) - bot_id = ctx.bot_info.id chat_id = chat.id typesense_photo = - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: "", - url: photo_url(bot_id, file.file_id), + file_id: file.file_id, belongs_to_id: chat_id }) photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.1, belongs_to_id: chat_id @@ -170,13 +169,12 @@ defmodule SaveIt.Bot do end # caption: contains /similar or /search -> search similar photos; otherwise, find same photos - def handle({:message, %{chat: chat, caption: caption, photo: photos}}, ctx) do + def handle({:message, %{chat: chat, caption: caption, photo: photos}}, _ctx) do photo = List.last(photos) file = ExGram.get_file!(photo.file_id) photo_file_content = Telegram.download_file_content!(file.file_path) - bot_id = ctx.bot_info.id chat_id = chat.id caption = @@ -187,17 +185,17 @@ defmodule SaveIt.Bot do end typesense_photo = - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: caption, - url: photo_url(bot_id, file.file_id), + file_id: file.file_id, belongs_to_id: chat_id }) case caption do "" -> photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.4, belongs_to_id: chat_id @@ -207,7 +205,7 @@ defmodule SaveIt.Bot do _ -> photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.1, belongs_to_id: chat_id @@ -310,18 +308,15 @@ defmodule SaveIt.Bot do end end - def handle( - {:update, - %ExGram.Model.Update{message: nil, edited_message: nil, channel_post: _channel_post}}, - _context - ) do - Logger.warning("this is a channel post, ignore it") + def handle({:edited_message, %{photo: nil}}, _context) do + Logger.warning("this is an edited message, ignore it") + # TODO: edit /search trigger re-search {:ok, nil} end - def handle({:edited_message, _msg}, _context) do - Logger.warning("this is an edited message, ignore it") - {:ok, nil} + def handle({:edited_message, %{chat: chat, caption: caption, photo: photos}}, _context) do + file_id = photos |> List.last() |> Map.get(:file_id) + PhotoService.update_photo_caption!(file_id, chat.id, caption) end def handle({:update, _update}, _context) do @@ -334,19 +329,6 @@ defmodule SaveIt.Bot do {:ok, nil} end - defp pick_file_id_from_photo_url(photo_url) do - captures = - Regex.named_captures(~r"/files/(?\d+)/(?.+)", photo_url) - - if captures == nil do - Logger.error("Invalid photo URL: #{photo_url}") - nil - else - %{"file_id" => file_id} = captures - file_id - end - end - defp answer_photos(chat_id, nil) do send_message(chat_id, "No photos found.") end @@ -360,7 +342,7 @@ defmodule SaveIt.Bot do Enum.map(similar_photos, fn photo -> %ExGram.Model.InputMediaPhoto{ type: "photo", - media: pick_file_id_from_photo_url(photo["url"]), + media: photo["file_id"], caption: "Found photos", show_caption_above_media: true } @@ -414,19 +396,19 @@ defmodule SaveIt.Bot do Enum.each(filenames, fn filename -> bot_send_file(chat_id, filename, {:file, filename}) end) end - defp bot_send_file(chat_id, file_name, file_content, _opts \\ []) do + defp bot_send_file(chat_id, file_name, file_content, opts \\ []) do content = case file_content do {:file, file} -> {:file, file} {:file_content, file_content, file_name} -> {:file_content, file_content, file_name} end - # caption = opts[:caption] + caption = Keyword.get(opts, :caption, "") case file_extension(file_name) do ext when ext in [".png", ".jpg", ".jpeg"] -> - {:ok, msg} = ExGram.send_photo(chat_id, content) - bot_id = msg.from.id + {:ok, msg} = ExGram.send_photo(chat_id, content, caption: caption) + file_id = get_file_id(msg) image_base64 = @@ -435,21 +417,21 @@ defmodule SaveIt.Bot do {:file_content, file_content, _file_name} -> Base.encode64(file_content) end - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: image_base64, - caption: file_name, - url: photo_url(bot_id, file_id), + caption: caption, + file_id: file_id, belongs_to_id: chat_id }) ".mp4" -> - ExGram.send_video(chat_id, content, supports_streaming: true) + ExGram.send_video(chat_id, content, supports_streaming: true, caption: caption) ".gif" -> - ExGram.send_animation(chat_id, content) + ExGram.send_animation(chat_id, content, caption: caption) _ -> - ExGram.send_document(chat_id, content) + ExGram.send_document(chat_id, content, caption: caption) end end @@ -487,12 +469,4 @@ defmodule SaveIt.Bot do """) end end - - defp photo_url(bot_id, file_id) do - proxy_url = Application.fetch_env!(:save_it, :web_url) <> "/telegram/files" - - encoded_bot_id = URI.encode(bot_id |> to_string()) - encoded_file_id = URI.encode(file_id) - "#{proxy_url}/#{encoded_bot_id}/#{encoded_file_id}" - end end diff --git a/lib/save_it/typesense_photo.ex b/lib/save_it/photo_service.ex similarity index 74% rename from lib/save_it/typesense_photo.ex rename to lib/save_it/photo_service.ex index 587b84d..931d8b8 100644 --- a/lib/save_it/typesense_photo.ex +++ b/lib/save_it/photo_service.ex @@ -1,4 +1,4 @@ -defmodule SaveIt.TypesensePhoto do +defmodule SaveIt.PhotoService do require Logger alias SmallSdk.Typesense @@ -19,13 +19,30 @@ defmodule SaveIt.TypesensePhoto do end def update_photo(photo) do - Typesense.update_document("photos", photo.id, photo) + Typesense.update_document!("photos", photo["id"], photo) + end + + def update_photo_caption!(file_id, belongs_to_id, caption) do + get_photo!(file_id, belongs_to_id) + |> Map.put("caption", caption) + |> update_photo() end def get_photo(photo_id) do Typesense.get_document("photos", photo_id) end + def get_photo!(file_id, belongs_to_id) do + case Typesense.search_documents!("photos", + q: "*", + query_by: "caption", + filter_by: "file_id:=#{file_id} && belongs_to_id:=#{belongs_to_id}" + ) do + [photo | _] -> photo + [] -> nil + end + end + def search_photos!(q, opts) do belongs_to_id = Keyword.get(opts, :belongs_to_id) @@ -46,7 +63,8 @@ defmodule SaveIt.TypesensePhoto do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) - res.body["results"] |> typesense_results_to_documents() + # FIXME: nil check + res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end def search_similar_photos!(photo_id, opts \\ []) when is_binary(photo_id) do @@ -69,11 +87,8 @@ defmodule SaveIt.TypesensePhoto do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) - res.body["results"] |> typesense_results_to_documents() - end - - defp typesense_results_to_documents(results) do - results |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + # FIXME: nil check + res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end defp get_env() do diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 6260713..ece7bad 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -1,25 +1,79 @@ defmodule SmallSdk.Typesense do require Logger + def handle_response(res) do + case res do + %Req.Response{status: 200} -> + res.body + + %Req.Response{status: 201} -> + res.body + + %Req.Response{status: 400} -> + Logger.error("Bad Request: #{inspect(res.body)}") + raise "Bad Request" + + %Req.Response{status: 401} -> + raise "Unauthorized" + + %Req.Response{status: 404} -> + nil + + %Req.Response{status: 409} -> + raise "Conflict" + + %Req.Response{status: 422} -> + raise "Unprocessable Entity" + + %Req.Response{status: 503} -> + raise "Service Unavailable" + + _ -> + raise "Unknown error" + end + end + def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") {:ok, res} = Req.post(req, json: document) - res.body + handle_response(res) + end + + def search_documents!(collection_name, opts) do + q = Keyword.get(opts, :q, "*") + query_by = Keyword.get(opts, :query_by, "") + filter_by = Keyword.get(opts, :filter_by, "") + + req = build_request("/collections/#{collection_name}/documents/search") + + {:ok, res} = + Req.get(req, + params: %{ + q: q, + query_by: query_by, + filter_by: filter_by, + exclude_fields: "image_embedding" + } + ) + + data = handle_response(res) + + data["hits"] |> Enum.map(&Map.get(&1, "document")) end def get_document(collection_name, document_id) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") {:ok, res} = Req.get(req) - res.body + handle_response(res) end - def update_document(collection_name, document_id, update_input) do + def update_document!(collection_name, document_id, update_input) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") {:ok, res} = Req.patch(req, json: update_input) - res.body + handle_response(res) end def create_search_key() do diff --git a/lib/small_sdk/typesense_admin.ex b/lib/small_sdk/typesense_admin.ex deleted file mode 100644 index 1b5fd6d..0000000 --- a/lib/small_sdk/typesense_admin.ex +++ /dev/null @@ -1,66 +0,0 @@ -defmodule SmallSdk.TypesenseAdmin do - @photos_schema %{ - "name" => "photos", - "fields" => [ - # image: base64 encoded string - %{"name" => "image", "type" => "image", "store" => false}, - %{ - "name" => "image_embedding", - "type" => "float[]", - "embed" => %{ - "from" => ["image"], - "model_config" => %{ - "model_name" => "ts/clip-vit-b-p32" - } - } - }, - %{"name" => "caption", "type" => "string", "optional" => true, "facet" => false}, - # "telegram:///" - %{"name" => "url", "type" => "string"}, - # chat.id -> string - %{"name" => "belongs_to_id", "type" => "string"}, - # unix timestamp - %{"name" => "inserted_at", "type" => "int64"} - ], - "default_sorting_field" => "inserted_at" - } - - def reset() do - delete_collection!(@photos_schema["name"]) - create_collection!(@photos_schema) - end - - def create_collection!(schema) do - req = build_request("/collections") - {:ok, res} = Req.post(req, json: schema) - - res.body - end - - def delete_collection!(collection_name) do - req = build_request("/collections/#{collection_name}") - {:ok, res} = Req.delete(req) - - res.body - end - - defp get_env() do - url = Application.fetch_env!(:save_it, :typesense_url) - api_key = Application.fetch_env!(:save_it, :typesense_api_key) - - {url, api_key} - end - - defp build_request(path) do - {url, api_key} = get_env() - - Req.new( - base_url: url, - url: path, - headers: [ - {"Content-Type", "application/json"}, - {"X-TYPESENSE-API-KEY", api_key} - ] - ) - end -end diff --git a/priv/typesense/2024-10-24_create_photos_collection.exs b/priv/typesense/2024-10-24_create_photos_collection.exs new file mode 100644 index 0000000..aa0aff4 --- /dev/null +++ b/priv/typesense/2024-10-24_create_photos_collection.exs @@ -0,0 +1 @@ +Migration.Typesense.Photo.create_collection!() diff --git a/priv/typesense/reset.exs b/priv/typesense/reset.exs index 00773d2..2c5c2d2 100644 --- a/priv/typesense/reset.exs +++ b/priv/typesense/reset.exs @@ -1,3 +1,3 @@ # mix run priv/typesense/reset.exs -SmallSdk.TypesenseAdmin.reset() +Migration.Typesense.Photo.reset!() diff --git a/zeabur/template.yaml b/zeabur/template.yaml new file mode 100644 index 0000000..d0c3d30 --- /dev/null +++ b/zeabur/template.yaml @@ -0,0 +1,83 @@ +# yaml-language-server: $schema=https://schema.zeabur.app/template.json +apiVersion: zeabur.com/v1 +kind: Template +metadata: + name: SaveIt +spec: + description: SaveIt is a telegram bot that helps you save photos. + icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + tags: + - Bot + - Telegram + - Photos Storage + - Search Engine + coverImage: + + readme: |- + # SaveIt + A telegram bot can Save photos and Search photos + + ## Features + - Save photos via a link + - Search photos using semantic search + - Find similar photos by photo + + ## Learn more + https://github.com/ThaddeusJiang/save_it + + + + services: + - name: save_it + icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + template: GIT + spec: + source: + source: GITHUB + repo: 831394769 + branch: main + rootDirectory: / + env: + TELEGRAM_BOT_TOKEN: + default: "" + expose: false + TYPESENSE_URL: + default: http://typesense.zeabur.internal:8108 + expose: false + TYPESENSE_API_KEY: + default: ${TYPESENSE_API_KEY} + expose: false + GOOGLE_OAUTH_CLIENT_ID: + default: "" + expose: false + GOOGLE_OAUTH_CLIENT_SECRET: + default: "" + expose: false + configs: [] + - name: typesense + icon: https://typesense.org/docs/images/typesense_logo.svg + template: PREBUILT_V2 + spec: + source: + image: typesense/typesense:27.1 + ports: + - id: web + port: 8108 + type: HTTP + volumes: + - id: data + dir: /data + instructions: + - type: PASSWORD + title: Typesense API Key + content: ${TYPESENSE_API_KEY} + category: Credentials + env: + TYPESENSE_API_KEY: + default: ${PASSWORD} + expose: true + TYPESENSE_DATA_DIR: + default: /data + expose: false + + configs: [] From 5d0a1fc7b5c86c7150005134beaa868f94ae0aaa Mon Sep 17 00:00:00 2001 From: TJ Date: Mon, 28 Oct 2024 14:12:37 +0900 Subject: [PATCH 02/10] refactor, show photo's caption --- lib/save_it/bot.ex | 2 +- lib/save_it/photo_service.ex | 36 +++++++++++++++- lib/small_sdk/typesense.ex | 83 +++++++++++++++++------------------- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index 3afba0f..801f8d8 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -343,7 +343,7 @@ defmodule SaveIt.Bot do %ExGram.Model.InputMediaPhoto{ type: "photo", media: photo["file_id"], - caption: "Found photos", + caption: photo["caption"], show_caption_above_media: true } end) diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index 931d8b8..e476a42 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -62,9 +62,9 @@ defmodule SaveIt.PhotoService do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) + data = handle_response(res) - # FIXME: nil check - res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + data["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end def search_similar_photos!(photo_id, opts \\ []) when is_binary(photo_id) do @@ -110,4 +110,36 @@ defmodule SaveIt.PhotoService do ] ) end + + defp handle_response(res) do + case res do + %Req.Response{status: 200} -> + res.body + + %Req.Response{status: 201} -> + res.body + + %Req.Response{status: 400} -> + Logger.error("Bad Request: #{inspect(res.body)}") + raise "Bad Request" + + %Req.Response{status: 401} -> + raise "Unauthorized" + + %Req.Response{status: 404} -> + nil + + %Req.Response{status: 409} -> + raise "Conflict" + + %Req.Response{status: 422} -> + raise "Unprocessable Entity" + + %Req.Response{status: 503} -> + raise "Service Unavailable" + + _ -> + raise "Unknown error" + end + end end diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index ece7bad..4c784d3 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -1,38 +1,6 @@ defmodule SmallSdk.Typesense do require Logger - def handle_response(res) do - case res do - %Req.Response{status: 200} -> - res.body - - %Req.Response{status: 201} -> - res.body - - %Req.Response{status: 400} -> - Logger.error("Bad Request: #{inspect(res.body)}") - raise "Bad Request" - - %Req.Response{status: 401} -> - raise "Unauthorized" - - %Req.Response{status: 404} -> - nil - - %Req.Response{status: 409} -> - raise "Conflict" - - %Req.Response{status: 422} -> - raise "Unprocessable Entity" - - %Req.Response{status: 503} -> - raise "Service Unavailable" - - _ -> - raise "Unknown error" - end - end - def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") {:ok, res} = Req.post(req, json: document) @@ -45,18 +13,15 @@ defmodule SmallSdk.Typesense do query_by = Keyword.get(opts, :query_by, "") filter_by = Keyword.get(opts, :filter_by, "") - req = build_request("/collections/#{collection_name}/documents/search") - - {:ok, res} = - Req.get(req, - params: %{ - q: q, - query_by: query_by, - filter_by: filter_by, - exclude_fields: "image_embedding" - } - ) + query_params = %{ + q: q, + query_by: query_by, + filter_by: filter_by, + exclude_fields: "image_embedding" + } + req = build_request("/collections/#{collection_name}/documents/search") + {:ok, res} = Req.get(req, params: query_params) data = handle_response(res) data["hits"] |> Enum.map(&Map.get(&1, "document")) @@ -114,4 +79,36 @@ defmodule SmallSdk.Typesense do ] ) end + + defp handle_response(res) do + case res do + %Req.Response{status: 200} -> + res.body + + %Req.Response{status: 201} -> + res.body + + %Req.Response{status: 400} -> + Logger.error("Bad Request: #{inspect(res.body)}") + raise "Bad Request" + + %Req.Response{status: 401} -> + raise "Unauthorized" + + %Req.Response{status: 404} -> + nil + + %Req.Response{status: 409} -> + raise "Conflict" + + %Req.Response{status: 422} -> + raise "Unprocessable Entity" + + %Req.Response{status: 503} -> + raise "Service Unavailable" + + _ -> + raise "Unknown error" + end + end end From 8f589d3e2883db4da97b2b63e266c08f0d18ed25 Mon Sep 17 00:00:00 2001 From: TJ Date: Mon, 28 Oct 2024 14:47:23 +0900 Subject: [PATCH 03/10] refactor, typesense sdk delete_document --- lib/save_it/photo_service.ex | 4 ++-- lib/small_sdk/typesense.ex | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index e476a42..14d4e30 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -86,9 +86,9 @@ defmodule SaveIt.PhotoService do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) + data = handle_response(res) - # FIXME: nil check - res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + data["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end defp get_env() do diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 4c784d3..c61d635 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -41,6 +41,13 @@ defmodule SmallSdk.Typesense do handle_response(res) end + def delete_document!(collection_name, document_id) do + req = build_request("/collections/#{collection_name}/documents/#{document_id}") + {:ok, res} = Req.delete(req) + + handle_response(res) + end + def create_search_key() do {url, _} = get_env() req = build_request("/keys") From 7bc2693fd2102e760337edcc425f5afae35dd03f Mon Sep 17 00:00:00 2001 From: TJ Date: Mon, 28 Oct 2024 15:10:48 +0900 Subject: [PATCH 04/10] refactor --- lib/save_it/photo_service.ex | 65 +++++++++++++++++++++++++----------- lib/small_sdk/typesense.ex | 29 ++++++++-------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index 14d4e30..e933249 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -23,16 +23,22 @@ defmodule SaveIt.PhotoService do end def update_photo_caption!(file_id, belongs_to_id, caption) do - get_photo!(file_id, belongs_to_id) - |> Map.put("caption", caption) - |> update_photo() + case get_photo(file_id, belongs_to_id) do + photo when is_map(photo) -> + photo + |> Map.put("caption", caption) + |> update_photo() + + nil -> + raise "Photo not found for file_id #{file_id} and belongs_to_id #{belongs_to_id}" + end end def get_photo(photo_id) do Typesense.get_document("photos", photo_id) end - def get_photo!(file_id, belongs_to_id) do + def get_photo(file_id, belongs_to_id) do case Typesense.search_documents!("photos", q: "*", query_by: "caption", @@ -64,7 +70,16 @@ defmodule SaveIt.PhotoService do {:ok, res} = Req.post(req, json: req_body) data = handle_response(res) - data["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + results = data["results"] + + if results != [] do + results + |> hd() + |> Map.get("hits") + |> Enum.map(&Map.get(&1, "document")) + else + [] + end end def search_similar_photos!(photo_id, opts \\ []) when is_binary(photo_id) do @@ -88,7 +103,16 @@ defmodule SaveIt.PhotoService do {:ok, res} = Req.post(req, json: req_body) data = handle_response(res) - data["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + results = data["results"] + + if results != [] do + results + |> hd() + |> Map.get("hits") + |> Enum.map(&Map.get(&1, "document")) + else + [] + end end defp get_env() do @@ -111,35 +135,36 @@ defmodule SaveIt.PhotoService do ) end - defp handle_response(res) do - case res do - %Req.Response{status: 200} -> - res.body + defp handle_response(%Req.Response{status: status, body: body}) do + case status do + 200 -> + body - %Req.Response{status: 201} -> - res.body + 201 -> + body - %Req.Response{status: 400} -> - Logger.error("Bad Request: #{inspect(res.body)}") + 400 -> + Logger.warning("Bad Request: #{inspect(body)}") raise "Bad Request" - %Req.Response{status: 401} -> + 401 -> raise "Unauthorized" - %Req.Response{status: 404} -> + 404 -> nil - %Req.Response{status: 409} -> + 409 -> raise "Conflict" - %Req.Response{status: 422} -> + 422 -> raise "Unprocessable Entity" - %Req.Response{status: 503} -> + 503 -> raise "Service Unavailable" _ -> - raise "Unknown error" + Logger.error("Unhandled status code #{status}: #{inspect(body)}") + raise "Unknown error: #{status}" end end end diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index c61d635..be782e7 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -87,35 +87,36 @@ defmodule SmallSdk.Typesense do ) end - defp handle_response(res) do - case res do - %Req.Response{status: 200} -> - res.body + defp handle_response(%Req.Response{status: status, body: body}) do + case status do + 200 -> + body - %Req.Response{status: 201} -> - res.body + 201 -> + body - %Req.Response{status: 400} -> - Logger.error("Bad Request: #{inspect(res.body)}") + 400 -> + Logger.warning("Bad Request: #{inspect(body)}") raise "Bad Request" - %Req.Response{status: 401} -> + 401 -> raise "Unauthorized" - %Req.Response{status: 404} -> + 404 -> nil - %Req.Response{status: 409} -> + 409 -> raise "Conflict" - %Req.Response{status: 422} -> + 422 -> raise "Unprocessable Entity" - %Req.Response{status: 503} -> + 503 -> raise "Service Unavailable" _ -> - raise "Unknown error" + Logger.error("Unhandled status code #{status}: #{inspect(body)}") + raise "Unknown error: #{status}" end end end From 6a97220ff0475ae1f36d3d99067307a9cae2ffbc Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 11:14:09 +0900 Subject: [PATCH 05/10] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20handle=5Frespo?= =?UTF-8?q?nse=20and=20handle=5Fresponse!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.template | 7 ++++++ .gitignore | 2 ++ README.md | 40 ++++++---------------------------- lib/migration/typesense.ex | 17 ++++++--------- lib/save_it/photo_service.ex | 42 ++++-------------------------------- lib/small_sdk/typesense.ex | 39 ++++++++++++++++++++++----------- start.sh | 5 +++++ 7 files changed, 57 insertions(+), 95 deletions(-) create mode 100644 .env.template create mode 100755 start.sh diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..43b0d21 --- /dev/null +++ b/.env.template @@ -0,0 +1,7 @@ +TELEGRAM_BOT_TOKEN= + +TYPESENSE_URL= +TYPESENSE_API_KEY= + +GOOGLE_OAUTH_CLIENT_ID= +GOOGLE_OAUTH_CLIENT_SECRET= diff --git a/.gitignore b/.gitignore index 1c7fa49..1cad362 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ _dev* _stag* _prod* nohup.out + +.env diff --git a/README.md b/README.md index 72bd025..feb5e88 100644 --- a/README.md +++ b/README.md @@ -50,50 +50,22 @@ https://t.me/save_it_playground ## Development ```sh -# install +# Install mix deps.get ``` ```sh -# start typesense +# Start typesense docker compose up ``` ```sh -# modify env -export TELEGRAM_BOT_TOKEN= -export TYPESENSE_URL= -export TYPESENSE_API_KEY= - -iex -S mix run --no-halt -``` - -Pro Tips: create shell script for fast run app - -1. touch start.sh - -```sh -#!/bin/sh - -export TELEGRAM_BOT_TOKEN= - -export TYPESENSE_URL= -export TYPESENSE_API_KEY= - -export GOOGLE_OAUTH_CLIENT_ID= -export GOOGLE_OAUTH_CLIENT_SECRET= - -iex -S mix run --no-halt -``` - -2. execute permission - -```sh -chmod +x start.sh +# Create .env file +cp .env.template .env ``` -3. run +Run app ```sh -./start.sh +sh start.sh ``` diff --git a/lib/migration/typesense.ex b/lib/migration/typesense.ex index 8bea87e..cf55841 100644 --- a/lib/migration/typesense.ex +++ b/lib/migration/typesense.ex @@ -1,23 +1,18 @@ defmodule Migration.Typesense do + alias SmallSdk.Typesense + def create_collection!(schema) do req = build_request("/collections") - {:ok, res} = Req.post(req, json: schema) + res = Req.post!(req, json: schema) - res.body + Typesense.handle_response!(res) end def delete_collection!(collection_name) do req = build_request("/collections/#{collection_name}") - {:ok, res} = Req.delete(req) - - res.body - end - - def list_collections() do - req = build_request("/collections") - {:ok, res} = Req.get(req) + res = Req.delete!(req) - res.body + Typesense.handle_response!(res) end defp get_env() do diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index e933249..6c6efc9 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -41,7 +41,6 @@ defmodule SaveIt.PhotoService do def get_photo(file_id, belongs_to_id) do case Typesense.search_documents!("photos", q: "*", - query_by: "caption", filter_by: "file_id:=#{file_id} && belongs_to_id:=#{belongs_to_id}" ) do [photo | _] -> photo @@ -67,8 +66,8 @@ defmodule SaveIt.PhotoService do } req = build_request("/multi_search") - {:ok, res} = Req.post(req, json: req_body) - data = handle_response(res) + res = Req.post(req, json: req_body) + data = Typesense.handle_response(res) results = data["results"] @@ -100,8 +99,8 @@ defmodule SaveIt.PhotoService do } req = build_request("/multi_search") - {:ok, res} = Req.post(req, json: req_body) - data = handle_response(res) + res = Req.post(req, json: req_body) + data = Typesense.handle_response(res) results = data["results"] @@ -134,37 +133,4 @@ defmodule SaveIt.PhotoService do ] ) end - - defp handle_response(%Req.Response{status: status, body: body}) do - case status do - 200 -> - body - - 201 -> - body - - 400 -> - Logger.warning("Bad Request: #{inspect(body)}") - raise "Bad Request" - - 401 -> - raise "Unauthorized" - - 404 -> - nil - - 409 -> - raise "Conflict" - - 422 -> - raise "Unprocessable Entity" - - 503 -> - raise "Service Unavailable" - - _ -> - Logger.error("Unhandled status code #{status}: #{inspect(body)}") - raise "Unknown error: #{status}" - end - end end diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index be782e7..3efb5ff 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -3,7 +3,7 @@ defmodule SmallSdk.Typesense do def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") - {:ok, res} = Req.post(req, json: document) + res = Req.post(req, json: document) handle_response(res) end @@ -21,7 +21,7 @@ defmodule SmallSdk.Typesense do } req = build_request("/collections/#{collection_name}/documents/search") - {:ok, res} = Req.get(req, params: query_params) + res = Req.get(req, params: query_params) data = handle_response(res) data["hits"] |> Enum.map(&Map.get(&1, "document")) @@ -29,21 +29,21 @@ defmodule SmallSdk.Typesense do def get_document(collection_name, document_id) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") - {:ok, res} = Req.get(req) + res = Req.get(req) handle_response(res) end def update_document!(collection_name, document_id, update_input) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") - {:ok, res} = Req.patch(req, json: update_input) + res = Req.patch(req, json: update_input) handle_response(res) end def delete_document!(collection_name, document_id) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") - {:ok, res} = Req.delete(req) + res = Req.delete(req) handle_response(res) end @@ -52,7 +52,7 @@ defmodule SmallSdk.Typesense do {url, _} = get_env() req = build_request("/keys") - {:ok, res} = + res = Req.post(req, json: %{ "description" => "Search-only photos key", @@ -61,9 +61,11 @@ defmodule SmallSdk.Typesense do } ) + data = handle_response(res) + %{ url: url, - api_key: res.body["value"] + api_key: data["value"] } end @@ -87,12 +89,9 @@ defmodule SmallSdk.Typesense do ) end - defp handle_response(%Req.Response{status: status, body: body}) do + def handle_response({:ok, %{status: status, body: body}}) do case status do - 200 -> - body - - 201 -> + status when status in 200..209 -> body 400 -> @@ -119,4 +118,20 @@ defmodule SmallSdk.Typesense do raise "Unknown error: #{status}" end end + + def handle_response({:error, reason}) do + Logger.error("Request failed: #{inspect(reason)}") + raise "Request failed" + end + + def handle_response!(%{status: status, body: body}) do + case status do + status when status in 200..209 -> + body + + status -> + Logger.warning("Request failed with status #{status}: #{inspect(body)}") + raise "Request failed with status #{status}" + end + end end diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..d65c5c4 --- /dev/null +++ b/start.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +source .env + +iex -S mix run --no-halt From d268df3bc7b687743a99e864d30b1059392ebbe4 Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 11:18:12 +0900 Subject: [PATCH 06/10] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20get=5Fenv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/small_sdk/telegram.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/small_sdk/telegram.ex b/lib/small_sdk/telegram.ex index ad4586c..6dcb760 100644 --- a/lib/small_sdk/telegram.ex +++ b/lib/small_sdk/telegram.ex @@ -6,7 +6,8 @@ defmodule SmallSdk.Telegram do plug(Tesla.Middleware.BaseUrl, "https://api.telegram.org") def download_file_content(file_path) when is_binary(file_path) do - url = "/file/bot#{Application.fetch_env!(:save_it, :telegram_bot_token)}/#{file_path}" + bot_token = get_env() + url = "/file/bot#{bot_token}/#{file_path}" case get(url) do {:ok, response} -> @@ -23,4 +24,10 @@ defmodule SmallSdk.Telegram do {:error, error} -> raise "Error: #{inspect(error)}" end end + + defp get_env() do + bot_token = Application.fetch_env!(:save_it, :telegram_bot_token) + + bot_token + end end From ac0f40236fc0509bf394ac366e06f1429249e5f0 Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 12:46:35 +0900 Subject: [PATCH 07/10] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20UrlHelper=20va?= =?UTF-8?q?lidate=5Furl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 -- README.md | 10 +++------- lib/migration/typesense.ex | 5 ++++- lib/save_it/photo_service.ex | 4 +++- lib/small_sdk/typesense.ex | 4 +++- lib/tj/url_helper.ex | 11 +++++++++++ 6 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 lib/tj/url_helper.ex diff --git a/.gitignore b/.gitignore index 1cad362..1c7fa49 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,3 @@ _dev* _stag* _prod* nohup.out - -.env diff --git a/README.md b/README.md index feb5e88..753d71e 100644 --- a/README.md +++ b/README.md @@ -60,12 +60,8 @@ docker compose up ``` ```sh -# Create .env file -cp .env.template .env -``` - -Run app +# Run +export TELEGRAM_BOT_TOKEN= -```sh -sh start.sh +iex -S mix run --no-halt ``` diff --git a/lib/migration/typesense.ex b/lib/migration/typesense.ex index cf55841..347064c 100644 --- a/lib/migration/typesense.ex +++ b/lib/migration/typesense.ex @@ -1,6 +1,8 @@ defmodule Migration.Typesense do alias SmallSdk.Typesense + import Tj.UrlHelper, only: [validate_url!: 1] + def create_collection!(schema) do req = build_request("/collections") res = Req.post!(req, json: schema) @@ -16,7 +18,8 @@ defmodule Migration.Typesense do end defp get_env() do - url = Application.fetch_env!(:save_it, :typesense_url) + url = Application.fetch_env!(:save_it, :typesense_url) |> validate_url!() + api_key = Application.fetch_env!(:save_it, :typesense_api_key) {url, api_key} diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index 6c6efc9..42ea6c0 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -2,6 +2,8 @@ defmodule SaveIt.PhotoService do require Logger alias SmallSdk.Typesense + import Tj.UrlHelper, only: [validate_url!: 1] + def create_photo!( %{ belongs_to_id: belongs_to_id @@ -115,7 +117,7 @@ defmodule SaveIt.PhotoService do end defp get_env() do - url = Application.fetch_env!(:save_it, :typesense_url) + url = Application.fetch_env!(:save_it, :typesense_url) |> validate_url!() api_key = Application.fetch_env!(:save_it, :typesense_api_key) {url, api_key} diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 3efb5ff..3517361 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -1,6 +1,8 @@ defmodule SmallSdk.Typesense do require Logger + import Tj.UrlHelper, only: [validate_url!: 1] + def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") res = Req.post(req, json: document) @@ -70,7 +72,7 @@ defmodule SmallSdk.Typesense do end defp get_env() do - url = Application.fetch_env!(:save_it, :typesense_url) + url = Application.fetch_env!(:save_it, :typesense_url) |> validate_url!() api_key = Application.fetch_env!(:save_it, :typesense_api_key) {url, api_key} diff --git a/lib/tj/url_helper.ex b/lib/tj/url_helper.ex new file mode 100644 index 0000000..3fab418 --- /dev/null +++ b/lib/tj/url_helper.ex @@ -0,0 +1,11 @@ +defmodule Tj.UrlHelper do + def validate_url!(url) do + uri = URI.parse(url) + + if uri.scheme in ["http", "https"] and uri.host do + url + else + raise ArgumentError, "Invalid URL: #{url}" + end + end +end From 0545ed5856fbf08bca5a139a5012307eb0d52bb7 Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 14:21:19 +0900 Subject: [PATCH 08/10] =?UTF-8?q?chore:=20=F0=9F=A4=96=20typesense=20migra?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SaveIt.Migration.Typesense.Photo.migrate_photos_2024_10_29!() --- docs/dev-logs/2024-10-25.md | 2 - lib/migration/typesense/photo.ex | 35 --------- lib/{ => save_it}/migration/typesense.ex | 9 ++- lib/save_it/migration/typesense/photo.ex | 78 +++++++++++++++++++ lib/small_sdk/typesense.ex | 18 +++++ .../2024-10-24_create_photos_collection.exs | 2 +- .../2024-10-29_photos_url_to_file_id.ex | 1 + priv/typesense/reset.exs | 2 +- test/save_it_test.exs | 2 + 9 files changed, 109 insertions(+), 40 deletions(-) delete mode 100644 lib/migration/typesense/photo.ex rename lib/{ => save_it}/migration/typesense.ex (78%) create mode 100644 lib/save_it/migration/typesense/photo.ex create mode 100644 priv/typesense/2024-10-29_photos_url_to_file_id.ex diff --git a/docs/dev-logs/2024-10-25.md b/docs/dev-logs/2024-10-25.md index ddc6e58..3b17b83 100644 --- a/docs/dev-logs/2024-10-25.md +++ b/docs/dev-logs/2024-10-25.md @@ -4,6 +4,4 @@ ```elixir ** (MatchError) no match of right hand side value: {:error, %Req.TransportError{reason: :timeout}} - (save_it 0.2.0-rc.1) lib/migration/typesense.ex:11: Migration.Typesense.create_collection!/1 - priv/typesense/reset.exs:3: (file) ``` diff --git a/lib/migration/typesense/photo.ex b/lib/migration/typesense/photo.ex deleted file mode 100644 index ebe686b..0000000 --- a/lib/migration/typesense/photo.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule Migration.Typesense.Photo do - alias Migration.Typesense - - @photos_schema %{ - "name" => "photos", - "fields" => [ - # image: base64 encoded string - %{"name" => "image", "type" => "image", "store" => false}, - %{ - "name" => "image_embedding", - "type" => "float[]", - "embed" => %{ - "from" => ["image"], - "model_config" => %{ - "model_name" => "ts/clip-vit-b-p32" - } - } - }, - %{"name" => "caption", "type" => "string", "optional" => true}, - %{"name" => "file_id", "type" => "string"}, - %{"name" => "belongs_to_id", "type" => "string"}, - %{"name" => "inserted_at", "type" => "int64"} - ], - "default_sorting_field" => "inserted_at" - } - - def create_collection!() do - Typesense.create_collection!(@photos_schema) - end - - def reset!() do - Typesense.delete_collection!(@photos_schema["name"]) - Typesense.create_collection!(@photos_schema) - end -end diff --git a/lib/migration/typesense.ex b/lib/save_it/migration/typesense.ex similarity index 78% rename from lib/migration/typesense.ex rename to lib/save_it/migration/typesense.ex index 347064c..747a13a 100644 --- a/lib/migration/typesense.ex +++ b/lib/save_it/migration/typesense.ex @@ -1,4 +1,4 @@ -defmodule Migration.Typesense do +defmodule SaveIt.Migration.Typesense do alias SmallSdk.Typesense import Tj.UrlHelper, only: [validate_url!: 1] @@ -10,6 +10,13 @@ defmodule Migration.Typesense do Typesense.handle_response!(res) end + def update_collection!(collection_name, schema) do + req = build_request("/collections/#{collection_name}") + res = Req.patch!(req, json: schema) + + Typesense.handle_response!(res) + end + def delete_collection!(collection_name) do req = build_request("/collections/#{collection_name}") res = Req.delete!(req) diff --git a/lib/save_it/migration/typesense/photo.ex b/lib/save_it/migration/typesense/photo.ex new file mode 100644 index 0000000..892d815 --- /dev/null +++ b/lib/save_it/migration/typesense/photo.ex @@ -0,0 +1,78 @@ +defmodule SaveIt.Migration.Typesense.Photo do + require Logger + alias SaveIt.Migration.Typesense + + alias SmallSdk.Typesense, as: TypesenseDataClient + + @collection_name "photos" + + def create_photos_20241024!() do + schema = %{ + "name" => @collection_name, + "fields" => [ + %{"name" => "image", "type" => "image", "store" => false}, + %{ + "name" => "image_embedding", + "type" => "float[]", + "embed" => %{ + "from" => ["image"], + "model_config" => %{ + "model_name" => "ts/clip-vit-b-p32" + } + } + }, + %{"name" => "caption", "type" => "string", "optional" => true}, + %{"name" => "url", "type" => "string"}, + %{"name" => "belongs_to_id", "type" => "string"}, + %{"name" => "inserted_at", "type" => "int64"} + ], + "default_sorting_field" => "inserted_at" + } + + Typesense.create_collection!(schema) + end + + def migrate_photos_2024_10_29!() do + Logger.info("updating photos collection") + + Typesense.update_collection!(@collection_name, %{ + "fields" => [ + %{"name" => "file_id", "type" => "string"} + ] + }) + + Logger.info("migrating photos data") + + TypesenseDataClient.list_documents(@collection_name, per_page: 10000) + |> Enum.each(fn doc -> + id = doc["id"] + + file_id = + doc["url"] + |> String.split("/") + |> List.last() + + TypesenseDataClient.update_document!(@collection_name, id, %{ + "file_id" => file_id + }) + end) + + Logger.info("dropping url field") + + Typesense.update_collection!(@collection_name, %{ + "fields" => [ + %{"name" => "url", "drop" => true} + ] + }) + end + + def drop_photos!() do + Typesense.delete_collection!(@collection_name) + end + + def reset!() do + drop_photos!() + create_photos_20241024!() + migrate_photos_2024_10_29!() + end +end diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 3517361..dcc2263 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -10,6 +10,24 @@ defmodule SmallSdk.Typesense do handle_response(res) end + def list_documents(collection_name, opts \\ []) do + page = Keyword.get(opts, :page, 1) + per_page = Keyword.get(opts, :per_page, 10) + req = build_request("/collections/#{collection_name}/documents") + + res = + Req.get(req, + params: %{ + q: "*", + query_by: "", + page: page, + per_page: per_page + } + ) + + handle_response(res) || [] + end + def search_documents!(collection_name, opts) do q = Keyword.get(opts, :q, "*") query_by = Keyword.get(opts, :query_by, "") diff --git a/priv/typesense/2024-10-24_create_photos_collection.exs b/priv/typesense/2024-10-24_create_photos_collection.exs index aa0aff4..d74f9eb 100644 --- a/priv/typesense/2024-10-24_create_photos_collection.exs +++ b/priv/typesense/2024-10-24_create_photos_collection.exs @@ -1 +1 @@ -Migration.Typesense.Photo.create_collection!() +SaveIt.Migration.Typesense.Photo.create_photos_20241024!() diff --git a/priv/typesense/2024-10-29_photos_url_to_file_id.ex b/priv/typesense/2024-10-29_photos_url_to_file_id.ex new file mode 100644 index 0000000..cbf4533 --- /dev/null +++ b/priv/typesense/2024-10-29_photos_url_to_file_id.ex @@ -0,0 +1 @@ +SaveIt.Migration.Typesense.Photo.migrate_photos_2024_10_29!() diff --git a/priv/typesense/reset.exs b/priv/typesense/reset.exs index 2c5c2d2..881cd8c 100644 --- a/priv/typesense/reset.exs +++ b/priv/typesense/reset.exs @@ -1,3 +1,3 @@ # mix run priv/typesense/reset.exs -Migration.Typesense.Photo.reset!() +SaveIt.Migration.Typesense.Photo.reset!() diff --git a/test/save_it_test.exs b/test/save_it_test.exs index 20cc620..227ebc8 100644 --- a/test/save_it_test.exs +++ b/test/save_it_test.exs @@ -2,6 +2,8 @@ defmodule SaveItTest do use ExUnit.Case doctest SaveIt + doctest SaveIt.Migration.Typesense.Photo + test "greets the world" do assert SaveIt.hello() == :world end From c35c1747d5381a2e4c9c78c01cbc42660a941653 Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 14:46:31 +0900 Subject: [PATCH 09/10] refactor --- .env.template | 7 ------- lib/save_it/migration/typesense/photo.ex | 4 ++-- priv/typesense/2024-10-29_photos_url_to_file_id.ex | 2 +- start.sh | 5 ----- 4 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 .env.template delete mode 100755 start.sh diff --git a/.env.template b/.env.template deleted file mode 100644 index 43b0d21..0000000 --- a/.env.template +++ /dev/null @@ -1,7 +0,0 @@ -TELEGRAM_BOT_TOKEN= - -TYPESENSE_URL= -TYPESENSE_API_KEY= - -GOOGLE_OAUTH_CLIENT_ID= -GOOGLE_OAUTH_CLIENT_SECRET= diff --git a/lib/save_it/migration/typesense/photo.ex b/lib/save_it/migration/typesense/photo.ex index 892d815..65f186f 100644 --- a/lib/save_it/migration/typesense/photo.ex +++ b/lib/save_it/migration/typesense/photo.ex @@ -32,7 +32,7 @@ defmodule SaveIt.Migration.Typesense.Photo do Typesense.create_collection!(schema) end - def migrate_photos_2024_10_29!() do + def migrate_photos_20241029!() do Logger.info("updating photos collection") Typesense.update_collection!(@collection_name, %{ @@ -73,6 +73,6 @@ defmodule SaveIt.Migration.Typesense.Photo do def reset!() do drop_photos!() create_photos_20241024!() - migrate_photos_2024_10_29!() + migrate_photos_20241029!() end end diff --git a/priv/typesense/2024-10-29_photos_url_to_file_id.ex b/priv/typesense/2024-10-29_photos_url_to_file_id.ex index cbf4533..1358964 100644 --- a/priv/typesense/2024-10-29_photos_url_to_file_id.ex +++ b/priv/typesense/2024-10-29_photos_url_to_file_id.ex @@ -1 +1 @@ -SaveIt.Migration.Typesense.Photo.migrate_photos_2024_10_29!() +SaveIt.Migration.Typesense.Photo.migrate_photos_20241029!() diff --git a/start.sh b/start.sh deleted file mode 100755 index d65c5c4..0000000 --- a/start.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -source .env - -iex -S mix run --no-halt From bd0b6370c71cc3a5044201264a2688615432b488 Mon Sep 17 00:00:00 2001 From: TJ Date: Tue, 29 Oct 2024 15:11:32 +0900 Subject: [PATCH 10/10] fix: typesense list_documents --- lib/save_it/migration/typesense.ex | 7 +++++ lib/save_it/migration/typesense/photo.ex | 38 ++++++++++++------------ lib/small_sdk/typesense.ex | 8 +++-- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/lib/save_it/migration/typesense.ex b/lib/save_it/migration/typesense.ex index 747a13a..04232fb 100644 --- a/lib/save_it/migration/typesense.ex +++ b/lib/save_it/migration/typesense.ex @@ -17,6 +17,13 @@ defmodule SaveIt.Migration.Typesense do Typesense.handle_response!(res) end + def list_collections!() do + req = build_request("/collections") + res = Req.get!(req) + + Typesense.handle_response!(res) + end + def delete_collection!(collection_name) do req = build_request("/collections/#{collection_name}") res = Req.delete!(req) diff --git a/lib/save_it/migration/typesense/photo.ex b/lib/save_it/migration/typesense/photo.ex index 65f186f..a401728 100644 --- a/lib/save_it/migration/typesense/photo.ex +++ b/lib/save_it/migration/typesense/photo.ex @@ -37,33 +37,33 @@ defmodule SaveIt.Migration.Typesense.Photo do Typesense.update_collection!(@collection_name, %{ "fields" => [ - %{"name" => "file_id", "type" => "string"} + %{"name" => "file_id", "type" => "string", "optional" => true} ] }) + end - Logger.info("migrating photos data") + def migrate_photos_data_20241029 do + Logger.info("migrating photos documents") - TypesenseDataClient.list_documents(@collection_name, per_page: 10000) - |> Enum.each(fn doc -> - id = doc["id"] + docs = + TypesenseDataClient.list_documents(@collection_name, per_page: 200) - file_id = - doc["url"] - |> String.split("/") - |> List.last() + count = + Enum.map(docs, fn doc -> + id = doc["id"] - TypesenseDataClient.update_document!(@collection_name, id, %{ - "file_id" => file_id - }) - end) + file_id = + doc["url"] + |> String.split("/") + |> List.last() - Logger.info("dropping url field") + TypesenseDataClient.update_document!(@collection_name, id, %{ + "file_id" => file_id + }) + end) + |> Enum.count() - Typesense.update_collection!(@collection_name, %{ - "fields" => [ - %{"name" => "url", "drop" => true} - ] - }) + Logger.info("migrated #{count} photos") end def drop_photos!() do diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index dcc2263..004a24f 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -13,19 +13,21 @@ defmodule SmallSdk.Typesense do def list_documents(collection_name, opts \\ []) do page = Keyword.get(opts, :page, 1) per_page = Keyword.get(opts, :per_page, 10) - req = build_request("/collections/#{collection_name}/documents") + req = build_request("/collections/#{collection_name}/documents/search") res = Req.get(req, params: %{ q: "*", - query_by: "", + query_by: "caption", page: page, per_page: per_page } ) - handle_response(res) || [] + data = handle_response(res) + + data["hits"] |> Enum.map(&Map.get(&1, "document")) end def search_documents!(collection_name, opts) do