Ajout client pour l'API de conversion EnRoute

apps/transport/lib/converters/gtfs_to_netex_enroute.ex

@@ -0,0 +1,85 @@
+defmodule Transport.Converters.GTFSToNeTExEnRoute do
+  @moduledoc """
+  A client of for the EnRoute's Conversions API.
+  Documentation: https://documenter.getpostman.com/view/9203997/SzmfXwrp
+  """
+  require Logger
+  @base_url "https://chouette-convert.enroute.mobi/api/conversions"
+
+  @doc """
+  Creates a GTFS to NeTEx conversion job on the EnRoute's API.
+  Returns the job UUID as a string.
+  """
+  @spec create_gtfs_to_netex_conversion(binary(), binary()) :: binary()
+  def create_gtfs_to_netex_conversion(filepath, profile \\ "french") do
+    Logger.info("Converting #{filepath} from GTFS to NeTEx with the #{profile} profile")
+
+    form =
+      {:multipart,
+       [
+         {"type", "gtfs-netex"},
+         {"options[profile]", profile},
+         {:file, filepath, {"form-data", [{:name, "file"}, {:filename, Path.basename(filepath)}]}, []}
+       ]}
+
+    %HTTPoison.Response{status_code: 201, body: body} = http_client().post!(base_url(), form, auth_headers())
+    body |> Jason.decode!() |> Map.fetch!("id")
+  end
+
+  @doc """
+  Polls the API to know if the conversion is finished.
+  """
+  @spec get_conversion(binary()) :: :error | {:success | :pending | :failed, map()}
+  def get_conversion(uuid) do
+    case http_client().get(Path.join(base_url(), uuid), auth_headers()) do
+      {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
+        json = Jason.decode!(body)
+
+        case Map.fetch!(json, "status") do
+          "success" ->
+            Logger.info("Conversion ##{uuid} is finished. #{inspect(json)}")
+            {:success, json}
+
+          value when value in ["pending", "running"] ->
+            Logger.info("Conversion ##{uuid} is #{value}. #{inspect(json)}")
+            {:pending, json}
+
+          "failed" ->
+            {:failed, json}
+        end
+
+      error ->
+        Logger.error("Conversion ##{uuid} has an error. #{inspect(error)}")
+        :error
+    end
+  end
+
+  @doc """
+  Downloads the conversion and save it to the local disk using a file stream.
+  """
+  @spec download_conversion(binary(), File.Stream.t()) :: :ok
+  def download_conversion(uuid, %File.Stream{path: path} = file_stream) do
+    url = Path.join([base_url(), uuid, "download"])
+    Logger.info("Downloading conversion ##{uuid} to file stream at #{path}")
+
+    Req.get!(url, compressed: false, headers: auth_headers(), into: file_stream)
+    :ok
+  end
+
+  defp base_url do
+    # Use Bypass with Req in the test environment, we need to change the base URL
+    bypass = Process.get(:req_bypass)
+
+    if Mix.env() == :test and not is_nil(bypass) do
+      "http://localhost:#{bypass.port}"
+    else
+      @base_url
+    end
+  end
+
+  defp http_client, do: Transport.Shared.Wrapper.HTTPoison.impl()
+
+  defp auth_headers do
+    [{"authorization", "Token token=#{Application.fetch_env!(:transport, :enroute_token)}"}]
+  end
+end

apps/transport/mix.exs

@@ -62,7 +62,8 @@ defmodule Transport.Mixfile do
       {:httpoison, "~> 2.1"},
       {:phoenix, "~> 1.6.2"},
       {:phoenix_html, "~> 3.1"},
-      {:phoenix_markdown, "~> 1.0"},
+      # Compilation issue for this dependency, see https://github.com/etalab/transport-site/issues/3499
+      {:phoenix_markdown, git: "https://github.com/pzingg/phoenix_markdown.git", ref: "b2e5ff67c9ce9160d7ef1f66d0c859dfa6284a53"},
       # Careful with the upgrade: https://github.com/etalab/transport-site/issues/3433
       {:phoenix_live_view, "~> 0.18.0"},
       {:html_sanitize_ex, "~> 1.4"},
@@ -73,7 +74,8 @@ defmodule Transport.Mixfile do
       {:timex, "~> 3.7"},
       {:sentry, "~> 8.1"},
       {:scrivener, "~> 2.5"},
-      {:scrivener_html, git: "https://github.com/mgwidmann/scrivener_html.git", ref: "9224d1"},
+      # Compilation issue for this dependency, see https://github.com/etalab/transport-site/issues/3499
+      {:scrivener_html, git: "https://github.com/etalab/scrivener_html.git", ref: "f0245703abf7d0ce2b48a0f7e96997def7649e5f"},
       {:scrivener_list, "~>2.0"},
       {:jason, "~> 1.1"},
       {:open_api_spex, "~> 3.8"},
@@ -132,7 +134,7 @@ defmodule Transport.Mixfile do
       {:appsignal, "~> 2.0"},
       {:appsignal_phoenix, "~> 2.0"},
       {:vega_lite, "~> 0.1.7"},
-      {:req, "~> 0.3.11", only: :dev},
+      {:req, "~> 0.4.4"},
       {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false},
       {:swoosh, "~> 1.11"},
       {:phoenix_swoosh, "~> 1.0"},

apps/transport/test/transport/converters/gtfs_to_netex_enroute_test.exs

@@ -0,0 +1,112 @@
+defmodule Transport.Converters.GTFSToNeTExEnRouteTest do
+  use ExUnit.Case, async: true
+  @moduletag :capture_log
+  import Mox
+  alias Transport.Converters.GTFSToNeTExEnRoute
+
+  setup :verify_on_exit!
+
+  setup do
+    {:ok, bypass: Bypass.open()}
+  end
+
+  test "create_gtfs_to_netex_conversion" do
+    uuid = Ecto.UUID.generate()
+    filepath = "/tmp/test"
+
+    Transport.HTTPoison.Mock
+    |> expect(:post!, fn "https://chouette-convert.enroute.mobi/api/conversions",
+                         {:multipart,
+                          [
+                            {"type", "gtfs-netex"},
+                            {"options[profile]", "french"},
+                            {:file, ^filepath, {"form-data", [name: "file", filename: "test"]}, []}
+                          ]},
+                         [{"authorization", "Token token=fake_enroute_token"}] ->
+      %HTTPoison.Response{status_code: 201, body: Jason.encode!(%{"id" => uuid})}
+    end)
+
+    assert uuid == GTFSToNeTExEnRoute.create_gtfs_to_netex_conversion(filepath)
+  end
+
+  describe "get_conversion" do
+    test "success case" do
+      uuid = Ecto.UUID.generate()
+      url = "https://chouette-convert.enroute.mobi/api/conversions/#{uuid}"
+
+      Transport.HTTPoison.Mock
+      |> expect(:get, fn ^url, [{"authorization", "Token token=fake_enroute_token"}] ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"id" => uuid, "status" => "success"})}}
+      end)
+
+      assert {:success, %{"id" => uuid, "status" => "success"}} == GTFSToNeTExEnRoute.get_conversion(uuid)
+    end
+
+    test "pending case" do
+      uuid = Ecto.UUID.generate()
+      url = "https://chouette-convert.enroute.mobi/api/conversions/#{uuid}"
+
+      Transport.HTTPoison.Mock
+      |> expect(:get, fn ^url, [{"authorization", "Token token=fake_enroute_token"}] ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"id" => uuid, "status" => "pending"})}}
+      end)
+
+      assert {:pending, %{"id" => uuid, "status" => "pending"}} == GTFSToNeTExEnRoute.get_conversion(uuid)
+    end
+
+    test "running case" do
+      uuid = Ecto.UUID.generate()
+      url = "https://chouette-convert.enroute.mobi/api/conversions/#{uuid}"
+
+      Transport.HTTPoison.Mock
+      |> expect(:get, fn ^url, [{"authorization", "Token token=fake_enroute_token"}] ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"id" => uuid, "status" => "running"})}}
+      end)
+
+      assert {:pending, %{"id" => uuid, "status" => "running"}} == GTFSToNeTExEnRoute.get_conversion(uuid)
+    end
+
+    test "failed case" do
+      uuid = Ecto.UUID.generate()
+      url = "https://chouette-convert.enroute.mobi/api/conversions/#{uuid}"
+
+      Transport.HTTPoison.Mock
+      |> expect(:get, fn ^url, [{"authorization", "Token token=fake_enroute_token"}] ->
+        {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"id" => uuid, "status" => "failed"})}}
+      end)
+
+      assert {:failed, %{"id" => uuid, "status" => "failed"}} == GTFSToNeTExEnRoute.get_conversion(uuid)
+    end
+
+    test "HTTP error case" do
+      uuid = Ecto.UUID.generate()
+      url = "https://chouette-convert.enroute.mobi/api/conversions/#{uuid}"
+
+      Transport.HTTPoison.Mock
+      |> expect(:get, fn ^url, [{"authorization", "Token token=fake_enroute_token"}] ->
+        {:error, %HTTPoison.Error{}}
+      end)
+
+      assert :error = GTFSToNeTExEnRoute.get_conversion(uuid)
+    end
+  end
+
+  test "download_conversion", %{bypass: bypass} do
+    uuid = Ecto.UUID.generate()
+    tmp_path = Path.join(System.tmp_dir!(), uuid)
+    url = "#{uuid}/download"
+
+    Process.put(:req_bypass, bypass)
+
+    Bypass.expect_once(bypass, "GET", url, fn %Plug.Conn{} = conn ->
+      assert ["Token token=fake_enroute_token"] = Plug.Conn.get_req_header(conn, "authorization")
+      Plug.Conn.send_resp(conn, 200, "File content")
+    end)
+
+    refute File.exists?(tmp_path)
+    assert :ok == GTFSToNeTExEnRoute.download_conversion(uuid, File.stream!(tmp_path))
+
+    assert "File content" == File.read!(tmp_path)
+    File.rm!(tmp_path)
+  end
+end

config/config.exs

@@ -188,6 +188,7 @@ config :gettext, :default_locale, "fr"
 config :transport,
   domain_name: System.get_env("DOMAIN_NAME", "transport.data.gouv.fr"),
   export_secret_key: System.get_env("EXPORT_SECRET_KEY"),
+  enroute_token: System.get_env("ENROUTE_TOKEN"),
   max_import_concurrent_jobs: (System.get_env("MAX_IMPORT_CONCURRENT_JOBS") || "1") |> String.to_integer(),
   nb_days_to_keep_validations: 60,
   join_our_slack_link: "https://join.slack.com/t/transportdatagouvfr/shared_invite/zt-2n1n92ye-sdGQ9SeMh5BkgseaIzV8kA",

config/test.exs

@@ -48,7 +48,8 @@ config :transport,
     gtfs_diff: "gtfs-diff-test"
   },
   workflow_notifier: Transport.Jobs.Workflow.ProcessNotifier,
-  export_secret_key: "fake_export_secret_key"
+  export_secret_key: "fake_export_secret_key",
+  enroute_token: "fake_enroute_token"
 
 config :ex_aws,
   cellar_organisation_id: "fake-cellar_organisation_id"