diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/chicago_electronic_distributors_us.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/chicago_electronic_distributors_us.ex index 53ff458..5899f70 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/chicago_electronic_distributors_us.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/chicago_electronic_distributors_us.ex @@ -30,6 +30,12 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.ChicagoElectroni sku: "RPI4-MODBP-8GB" }, + # RPi Zero + %ProductUpdate{ + url: "https://chicagodist.com/products/raspberry-pi-zero-w", + sku: "SC0020" + }, + # RPi Zero 2 %ProductUpdate{url: "https://chicagodist.com/products/raspberry-pi-zero-2", sku: "SC0510"} ] diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/elektor_nl.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/elektor_nl.ex index 241ab3b..676f632 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/elektor_nl.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/elektor_nl.ex @@ -2,6 +2,24 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.ElektorNl do alias NervesMetalDetector.Vendors.ElektorNl.ProductUpdate @items [ + # RPi 4 + %ProductUpdate{ + url: "https://www.elektor.nl/raspberry-pi-4-b-1-gb-ram", + sku: "RPI4-MODBP-1GB" + }, + %ProductUpdate{ + url: "https://www.elektor.nl/raspberry-pi-4-b-2-gb-ram", + sku: "RPI4-MODBP-1GB" + }, + %ProductUpdate{ + url: "https://www.elektor.nl/raspberry-pi-4-b-4-gb-ram", + sku: "RPI4-MODBP-1GB" + }, + %ProductUpdate{ + url: "https://www.elektor.nl/raspberry-pi-4-b-8-gb-ram", + sku: "RPI4-MODBP-8GB" + }, + # RPi Zero %ProductUpdate{url: "https://www.elektor.nl/raspberry-pi-zero-w", sku: "SC0020"}, diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/kamami_pl.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/kamami_pl.ex index 5a0816f..1bd0534 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/kamami_pl.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/kamami_pl.ex @@ -36,6 +36,13 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.KamamiPl do sku: "RPI4-MODBP-8GB" }, + # RPi Zero + %ProductUpdate{ + url: + "https://kamami.pl/raspberry-pi-zero/1183083-sc0020-raspberry-pi-zero-w--707565827164.html", + sku: "SC0020" + }, + # RPi Zero 2 %ProductUpdate{ url: diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ca.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ca.ex index a8742bc..0f66c6d 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ca.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ca.ex @@ -176,7 +176,11 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.PiShopCa do %ProductUpdate{url: "https://www.pishop.ca/product/raspberry-pi-zero-w/", sku: "SC0020"}, # RPi Zero 2 - %ProductUpdate{url: "https://www.pishop.ca/product/raspberry-pi-zero-2-w/", sku: "SC0510"} + %ProductUpdate{url: "https://www.pishop.ca/product/raspberry-pi-zero-2-w/", sku: "SC0510"}, + %ProductUpdate{ + url: "https://www.pishop.ca/product/raspberry-pi-zero-2-w-with-pre-soldered-headers/", + sku: "SC0510WH" + } ] def all(), do: @items diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ch.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ch.ex index fc23734..29f162e 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ch.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_ch.ex @@ -66,6 +66,10 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.PiShopCh do url: "https://www.pi-shop.ch/raspberry-pi-compute-module-4-and-io-board-2-gb-ram-lite", sku: "CM4002000" }, + %ProductUpdate{ + url: "https://www.pi-shop.ch/raspberry-pi-compute-module-4-and-io-board-2-gb-ram-8gb-emmc", + sku: "CM4002008" + }, %ProductUpdate{ url: "https://www.pi-shop.ch/raspberry-pi-compute-module-4-and-io-board-4-gb-ram-lite-wireless", diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_us.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_us.ex index 0cfef9c..ea81573 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_us.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/pi_shop_us.ex @@ -184,7 +184,11 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.PiShopUs do }, # RPi Zero 2 - %ProductUpdate{url: "https://www.pishop.us/product/raspberry-pi-zero-2-w/", sku: "SC0510"} + %ProductUpdate{url: "https://www.pishop.us/product/raspberry-pi-zero-2-w/", sku: "SC0510"}, + %ProductUpdate{ + url: "https://www.pishop.us/product/raspberry-pi-zero-2-w-with-pre-soldered-headers/", + sku: "SC0510WH" + } ] def all(), do: @items diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_pi_dk.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_pi_dk.ex index 4e5f343..477f884 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_pi_dk.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_pi_dk.ex @@ -2,6 +2,156 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.RaspberryPiDk do alias NervesMetalDetector.Vendors.RaspberryPiDk.ProductUpdate @items [ + # RPi CM3 + # These urls cannot be properly scraped because the metadata on the shop + # does not have info for each individual configuration + # %ProductUpdate{ + # url: "https://raspberrypi.dk/en/product/raspberry-pi-compute-module-3-plus/?attribute_pa_pi-cm3-plus-udgave=compute-module-3-lite", + # sku: "CM3+Lite" + # }, + # %ProductUpdate{ + # url: "https://raspberrypi.dk/en/product/raspberry-pi-compute-module-3-plus/?attribute_pa_pi-cm3-plus-udgave=compute-module-3-8gb", + # sku: "CM3+8GB" + # }, + # %ProductUpdate{ + # url: "https://raspberrypi.dk/en/product/raspberry-pi-compute-module-3-plus/?attribute_pa_pi-cm3-plus-udgave=compute-module-3-16gb", + # sku: "CM3+16GB" + # }, + # %ProductUpdate{ + # url: "https://raspberrypi.dk/en/product/raspberry-pi-compute-module-3-plus/?attribute_pa_pi-cm3-plus-udgave=compute-module-3-32gb", + # sku: "CM3+32GB" + # }, + + # RPi CM4 + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4001000-sc0695/", + sku: "CM4001000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4001008-sc0696/", + sku: "CM4001008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4001016-sc0697/", + sku: "CM4001016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4001032-sc0698/", + sku: "CM4001032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4002000-sc0679/", + sku: "CM4002000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4002008-sc0680/", + sku: "CM4002008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4002016-sc0681/", + sku: "CM4002016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4002032-sc0682/", + sku: "CM4002032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4008032-sc0690/", + sku: "CM4008032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4004000-sc0683/", + sku: "CM4004000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4101000-sc0691/", + sku: "CM4101000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4004008-sc0684/", + sku: "CM4004008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4101008-sc0692/", + sku: "CM4101008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4004016-sc0685/", + sku: "CM4004016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4101016-sc0693/", + sku: "CM4101016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4004032-sc0686/", + sku: "CM4004032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4101032-sc0694/", + sku: "CM4101032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4008000-sc0687/", + sku: "CM4008000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4102000-sc0667/", + sku: "CM4102000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4008008-sc0688/", + sku: "CM4008008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4102008-sc0668/", + sku: "CM4102008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4008016-sc0689/", + sku: "CM4008016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4102016-sc0669/", + sku: "CM4102016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4102032-sc0670/", + sku: "CM4102032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4104000-sc0671/", + sku: "CM4104000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4104008-sc0672/", + sku: "CM4104008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4104016-sc0673/", + sku: "CM4104016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4104032-sc0674/", + sku: "CM4104032" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4108000-sc0675/", + sku: "CM4108000" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4108008-sc0676/", + sku: "CM4108008" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4108016-sc0677/", + sku: "CM4108016" + }, + %ProductUpdate{ + url: "https://raspberrypi.dk/en/product/cm4108032-sc0678/", + sku: "CM4108032" + }, + # RPi 3 %ProductUpdate{ url: "https://raspberrypi.dk/produkt/raspberry-pi-3-model-a-plus/", diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_store_nl.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_store_nl.ex index 11b7cfb..82777d3 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_store_nl.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/raspberry_store_nl.ex @@ -31,6 +31,13 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.RaspberryStoreNl sku: "RPI4-MODBP-8GB" }, + # RPi Zero + %ProductUpdate{ + url: + "https://www.raspberrystore.nl/PrestaShop/en/raspberry-pi-zero-v1-en-v2/164-raspberry-pi-zero-707565827164.html", + sku: "SC0020" + }, + # RPi Zero 2 %ProductUpdate{ url: diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/reichelt_de.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/reichelt_de.ex index ca44acb..1b4ceca 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/reichelt_de.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/reichelt_de.ex @@ -2,6 +2,28 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.ReicheltDe do alias NervesMetalDetector.Vendors.ReicheltDe.ProductUpdate @items [ + # RPi CM4 + %ProductUpdate{ + url: + "https://www.reichelt.de/de/en/raspberry-pi-compute-module-4-1gb-ram-16gb-emmc-wlan-rpi-cm4w-1gb16gb-p290501.html", + sku: "CM4101016" + }, + %ProductUpdate{ + url: + "https://www.reichelt.de/de/en/raspberry-pi-compute-module-4-1gb-ram-32gb-emmc-rpi-cm4-1gb32gb-p290498.html", + sku: "CM4001032" + }, + %ProductUpdate{ + url: + "https://www.reichelt.de/de/en/raspberry-pi-compute-module-4-1gb-ram-8gb-emmc-rpi-cm4-1gb8gb-p290496.html?&trstct=pol_11&nbc=1", + sku: "CM4001008" + }, + %ProductUpdate{ + url: + "https://www.reichelt.de/de/en/raspberry-pi-compute-module-4-4gb-ram-8gb-emmc-wlan-rpi-cm4w-4gb8gb-p290540.html", + sku: "CM4104008" + }, + # RPi 3 %ProductUpdate{ url: diff --git a/lib/nerves_metal_detector/inventory/data/product_update_items/seeed_studio_cn.ex b/lib/nerves_metal_detector/inventory/data/product_update_items/seeed_studio_cn.ex index 65de248..2715e23 100644 --- a/lib/nerves_metal_detector/inventory/data/product_update_items/seeed_studio_cn.ex +++ b/lib/nerves_metal_detector/inventory/data/product_update_items/seeed_studio_cn.ex @@ -34,6 +34,12 @@ defmodule NervesMetalDetector.Inventory.Data.ProductUpdateItems.SeeedStudioCn do %ProductUpdate{ url: "https://www.seeedstudio.com/raspberry-pi-compute-module-cm4108032-p-5220.html", sku: "CM4108032" + }, + + # RPi 4 + %ProductUpdate{ + url: "https://www.seeedstudio.com/Raspberry-Pi-4-Computer-Model-B-4GB-p-4077.html", + sku: "RPI4-MODBP-4GB" } ] diff --git a/lib/nerves_metal_detector_dev/rpi_locator.ex b/lib/nerves_metal_detector_dev/rpi_locator.ex index 5f2d8f6..457a8c5 100644 --- a/lib/nerves_metal_detector_dev/rpi_locator.ex +++ b/lib/nerves_metal_detector_dev/rpi_locator.ex @@ -33,26 +33,16 @@ defmodule NervesMetalDetectorDev.RpiLocator do end def fetch_items(vendor) do - scrape_url = "https://rpilocator.com/?vendor=#{vendor}" - - second_request_headers = [ + headers = [ {"Accept", "application/json, text/javascript, */*; q=0.01"}, {"X-Requested-With", "XMLHttpRequest"}, - {"referer", scrape_url} + {"referer", "https://rpilocator.com/?vendor=#{vendor}"} ] - with {:ok, %{status_code: 200, body: body, headers: headers}} <- HTTPoison.get(scrape_url), - {:ok, parsed} <- Floki.parse_document(body), - {:ok, info} <- find_js_block(parsed), - {:ok, local_token} <- parse_local_token(info), - {:ok, query_filter} <- parse_query_filter(info), - ts <- System.os_time(:millisecond), - url <- - "https://rpilocator.com/data.cfm?method=getProductTable&token=#{local_token}&#{query_filter}&_=#{ts}", - cookies <- extract_cookies_from_headers(headers), - options <- build_request_options(cookies), - {:ok, %{status_code: 200, body: body}} <- - HTTPoison.get(url, second_request_headers, options), + ts = System.os_time(:millisecond) + url = "https://rpilocator.com/data.cfm?method=getProductTable&vendor=#{vendor}&_=#{ts}" + + with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url, headers), {:ok, parsed_body} <- Jason.decode(body) do items = Enum.map(parsed_body["data"], fn item -> @@ -68,67 +58,4 @@ defmodule NervesMetalDetectorDev.RpiLocator do :error end end - - defp find_js_block(parsed) do - content = - parsed - |> Floki.find("script[type=\"text/javascript\"]") - |> Enum.at(1) - |> Floki.children() - |> Floki.text() - - cond do - content not in [nil, ""] -> {:ok, content} - true -> :error - end - end - - defp parse_local_token(info) do - case Regex.run(~r/localToken="([[:alnum:]]+)"/, info) do - [_, local_token] -> {:ok, local_token} - _ -> :error - end - end - - defp parse_query_filter(info) do - case Regex.run(~r/queryFilter="([[:alnum:]|=]*)"/, info) do - [_, query_filter] -> {:ok, query_filter} - _ -> :error - end - end - - defp extract_cookies_from_headers(headers) do - headers - |> Enum.filter(fn - {"set-cookie", _} -> true - _ -> false - end) - |> Enum.map(&parse_cookie_from_header/1) - end - - defp parse_cookie_from_header({"set-cookie", cookie_string}) do - [name_value | _] = String.split(cookie_string, ";") - [name, value] = String.split(name_value, "=", parts: 2) - - {name, value} - end - - defp build_request_options(cookies) do - case cookies do - nil -> - [] - - cookies -> - cookies_formatted = - cookies - |> Enum.map(&format_cookie/1) - |> Enum.join("; ") - - [hackney: [cookie: [cookies_formatted]]] - end - end - - defp format_cookie({name, value}) do - name <> "=" <> value - end end