From 314cf09c2e0b17b9aeddf00813bf34dfda002c2e Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:48:22 -0400 Subject: [PATCH 01/11] Working ish with new url, does not scrape json yet and formatting --- .gitattributes | 2 + .gitignore | 3 + app/models/curiosity_scraper.rb | 51 ++++++---- db/schema.rb | 3 +- folder_structure.txt | 164 ++++++++++++++++++++++++++++++++ lib/tasks/scrapers.rake | 26 +++++ 6 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 .gitattributes create mode 100644 folder_structure.txt create mode 100644 lib/tasks/scrapers.rake diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3415e11 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +bin/* text eol=lf diff --git a/.gitignore b/.gitignore index 380850e..2137d68 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ /tmp /public/assets /coverage + +Dockerfile +docker-compose.yml \ No newline at end of file diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index fe32e76..7288ecd 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -17,31 +17,44 @@ def main_page Nokogiri::HTML(URI.open("https://mars.nasa.gov/msl/multimedia/raw-images/?order=sol+desc%2Cinstrument_sort+asc%2Csample_type_sort+asc%2C+date_taken+desc&per_page=50&page=0&mission=msl")) end - def collect_links - latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i - latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = latest_sol_scraped..latest_sol_available - sols_to_scrape.map { |sol| - "https://mars.nasa.gov/msl/raw/listimagesraw.cfm?&s=#{sol}" - } - end + def collect_links + latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i + latest_sol_scraped = rover.photos.maximum(:sol).to_i + sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Limit to the last 50 sols + sols_to_scrape.map { |sol| "https://mars.nasa.gov/api/v1/raw_image_items/?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=2024-10-07T17:30:51.000Z:date_received:gte&condition_3=#{sol}:sol:in&search=&extended=thumbnail::sample_type::noteq" } + + end private - def create_photos - collect_links.each do |url| - scrape_photo_page(url) - end - end + def create_photos + collect_links.each do |url| + puts "Starting to scrape photos for URL: #{url}" + scrape_photo_page(url) + puts "Finished scraping photos for URL: #{url}" + end + end + +def scrape_photo_page(url) + begin + Timeout.timeout(30) do # Set a 30-second timeout for each page request + image_page = Nokogiri::HTML(URI.open(url, open_timeout: 10, read_timeout: 20)) + image_array = image_page.css("div.RawImageCaption a").map { |link| link["href"] } - def scrape_photo_page(url) - image_page = Nokogiri::HTML(URI.open url) - image_array = image_page.css("div.RawImageCaption a") - .map { |link| link["href"] } - image_array.each do |image| - create_photo(image, url) + puts "Found #{image_array.count} photos for URL: #{url}" + + image_array.each do |image| + create_photo(image, url) + end end + rescue Timeout::Error + puts "Timeout occurred when scraping URL: #{url}. Moving on to the next one." + rescue OpenURI::HTTPError => e + puts "HTTP error occurred: #{e.message} for URL: #{url}. Moving on to the next one." + rescue StandardError => e + puts "An error occurred: #{e.message} for URL: #{url}. Moving on to the next one." end +end def create_photo(image, url) if !thumbnail?(image) diff --git a/db/schema.rb b/db/schema.rb index e0b5ed0..07e0c70 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2016_09_29_212810) do - +ActiveRecord::Schema[7.0].define(version: 2016_09_29_212810) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/folder_structure.txt b/folder_structure.txt new file mode 100644 index 0000000..f40e872 --- /dev/null +++ b/folder_structure.txt @@ -0,0 +1,164 @@ +Folder PATH listing +Volume serial number is EEAD-45D2 +C:. +³ .codeclimate.yml +³ .gitattributes +³ .gitignore +³ .rspec +³ .ruby-version +³ config.ru +³ docker-compose.yml +³ Dockerfile +³ folder_structure.txt +³ Gemfile +³ Gemfile.lock +³ License.md +³ Procfile +³ Rakefile +³ README.md +³ +ÃÄÄÄ.circleci +³ config.yml +³ +ÃÄÄÄapp +³ ÃÄÄÄassets +³ ³ ÀÄÄÄconfig +³ ³ manifest.js +³ ³ +³ ÃÄÄÄcontrollers +³ ³ ³ application_controller.rb +³ ³ ³ static_controller.rb +³ ³ ³ +³ ³ ÀÄÄÄapi +³ ³ ÀÄÄÄv1 +³ ³ latest_photos_controller.rb +³ ³ manifests_controller.rb +³ ³ photos_controller.rb +³ ³ rovers_controller.rb +³ ³ +³ ÃÄÄÄhelpers +³ ³ application_helper.rb +³ ³ +³ ÃÄÄÄmodels +³ ³ camera.rb +³ ³ curiosity_scraper.rb +³ ³ opportunity_spirit_scraper.rb +³ ³ perseverance_scraper.rb +³ ³ photo.rb +³ ³ photo_manifest.rb +³ ³ rover.rb +³ ³ +³ ÀÄÄÄserializers +³ camera_serializer.rb +³ photo_manifest_serializer.rb +³ photo_serializer.rb +³ rover_serializer.rb +³ +ÃÄÄÄbin +³ bundle +³ rails +³ rake +³ setup +³ spring +³ +ÃÄÄÄconfig +³ ³ application.rb +³ ³ boot.rb +³ ³ database.yml +³ ³ environment.rb +³ ³ newrelic.yml +³ ³ puma.rb +³ ³ routes.rb +³ ³ secrets.yml +³ ³ storage.yml +³ ³ +³ ÃÄÄÄenvironments +³ ³ development.rb +³ ³ production.rb +³ ³ test.rb +³ ³ +³ ÃÄÄÄinitializers +³ ³ backtrace_silencers.rb +³ ³ cookies_serializer.rb +³ ³ filter_parameter_logging.rb +³ ³ inflections.rb +³ ³ mime_types.rb +³ ³ redis.rb +³ ³ session_store.rb +³ ³ wrap_parameters.rb +³ ³ +³ ÀÄÄÄlocales +³ en.yml +³ +ÃÄÄÄdb +³ ³ schema.rb +³ ³ seeds.rb +³ ³ +³ ÀÄÄÄmigrate +³ 20150111223524_create_photos.rb +³ 20150326195232_add_index_to_photos.rb +³ 20150610043755_add_earth_date_to_photos.rb +³ 20150611024520_change_earth_date_to_date.rb +³ 20150613144825_create_rovers.rb +³ 20150613145324_add_rover_id_to_photos.rb +³ 20150613161623_create_cameras.rb +³ 20150613162330_add_camera_id_to_photos.rb +³ 20150613162909_remove_null_false_from_photos.rb +³ 20150613172554_rename_camera_on_photos.rb +³ 20150613230252_add_indexes.rb +³ 20150621054555_add_full_name_to_cameras.rb +³ 20150624004521_add_index_on_earth_date.rb +³ 20151024040622_fix_navcam_null_camera_ids_for_curiosity_photos.rb +³ 20160929175428_add_launch_date_and_status_to_rover.rb +³ 20160929212810_add_index_on_sol_to_photos.rb +³ +ÃÄÄÄlog +³ development.log +³ newrelic_agent.log +³ +ÃÄÄÄpublic +³ ³ crossdomain.xml +³ ³ favicon.ico +³ ³ index.html +³ ³ robots.txt +³ ³ +³ ÀÄÄÄexplore +³ ³ mars-photos-ember-b5f0177a34d407192d3d78d2ec77b6f4.js +³ ³ mars-photos-ember-fc9f28ea6e98f5cf721a0009248bfe25.css +³ ³ vendor-a5d6b9b8755084f4802ef87949fbbc4d.js +³ ³ vendor-d41d8cd98f00b204e9800998ecf8427e.css +³ ³ +³ ÀÄÄÄimages +³ Curiosity_rover.jpg +³ Opportunity_rover.jpg +³ Perseverance_rover.jpg +³ Spirit_rover.jpg +³ +ÃÄÄÄspec +³ ³ factories.rb +³ ³ rails_helper.rb +³ ³ spec_helper.rb +³ ³ +³ ÃÄÄÄcontrollers +³ ³ ÀÄÄÄapi +³ ³ ÀÄÄÄv1 +³ ³ manifests_controller_spec.rb +³ ³ photos_controller_spec.rb +³ ³ rovers_controller_spec.rb +³ ³ +³ ÃÄÄÄmodels +³ ³ curiosity_scraper_spec.rb +³ ³ perseverance_scraper_spec.rb +³ ³ photo_manifest_spec.rb +³ ³ photo_spec.rb +³ ³ rover_spec.rb +³ ³ +³ ÀÄÄÄsupport +³ request_helpers.rb +³ +ÀÄÄÄtmp + ÃÄÄÄcache + ÃÄÄÄpids + ³ server.pid + ³ + ÀÄÄÄsockets diff --git a/lib/tasks/scrapers.rake b/lib/tasks/scrapers.rake new file mode 100644 index 0000000..75d485e --- /dev/null +++ b/lib/tasks/scrapers.rake @@ -0,0 +1,26 @@ +# lib/tasks/scrapers.rake +namespace :scraper do + desc "Run the Curiosity rover scraper to populate photos" + task curiosity: :environment do + puts "Starting Curiosity Scraper..." + scraper = CuriosityScraper.new + scraper.scrape + puts "Curiosity Scraper completed." + end + + desc "Run the Perseverance rover scraper to populate photos" + task perseverance: :environment do + puts "Starting Perseverance Scraper..." + scraper = PerseveranceScraper.new + scraper.scrape + puts "Perseverance Scraper completed." + end + + desc "Run the Opportunity and Spirit rover scraper to populate photos" + task opportunity_spirit: :environment do + puts "Starting Opportunity and Spirit Scraper..." + scraper = OpportunitySpiritScraper.new + scraper.scrape + puts "Opportunity and Spirit Scraper completed." + end +end From 97b68f4b61b070afd22dcd8d248999e03f8cdbff Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:37:20 -0400 Subject: [PATCH 02/11] Working without DB? --- app/models/curiosity_scraper.rb | 100 ++++++++++++-------------------- 1 file changed, 37 insertions(+), 63 deletions(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 7288ecd..596895c 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -1,7 +1,7 @@ class CuriosityScraper require "open-uri" require 'json' - BASE_URL = "https://mars.nasa.gov/msl/multimedia/raw/" + BASE_URL = "https://mars.nasa.gov/api/v1/raw_image_items/" attr_reader :rover def initialize @@ -12,80 +12,54 @@ def scrape create_photos end - # grabs the HTML from the main page of the curiosity rover image gallery - def main_page - Nokogiri::HTML(URI.open("https://mars.nasa.gov/msl/multimedia/raw-images/?order=sol+desc%2Cinstrument_sort+asc%2Csample_type_sort+asc%2C+date_taken+desc&per_page=50&page=0&mission=msl")) - end - - def collect_links - latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i - latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Limit to the last 50 sols - sols_to_scrape.map { |sol| "https://mars.nasa.gov/api/v1/raw_image_items/?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=2024-10-07T17:30:51.000Z:date_received:gte&condition_3=#{sol}:sol:in&search=&extended=thumbnail::sample_type::noteq" } + def collect_links + # Fetch the latest sol available through the API + response = JSON.parse(URI.open(BASE_URL + "?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=1&page=0&condition_1=msl:mission").read) + latest_sol_available = response["items"].first["sol"].to_i + latest_sol_scraped = rover.photos.maximum(:sol).to_i + sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Only fetch the last 10 sols - end + sols_to_scrape.map { |sol| + "#{BASE_URL}?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=#{sol}:sol:in" + } + end private - def create_photos - collect_links.each do |url| - puts "Starting to scrape photos for URL: #{url}" - scrape_photo_page(url) - puts "Finished scraping photos for URL: #{url}" - end - end - -def scrape_photo_page(url) - begin - Timeout.timeout(30) do # Set a 30-second timeout for each page request - image_page = Nokogiri::HTML(URI.open(url, open_timeout: 10, read_timeout: 20)) - image_array = image_page.css("div.RawImageCaption a").map { |link| link["href"] } - - puts "Found #{image_array.count} photos for URL: #{url}" + def create_photos + collect_links.each do |url| + scrape_photo_page(url) + end + end - image_array.each do |image| - create_photo(image, url) + def scrape_photo_page(url) + begin + response = JSON.parse(URI.open(url).read) + response['items'].each do |image| + create_photo(image) if image['extended'] && image['extended']['sample_type'] == 'full' end + rescue OpenURI::HTTPError => e + puts "HTTP error occurred: #{e.message} for URL: #{url}. Skipping." + rescue StandardError => e + puts "Error occurred: #{e.message} for URL: #{url}. Skipping." end - rescue Timeout::Error - puts "Timeout occurred when scraping URL: #{url}. Moving on to the next one." - rescue OpenURI::HTTPError => e - puts "HTTP error occurred: #{e.message} for URL: #{url}. Moving on to the next one." - rescue StandardError => e - puts "An error occurred: #{e.message} for URL: #{url}. Moving on to the next one." end -end - def create_photo(image, url) - if !thumbnail?(image) - sol = url.scan(/(?<==)\d+/).first - camera = camera_from_url image - fail "Camera not found. Name: #{camera}" if camera.is_a?(String) - photo = Photo.find_or_initialize_by(sol: sol, camera: camera, - img_src: image, rover: rover) + def create_photo(image) + sol = image['sol'] + camera = camera_from_json(image) + link = image['https_url'] + + if camera.is_a?(String) + puts "WARNING: Camera not found. Name: #{camera}" + else + photo = Photo.find_or_initialize_by(sol: sol, camera: camera, img_src: link, rover: rover) photo.log_and_save_if_new end end - def camera_abbreviations - { - fcam: "FHAZ", - rcam: "RHAZ", - ccam: "CHEMCAM", - mcam: "MAST", - ncam: "NAVCAM", - mhli: "MAHLI", - mrdi: "MARDI" - } - end - - def thumbnail?(image_url) - image_url.to_s.include?("_T") - end - - def camera_from_url(image_url) - camera_abbreviation = image_url.match(/\/(?\w{4})\/\w+.(JPG|jpg|PNG|png)/)[:camera] - camera_name = camera_abbreviations[camera_abbreviation.to_sym] - rover.cameras.find_by(name: camera_name) || camera_name || camera_abbreviation + def camera_from_json(image) + camera_name = image['instrument'] + rover.cameras.find_by(name: camera_name) || camera_name end end From 827662d528f5a3bcc002094f9c50a95e57aa260d Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:39:19 -0400 Subject: [PATCH 03/11] Working I think? --- app/models/curiosity_scraper.rb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 596895c..885b2d3 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -60,6 +60,22 @@ def create_photo(image) def camera_from_json(image) camera_name = image['instrument'] - rover.cameras.find_by(name: camera_name) || camera_name + camera = rover.cameras.find_by(name: camera_name) || rover.cameras.find_by(full_name: camera_name) + + if camera.nil? + # Log a warning + puts "WARNING: Camera not found. Name: #{camera_name}. Adding to database." + + # Add the new camera to the database + camera = rover.cameras.create(name: camera_name, full_name: camera_name) + + if camera.persisted? + puts "New camera added to database: #{camera_name}" + else + puts "Failed to add camera to the database: #{camera_name}" + end + end + + camera end end From 5efa97d41f49cef28f545801c37400b4dbe35128 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:39:53 -0400 Subject: [PATCH 04/11] Docker file un-ignore --- .gitignore | 3 --- Dockerfile | 26 ++++++++++++++++++++++++++ docker-compose.yml | 26 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 2137d68..380850e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,3 @@ /tmp /public/assets /coverage - -Dockerfile -docker-compose.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8fbc454 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Use an official Ruby image that includes Ruby 3.2.2 +FROM ruby:3.2.2 + +# Set the working directory in the container +WORKDIR /app + +# Copy the Gemfile and Gemfile.lock into the container +COPY Gemfile Gemfile.lock /app/ + +# Install dependencies, such as gems listed in the Gemfile +RUN bundle install + +# Copy the rest of the application code into the container +COPY . /app + +# Install Node.js and Yarn to handle JavaScript dependencies +RUN apt-get update -qq && apt-get install -y nodejs npm && npm install --global yarn + +# Set up PostgreSQL client dependencies +RUN apt-get install -y postgresql-client + +# Expose port 3000 to the outside world +EXPOSE 3000 + +# Run the Rails server when the container starts +CMD ["rails", "server", "-b", "0.0.0.0"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..93aedd5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + db: + image: postgres:13 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: mars_photo_api_development + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + + web: + build: . + command: bash -c "rm -f tmp/pids/server.pid && rails server -b 0.0.0.0" + volumes: + - .:/app + ports: + - "3000:3000" + depends_on: + - db + environment: + DATABASE_URL: "postgres://postgres:password@db:5432/mars_photo_api_development" + +volumes: + pgdata: From 693ff47268929477bd50fa9cd9a4db851fbaf0b2 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:51:15 -0400 Subject: [PATCH 05/11] Removes scrape limit --- app/models/curiosity_scraper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 885b2d3..da8f1b2 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -17,7 +17,7 @@ def collect_links response = JSON.parse(URI.open(BASE_URL + "?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=1&page=0&condition_1=msl:mission").read) latest_sol_available = response["items"].first["sol"].to_i latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Only fetch the last 10 sols + sols_to_scrape = (latest_sol_scraped..latest_sol_available) #.to_a.last(10) # Only fetch the last 10 sols sols_to_scrape.map { |sol| "#{BASE_URL}?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=#{sol}:sol:in" From 6c859d9206667a062b93c9fe874939b65b639107 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:57:40 -0400 Subject: [PATCH 06/11] Revert "Removes scrape limit" This reverts commit 693ff47268929477bd50fa9cd9a4db851fbaf0b2. --- app/models/curiosity_scraper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index da8f1b2..885b2d3 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -17,7 +17,7 @@ def collect_links response = JSON.parse(URI.open(BASE_URL + "?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=1&page=0&condition_1=msl:mission").read) latest_sol_available = response["items"].first["sol"].to_i latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = (latest_sol_scraped..latest_sol_available) #.to_a.last(10) # Only fetch the last 10 sols + sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Only fetch the last 10 sols sols_to_scrape.map { |sol| "#{BASE_URL}?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=#{sol}:sol:in" From 71152dc4fd4a189eb28324c291ad087fd70a7e27 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:57:52 -0400 Subject: [PATCH 07/11] Revert "Docker file un-ignore" This reverts commit 5efa97d41f49cef28f545801c37400b4dbe35128. --- .gitignore | 3 +++ Dockerfile | 26 -------------------------- docker-compose.yml | 26 -------------------------- 3 files changed, 3 insertions(+), 52 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 380850e..2137d68 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ /tmp /public/assets /coverage + +Dockerfile +docker-compose.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8fbc454..0000000 --- a/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use an official Ruby image that includes Ruby 3.2.2 -FROM ruby:3.2.2 - -# Set the working directory in the container -WORKDIR /app - -# Copy the Gemfile and Gemfile.lock into the container -COPY Gemfile Gemfile.lock /app/ - -# Install dependencies, such as gems listed in the Gemfile -RUN bundle install - -# Copy the rest of the application code into the container -COPY . /app - -# Install Node.js and Yarn to handle JavaScript dependencies -RUN apt-get update -qq && apt-get install -y nodejs npm && npm install --global yarn - -# Set up PostgreSQL client dependencies -RUN apt-get install -y postgresql-client - -# Expose port 3000 to the outside world -EXPOSE 3000 - -# Run the Rails server when the container starts -CMD ["rails", "server", "-b", "0.0.0.0"] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 93aedd5..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -services: - db: - image: postgres:13 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - POSTGRES_DB: mars_photo_api_development - ports: - - "5432:5432" - volumes: - - pgdata:/var/lib/postgresql/data - - web: - build: . - command: bash -c "rm -f tmp/pids/server.pid && rails server -b 0.0.0.0" - volumes: - - .:/app - ports: - - "3000:3000" - depends_on: - - db - environment: - DATABASE_URL: "postgres://postgres:password@db:5432/mars_photo_api_development" - -volumes: - pgdata: From feb076d4630c229a0254e45198e9865c2b6e7ba4 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:58:00 -0400 Subject: [PATCH 08/11] Revert "Working I think?" This reverts commit 827662d528f5a3bcc002094f9c50a95e57aa260d. --- app/models/curiosity_scraper.rb | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 885b2d3..596895c 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -60,22 +60,6 @@ def create_photo(image) def camera_from_json(image) camera_name = image['instrument'] - camera = rover.cameras.find_by(name: camera_name) || rover.cameras.find_by(full_name: camera_name) - - if camera.nil? - # Log a warning - puts "WARNING: Camera not found. Name: #{camera_name}. Adding to database." - - # Add the new camera to the database - camera = rover.cameras.create(name: camera_name, full_name: camera_name) - - if camera.persisted? - puts "New camera added to database: #{camera_name}" - else - puts "Failed to add camera to the database: #{camera_name}" - end - end - - camera + rover.cameras.find_by(name: camera_name) || camera_name end end From bd26a5cfd987481a07602dd3a9dcd7bf0d842a6b Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:58:08 -0400 Subject: [PATCH 09/11] Revert "Working without DB?" This reverts commit 97b68f4b61b070afd22dcd8d248999e03f8cdbff. --- app/models/curiosity_scraper.rb | 100 ++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 37 deletions(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 596895c..7288ecd 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -1,7 +1,7 @@ class CuriosityScraper require "open-uri" require 'json' - BASE_URL = "https://mars.nasa.gov/api/v1/raw_image_items/" + BASE_URL = "https://mars.nasa.gov/msl/multimedia/raw/" attr_reader :rover def initialize @@ -12,54 +12,80 @@ def scrape create_photos end - def collect_links - # Fetch the latest sol available through the API - response = JSON.parse(URI.open(BASE_URL + "?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=1&page=0&condition_1=msl:mission").read) - latest_sol_available = response["items"].first["sol"].to_i - latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Only fetch the last 10 sols - - sols_to_scrape.map { |sol| - "#{BASE_URL}?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=#{sol}:sol:in" - } + # grabs the HTML from the main page of the curiosity rover image gallery + def main_page + Nokogiri::HTML(URI.open("https://mars.nasa.gov/msl/multimedia/raw-images/?order=sol+desc%2Cinstrument_sort+asc%2Csample_type_sort+asc%2C+date_taken+desc&per_page=50&page=0&mission=msl")) end + def collect_links + latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i + latest_sol_scraped = rover.photos.maximum(:sol).to_i + sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Limit to the last 50 sols + sols_to_scrape.map { |sol| "https://mars.nasa.gov/api/v1/raw_image_items/?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=2024-10-07T17:30:51.000Z:date_received:gte&condition_3=#{sol}:sol:in&search=&extended=thumbnail::sample_type::noteq" } + + end + private - def create_photos - collect_links.each do |url| - scrape_photo_page(url) - end - end + def create_photos + collect_links.each do |url| + puts "Starting to scrape photos for URL: #{url}" + scrape_photo_page(url) + puts "Finished scraping photos for URL: #{url}" + end + end + +def scrape_photo_page(url) + begin + Timeout.timeout(30) do # Set a 30-second timeout for each page request + image_page = Nokogiri::HTML(URI.open(url, open_timeout: 10, read_timeout: 20)) + image_array = image_page.css("div.RawImageCaption a").map { |link| link["href"] } - def scrape_photo_page(url) - begin - response = JSON.parse(URI.open(url).read) - response['items'].each do |image| - create_photo(image) if image['extended'] && image['extended']['sample_type'] == 'full' + puts "Found #{image_array.count} photos for URL: #{url}" + + image_array.each do |image| + create_photo(image, url) end - rescue OpenURI::HTTPError => e - puts "HTTP error occurred: #{e.message} for URL: #{url}. Skipping." - rescue StandardError => e - puts "Error occurred: #{e.message} for URL: #{url}. Skipping." end + rescue Timeout::Error + puts "Timeout occurred when scraping URL: #{url}. Moving on to the next one." + rescue OpenURI::HTTPError => e + puts "HTTP error occurred: #{e.message} for URL: #{url}. Moving on to the next one." + rescue StandardError => e + puts "An error occurred: #{e.message} for URL: #{url}. Moving on to the next one." end +end - def create_photo(image) - sol = image['sol'] - camera = camera_from_json(image) - link = image['https_url'] - - if camera.is_a?(String) - puts "WARNING: Camera not found. Name: #{camera}" - else - photo = Photo.find_or_initialize_by(sol: sol, camera: camera, img_src: link, rover: rover) + def create_photo(image, url) + if !thumbnail?(image) + sol = url.scan(/(?<==)\d+/).first + camera = camera_from_url image + fail "Camera not found. Name: #{camera}" if camera.is_a?(String) + photo = Photo.find_or_initialize_by(sol: sol, camera: camera, + img_src: image, rover: rover) photo.log_and_save_if_new end end - def camera_from_json(image) - camera_name = image['instrument'] - rover.cameras.find_by(name: camera_name) || camera_name + def camera_abbreviations + { + fcam: "FHAZ", + rcam: "RHAZ", + ccam: "CHEMCAM", + mcam: "MAST", + ncam: "NAVCAM", + mhli: "MAHLI", + mrdi: "MARDI" + } + end + + def thumbnail?(image_url) + image_url.to_s.include?("_T") + end + + def camera_from_url(image_url) + camera_abbreviation = image_url.match(/\/(?\w{4})\/\w+.(JPG|jpg|PNG|png)/)[:camera] + camera_name = camera_abbreviations[camera_abbreviation.to_sym] + rover.cameras.find_by(name: camera_name) || camera_name || camera_abbreviation end end From cb51fe5196dca2c8e6b379244516c1ac9cf56862 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:58:13 -0400 Subject: [PATCH 10/11] Revert "Working ish with new url, does not scrape json yet" This reverts commit 314cf09c2e0b17b9aeddf00813bf34dfda002c2e. --- .gitattributes | 2 - .gitignore | 3 - app/models/curiosity_scraper.rb | 51 ++++------ db/schema.rb | 3 +- folder_structure.txt | 164 -------------------------------- lib/tasks/scrapers.rake | 26 ----- 6 files changed, 21 insertions(+), 228 deletions(-) delete mode 100644 .gitattributes delete mode 100644 folder_structure.txt delete mode 100644 lib/tasks/scrapers.rake diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 3415e11..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -* text=auto -bin/* text eol=lf diff --git a/.gitignore b/.gitignore index 2137d68..380850e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,3 @@ /tmp /public/assets /coverage - -Dockerfile -docker-compose.yml \ No newline at end of file diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index 7288ecd..fe32e76 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -17,44 +17,31 @@ def main_page Nokogiri::HTML(URI.open("https://mars.nasa.gov/msl/multimedia/raw-images/?order=sol+desc%2Cinstrument_sort+asc%2Csample_type_sort+asc%2C+date_taken+desc&per_page=50&page=0&mission=msl")) end - def collect_links - latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i - latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = (latest_sol_scraped..latest_sol_available).to_a.last(10) # Limit to the last 50 sols - sols_to_scrape.map { |sol| "https://mars.nasa.gov/api/v1/raw_image_items/?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=2024-10-07T17:30:51.000Z:date_received:gte&condition_3=#{sol}:sol:in&search=&extended=thumbnail::sample_type::noteq" } - - end + def collect_links + latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i + latest_sol_scraped = rover.photos.maximum(:sol).to_i + sols_to_scrape = latest_sol_scraped..latest_sol_available + sols_to_scrape.map { |sol| + "https://mars.nasa.gov/msl/raw/listimagesraw.cfm?&s=#{sol}" + } + end private - def create_photos - collect_links.each do |url| - puts "Starting to scrape photos for URL: #{url}" - scrape_photo_page(url) - puts "Finished scraping photos for URL: #{url}" - end - end - -def scrape_photo_page(url) - begin - Timeout.timeout(30) do # Set a 30-second timeout for each page request - image_page = Nokogiri::HTML(URI.open(url, open_timeout: 10, read_timeout: 20)) - image_array = image_page.css("div.RawImageCaption a").map { |link| link["href"] } - - puts "Found #{image_array.count} photos for URL: #{url}" + def create_photos + collect_links.each do |url| + scrape_photo_page(url) + end + end - image_array.each do |image| - create_photo(image, url) - end + def scrape_photo_page(url) + image_page = Nokogiri::HTML(URI.open url) + image_array = image_page.css("div.RawImageCaption a") + .map { |link| link["href"] } + image_array.each do |image| + create_photo(image, url) end - rescue Timeout::Error - puts "Timeout occurred when scraping URL: #{url}. Moving on to the next one." - rescue OpenURI::HTTPError => e - puts "HTTP error occurred: #{e.message} for URL: #{url}. Moving on to the next one." - rescue StandardError => e - puts "An error occurred: #{e.message} for URL: #{url}. Moving on to the next one." end -end def create_photo(image, url) if !thumbnail?(image) diff --git a/db/schema.rb b/db/schema.rb index 07e0c70..e0b5ed0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,8 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2016_09_29_212810) do +ActiveRecord::Schema.define(version: 2016_09_29_212810) do + # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/folder_structure.txt b/folder_structure.txt deleted file mode 100644 index f40e872..0000000 --- a/folder_structure.txt +++ /dev/null @@ -1,164 +0,0 @@ -Folder PATH listing -Volume serial number is EEAD-45D2 -C:. -³ .codeclimate.yml -³ .gitattributes -³ .gitignore -³ .rspec -³ .ruby-version -³ config.ru -³ docker-compose.yml -³ Dockerfile -³ folder_structure.txt -³ Gemfile -³ Gemfile.lock -³ License.md -³ Procfile -³ Rakefile -³ README.md -³ -ÃÄÄÄ.circleci -³ config.yml -³ -ÃÄÄÄapp -³ ÃÄÄÄassets -³ ³ ÀÄÄÄconfig -³ ³ manifest.js -³ ³ -³ ÃÄÄÄcontrollers -³ ³ ³ application_controller.rb -³ ³ ³ static_controller.rb -³ ³ ³ -³ ³ ÀÄÄÄapi -³ ³ ÀÄÄÄv1 -³ ³ latest_photos_controller.rb -³ ³ manifests_controller.rb -³ ³ photos_controller.rb -³ ³ rovers_controller.rb -³ ³ -³ ÃÄÄÄhelpers -³ ³ application_helper.rb -³ ³ -³ ÃÄÄÄmodels -³ ³ camera.rb -³ ³ curiosity_scraper.rb -³ ³ opportunity_spirit_scraper.rb -³ ³ perseverance_scraper.rb -³ ³ photo.rb -³ ³ photo_manifest.rb -³ ³ rover.rb -³ ³ -³ ÀÄÄÄserializers -³ camera_serializer.rb -³ photo_manifest_serializer.rb -³ photo_serializer.rb -³ rover_serializer.rb -³ -ÃÄÄÄbin -³ bundle -³ rails -³ rake -³ setup -³ spring -³ -ÃÄÄÄconfig -³ ³ application.rb -³ ³ boot.rb -³ ³ database.yml -³ ³ environment.rb -³ ³ newrelic.yml -³ ³ puma.rb -³ ³ routes.rb -³ ³ secrets.yml -³ ³ storage.yml -³ ³ -³ ÃÄÄÄenvironments -³ ³ development.rb -³ ³ production.rb -³ ³ test.rb -³ ³ -³ ÃÄÄÄinitializers -³ ³ backtrace_silencers.rb -³ ³ cookies_serializer.rb -³ ³ filter_parameter_logging.rb -³ ³ inflections.rb -³ ³ mime_types.rb -³ ³ redis.rb -³ ³ session_store.rb -³ ³ wrap_parameters.rb -³ ³ -³ ÀÄÄÄlocales -³ en.yml -³ -ÃÄÄÄdb -³ ³ schema.rb -³ ³ seeds.rb -³ ³ -³ ÀÄÄÄmigrate -³ 20150111223524_create_photos.rb -³ 20150326195232_add_index_to_photos.rb -³ 20150610043755_add_earth_date_to_photos.rb -³ 20150611024520_change_earth_date_to_date.rb -³ 20150613144825_create_rovers.rb -³ 20150613145324_add_rover_id_to_photos.rb -³ 20150613161623_create_cameras.rb -³ 20150613162330_add_camera_id_to_photos.rb -³ 20150613162909_remove_null_false_from_photos.rb -³ 20150613172554_rename_camera_on_photos.rb -³ 20150613230252_add_indexes.rb -³ 20150621054555_add_full_name_to_cameras.rb -³ 20150624004521_add_index_on_earth_date.rb -³ 20151024040622_fix_navcam_null_camera_ids_for_curiosity_photos.rb -³ 20160929175428_add_launch_date_and_status_to_rover.rb -³ 20160929212810_add_index_on_sol_to_photos.rb -³ -ÃÄÄÄlog -³ development.log -³ newrelic_agent.log -³ -ÃÄÄÄpublic -³ ³ crossdomain.xml -³ ³ favicon.ico -³ ³ index.html -³ ³ robots.txt -³ ³ -³ ÀÄÄÄexplore -³ ³ mars-photos-ember-b5f0177a34d407192d3d78d2ec77b6f4.js -³ ³ mars-photos-ember-fc9f28ea6e98f5cf721a0009248bfe25.css -³ ³ vendor-a5d6b9b8755084f4802ef87949fbbc4d.js -³ ³ vendor-d41d8cd98f00b204e9800998ecf8427e.css -³ ³ -³ ÀÄÄÄimages -³ Curiosity_rover.jpg -³ Opportunity_rover.jpg -³ Perseverance_rover.jpg -³ Spirit_rover.jpg -³ -ÃÄÄÄspec -³ ³ factories.rb -³ ³ rails_helper.rb -³ ³ spec_helper.rb -³ ³ -³ ÃÄÄÄcontrollers -³ ³ ÀÄÄÄapi -³ ³ ÀÄÄÄv1 -³ ³ manifests_controller_spec.rb -³ ³ photos_controller_spec.rb -³ ³ rovers_controller_spec.rb -³ ³ -³ ÃÄÄÄmodels -³ ³ curiosity_scraper_spec.rb -³ ³ perseverance_scraper_spec.rb -³ ³ photo_manifest_spec.rb -³ ³ photo_spec.rb -³ ³ rover_spec.rb -³ ³ -³ ÀÄÄÄsupport -³ request_helpers.rb -³ -ÀÄÄÄtmp - ÃÄÄÄcache - ÃÄÄÄpids - ³ server.pid - ³ - ÀÄÄÄsockets diff --git a/lib/tasks/scrapers.rake b/lib/tasks/scrapers.rake deleted file mode 100644 index 75d485e..0000000 --- a/lib/tasks/scrapers.rake +++ /dev/null @@ -1,26 +0,0 @@ -# lib/tasks/scrapers.rake -namespace :scraper do - desc "Run the Curiosity rover scraper to populate photos" - task curiosity: :environment do - puts "Starting Curiosity Scraper..." - scraper = CuriosityScraper.new - scraper.scrape - puts "Curiosity Scraper completed." - end - - desc "Run the Perseverance rover scraper to populate photos" - task perseverance: :environment do - puts "Starting Perseverance Scraper..." - scraper = PerseveranceScraper.new - scraper.scrape - puts "Perseverance Scraper completed." - end - - desc "Run the Opportunity and Spirit rover scraper to populate photos" - task opportunity_spirit: :environment do - puts "Starting Opportunity and Spirit Scraper..." - scraper = OpportunitySpiritScraper.new - scraper.scrape - puts "Opportunity and Spirit Scraper completed." - end -end From e03bcd74b9f8fdff7db5d94cd8e1662ceb94d5d1 Mon Sep 17 00:00:00 2001 From: Aidan Powers <47310537+AidanPowers@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:00:13 -0400 Subject: [PATCH 11/11] New Scraper based on PerseveranceScraper --- app/models/curiosity_scraper.rb | 81 +++++++++++++++++---------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/app/models/curiosity_scraper.rb b/app/models/curiosity_scraper.rb index fe32e76..da8f1b2 100644 --- a/app/models/curiosity_scraper.rb +++ b/app/models/curiosity_scraper.rb @@ -1,7 +1,7 @@ class CuriosityScraper require "open-uri" require 'json' - BASE_URL = "https://mars.nasa.gov/msl/multimedia/raw/" + BASE_URL = "https://mars.nasa.gov/api/v1/raw_image_items/" attr_reader :rover def initialize @@ -12,17 +12,15 @@ def scrape create_photos end - # grabs the HTML from the main page of the curiosity rover image gallery - def main_page - Nokogiri::HTML(URI.open("https://mars.nasa.gov/msl/multimedia/raw-images/?order=sol+desc%2Cinstrument_sort+asc%2Csample_type_sort+asc%2C+date_taken+desc&per_page=50&page=0&mission=msl")) - end - def collect_links - latest_sol_available = JSON.parse(main_page.css('[data-react-props]').last.attr('data-react-props'))["header_counts"]["latest_sol"].to_i + # Fetch the latest sol available through the API + response = JSON.parse(URI.open(BASE_URL + "?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=1&page=0&condition_1=msl:mission").read) + latest_sol_available = response["items"].first["sol"].to_i latest_sol_scraped = rover.photos.maximum(:sol).to_i - sols_to_scrape = latest_sol_scraped..latest_sol_available + sols_to_scrape = (latest_sol_scraped..latest_sol_available) #.to_a.last(10) # Only fetch the last 10 sols + sols_to_scrape.map { |sol| - "https://mars.nasa.gov/msl/raw/listimagesraw.cfm?&s=#{sol}" + "#{BASE_URL}?order=sol%20desc,instrument_sort%20asc,sample_type_sort%20asc,%20date_taken%20desc&per_page=200&page=0&condition_1=msl:mission&condition_2=#{sol}:sol:in" } end @@ -35,44 +33,49 @@ def create_photos end def scrape_photo_page(url) - image_page = Nokogiri::HTML(URI.open url) - image_array = image_page.css("div.RawImageCaption a") - .map { |link| link["href"] } - image_array.each do |image| - create_photo(image, url) + begin + response = JSON.parse(URI.open(url).read) + response['items'].each do |image| + create_photo(image) if image['extended'] && image['extended']['sample_type'] == 'full' + end + rescue OpenURI::HTTPError => e + puts "HTTP error occurred: #{e.message} for URL: #{url}. Skipping." + rescue StandardError => e + puts "Error occurred: #{e.message} for URL: #{url}. Skipping." end end - def create_photo(image, url) - if !thumbnail?(image) - sol = url.scan(/(?<==)\d+/).first - camera = camera_from_url image - fail "Camera not found. Name: #{camera}" if camera.is_a?(String) - photo = Photo.find_or_initialize_by(sol: sol, camera: camera, - img_src: image, rover: rover) + def create_photo(image) + sol = image['sol'] + camera = camera_from_json(image) + link = image['https_url'] + + if camera.is_a?(String) + puts "WARNING: Camera not found. Name: #{camera}" + else + photo = Photo.find_or_initialize_by(sol: sol, camera: camera, img_src: link, rover: rover) photo.log_and_save_if_new end end - def camera_abbreviations - { - fcam: "FHAZ", - rcam: "RHAZ", - ccam: "CHEMCAM", - mcam: "MAST", - ncam: "NAVCAM", - mhli: "MAHLI", - mrdi: "MARDI" - } - end + def camera_from_json(image) + camera_name = image['instrument'] + camera = rover.cameras.find_by(name: camera_name) || rover.cameras.find_by(full_name: camera_name) - def thumbnail?(image_url) - image_url.to_s.include?("_T") - end + if camera.nil? + # Log a warning + puts "WARNING: Camera not found. Name: #{camera_name}. Adding to database." + + # Add the new camera to the database + camera = rover.cameras.create(name: camera_name, full_name: camera_name) + + if camera.persisted? + puts "New camera added to database: #{camera_name}" + else + puts "Failed to add camera to the database: #{camera_name}" + end + end - def camera_from_url(image_url) - camera_abbreviation = image_url.match(/\/(?\w{4})\/\w+.(JPG|jpg|PNG|png)/)[:camera] - camera_name = camera_abbreviations[camera_abbreviation.to_sym] - rover.cameras.find_by(name: camera_name) || camera_name || camera_abbreviation + camera end end