Perseverance scraper spec

.ruby-version

@@ -1 +1 @@
-2.7.1
+3.0.0

.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 
 rvm:
-  - 2.7.1
+  - 3.0.0
 
 before_script:
   - psql -c 'create database mars_curiosity_test;' -U postgres

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-ruby '2.7.1'
+ruby '3.0.0'
 
 gem 'pg', '~> 1.0'
 gem 'rake'

Gemfile.lock

@@ -1,63 +1,63 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actioncable (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      activejob (= 6.1.2.1)
-      activerecord (= 6.1.2.1)
-      activestorage (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actionmailbox (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      activejob (= 6.1.3.1)
+      activerecord (= 6.1.3.1)
+      activestorage (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       mail (>= 2.7.1)
-    actionmailer (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      actionview (= 6.1.2.1)
-      activejob (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actionmailer (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      actionview (= 6.1.3.1)
+      activejob (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.2.1)
-      actionview (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actionpack (6.1.3.1)
+      actionview (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      activerecord (= 6.1.2.1)
-      activestorage (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actiontext (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      activerecord (= 6.1.3.1)
+      activestorage (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       nokogiri (>= 1.8.5)
-    actionview (6.1.2.1)
-      activesupport (= 6.1.2.1)
+    actionview (6.1.3.1)
+      activesupport (= 6.1.3.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
     active_model_serializers (0.9.8)
       activemodel (>= 3.2)
       concurrent-ruby (~> 1.0)
-    activejob (6.1.2.1)
-      activesupport (= 6.1.2.1)
+    activejob (6.1.3.1)
+      activesupport (= 6.1.3.1)
       globalid (>= 0.3.6)
-    activemodel (6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activerecord (6.1.2.1)
-      activemodel (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activestorage (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      activejob (= 6.1.2.1)
-      activerecord (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
-      marcel (~> 0.3.1)
-      mimemagic (~> 0.3.2)
-    activesupport (6.1.2.1)
+    activemodel (6.1.3.1)
+      activesupport (= 6.1.3.1)
+    activerecord (6.1.3.1)
+      activemodel (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
+    activestorage (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      activejob (= 6.1.3.1)
+      activerecord (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
+      marcel (~> 1.0.0)
+      mini_mime (~> 1.0.2)
+    activesupport (6.1.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -82,7 +82,7 @@ GEM
       redis (~> 4.1)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    i18n (1.8.8)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     json (2.3.0)
     kaminari (1.2.1)
@@ -102,19 +102,17 @@ GEM
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (0.3.3)
-      mimemagic (~> 0.3.2)
+    marcel (1.0.1)
     method_source (1.0.0)
-    mimemagic (0.3.5)
-    mini_mime (1.0.2)
+    mini_mime (1.0.3)
     mini_portile2 (2.5.0)
-    minitest (5.14.3)
+    minitest (5.14.4)
     newrelic_rpm (6.11.0.365)
-    nio4r (2.5.5)
-    nokogiri (1.11.1)
+    nio4r (2.5.7)
+    nokogiri (1.11.2)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    nokogiri (1.11.1-x64-mingw32)
+    nokogiri (1.11.2-x64-mingw32)
       racc (~> 1.4)
     pg (1.2.3)
     pg (1.2.3-x64-mingw32)
@@ -129,20 +127,20 @@ GEM
       rack (>= 2.0.0)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.1.2.1)
-      actioncable (= 6.1.2.1)
-      actionmailbox (= 6.1.2.1)
-      actionmailer (= 6.1.2.1)
-      actionpack (= 6.1.2.1)
-      actiontext (= 6.1.2.1)
-      actionview (= 6.1.2.1)
-      activejob (= 6.1.2.1)
-      activemodel (= 6.1.2.1)
-      activerecord (= 6.1.2.1)
-      activestorage (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    rails (6.1.3.1)
+      actioncable (= 6.1.3.1)
+      actionmailbox (= 6.1.3.1)
+      actionmailer (= 6.1.3.1)
+      actionpack (= 6.1.3.1)
+      actiontext (= 6.1.3.1)
+      actionview (= 6.1.3.1)
+      activejob (= 6.1.3.1)
+      activemodel (= 6.1.3.1)
+      activerecord (= 6.1.3.1)
+      activestorage (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       bundler (>= 1.15.0)
-      railties (= 6.1.2.1)
+      railties (= 6.1.3.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
@@ -154,9 +152,9 @@ GEM
       rails_stdout_logging
     rails_serve_static_assets (0.0.5)
     rails_stdout_logging (0.0.5)
-    railties (6.1.2.1)
-      actionpack (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
+    railties (6.1.3.1)
+      actionpack (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
       method_source
       rake (>= 0.8.7)
       thor (~> 1.0)
@@ -229,7 +227,7 @@ DEPENDENCIES
   uglifier
 
 RUBY VERSION
-   ruby 2.7.1p83
+   ruby 3.0.0p0
 
 BUNDLED WITH
-   2.1.4
+   2.2.3

README.md

@@ -32,6 +32,8 @@ Along with querying by date, results can also be filtered by the camera with whi
    FRONT_HAZCAM_RIGHT_A|Front Hazard Avoidance Camera - Right
    REAR_HAZCAM_LEFT|Rear Hazard Avoidance Camera - Left
    REAR_HAZCAM_RIGHT|Rear Hazard Avoidance Camera - Right
+   SKYCAM|MEDA Skycam
+   SHERLOC_WATSON|SHERLOC WATSON Camera
 
 #### Other rovers
 

app/controllers/api/v1/latest_photos_controller.rb

@@ -13,7 +13,7 @@ def index
   private
 
   def photo_params
-    params.permit(:camera, :earth_date).merge(sol: @rover.photos.maximum(:sol))
+    params.permit(:camera, :earth_date, :rover_id).merge(sol: @rover.photos.maximum(:sol))
   end
 
   def search_photos

app/controllers/api/v1/photos_controller.rb

@@ -16,7 +16,7 @@ def index
   private
 
   def photo_params
-    params.permit :sol, :camera, :earth_date
+    params.permit :sol, :camera, :earth_date, :rover_id
   end
 
   def photos(rover)

app/models/perseverance_scraper.rb

@@ -44,10 +44,13 @@ def create_photo(image)
     sol = image['sol']
     camera = camera_from_json image
     link = image['image_files']['large']
-    fail "Camera not found. Name: #{camera}" if camera.is_a?(String)
-    photo = Photo.find_or_initialize_by(sol: sol, camera: camera,
-                                        img_src: link, rover: rover)
-    photo.log_and_save_if_new
+    if camera.is_a?(String)
+      puts "WARNING: Camera not found. Name: #{camera}"
+    else
+      photo = Photo.find_or_initialize_by(sol: sol, camera: camera,
+                                          img_src: link, rover: rover)
+      photo.log_and_save_if_new
+    end
   end
 
   def camera_abbreviations
@@ -58,15 +61,17 @@ def camera_abbreviations
       epu1cam: "EDL_PUCAM1",
       epu2cam: "EDL_PUCAM2",
       navlcam: "NAVCAM_LEFT",
-      nacrcam: "NAVCAM_RIGHT",
+      navrcam: "NAVCAM_RIGHT",
       mczlcam: "MCZ_LEFT",
       mczrcam: "MCZ_RIGHT",
       fhlacam: "FRONT_HAZCAM_LEFT_A",
       fhracam: "FRONT_HAZCAM_RIGHT_A",
       fhlbcam: "FRONT_HAZCAM_LEFT_B",
       fhrbcam: "FRONT_HAZCAM_RIGHT_B",
       rhlcam:  "REAR_HAZCAM_LEFT",
-      rhrcam:  "REAR_HAZCAM_RIGHT"
+      rhrcam:  "REAR_HAZCAM_RIGHT",
+      skycam:  "SKYCAM",
+      watson:  "SHERLOC_WATSON"
     }
   end
 

db/seeds.rb

@@ -20,6 +20,10 @@
 perseverance.cameras.find_or_create_by(name: "FRONT_HAZCAM_RIGHT_A", full_name: "Front Hazard Avoidance Camera - Right")
 perseverance.cameras.find_or_create_by(name: "REAR_HAZCAM_LEFT", full_name: "Rear Hazard Avoidance Camera - Left")
 perseverance.cameras.find_or_create_by(name: "REAR_HAZCAM_RIGHT", full_name: "Rear Hazard Avoidance Camera - Right")
+perseverance.cameras.find_or_create_by(name: "SKYCAM", full_name: "MEDA Skycam")
+perseverance.cameras.find_or_create_by(name: "SHERLOC_WATSON", full_name: "SHERLOC WATSON Camera")
+perseverance.cameras.find_or_create_by(name: "SUPERCAM_RMI", full_name: "SuperCam Remote Micro Imager")
+perseverance.cameras.find_or_create_by(name: "LCAM", full_name: "Lander Vision System Camera")
 
 opportunity.cameras.find_or_create_by(name: "FHAZ", full_name: "Front Hazard Avoidance Camera")
 opportunity.cameras.find_or_create_by(name: "RHAZ", full_name: "Read Hazard Avoidance Camera")

spec/models/perseverance_scraper_spec.rb

@@ -22,7 +22,7 @@
     let!(:epu1cam) {create :camera, rover: perseverance, name: "EDL_PUCAM1" }
     let!(:epu2cam) {create :camera, rover: perseverance, name: "EDL_PUCAM2" }
     let!(:navlcam) {create :camera, rover: perseverance, name: "NAVCAM_LEFT" }
-    let!(:nacrcam) {create :camera, rover: perseverance, name: "NAVCAM_RIGHT" }
+    let!(:navrcam) {create :camera, rover: perseverance, name: "NAVCAM_RIGHT" }
     let!(:mczlcam) {create :camera, rover: perseverance, name: "MCZ_LEFT" }
     let!(:mczrcam) {create :camera, rover: perseverance, name: "MCZ_RIGHT" }
     let!(:fhlacam) {create :camera, rover: perseverance, name: "FRONT_HAZCAM_LEFT_A" }
@@ -31,11 +31,14 @@
     let!(:fhrbcam) {create :camera, rover: perseverance, name: "FRONT_HAZCAM_RIGHT_B" }
     let!(:rhlcam) {create :camera, rover: perseverance, name:  "REAR_HAZCAM_LEFT" }
     let!(:rhrcam) {create :camera, rover: perseverance, name:  "REAR_HAZCAM_RIGHT" }
+    let!(:skycam) {create :camera, rover: perseverance, name:  "SKYCAM" }
+    let!(:waston) {create :camera, rover: perseverance, name:  "SHERLOC_WATSON" }
+    let!(:supercam) { create :camera, rover: perseverance, name: "SUPERCAM_RMI" }
 
     it "should create photo objects" do
       allow(scraper).to receive(:collect_links).and_return ["https://mars.nasa.gov/rss/api/?feed=raw_images&category=mars2020&feedtype=json&sol=1"]
 
-      expect{ scraper.scrape }.to change { Photo.count }.by(191)
+      expect{ scraper.scrape }.to change { Photo.count }.by(34)
     end
   end
 end