diff --git a/Gemfile b/Gemfile index 845d443f..a505715c 100644 --- a/Gemfile +++ b/Gemfile @@ -8,7 +8,6 @@ gem 'doorkeeper', '~> 5' # To resize active storage images: # Revise if this is needed after Rails 6.0 gem 'image_processing' - gem 'ancestry' gem 'api-pagination' gem 'api_cache' @@ -87,6 +86,7 @@ end group :development, :test do # gem 'rspec_api_blueprint', require: false + gem "pry" gem 'brakeman', github: 'presidentbeef/brakeman', require: false gem 'byebug' gem 'cane' diff --git a/Gemfile.lock b/Gemfile.lock index feae9900..f5b9678b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -572,6 +572,7 @@ DEPENDENCIES pg pg_search premailer-rails + pry pry-rails puma pundit @@ -615,4 +616,4 @@ RUBY VERSION ruby 3.0.6p216 BUNDLED WITH - 2.4.13 + 2.5.6 diff --git a/app/controllers/v0/application_controller.rb b/app/controllers/v0/application_controller.rb index cc15a895..5d323a39 100644 --- a/app/controllers/v0/application_controller.rb +++ b/app/controllers/v0/application_controller.rb @@ -6,6 +6,7 @@ class ApplicationController < ActionController::API include ActionController::HttpAuthentication::Token::ControllerMethods include ActionController::Helpers include ActionController::ImplicitRender + include ActionController::Caching include Pundit::Authorization include PrettyJSON diff --git a/app/controllers/v0/components_controller.rb b/app/controllers/v0/components_controller.rb index 9b525c4a..12906d8f 100644 --- a/app/controllers/v0/components_controller.rb +++ b/app/controllers/v0/components_controller.rb @@ -7,7 +7,7 @@ def index end def show - @component = Component.includes(:board, :sensor).find(params[:id]) + @component = Component.includes(:device, :sensor).find(params[:id]) authorize @component end @@ -15,10 +15,8 @@ def show def component_params params.permit( - :board_id, - :board_type, - :sensor_id, - :equation + :device_id, + :sensor_id ) end diff --git a/app/controllers/v0/devices_controller.rb b/app/controllers/v0/devices_controller.rb index 07affcc4..a993399f 100644 --- a/app/controllers/v0/devices_controller.rb +++ b/app/controllers/v0/devices_controller.rb @@ -1,13 +1,12 @@ module V0 class DevicesController < ApplicationController - before_action :check_if_authorized!, only: [:create] after_action :verify_authorized, except: [:index, :world_map, :fresh_world_map] def show @device = Device.includes( - :kit, :owner, :sensors,:tags).find(params[:id]) + :owner,:tags, {sensors: :measurement}).find(params[:id]) authorize @device @device end @@ -15,8 +14,9 @@ def show def index raise_ransack_errors_as_bad_request do @q = policy_scope(Device) - .includes(:owner, :tags, kit: [:components, :sensors]) + .includes(:owner, :tags, :components, {sensors: :measurement}) .ransack(params[:q], auth_object: (current_user&.is_admin? ? :admin : nil)) + # We are here customly adding multiple tags into the Ransack query. # Ransack supports this, but how do we add multiple tag names in URL string? Which separator to use? # See Issue #186 https://github.com/fablabbcn/smartcitizen-api/issues/186 @@ -75,36 +75,9 @@ def destroy # debug method, must be refactored def fresh_world_map - @devices = Device.where.not(latitude: nil).where.not(data: nil).includes(:owner,:tags).map do |device| - { - id: device.id, - name: device.name, - description: (device.description.present? ? device.description : nil), - owner_id: device.owner_id, - owner_username: device.owner_id ? device.owner_username : nil, - latitude: device.latitude, - longitude: device.longitude, - city: device.city, - country_code: device.country_code, - is_private: device.is_private, - kit_id: device.kit_id, - state: device.state, - system_tags: device.system_tags, - user_tags: device.user_tags, - added_at: device.added_at, - updated_at: device.updated_at, - last_reading_at: (device.last_reading_at.present? ? device.last_reading_at : nil) - } - end - render json: @devices end def world_map - unless params[:cachebuster] - expires_in 30.seconds, public: true # CRON cURL every 60 seconds to cache - end - - render json: Device.for_world_map end private @@ -113,6 +86,10 @@ def device_params params_to_permit = [ :name, :description, + :hardware_name_override, + :hardware_version_override, + :hardware_type_override, + :hardware_slug_override, :mac_address, :latitude, :longitude, @@ -122,7 +99,6 @@ def device_params :notify_stopped_publishing, :exposure, :meta, - :kit_id, :user_tags, postprocessing_attributes: [:blueprint_url, :hardware_url, :latest_postprocessing, :meta, :forwarding_params], ] diff --git a/app/controllers/v0/kits_controller.rb b/app/controllers/v0/kits_controller.rb deleted file mode 100644 index 0de3cdbd..00000000 --- a/app/controllers/v0/kits_controller.rb +++ /dev/null @@ -1,48 +0,0 @@ -module V0 - class KitsController < ApplicationController - - def show - @kit = Kit.includes(:sensors).friendly.find(params[:id]) - authorize @kit - @kit - end - - def index - @kits = Kit.includes(:sensors, sensors: [:measurement, :tag_sensors]) - @kits = paginate(@kits) - end - - def create - @kit = Kit.new(kit_params) - authorize @kit - if @kit.save - render :show, status: :created - else - raise Smartcitizen::UnprocessableEntity.new @kit.errors - end - end - - def update - @kit = Kit.find(params[:id]) - authorize @kit - if @kit.update(kit_params) - render :show, status: :ok - else - raise Smartcitizen::UnprocessableEntity.new @kit.errors - end - end - -private - - def kit_params - params.permit( - :name, - :description, - :slug, - :sensor_map, - :sensor_ids => [] - ) - end - - end -end diff --git a/app/controllers/v0/onboarding/orphan_devices_controller.rb b/app/controllers/v0/onboarding/orphan_devices_controller.rb index 742dcdcd..7f46f8a0 100644 --- a/app/controllers/v0/onboarding/orphan_devices_controller.rb +++ b/app/controllers/v0/onboarding/orphan_devices_controller.rb @@ -43,7 +43,7 @@ def update private def orphan_device_params - params.permit(:name, :description, :kit_id, :exposure, :latitude, :longitude, :user_tags) + params.permit(:name, :description, :exposure, :latitude, :longitude, :user_tags) end def set_orphan_device diff --git a/app/controllers/v0/readings_controller.rb b/app/controllers/v0/readings_controller.rb index 397a4ceb..71ae774f 100644 --- a/app/controllers/v0/readings_controller.rb +++ b/app/controllers/v0/readings_controller.rb @@ -57,11 +57,6 @@ def csv_archive @device = Device.find(params[:id]) authorize @device, :update? - if @device.kit.nil? - render json: { id: "error", message: "Device does not have a kit", url: "", errors: "" }, status: 420 - return - end - if !@device.csv_export_requested_at or (@device.csv_export_requested_at < 15.minutes.ago) @device.update_column(:csv_export_requested_at, Time.now.utc) if Rails.env.test? diff --git a/app/controllers/v0/static_controller.rb b/app/controllers/v0/static_controller.rb index 23b2b7e8..b9d24667 100644 --- a/app/controllers/v0/static_controller.rb +++ b/app/controllers/v0/static_controller.rb @@ -15,7 +15,6 @@ def home current_user_url: [request.base_url, v0_me_index_path].join, components_url: [request.base_url, v0_components_path].join, devices_url: [request.base_url, v0_devices_path].join, - kits_url: [request.base_url, v0_kits_path].join, measurements_url: [request.base_url, v0_measurements_path].join, sensors_url: [request.base_url, v0_sensors_path].join, users_url: [request.base_url, v0_users_path].join, @@ -32,12 +31,12 @@ def metrics private: Device.where(is_private: true).count, test: Device.where(is_test: true).count, online: { - now: Device.where('last_recorded_at > ?', 10.minutes.ago).count, - last_hour: Device.where('last_recorded_at > ?', 1.hour.ago).count, - today: Device.where('last_recorded_at > ?', Time.now.beginning_of_day).count, - this_month: Device.where('last_recorded_at > ?', Time.now.beginning_of_month).count, - this_year: Device.where('last_recorded_at > ?', Time.now.beginning_of_year).count, - all_time: Device.where.not(last_recorded_at: nil).count + now: Device.where('last_reading_at > ?', 10.minutes.ago).count, + last_hour: Device.where('last_reading_at > ?', 1.hour.ago).count, + today: Device.where('last_reading_at > ?', Time.now.beginning_of_day).count, + this_month: Device.where('last_reading_at > ?', Time.now.beginning_of_month).count, + this_year: Device.where('last_reading_at > ?', Time.now.beginning_of_year).count, + all_time: Device.where.not(last_reading_at: nil).count }, readings: { good: { diff --git a/app/jobs/check_battery_level_below_job.rb b/app/jobs/check_battery_level_below_job.rb index 09695be1..706725c7 100644 --- a/app/jobs/check_battery_level_below_job.rb +++ b/app/jobs/check_battery_level_below_job.rb @@ -18,7 +18,7 @@ def perform(*args) if device.data["10"].to_i < 15 && device.data["10"].to_i > 1 #p "Sending email to: #{device.owner.email} - device: #{device}" - device.update(notify_low_battery_timestamp: Time.now) + device.update_column(:notify_low_battery_timestamp, Time.now) UserMailer.device_battery_low(device.id).deliver_now end diff --git a/app/jobs/check_device_stopped_publishing_job.rb b/app/jobs/check_device_stopped_publishing_job.rb index 8b65c65b..8b97cf65 100644 --- a/app/jobs/check_device_stopped_publishing_job.rb +++ b/app/jobs/check_device_stopped_publishing_job.rb @@ -4,12 +4,12 @@ class CheckDeviceStoppedPublishingJob < ApplicationJob def perform(*args) # Do something later - devices = Device.where(notify_stopped_publishing: true).where("last_recorded_at < ?", 60.minutes.ago) + devices = Device.where(notify_stopped_publishing: true).where("last_reading_at < ?", 60.minutes.ago) CheckupNotifyJob.perform_now("#{devices.count} devices with notification on: stopped_publishing at least an hour ago. Ids: #{devices.pluck(:id)}") devices.each do |device| if device.notify_stopped_publishing_timestamp < 24.hours.ago - device.update notify_stopped_publishing_timestamp: Time.now + device.update_column(:notify_stopped_publishing_timestamp, Time.now) UserMailer.device_stopped_publishing(device.id).deliver_now end end diff --git a/app/lib/mqtt_messages_handler.rb b/app/lib/mqtt_messages_handler.rb index b972bb4f..86fd867c 100644 --- a/app/lib/mqtt_messages_handler.rb +++ b/app/lib/mqtt_messages_handler.rb @@ -11,14 +11,11 @@ def self.handle_topic(topic, message) return if topic.nil? + handshake_device(topic) + # The following do NOT need a device if topic.to_s.include?('inventory') DeviceInventory.create({ report: (message rescue nil) }) - elsif topic.to_s.include?('hello') - orphan_device = OrphanDevice.find_by(device_token: device_token(topic)) - return if orphan_device.nil? - - handle_hello(orphan_device) end device = Device.find_by(device_token: device_token(topic)) @@ -41,7 +38,7 @@ def self.handle_topic(topic, message) } ) Sentry.add_breadcrumb(crumb) - device.update hardware_info: json_message + device.update_column(:hardware_info, json_message) end end @@ -55,6 +52,7 @@ def self.handle_readings(device, message) end rescue Exception => e Sentry.capture_exception(e) + raise e if Rails.env.test? #puts e.inspect #puts message end @@ -88,11 +86,13 @@ def self.parse_raw_readings(message, device_id=nil) JSON[reading] end - def self.handle_hello(orphan_device) - payload = {} - orphan_device.update(device_handshake: true) - payload[:onboarding_session] = orphan_device.onboarding_session - Redis.current.publish('token-received', payload.to_json) + def self.handshake_device(topic) + orphan_device = OrphanDevice.find_by(device_token: device_token(topic)) + return if orphan_device.nil? + orphan_device.update!(device_handshake: true) + Redis.current.publish('token-received', { + onboarding_session: orphan_device.onboarding_session + }.to_json) end # takes a packet and returns 'device token' from topic diff --git a/app/models/component.rb b/app/models/component.rb index 55350bd4..fa4b9ae5 100644 --- a/app/models/component.rb +++ b/app/models/component.rb @@ -1,11 +1,16 @@ # This joins a device with its sensors. class Component < ActiveRecord::Base - belongs_to :board, polymorphic: true + belongs_to :device belongs_to :sensor - validates_presence_of :board, :sensor - validates :sensor_id, :uniqueness => { :scope => [:board_id, :board_type] } + validates_presence_of :device, :sensor + validates :sensor_id, :uniqueness => { :scope => [:device_id] } + validates :key, :uniqueness => { :scope => [:device_id] } + + before_validation :set_key, on: :create + + delegate :equation, :reverse_equation, to: :sensor # Accepts a raw sensor reading and uses its equation to process and return # a calibrated version @@ -19,4 +24,19 @@ def normalized_value x reverse_equation ? eval( ['->x{',reverse_equation,'}'].join ).call(x) : x end + def get_unique_key(default_key, other_keys) + matching_keys = other_keys.select { |k| k =~ /^#{default_key}/ } + ix = matching_keys.length + ix == 0 ? default_key : "#{default_key}_#{ix}" + end + + private + + def set_key + if sensor && device && !key + default_key = sensor.default_key + other_component_keys = device.components.map(&:key) + self.key = get_unique_key(default_key, other_component_keys) + end + end end diff --git a/app/models/concerns/data_parser/storer.rb b/app/models/concerns/data_parser/storer.rb index 9d09dd0e..721ddd10 100644 --- a/app/models/concerns/data_parser/storer.rb +++ b/app/models/concerns/data_parser/storer.rb @@ -54,22 +54,12 @@ def timestamp_parse(timestamp) end def sensor_reading(device, sensor) - begin - id = Integer(sensor['id']) - key = device.find_sensor_key_by_id(id) - rescue - key = sensor['id'] - id = device.find_sensor_id_by_key(key) - end - component = device.components.detect{ |c| c["sensor_id"] == id } - - #raise "This component does not have sensor_id: #{id}" if component.nil? + component = device.find_or_create_component_for_sensor_reading(sensor) return nil if component.nil? - value = component.normalized_value( (Float(sensor['value']) rescue sensor['value']) ) { - id: id, - key: key, + id: component.sensor_id, + key: component.key, component: component, value: value } diff --git a/app/models/device.rb b/app/models/device.rb index 48a484d6..92fc8cd9 100644 --- a/app/models/device.rb +++ b/app/models/device.rb @@ -17,18 +17,18 @@ class Device < ActiveRecord::Base multisearchable :against => [:name, :description, :city, :country_name], if: :active? - belongs_to :kit, optional: :true belongs_to :owner, class_name: 'User' has_many :devices_tags, dependent: :destroy has_many :tags, through: :devices_tags - has_many :components, as: :board + has_many :components, dependent: :destroy has_many :sensors, through: :components has_one :postprocessing, dependent: :destroy accepts_nested_attributes_for :postprocessing, update_only: true - validates_presence_of :name, :owner, on: :create + validates_presence_of :name + validates_presence_of :owner, on: :create #validates_uniqueness_of :name, scope: :owner_id, on: :create validates_uniqueness_of :device_token, allow_nil: true @@ -56,9 +56,6 @@ class Device < ActiveRecord::Base :debug_push, :enclosure_type - alias_attribute :added_at, :created_at - alias_attribute :last_reading_at, :last_recorded_at - before_save :set_state reverse_geocoded_by :latitude, :longitude do |obj, results| @@ -73,26 +70,55 @@ class Device < ActiveRecord::Base end end + scope :for_world_map, -> { + where.not(latitude: nil).where.not(last_reading_at: nil).where(is_test: false).includes(:owner, :tags) + } + def self.ransackable_attributes(auth_object = nil) if auth_object == :admin # admin can ransack on every attribute self.authorizable_ransackable_attributes else - ["id", "name", "description", "created_at", "updated_at", "last_recorded_at", "state","geohash", "uuid", "kit_id"] + ["id", "name", "description", "created_at", "updated_at", "last_reading_at", "state","geohash", "uuid", "kit_id"] end end def self.ransackable_associations(auth_object = nil) [ - "components", "devices_tags", "kit", "owner", + "components", "devices_tags", "owner", "pg_search_document" , "postprocessing", "sensors", "tags" ] end def sensor_keys - # will be changed when different kinds of device added - %w(temp bat co hum light nets no2 noise panel) + sensor_map.keys + end + + def sensor_map + components.map { |c| [c.key, c.sensor.id]}.to_h + end + + def find_or_create_component_by_sensor_id(sensor_id) + return nil if sensor_id.nil? || !Sensor.exists?(id: sensor_id) + components.find_or_create_by(sensor_id: sensor_id) + end + + def find_or_create_component_by_sensor_key(sensor_key) + return nil if sensor_key.nil? + sensor = Sensor.find_by(default_key: sensor_key) + return nil if sensor.nil? + components.find_or_create_by(sensor_id: sensor.id) + end + + def find_or_create_component_for_sensor_reading(reading) + key_or_id = reading["id"] + if key_or_id.is_a?(Integer) || key_or_id =~ /\d+/ + # It's an integer and therefore a sensor id + find_or_create_component_by_sensor_id(key_or_id) + else + find_or_create_component_by_sensor_key(key_or_id) + end end def find_component_by_sensor_id sensor_id @@ -100,11 +126,11 @@ def find_component_by_sensor_id sensor_id end def find_sensor_id_by_key sensor_key - kit.sensor_map[sensor_key.to_s] rescue nil + sensor_map[sensor_key.to_s] rescue nil end def find_sensor_key_by_id sensor_id - kit.sensor_map.invert[sensor_id] rescue nil + sensor_map.invert[sensor_id] rescue nil end def user_tags @@ -117,25 +143,6 @@ def user_tags=(tag_names) end end - # temporary kit getter/setter - def kit_version - if self.kit_id - if self.kit_id == 2 - "1.0" - elsif self.kit_id == 3 - "1.1" - end - end - end - - def kit_version=(kv) - if kv == "1.0" - self.kit_id = 2 - elsif kv == "1.1" - self.kit_id = 3 - end - end - def owner_username owner.username if owner end @@ -145,7 +152,7 @@ def system_tags exposure, # indoor / outdoor ('new' if created_at > 1.week.ago), # new ('test_device' if is_test?), - ((last_recorded_at.present? and last_recorded_at > 60.minutes.ago) ? 'online' : 'offline') # state + ((last_reading_at.present? and last_reading_at > 60.minutes.ago) ? 'online' : 'offline') # state ].reject(&:blank?).sort end @@ -171,14 +178,6 @@ def firmware end end - def components - kit ? kit.components : super - end - - def sensors - kit ? kit.sensors : super - end - def status data.present? ? state : 'new' end @@ -193,32 +192,34 @@ def soft_state end end + def formatted_location + { + ip: nil, + exposure: exposure, + elevation: elevation.try(:to_i) , + latitude: latitude, + longitude: longitude, + geohash: geohash, + city: city, + country_code: country_code, + country: country_name + } + end + def formatted_data s = { - recorded_at: last_recorded_at, - added_at: last_recorded_at, - # calibrated_at: updated_at, - location: { - ip: nil, - exposure: exposure, - elevation: elevation.try(:to_i) , - latitude: latitude, - longitude: longitude, - geohash: geohash, - city: city, - country_code: country_code, - country: country_name - }, sensors: [] } - sensors.sort_by(&:name).each do |sensor| - sa = sensor.attributes + components.sort_by {|c| c.sensor.name }.each do |component| + sensor = component.sensor + sa = sensor.attributes.except(*%w{key equation reverse_equation measurement_id}) sa = sa.merge( + measurement: sensor.measurement&.for_sensor_json, value: (data ? data["#{sensor.id}"] : nil), - raw_value: (data ? data["#{sensor.id}_raw"] : nil), prev_value: (old_data ? old_data["#{sensor.id}"] : nil), - prev_raw_value: (old_data ? old_data["#{sensor.id}_raw"] : nil) + last_reading_at: component.last_reading_at, + tags: sensor.tags ) s[:sensors] << sa end @@ -234,47 +235,40 @@ def self.geocode_all_without_location end end - def set_version_if_required! identifier - if identifier and (identifier == "1.1" or identifier == "1.0") # and !device.kit_id - if self.kit_version.blank? or self.kit_version != identifier - self.kit_version = identifier - self.save validate: false - end - end + def remove_mac_address_for_newly_registered_device! + update(old_mac_address: mac_address, mac_address: nil) end - def self.for_world_map - Rails.cache.fetch("world_map", expires_in: 10.seconds) do - where - .not(latitude: nil) - .where.not(data: nil) - .where(is_test: false) - .includes(:owner,:tags) - .map do |device| - { - id: device.id, - name: device.name, - description: (device.description.present? ? device.description : nil), - owner_id: device.owner_id, - owner_username: device.owner_id ? device.owner_username : nil, - latitude: device.latitude, - longitude: device.longitude, - city: device.city, - country_code: device.country_code, - kit_id: device.kit_id, - state: device.state, - system_tags: device.system_tags, - user_tags: device.user_tags, - added_at: device.added_at, - updated_at: device.updated_at, - last_reading_at: (device.last_reading_at.present? ? device.last_reading_at : nil) - } - end + def update_component_timestamps(timestamp, sensor_ids) + components.select {|c| sensor_ids.include?(c.sensor_id) }.each do |component| + component.update_column(:last_reading_at, timestamp) end end - def remove_mac_address_for_newly_registered_device! - update(old_mac_address: mac_address, mac_address: nil) + def hardware(authorized=false) + { + name: hardware_name, + type: hardware_type, + version: hardware_version, + slug: hardware_slug, + last_status_message: authorized ? hardware_info : "[FILTERED]", + } + end + + def hardware_name + hardware_name_override || [hardware_version ? "SmartCitizen Kit" : "Unknown", hardware_version].compact.join(" ") + end + + def hardware_type + hardware_type_override || (hardware_version ? "SCK" : "Unknown") + end + + def hardware_version + hardware_version_override || hardware_info&.fetch('hw_ver', nil) + end + + def hardware_slug + hardware_slug_override || [hardware_type.downcase, hardware_version&.gsub(".", ",")].compact.join(":") end private diff --git a/app/models/kit.rb b/app/models/kit.rb deleted file mode 100644 index 5f39e932..00000000 --- a/app/models/kit.rb +++ /dev/null @@ -1,19 +0,0 @@ -# There is a naming conflict between frontend and backend. Here, a Kit is -# a template for Device, that includes all of its sensors. Instead of every -# Device having 9 sensors, a Device has_one Kit, and that Kit has 9 sensors. -# Doing this means 1000 SCKs == 1000 Devices, 1 Kit, 9 components and 9 sensors. -# Without Kit it would require 1000 Devices, 9000 components and 9 sensors. - - # See device.rb for more information regarding the name conflict. - -class Kit < ActiveRecord::Base - - extend FriendlyId - friendly_id :slug - - has_many :devices - has_many :components, as: :board - has_many :sensors, through: :components - validates_presence_of :name, :description - -end diff --git a/app/models/measurement.rb b/app/models/measurement.rb index c01c630a..d6865bbf 100644 --- a/app/models/measurement.rb +++ b/app/models/measurement.rb @@ -3,4 +3,8 @@ class Measurement < ActiveRecord::Base has_many :sensors validates_presence_of :name, :description validates_uniqueness_of :name + + def for_sensor_json + attributes.except(*%w{created_at updated_at}) + end end diff --git a/app/models/orphan_device.rb b/app/models/orphan_device.rb index 2fdcf353..a254f9ae 100644 --- a/app/models/orphan_device.rb +++ b/app/models/orphan_device.rb @@ -15,7 +15,6 @@ def device_attributes { name: name, description: description, - kit_id: kit_id, user_tags: user_tags, exposure: exposure, latitude: latitude, diff --git a/app/models/raw_storer.rb b/app/models/raw_storer.rb index 59436ed3..d1037329 100644 --- a/app/models/raw_storer.rb +++ b/app/models/raw_storer.rb @@ -4,7 +4,7 @@ class RawStorer - def initialize data, mac, version, ip + def initialize data, mac, version, ip, raise_errors=false success = true @@ -17,8 +17,6 @@ def initialize data, mac, version, ip identifier = version.split('-').first - device.set_version_if_required!(identifier) - ts = data['timestamp'] || data[:timestamp] parsed_ts = Time.parse(ts) raise "timestamp error (raw)" if parsed_ts > 1.day.from_now or parsed_ts < 3.years.ago @@ -31,7 +29,8 @@ def initialize data, mac, version, ip metric = sensor metric_id = device.find_sensor_id_by_key(metric) - component = device.components.detect{|c|c["sensor_id"] == metric_id} #find_component_by_sensor_id(metric_id) + + component = device.find_or_create_component_by_sensor_id(metric_id) next if component.nil? value = component.normalized_value( (Float(value) rescue value) ) @@ -55,16 +54,18 @@ def initialize data, mac, version, ip #Kairos.http_post_to("/datapoints", _data) Redis.current.publish('telnet_queue', _data.to_json) + sensor_ids = sql_data.select {|k, v| k.is_a?(Integer) }.keys.compact.uniq + device.update_component_timestamps(parsed_ts, sensor_ids) - if parsed_ts > (device.last_recorded_at || Time.at(0)) + if parsed_ts > (device.last_reading_at || Time.at(0)) # update without touching updated_at - device.update_columns(last_recorded_at: parsed_ts, data: sql_data, state: 'has_published') + device.update_columns(last_reading_at: parsed_ts, data: sql_data, state: 'has_published') end rescue Exception => e success = false - + raise e if raise_errors end if !Rails.env.test? and device diff --git a/app/models/sensor.rb b/app/models/sensor.rb index 10192832..4bb14587 100644 --- a/app/models/sensor.rb +++ b/app/models/sensor.rb @@ -1,12 +1,9 @@ -# Every Device has one or more sensors. A Kit is a blueprint/group of sensors. -# A Kit is not an SCK. There is a naming conflict with the frontend, please see -# app/models/kit.rb for more information. +# Every Device has one or more sensors. class Sensor < ActiveRecord::Base has_many :components - has_many :boards, through: :components - has_many :kits, through: :components + has_many :devices, through: :components has_many :sensor_tags has_many :tag_sensors, through: :sensor_tags diff --git a/app/models/storer.rb b/app/models/storer.rb index c8f499df..efa5762a 100644 --- a/app/models/storer.rb +++ b/app/models/storer.rb @@ -12,6 +12,7 @@ def initialize device, reading, do_update = true rescue Exception => e Sentry.capture_exception(e) + raise e if Rails.env.test? end raise e unless e.nil? @@ -20,15 +21,17 @@ def initialize device, reading, do_update = true def update_device(parsed_ts, sql_data) return if parsed_ts <= Time.at(0) - if @device.last_recorded_at.present? - # Comparison errors if @device.last_recorded_at is nil (new devices). + if @device.last_reading_at.present? + # Comparison errors if @device.last_reading_at is nil (new devices). # Devices can post multiple readings, in a non-sorted order. # Do not update data with an older timestamp. - return if parsed_ts < @device.last_recorded_at + return if parsed_ts < @device.last_reading_at end sql_data = @device.data.present? ? @device.data.merge(sql_data) : sql_data - @device.update_columns(last_recorded_at: parsed_ts, data: sql_data, state: 'has_published') + @device.update_columns(last_reading_at: parsed_ts, data: sql_data, state: 'has_published') + sensor_ids = sql_data.select { |k, v| k.is_a?(Integer) }.keys.compact.uniq + @device.update_component_timestamps(parsed_ts, sensor_ids) ws_publish() end diff --git a/app/models/user.rb b/app/models/user.rb index 1954ad79..fd99874b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -28,7 +28,7 @@ class User < ActiveRecord::Base validates :username, :email, presence: true validates :username, uniqueness: true, if: :username? validates :username, length: { in: 3..30 }, allow_nil: true - validates :email, format: { with: /@/ }, uniqueness: true, if: :email?, on: :create + validates :email, format: { with: /@/ }, uniqueness: true validates :url, format: URI::regexp(%w(http https)), allow_nil: true, allow_blank: true, on: :create has_many :devices, foreign_key: 'owner_id', after_add: :update_cached_device_ids!, after_remove: :update_cached_device_ids! @@ -39,8 +39,6 @@ class User < ActiveRecord::Base before_create :generate_legacy_api_key - alias_attribute :joined_at, :created_at - def self.ransackable_attributes(auth_object = nil) [ "city", "country_code", "id", "username", "uuid", "created_at", "updated_at"] end @@ -125,7 +123,7 @@ def role def location { city: city, - country: country.try(:name), + country: country_name, country_code: country_code } end diff --git a/app/policies/kit_policy.rb b/app/policies/kit_policy.rb deleted file mode 100644 index 29e37ae1..00000000 --- a/app/policies/kit_policy.rb +++ /dev/null @@ -1,2 +0,0 @@ -class KitPolicy < AdminPolicy -end diff --git a/app/services/device_archive.rb b/app/services/device_archive.rb index 06c8c498..cdb72fef 100644 --- a/app/services/device_archive.rb +++ b/app/services/device_archive.rb @@ -45,18 +45,18 @@ def self.csv_file device_id def self.data_array device data = {} - keys = Array.new(device.kit.sensor_map.keys.length) - device.kit.sensor_map.keys.each_with_index do |key, index| + keys = Array.new(device.sensor_map.keys.length) + device.sensor_map.keys.each_with_index do |key, index| metric_id = device.find_sensor_id_by_key(key) return unless component = device.components.detect{ |c| c["sensor_id"] == metric_id } values = self.sensor_data(device, key) values.each do |v| time = Time.at(v[0]/1000).utc - data[time] ||= Array.new(device.kit.sensor_map.keys.length) + data[time] ||= Array.new(device.sensor_map.keys.length) data[time][index] = component.calibrated_value(v[1]) end - sensor = Sensor.find(device.kit.sensor_map[key]) + sensor = Sensor.find(device.sensor_map[key]) keys[index] = "#{sensor.measurement.name} in #{sensor.unit} (#{sensor.name})" end return [["timestamp"] + keys] + data.sort_by { |k, _| k.to_i } diff --git a/app/views/v0/components/_component.jbuilder b/app/views/v0/components/_component.jbuilder index 57c93afd..354a33fc 100644 --- a/app/views/v0/components/_component.jbuilder +++ b/app/views/v0/components/_component.jbuilder @@ -1 +1 @@ -json.(component, :id, :uuid, :board_id, :board_type, :sensor_id, :created_at, :updated_at) +json.(component, :id, :uuid, :device_id, :sensor_id, :created_at, :updated_at) diff --git a/app/views/v0/devices/_device.jbuilder b/app/views/v0/devices/_device.jbuilder index af8cc0aa..49235b16 100644 --- a/app/views/v0/devices/_device.jbuilder +++ b/app/views/v0/devices/_device.jbuilder @@ -1,5 +1,9 @@ -with_owner = true unless local_assigns.has_key?(:with_owner) -with_data = true unless local_assigns.has_key?(:with_data) +local_assigns[:with_owner] = true unless local_assigns.has_key?(:with_owner) +local_assigns[:with_data] = true unless local_assigns.has_key?(:with_data) +local_assigns[:with_postprocessing] = true unless local_assigns.has_key?(:with_postprocessing) +local_assigns[:with_location] = true unless local_assigns.has_key?(:with_location) +local_assigns[:slim_owner] = false unless local_assigns.has_key?(:slim_owner) +local_assigns[:never_authorized] = false unless local_assigns.has_key?(:never_authorized) json.( device, @@ -8,46 +12,47 @@ json.( :name, :description, :state, - :postprocessing, - :hardware_info, :system_tags, :user_tags, :is_private, - :notify_low_battery, - :notify_stopped_publishing, :last_reading_at, - :added_at, + :created_at, :updated_at ) -if current_user and (current_user.is_admin? or (device.owner_id and current_user.id == device.owner_id)) - json.merge! mac_address: device.mac_address + json.merge!(notify: { + stopped_publishing: device.notify_stopped_publishing, + low_battery: device.notify_low_battery + }) + +authorized = !local_assigns[:never_authorized] && (current_user && (current_user.is_admin? || (device.owner_id && current_user.id == device.owner_id))) + +if authorized json.merge! device_token: device.device_token + json.merge! mac_address: device.mac_address if device.mac_address else - json.merge! mac_address: '[FILTERED]' json.merge! device_token: '[FILTERED]' end +json.merge!(postprocessing: device.postprocessing) if local_assigns[:with_postprocessing] +json.merge!(location: device.formatted_location) if local_assigns[:with_location] +json.merge!(hardware: device.hardware(authorized)) -if with_owner && device.owner +if local_assigns[:with_owner] && device.owner json.owner do json.id device.owner.id json.uuid device.owner.uuid json.username device.owner.username - json.avatar device.owner.avatar - - json.profile_picture profile_picture_url(device.owner) - json.url device.owner.url - json.joined_at device.owner.joined_at - json.location device.owner.location - json.device_ids device.owner.cached_device_ids + + unless local_assigns[:slim_owner] + json.avatar device.owner.avatar + json.profile_picture profile_picture_url(device.owner) + json.location device.owner.location + json.device_ids device.owner.cached_device_ids + end end end -json.data device.formatted_data if with_data +json.data device.formatted_data if local_assigns[:with_data] + -if device.kit - json.kit device.kit, :id, :uuid, :slug, :name, :description, :created_at, :updated_at -else - json.merge! kit: nil -end diff --git a/app/views/v0/devices/_world_map_list.jbuilder b/app/views/v0/devices/_world_map_list.jbuilder new file mode 100644 index 00000000..b690b676 --- /dev/null +++ b/app/views/v0/devices/_world_map_list.jbuilder @@ -0,0 +1 @@ +json.array! Device.for_world_map, partial: 'device', as: :device, local_assigns: { with_data: false, with_postprocessing: false, slim_owner: true, never_authorized: never_authorized } \ No newline at end of file diff --git a/app/views/v0/devices/fresh_world_map.jbuilder b/app/views/v0/devices/fresh_world_map.jbuilder new file mode 100644 index 00000000..bb646244 --- /dev/null +++ b/app/views/v0/devices/fresh_world_map.jbuilder @@ -0,0 +1 @@ +json.partial! 'devices/world_map_list', { never_authorized: false } \ No newline at end of file diff --git a/app/views/v0/devices/world_map.jbuilder b/app/views/v0/devices/world_map.jbuilder new file mode 100644 index 00000000..f72cd4a4 --- /dev/null +++ b/app/views/v0/devices/world_map.jbuilder @@ -0,0 +1,3 @@ +json.cache! ["world_map"], expires_in: 1.minute do + json.partial! 'devices/world_map_list', { never_authorized: true} +end \ No newline at end of file diff --git a/app/views/v0/kits/_kit.jbuilder b/app/views/v0/kits/_kit.jbuilder deleted file mode 100644 index 11d8d222..00000000 --- a/app/views/v0/kits/_kit.jbuilder +++ /dev/null @@ -1,14 +0,0 @@ -json.(kit, - :id, :uuid, :slug, :name, :description, :created_at, :updated_at -) - -# json . kit do -# json.id kit.id -# json.slug kit.slug -# json.name kit.name -# json.description kit.description -# json.created_at kit.created_at.utc.iso8601 -# json.updated_at kit.updated_at.utc.iso8601 -# end - -json.sensors kit.sensors, partial: 'sensors/sensor', as: :sensor diff --git a/app/views/v0/kits/index.jbuilder b/app/views/v0/kits/index.jbuilder deleted file mode 100644 index 92e7c722..00000000 --- a/app/views/v0/kits/index.jbuilder +++ /dev/null @@ -1 +0,0 @@ -json.array! @kits, partial: 'kit', as: :kit diff --git a/app/views/v0/kits/show.jbuilder b/app/views/v0/kits/show.jbuilder deleted file mode 100644 index b96b0470..00000000 --- a/app/views/v0/kits/show.jbuilder +++ /dev/null @@ -1 +0,0 @@ -json.partial! "kit", kit: @kit diff --git a/app/views/v0/users/_user.jbuilder b/app/views/v0/users/_user.jbuilder index d3f54d33..ed70c911 100644 --- a/app/views/v0/users/_user.jbuilder +++ b/app/views/v0/users/_user.jbuilder @@ -12,7 +12,9 @@ json.(user, json.profile_picture profile_picture_url(user) -if current_user and (current_user.is_admin? or current_user == user) +authorized = current_user && current_user == user || current_user&.is_admin? + +if authorized json.merge! email: user.email json.merge! legacy_api_key: user.legacy_api_key else @@ -21,15 +23,7 @@ else end json.devices user.devices.filter { |d| - !d.is_private? || current_user == user || current_user&.is_admin? + !d.is_private? || authorized }.map do |device| - json.partial! "devices/device", device: device, with_data: false, with_owner: false - json.merge!(kit_id: device.kit_id) - if current_user == user || current_user&.is_admin? - json.merge!( - location: device.location, - latitude: device.latitude, - longitude: device.longitude, - ) - end + json.partial! "devices/device", device: device, with_data: false, with_owner: false, with_location: authorized end diff --git a/compose/cassandra.yml b/compose/cassandra.yml index 642b83df..e12ad25b 100644 --- a/compose/cassandra.yml +++ b/compose/cassandra.yml @@ -16,9 +16,10 @@ services: - "7199:7199" # JMX - "9042:9042" # CQL native transport - "9160:9160" # Thrift client API - #volumes: - #- ../sck-cassandra:/var/lib/cassandra - + volumes: + - ../sck-cassandra:/var/lib/cassandra +volumes: + sck-cassandra: # cassandra-2: # image: cassandra:3.11.4 # ports: diff --git a/config/routes.rb b/config/routes.rb index a23621fe..67148ffe 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -33,7 +33,6 @@ resources :tag_sensors resources :tags resources :measurements - resources :kits, except: [:destroy] resources :users resources :password_resets, only: [:show, :create, :update] resources :oauth_applications, path: 'applications' diff --git a/db/data/kits.csv b/db/data/kits.csv new file mode 100644 index 00000000..12eded23 --- /dev/null +++ b/db/data/kits.csv @@ -0,0 +1,46 @@ +id,name,description,slug,hardware_type,hardware_name,hardware_version,hardware_description,hardware_slug +2,SmartCitizen Kit 1.0,SCK 1.0 - Ambient Board,"sck:1,0",SCK,SmartCitizen Kit 1.0,1.0,SCK 1.0 - Ambient Board, +3,SmartCitizen Kit 1.1,SCK 1.1 - Ambient Board,"sck:1,1",SCK,SmartCitizen Kit 1.1,1.1,SCK 1.1 - Ambient Board, +4,NO2 Diffusion Tubes,Nitrogen Oxides Diffusion Tubes Spreadsheet,"ms:0,5",MS,NO2 Diffusion Tubes,0.5,Nitrogen Oxides Diffusion Tubes Spreadsheet, +5,Noise Sensor TA120,Noise Sensor CESVA TA120,"ms:0,6",CESVA,Noise Sensor TA120,TA210,Noise Sensor CESVA TA120, +6,Processing Open CV,Open CV based sensor running built on Processing.org and running on a Raspberry Pi,"ms:0,3",MS,Processing Open CV,0.3,Open CV based sensor running built on Processing.org and running on a Raspberry Pi, +7,Smart Citizen Kit 1.5 Expanded,SCK 1.5 + Grove Analog Sensor,"ms:0,4",SCK,Smart Citizen Kit 1.5 Expanded,1.5,SCK 1.5 + Grove Analog Sensor, +8,Making Sense WAAG #1,AQM sensor by WAAG,"ms:1,0",MS,Making Sense WAAG #1,1.0,AQM sensor by WAAG, +9,Making Sense WAAG #2,AQM sensor by WAAG 2,"ms:1,1",MS,Making Sense WAAG #2,1.0,AQM sensor by WAAG 2, +10,Making Sense WAAG #3,AQM sensor by WAAG 3,"ms:1,3",MS,Making Sense WAAG #3,1.0,AQM sensor by WAAG 3, +11,SCK 1.5 MS,SCK 1.5 - MS Pilot,"sck:1,5",SCK,SCK 1.5 MS,1.5,SCK 1.5 - MS Pilot, +12,Bora Kit,Bora Air Quality Sensors Kit,"ms:2,0",BK,Bora Kit,1.0,Bora Air Quality Sensors Kit, +13,GammaSense Kit,GammaSense web app that uses device camera via WebRTC,"ms:3,0",GSK,GammaSense Kit,1.0,GammaSense web app that uses device camera via WebRTC, +14,SCK 1.5,SCK 1.5 - Urban Board,"sck:1,5,1",SCK,SCK 1.5,1.5,SCK 1.5 - Urban Board, +15,SCK Aquapioneers 1.5 ,SCK 1.5 - Urban Board + Aquapioneers Board ,"sck:1,5:aq",SCK,SCK Aquapioneers 1.5 ,1.5,SCK 1.5 - Urban Board + Aquapioneers Board , +16,SCK Grow Soil Probes,SCK 1.5 + Atlas Scientific Sensor Probes,"sck:1,5:grow:as",SCK,SCK Grow Soil Probes,1.5,SCK 1.5 + Atlas Scientific Sensor Probes, +17,SCK Grow Soil Moisture,SCK 1.5 + Urban Board + Chirp Moisture Sensor,"sck:1,5:grow:moisture",SCK,SCK Grow Soil Moisture,1.5,SCK 1.5 + Urban Board + Chirp Moisture Sensor, +18,SCK 2.0 - Dev,SCK 2.0 - Urban Board - Dev,"sck:2,0,1:dev",SCK,SCK 2.0 - Dev,2.0,SCK 2.0 - Urban Board - Dev, +19,iSCAPE Station - Dev,iSCAPE Stations Development Kit,"sck:2,0:dev:iscape:station",SCK,iSCAPE Station - Dev,2.0,iSCAPE Stations Development Kit, +20,iSCAPE Citizen Kit,iSCAPE Citizen Kit ,"sck:2,0,1",SCK,iSCAPE Citizen Kit,2.0,iSCAPE Citizen Kit , +21,iSCAPE Station,iSCAPE Living Labs Station,"sck:2,0:iscape:station",SCK,iSCAPE Station,2.0,iSCAPE Living Labs Station, +22,BioPV Kit,SCK 2.0 - Urban Board + Atlas Scientific Sensors + Chirp Moisture + ADC,"sck:1,5:biopv",SCK,BioPV Kit,2.0,SCK 2.0 - Urban Board + Atlas Scientific Sensors + Chirp Moisture + ADC, +23,SCK Aquapioneers Lite 1.5,SCK 1.5 - Aquapioneers Lite Board ,"sck:1,5:aq:lite",SCK,SCK Aquapioneers Lite 1.5,1.5,SCK 1.5 - Aquapioneers Lite Board , +24,SCK Wine Makers,SCK 2.0 + Waterproof Temperature Sensor,"sck:2,0:winemakers",SCK,SCK Wine Makers,2.0,SCK 2.0 + Waterproof Temperature Sensor, +25,SCK 2.1 for DECODE,DECODE Citizen Kit ,"sck:2,0,1:decode",SCK,SCK 2.1 for DECODE,2.1,DECODE Citizen Kit , +26,SCK 2.1,Smart Citizen Kit 2.1 with Urban Sensor Board,"sck:2,1",SCK,SCK 2.1,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board, +27,SCK Grow Soil Moisture 2.1,Smart Citizen Kit 2.1 with Urban Sensor Board and Chirp Soil Moisture,"sck:2,1:grow:moisture",SCK,SCK Grow Soil Moisture 2.1,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board and Chirp Soil Moisture, +28,iSCAPE Station 2.1,iSCAPE Living Labs Station 2.1,"sck:2,1:iscape:station",SCK,iSCAPE Station 2.1,2.1,iSCAPE Living Labs Station 2.1, +29,SCK 2.1 Formaldehyde,Smart Citizen Kit 2.1 with Urban Sensor Board with Formaldehyde,"sck:2,1:form",SCK,SCK 2.1 Formaldehyde,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board with Formaldehyde, +30,SCK 2.1 PM Board Addons,Smart Citizen Kit 2.1 with PM Board for Experimental I/O,"sck:2,1:pmboard:experiments",SCK,SCK 2.1 PM Board Addons,2.1,Smart Citizen Kit 2.1 with PM Board for Experimental I/O, +31,SCK 2.1 Sea Water,Smart Citizen Kit 2.1 with Sea Water Sensors,"sck:2,1:seawater",SCK,SCK 2.1 Sea Water,2.1,Smart Citizen Kit 2.1 with Sea Water Sensors, +32,SCK 2.1 GPS,Smart Citizen Kit 2.1 with GPS,"sck:2,1:gps",SCK,SCK 2.1 GPS,2.1,Smart Citizen Kit 2.1 with GPS, +33,Smart Citizen Station 2.1 rev3,Smart Citizen Station 2.1 rev3,"sck:2,1:station:rev3",SCK,Smart Citizen Station 2.1 rev3,2.1,Smart Citizen Station 2.1 rev3, +34,BioPV Kit 2.1,SCK 2.1 - Urban Board + PM + Atlas + Soil Moisture + ADC ADS1115,"sck:2,1:biopv",SCK,BioPV Kit 2.1,2.1,SCK 2.1 - Urban Board + PM + Atlas + Soil Moisture + ADC ADS1115, +35,SCK 2.1 CO2,SCK 2.1 - Urban Board + PM + CO2,"sck:2,1:co2",SCK,SCK 2.1 CO2,2.1,SCK 2.1 - Urban Board + PM + CO2, +36,"SCK 2.1 CO2, CO and NO2",SCK 2.1 - Urban Board + PM + CO2 + CO + NO2,"sck:2,1:co2:co:no2",SCK,"SCK 2.1 CO2, CO and NO2",2.1,SCK 2.1 - Urban Board + PM + CO2 + CO + NO2, +37,"SCK 2.1 NO2, O3",SCK 2.1 - Urban Board + PM + NO2 + O3,"sck:2,1:no2:o3",SCK,"SCK 2.1 NO2, O3",2.1,SCK 2.1 - Urban Board + PM + NO2 + O3, +38,SCK 2.1 Soil and Air,"SCK 2.1 - Soil Moisture, Soil Temperature and Air","sck:2,1:soil:air",SCK,SCK 2.1 Soil and Air,2.1,"SCK 2.1 - Soil Moisture, Soil Temperature and Air", +39,SCK 2.1 T/RH,SCK 2.1 - Urban Board + PM + T + RH,"sck:2,1:t:rh",SCK,SCK 2.1 T/RH,2.1,SCK 2.1 - Urban Board + PM + T + RH, +40,DIY Sensor Kit PCTO @ LD22,"DIY Sensor Kit PCTO for air quality, environment and GPS readings",dyi:pm:co2:gps,DIY,DIY Sensor Kit PCTO @ LD22,1.0,"DIY Sensor Kit PCTO for air quality, environment and GPS readings", +41,SCK 2.1 SPS30,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SPS30,"sck:2,1",SCK,SCK 2.1 SPS30,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SPS30, +42,SCK 2.1 SEN5X,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SEN5X,"sck:2,1",SCK,SCK 2.1 SEN5X,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SEN5X, +43,SCK 2.1 SFA30,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SFA30,"sck:2,1",SCK,SCK 2.1 SFA30,2.1,Smart Citizen Kit 2.1 with Urban Sensor Board and Sensirion SFA30, +44,SCK 2.1 SFA30 and SCD30,"Smart Citizen Kit 2.1 with Urban Sensor Board, Sensirion SCD30 and Sensirion SFA30","sck:2,1",SCK,SCK 2.1 SFA30 and SCD30,2.1,"Smart Citizen Kit 2.1 with Urban Sensor Board, Sensirion SCD30 and Sensirion SFA30", +45,SCK 2.1 Debug,Smart Citizen Kit 2.1 internal development,"sck:2,1",SCK,SCK 2.1 Debug,2.1,Smart Citizen Kit 2.1 internal development, +,,,,,,,, diff --git a/db/data/sensors_without_default_keys.csv b/db/data/sensors_without_default_keys.csv new file mode 100644 index 00000000..75cc789c --- /dev/null +++ b/db/data/sensors_without_default_keys.csv @@ -0,0 +1,56 @@ +id,ancestry,count_children,name,description,unit,created_at,key +3,,2,DHT22,"A digital temperature and humidity sensor. It uses a capacitive humidity sensor and a thermistor to measure the surrounding air, and spits out a digital signal on the data pin (no analog input pins needed)",,2015-02-02 18:14:15.199342, +19,,2,HPP828E031 (SHT21),Digital Temperature and Relative Humidity Sensor,,2015-02-02 18:27:20.701211, +20,,4,MiCS4514,Gas Sensor,,2015-02-02 18:31:50.485383, +37,,4,Camera - Ionizing Radiation,CMOS camera embedded in Smart phones and laptops measuring ionizing radiation using the GammaSense web app,CPM,2017-05-17 10:34:20.744418, +44,,4,AS EZO Conductivity,Atlas Scientific EZO™ Conductivity Probe,,2017-08-04 12:05:30.666055, +47,,2,AS EZO DO,Atlas Scientific EZO™ Dissolved Oxygen Probe,,2017-08-04 12:05:30.666055, +52,NULL,3,ICS43432,I2S Digital Mems Microphone with custom Audio Processing Algorithm,NULL,2018-05-03 10:34:06.367516, +54,,2,Sensirion SHT31,Digital relative humidity and air temperature sensor,,2018-05-03 10:46:15.286301, +57,NULL,2,MPL3115A2,Digital Barometric Pressure Sensor,,2018-05-03 10:48:17.286301, +59,,3,SCK Gases Pro Board,SCK Gases Pro Board: Alphasense Series B 3 Channel Driver ,,2018-05-03 13:58:57.722415, +60,59,3,NO2-B43F,Alphasense Nitrogen Dioxide + Ozone,,2018-05-03 13:58:57.722415, +63,59,3,CO-B4F,Alphasense Carbon Monoxide + Ozone,,2018-05-03 14:16:33.478454, +66,59,3,OXB431,Alphasense Ozone,,2018-05-03 14:16:33.478454, +69,,3,SCK PM Board,SCK PM Driver: Dual Platfower PMSX003 driver,,2018-05-03 14:22:12.270653, +70,69,9,PMS5003_A,Plantower PMS5003 PM 10/2.5/1 sensor,,2018-05-03 14:22:12.270653, +74,69,9,PMS5003_B,Plantower PMS5003 PM 10/2.5/1 sensor,,2018-05-03 14:22:12.270653, +78,NULL,2,Ext SHT31,External digital relative humidity and air temperature sensor,,2018-05-03 16:56:15.286301, +81,20,0,MiCS-4514-NO2-CON,MICS Nitrogen Dioxide,ppb,2018-05-22 12:58:05.741005,mics_4514_no2 +82,20,0,MiCS-4514-CO-CON,MICS Carbon Monoxide,ppm,2018-05-22 12:58:05.741005,mics_4514_co +86,69,6,PMS5003_AVG,PM Board with dual PMS5003 PM 10/2.5/1 sensors,,2018-05-22 13:20:34.385306, +90,,4,Ultrasonic Ranger,Ping Ultrasonic (40KHz) non-contact distance ranger (Ping with Grove connector),,2018-07-20 11:10:48.578866, +111,,2,AMS CCS811,Digital gas sensor for monitoring indoor air quality,,2019-03-21 16:43:37.814294, +114,,3,PMS5003,PM PMS5003 PM 10/2.5/1 sensors,,2019-03-21 17:09:09.164676, +115,114,0,PMS5003-PM2.5,Particle Matter PM 2.5,ug/m3,2019-03-21 17:09:09.164676,pm25 +116,114,0,PMS5003-PM10,Particle Matter PM 10,ug/m3,2019-03-21 17:09:09.164676,pm10 +117,114,0,PMS5003-PM1,Particle Matter PM 1,ug/m3,2019-03-21 17:09:09.164676,pm1 +124,,7,GPS,Location data frm a GPS device,,2020-11-11 15:19:06.495725, +132,,4,ADC_48,16-bit Analog to Digital Converter - ADS1X15 addr. 48,V,2020-12-11 15:35:48.93594, +137,,4,ADC_49,16-bit Analog to Digital Converter - ADS1X15 addr. 49,V,2020-12-11 15:37:12.86794, +142,,4,ADC_4A,16-bit Analog to Digital Converter - ADS1X15 addr. 4A,V,2020-12-11 15:38:01.76852, +147,,4,ADC_4B,16-bit Analog to Digital Converter - ADS1X15 addr. 4B,V,2020-12-11 15:38:55.359661, +162,,2,SCD30,"Digital NDIR CO2, temperature and humidity sensor",,2021-05-07 16:14:52.627648, +163,,1,AS EZO ORP,Atlas Scientific EZO™ ORP Probe,,2022-09-06 11:58:58.089711, +171,,1,Winsen MH-Z16,Digital NDIR CO2 sensor,,2022-11-17 16:44:34.791725, +173,,2,Bosch BMP280,Digital ambient temperature and pressure sensor,,2022-11-17 16:45:47.510406, +176,,1,Electret Microphone,Electret Microphone with MAX9814 Amplifier,,2022-11-17 16:47:19.59493, +178,,2,NovaFitness SDS011,Light Scattering Particle Matter Sensor,,2022-11-17 16:48:07.440543, +181,,10,Sensirion SPS30,Light Scattering Particle Matter Sensor,,2023-05-16 16:13:32.451798, +192,,16,Sensirion SEN5X,Light Scattering Particle Matter Sensor with additional NOx and VOCs indexes,,2023-05-23 11:34:29.863582, +209,,3,Sensirion SFA30,Formaldehyde electrochemical sensor,,2023-05-30 11:51:02.871029, +213,,3,AMS AS7731,3 channel spectral UV-A/B/C sensor,,2023-07-14 16:34:12.947438, +214,213,0,AMS AS7731 - UVA,AMS AS7731 UVA Channel,uW/cm2,2023-07-14 16:33:21.304504,uv_a +215,213,0,AMS AS7731 - UVB,AMS AS7731 UVB Channel,uW/cm2,2023-07-14 16:33:21.320123,uv_b +216,213,0,AMS AS7731 - UVC,AMS AS7731 UVC Channel,uW/cm2,2023-07-14 16:33:21.328074,uv_c +217,52,0,TDK ICS43432 - Noise C,C-Scale RMS from I2S Digital Mems Microphone with custom Audio Processing Algorithm,dBC,2023-07-14 16:45:26.938851,noise_c +218,52,0,TDK ICS43432 - Noise Z,Z-Scale RMS from I2S Digital Mems Microphone with custom Audio Processing Algorithm,dBZ,2023-07-14 16:45:37.793749,noise_z +219,57,0,NXP MPL3115A2 - Altitude,Altitude from digital barometric pressure sensor,m,2023-07-14 16:53:03.159593,mpl_alt +222,,0,Battery Voltage,Battery voltage,V,2023-07-14 17:08:28.702493,batt_volt +223,,2,Sensirion SHT35,Digital relative humidity and air temperature sensor,,2023-07-14 17:12:58.415308, +224,223,0,Sensirion SHT35 - Temperature,Temperature from digital relative humidity and air temperature sensor,ºC,2023-07-14 17:14:02.042413,sht35_t +225,223,0,Sensirion SHT35 - Humidity,Humidity from digital relative humidity and air temperature sensor,%,2023-07-14 17:14:38.432689,sht35_h +226,,1,ST LPS33,Digital Barometric Pressure Sensor,,2023-07-24 16:39:27.885491, +227,226,0,ST LPS33 - Barometric Pressure,Digital Barometric Pressure Sensor,kPa,2023-07-24 16:41:36.559386,lps_p +,,,,,,, +,,,,,,, diff --git a/db/migrate/20230704150532_refactor_kits.rb b/db/migrate/20230704150532_refactor_kits.rb new file mode 100644 index 00000000..41420610 --- /dev/null +++ b/db/migrate/20230704150532_refactor_kits.rb @@ -0,0 +1,218 @@ +require 'csv' +class RefactorKits < ActiveRecord::Migration[6.0] + + def execute(query, args=[]) + args = [args] unless args.is_a?(Array) + sanitized = ActiveRecord::Base.send(:sanitize_sql_array, [query] + args) + ActiveRecord::Base.connection.execute(sanitized) + end + + def change + + # To start, create a hash of all devices to their sensors, to use as a check later on: + device_sensors_pre = Hash.new {|h, k| h[k] = []; } + execute(""" + SELECT devices.id AS device_id, components.sensor_id AS sensor_id + FROM devices + INNER JOIN components on components.board_id = devices.kit_id + AND components.board_type = 'Kit' + ORDER BY device_id, sensor_id ASC + """).each do |row| + device_sensors_pre[row["device_id"]] << row["sensor_id"] + end + + + # Add columns needed in new schema: + change_table :components do |t| + t.column :device_id, :integer + t.column :key, :string, null: true + end + change_column_null :components, :board_id, true + change_column_null :components, :board_type, true + + change_table :sensors do |t| + t.column :default_key, :string, null: true + t.column :equation, :string, null: true + t.column :reverse_equation, :string, null: true + end + + change_table :devices do |t| + t.column :hardware_type_override, :string, null: true + t.column :hardware_name_override, :string, null: true + t.column :hardware_version_override, :string, null: true + t.column :hardware_description_override, :string, null: true + t.column :hardware_slug_override, :string, null: true + end + + # Add default key to sensors to be used when new components are created: + puts "-- setting default keys for sensors" + + key_counter = Hash.new { |h, k| + h[k] = Hash.new {|h2, k2| + h2[k2] = 0 + } + } + + execute("SELECT sensor_map FROM kits").each { |row| + m = JSON.parse(row["sensor_map"]) + m.each { |k, v| key_counter[v][k] += 1 } + } + + key_counter.each do |id, keys| + key = keys.max_by {|k, v| v }[0] + execute("UPDATE sensors SET default_key=? WHERE id=?", [key, id]) + end + + + # Add equations to sensors: + puts "-- setting equations for sensors" + + execute(""" + SELECT sensor_id, + COUNT(DISTINCT equation) AS count_equation, + COUNT(DISTINCT reverse_equation) AS count_reverse_equation + FROM components + GROUP BY sensor_id + """).each do |row| + sensor_id = row["sensor_id"] + if row["count_equation"] == 1 && row["count_reverse_equation"] == 1 + equations = execute(""" + SELECT equation, reverse_equation + FROM components + WHERE sensor_id = ? + LIMIT 1 + """, sensor_id)[0] + execute(""" + UPDATE sensors + SET equation = ?, + reverse_equation = ? + WHERE id = ? + """, [equations["equation"], equations["reverse_equation"], sensor_id]) + end + end + + # Set default key for sensors which are not part of a kit: + sensor_info = CSV.foreach("db/data/sensors_without_default_keys.csv", headers:true).map(&:to_h).reduce({}) { |h, r| h[r["id"].to_i] = r; h } + + sensor_info.each do |sensor_id, sensor_data| + if sensor_data["key"] + execute("UPDATE sensors SET default_key = ? WHERE id = ?", [sensor_data["key"], sensor_id]) + end + end + + + # For each existing device. Look up its kit, set its hardware_info, and create a component for each of that kit's components, with reference to the device itself. + + kits_info = CSV.foreach("db/data/kits.csv", headers:true).map(&:to_h).reduce({}) { |h, r| h[r["id"].to_i] = r; h } + + puts "-- setting hardware info and creating components for devices" + + execute("SELECT * FROM devices").each do |device_row| + execute("SELECT * FROM kits WHERE id = ? LIMIT 1", device_row["kit_id"]).each do |kit_row| + + device_id = device_row["id"] + + kit_id = kit_row["id"] + kit_info = kits_info[kit_id] + + hardware_version = kit_info["hardware_version"] + + unless kit_info["hardware_type"] == "SCK" + hardware_type = kit_info["hardware_type"] + end + + default_name = "SmartCitizen Kit #{hardware_version}" + unless kit_info["hardware_name"] == default_name + hardware_name = kit_info["hardware_name"] + end + + default_slug = "#{kit_info["hardware_type"].downcase}:#{hardware_version.gsub(".", ",")}" + unless kit_info["slug"] == default_slug + hardware_slug = kit_info["slug"] + end + + unless kit_info["hardware_description"] == kit_info["hardware_name"] + hardware_description = kit_info["hardware_description"] + end + + execute(""" + UPDATE devices + SET + hardware_version_override = ?, + hardware_type_override = ?, + hardware_name_override = ?, + hardware_description_override = ?, + hardware_slug_override = ? + WHERE id = ? + """, [hardware_version, hardware_type, hardware_name, hardware_description, hardware_slug, device_id]) + + + kit_component_rows = execute(""" + SELECT * FROM components + WHERE board_type = 'Kit' + AND board_id = ? + """, kit_row["id"]) + sensor_map = JSON.parse(kit_row["sensor_map"]) + kit_component_rows.each do |kit_component_row| + sensor_row = execute("SELECT * FROM sensors WHERE id = ?", kit_component_row["sensor_id"])[0] + device_id = device_row["id"] + sensor_id = sensor_row["id"] + created_at = kit_component_row["created_at"] + updated_at = kit_component_row["updated_at"] + key = sensor_map.invert[sensor_id] + execute(""" + INSERT INTO components + (device_id, sensor_id, key, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + """, [device_id, sensor_id, key, created_at, updated_at]) + end + end + end + + # Check the devices / sensors mapping we created before against the current + # state: + + puts "-- checking device sensors state matches previous" + + device_sensors_post = Hash.new {|h, k| h[k] = [] } + execute(""" + SELECT devices.id AS device_id, components.sensor_id AS sensor_id + FROM devices + INNER JOIN components on components.device_id = devices.id + ORDER BY device_id, sensor_id ASC + """).each do |row| + device_sensors_post[row["device_id"]] << row["sensor_id"] + end + + unless device_sensors_pre == device_sensors_post + raise "Check failed - device sensors not the same before and after migration!" + end + + # Delete deprecated columns and tables: + change_table :orphan_devices do |t| + t.remove :kit_id + end + + change_table :components do |t| + t.remove :board_id + t.remove :board_type + t.remove :equation + t.remove :reverse_equation + end + + change_table :devices do |t| + t.remove :kit_id + end + + drop_table :kits + + # Remove the old component records: + execute "DELETE FROM components WHERE device_id IS NULL" + + # Set constraints on components and sensors for the new device relationshop: + change_column_null :components, :device_id, false + add_foreign_key :components, :devices + change_column_null :components, :key, null: false + change_column_null :sensors, :default_key, null: false + end +end diff --git a/db/migrate/20230929114837_further_kits_refactor_changes.rb b/db/migrate/20230929114837_further_kits_refactor_changes.rb new file mode 100644 index 00000000..504abea3 --- /dev/null +++ b/db/migrate/20230929114837_further_kits_refactor_changes.rb @@ -0,0 +1,12 @@ +class FurtherKitsRefactorChanges < ActiveRecord::Migration[6.1] + def change + rename_column :devices, :last_recorded_at, :last_reading_at + add_column :components, :location, :integer, default: 1 + connection.execute(%{ + UPDATE components + SET location=1 + WHERE location IS NULL + }) + change_column_null :components, :location, false + end +end diff --git a/db/migrate/20231005153412_rename_component_location_to_bus.rb b/db/migrate/20231005153412_rename_component_location_to_bus.rb new file mode 100644 index 00000000..7c0df694 --- /dev/null +++ b/db/migrate/20231005153412_rename_component_location_to_bus.rb @@ -0,0 +1,5 @@ +class RenameComponentLocationToBus < ActiveRecord::Migration[6.1] + def change + rename_column :components, :location, :bus + end +end diff --git a/db/migrate/20231006064514_add_last_reading_at_to_components.rb b/db/migrate/20231006064514_add_last_reading_at_to_components.rb new file mode 100644 index 00000000..fdf7aba2 --- /dev/null +++ b/db/migrate/20231006064514_add_last_reading_at_to_components.rb @@ -0,0 +1,11 @@ +class AddLastReadingAtToComponents < ActiveRecord::Migration[6.1] + def change + add_column :components, :last_reading_at, :datetime + execute %{ + UPDATE components + SET last_reading_at = devices.last_reading_at + FROM devices + WHERE components.device_id = devices.id + } + end +end diff --git a/db/migrate/20240228125910_remove_hardware_description_override_from_devices.rb b/db/migrate/20240228125910_remove_hardware_description_override_from_devices.rb new file mode 100644 index 00000000..13d091e6 --- /dev/null +++ b/db/migrate/20240228125910_remove_hardware_description_override_from_devices.rb @@ -0,0 +1,5 @@ +class RemoveHardwareDescriptionOverrideFromDevices < ActiveRecord::Migration[6.1] + def change + remove_column :devices, :hardware_description_override, :string + end +end diff --git a/db/migrate/20240318110256_add_world_map_indexes.rb b/db/migrate/20240318110256_add_world_map_indexes.rb new file mode 100644 index 00000000..2a46e93b --- /dev/null +++ b/db/migrate/20240318110256_add_world_map_indexes.rb @@ -0,0 +1,5 @@ +class AddWorldMapIndexes < ActiveRecord::Migration[6.1] + def change + add_index :devices, [:workflow_state, :is_test, :last_reading_at, :latitude], name: "world_map_request" + end +end diff --git a/db/migrate/20240318171656_add_component_device_sensor_index.rb b/db/migrate/20240318171656_add_component_device_sensor_index.rb new file mode 100644 index 00000000..e5b74064 --- /dev/null +++ b/db/migrate/20240318171656_add_component_device_sensor_index.rb @@ -0,0 +1,6 @@ +class AddComponentDeviceSensorIndex < ActiveRecord::Migration[6.1] + def change + remove_index :components, [:sensor_id] + add_index :components, [:device_id, :sensor_id] + end +end diff --git a/db/schema.rb b/db/schema.rb index 02382447..2c934353 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_07_05_095430) do +ActiveRecord::Schema.define(version: 2024_03_18_171656) do # These are extensions that must be enabled in order to support this database enable_extension "adminpack" @@ -58,16 +58,15 @@ end create_table "components", id: :serial, force: :cascade do |t| - t.integer "board_id" - t.string "board_type" t.integer "sensor_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.uuid "uuid", default: -> { "uuid_generate_v4()" } - t.text "equation" - t.text "reverse_equation" - t.index ["board_type", "board_id"], name: "index_components_on_board_type_and_board_id" - t.index ["sensor_id"], name: "index_components_on_sensor_id" + t.integer "device_id", null: false + t.string "key" + t.integer "bus", default: 1, null: false + t.datetime "last_reading_at" + t.index ["device_id", "sensor_id"], name: "index_components_on_device_id_and_sensor_id" end create_table "devices", id: :serial, force: :cascade do |t| @@ -79,10 +78,9 @@ t.float "longitude" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "kit_id" t.hstore "latest_data" t.string "geohash" - t.datetime "last_recorded_at" + t.datetime "last_reading_at" t.jsonb "meta" t.jsonb "location" t.jsonb "data" @@ -103,12 +101,16 @@ t.boolean "is_private", default: false t.boolean "is_test", default: false, null: false t.datetime "archived_at" + t.string "hardware_type_override" + t.string "hardware_name_override" + t.string "hardware_version_override" + t.string "hardware_slug_override" t.index ["device_token"], name: "index_devices_on_device_token", unique: true t.index ["geohash"], name: "index_devices_on_geohash" - t.index ["kit_id"], name: "index_devices_on_kit_id" - t.index ["last_recorded_at"], name: "index_devices_on_last_recorded_at" + t.index ["last_reading_at"], name: "index_devices_on_last_reading_at" t.index ["owner_id"], name: "index_devices_on_owner_id" t.index ["state"], name: "index_devices_on_state" + t.index ["workflow_state", "is_test", "last_reading_at", "latitude"], name: "world_map_request" t.index ["workflow_state"], name: "index_devices_on_workflow_state" end @@ -137,17 +139,6 @@ t.index ["sluggable_type"], name: "index_friendly_id_slugs_on_sluggable_type" end - create_table "kits", id: :serial, force: :cascade do |t| - t.string "name" - t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.string "slug" - t.uuid "uuid", default: -> { "uuid_generate_v4()" } - t.jsonb "sensor_map" - t.index ["slug"], name: "index_kits_on_slug" - end - create_table "measurements", id: :serial, force: :cascade do |t| t.string "name" t.text "description" @@ -201,7 +192,6 @@ create_table "orphan_devices", id: :serial, force: :cascade do |t| t.string "name" t.text "description" - t.integer "kit_id" t.string "exposure" t.float "latitude" t.float "longitude" @@ -253,6 +243,9 @@ t.datetime "updated_at", null: false t.integer "measurement_id" t.uuid "uuid", default: -> { "uuid_generate_v4()" } + t.string "default_key" + t.string "equation" + t.string "reverse_equation" t.index ["ancestry"], name: "index_sensors_on_ancestry" t.index ["measurement_id"], name: "index_sensors_on_measurement_id" end @@ -310,8 +303,8 @@ add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" add_foreign_key "api_tokens", "users", column: "owner_id" + add_foreign_key "components", "devices" add_foreign_key "components", "sensors" - add_foreign_key "devices", "kits" add_foreign_key "devices_tags", "devices" add_foreign_key "devices_tags", "tags" add_foreign_key "postprocessings", "devices" diff --git a/db/seeds.rb b/db/seeds.rb index 02b16daa..bb3ac5a4 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -11,32 +11,21 @@ exit end -p '---- Seeding for development environment ----' +p "---- Seeding for development environment ----" User.create( - username: 'user1', - email: 'email@example.com', - password: 'password', + username: "user1", + email: "email@example.com", + password: "password", country_code: Faker::Address.country_code, - city: Faker::Address.city + city: Faker::Address.city, ) -#Kit.create(name: 'Making Sense WAAG #1', description: 'AQM sensor by WAAG', slug:'makingSenseSlug') -# Kits need to have a sensor_map -3.times do - Kit.create( - name: "Kit #{Faker::Educator.campus}", - description: Faker::Lorem.sentence, - slug: 'sck:1,1', - sensor_map: {"temp": 12, "hum": 13, "light": 14} - ) -end - Measurement.create( [ - { name: 'air temperature', description: 'How hot is the air', unit: 'C' }, - { name: 'air humidity', description: 'How humit is the air', unit: '% Rel' }, - { name: 'light', description: 'Lux is a measure', unit: 'lux' } + { name: "air temperature", description: "How hot is the air", unit: "C" }, + { name: "air humidity", description: "How humit is the air", unit: "% Rel" }, + { name: "light", description: "Lux is a measure", unit: "lux" }, ] ) @@ -45,25 +34,25 @@ [ { id: 12, - name: 'My temp sensor', - unit: 'temperature unit', + name: "My temp sensor", + unit: "temperature unit", measurement: Measurement.first, - description: 'temp sens descript' + description: "temp sens descript", }, { id: 13, - name: 'My hum sensor', - unit: 'hum unit', + name: "My hum sensor", + unit: "hum unit", measurement: Measurement.second, - description: 'light sens descript' + description: "light sens descript", }, { id: 14, - name: 'My light sensor', - unit: 'light unit', + name: "My light sensor", + unit: "light unit", measurement: Measurement.third, - description: 'light sens descript' - } + description: "light sens descript", + }, ] ) end @@ -72,26 +61,23 @@ Sensor.find(14).tag_sensors.create( [ { - name: 'environmental seed 1', - description: 'environmental sensor tag' + name: "environmental seed 1", + description: "environmental sensor tag", }, { - name: 'light seed', - description: 'Light sensor tag' + name: "light seed", + description: "Light sensor tag", }, { - name: 'digital seed', - description: 'Digital sensor tag' - } + name: "digital seed", + description: "Digital sensor tag", + }, ] ) end -#device has many sensors through components -#has_many :components, as: :board - 10.times do - Device.create( + device = Device.create( { owner: User.all.sample, name: Faker::Address.city, @@ -102,69 +88,62 @@ # reverse_geocode will FAIL if it receives a location at sea latitude: 42.385, longitude: 2.173, - device_token: Faker::Crypto.sha1[0,6], + device_token: Faker::Crypto.sha1[0, 6], is_private: [true, false].sample, notify_low_battery: [true, false].sample, notify_low_battery_timestamp: Time.now, notify_stopped_publishing: [true, false].sample, notify_stopped_publishing_timestamp: Time.now, data: { - 7 => 50, + 7 => 50, 10 => rand(20), #battery level below 15 get emails 12 => -0.629348144531249, 13 => 131.992370605469, 14 => 37.8, 15 => 27.384, 16 => 275.303, - 17 => 100 + 17 => 100, }, - kit: Kit.all.sample } ) + + Component.create( + device: device, sensor: Sensor.find(12), + ) + Component.create( + device: device, sensor: Sensor.find(13), + ) + Component.create( + device: device, sensor: Sensor.find(14), + ) + Component.create( + device: device, sensor: Sensor.find(12), + ) + Component.create( + device: device, sensor: Sensor.find(13), + ) + Component.create( + device: device, sensor: Sensor.find(14), + ) + Component.create( + device: device, sensor: Sensor.find(12), + ) + Component.create( + device: device, sensor: Sensor.find(13), + ) + Component.create( + device: device, sensor: Sensor.find(14), + ) end # Make the last Device an archived one? Device.last.archive! -# belongs_to :board, polymorphic: true -# belongs_to :sensor -# Kit and Device have many Components, as: :board - - -Component.create( - board: Kit.first, sensor: Sensor.find(12) -) -Component.create( - board: Kit.first, sensor: Sensor.find(13) -) -Component.create( - board: Kit.first, sensor: Sensor.find(14) -) -Component.create( - board: Kit.second, sensor: Sensor.find(12) -) -Component.create( - board: Kit.second, sensor: Sensor.find(13) -) -Component.create( - board: Kit.second, sensor: Sensor.find(14) -) -Component.create( - board: Kit.third, sensor: Sensor.find(12) -) -Component.create( - board: Kit.third, sensor: Sensor.find(13) -) -Component.create( - board: Kit.third, sensor: Sensor.find(14) -) - - Tag.create( [ - { name: 'Amsterdam', description: 'SCK in Adam' }, - { name: 'Barcelona', description: 'SCK in Barcelona' }, - { name: 'Manchester', description: 'SCK in Manchester' } + { name: "Amsterdam", description: "SCK in Adam" }, + { name: "Barcelona", description: "SCK in Barcelona" }, + { name: "Manchester", description: "SCK in Manchester" }, ] ) @@ -173,15 +152,15 @@ [ { device: Device.first, tag: Tag.first }, { device: Device.first, tag: Tag.second }, - { device: Device.second, tag: Tag.second } + { device: Device.second, tag: Tag.second }, ] ) rescue - p 'DevicesTags already created' + p "DevicesTags already created" end DeviceInventory.create( - report: {"random_property":"random_result"}, + report: { "random_property": "random_result" }, ) d = Device.first @@ -193,18 +172,18 @@ "description": "iSCAPE Station Lab test unit", "state": "has_published", "info": { - "time":"2018-07-17T06:55:06Z", - "hw_ver":"2.0", - "id":"6C4C1AF4504E4B4B372E314AFF031619", - "sam_ver":"0.3.0-ce87e64", - "sam_bd":"2018-07-17T06:55:06Z", - "esp_ver":"0.3.0-ce87e64", - "esp_bd":"2018-07-17T06:55:06Z" - } - } + "time": "2018-07-17T06:55:06Z", + "hw_ver": "2.0", + "id": "6C4C1AF4504E4B4B372E314AFF031619", + "sam_ver": "0.3.0-ce87e64", + "sam_bd": "2018-07-17T06:55:06Z", + "esp_ver": "0.3.0-ce87e64", + "esp_bd": "2018-07-17T06:55:06Z", + }, + }, ) #ApiToken.create( # owner_id: User.first, # token: 'random token' #) -p '---- Seeding complete! ----' +p "---- Seeding complete! ----" diff --git a/docs/onboarding.md b/docs/onboarding.md index 2886a2c2..a3ea3cc5 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -5,7 +5,7 @@ ### POST /v0/onboarding/device Method creates orphan_device and returns unique 'onboarding_session' and 'device_token'. -Requires no params at all, however, it can take any of the following: 'name', 'description', 'kit_id', 'exposure', 'latitude', 'longitude', 'user_tags'. Any passed params will be used on the creation of the 'orphan_device'. +Requires no params at all, however, it can take any of the following: 'name', 'description', 'exposure', 'latitude', 'longitude', 'user_tags'. Any passed params will be used on the creation of the 'orphan_device'. Note that both 'device_token' and 'onboarding_session' are unique for each 'orphan_device'. ``` @@ -14,9 +14,7 @@ request example: Content-Type: application/json Accept: application/json ------------------------------ -{ - "kit_id": 1 -} +{} ``` ``` @@ -46,7 +44,6 @@ OnboardingSession: a71095a2-e99c-4664-82d8-b4c1c9bbc531 { "name": "Owner", "description": "device description", - "kit_id": 1, "exposure": "indoor", "latitude": 41.3966908, "longitude": 2.1921909, @@ -61,7 +58,6 @@ response example: "id": 7, "name": "Owner", "description": "device description", - "kit_id": 1, "exposure": "indoor", "latitude": 41.3966908, "longitude": 2.1921909, @@ -114,7 +110,7 @@ If 'Onboarding-Session' is not valid, (404) "Invalid onboarding_session". Requires user authentication, otherwise (401) "Authorization required" is returned. -Requires all the `/onboarding/device` parameters to be provided (`name`, `description`, `kit_id`, `exposure`, `latitude`, `longitude`, `user_tags`), otherwise results in a 422, "Missing Params". +Requires all the `/onboarding/device` parameters to be provided (`name`, `description`, `exposure`, `latitude`, `longitude`, `user_tags`), otherwise results in a 422, "Missing Params". ``` POST v0/onboarding/register request example: @@ -141,10 +137,9 @@ response example: "longitude": 2.1921909, "created_at": "2016-10-29T12:31:25+02:00", "updated_at": "2016-10-29T12:31:25+02:00", - "kit_id": 1, "latest_data": nil, "geohash": "sp3e9bh31y", - "last_recorded_at": nil, + "last_reading_at": nil, "meta": { "exposure": "indoor" }, @@ -171,7 +166,7 @@ This is the end of the onboarding process. ## Token notification -This is tiggered when the platform receives the first **"Hello World"** from the Kit after the *light setup* process. +This is tiggered when the platform receives the first **"Hello World"** from the Kit after the *light setup* process. `io.connect(‘wss://smartcitizen.xyz’).on('token-received’, doSomething);`` diff --git a/lib/tasks/i.rake b/lib/tasks/i.rake index 730fd02f..468d957c 100644 --- a/lib/tasks/i.rake +++ b/lib/tasks/i.rake @@ -54,7 +54,7 @@ namespace :i do else next end - if device.last_recorded_at < 8.hours_ago + if device.last_reading_at < 8.hours_ago device.update_column(:data, a[0]) end end diff --git a/lib/tasks/mqtt_subscriber.rake b/lib/tasks/mqtt_subscriber.rake index b5f77a40..0099587c 100644 --- a/lib/tasks/mqtt_subscriber.rake +++ b/lib/tasks/mqtt_subscriber.rake @@ -1,3 +1,4 @@ +require 'benchmark' namespace :mqtt do task sub: :environment do pid_file = Rails.root.join('tmp/pids/mqtt_subscriber.pid') @@ -44,7 +45,11 @@ namespace :mqtt do client.get do |topic, message| Sentry.with_scope do begin - MqttMessagesHandler.handle_topic(topic, message) + time = Benchmark.measure do + MqttMessagesHandler.handle_topic(topic, message) + end + mqtt_log.info "Processed MQTT message in #{time}" + mqtt_log.info "MQTT queue length: #{client.queue_length}" rescue Exception => e mqtt_log.info e Sentry.capture_exception(e) diff --git a/lib/tasks/pgsequences.rake b/lib/tasks/pgsequences.rake deleted file mode 100644 index adf01c30..00000000 --- a/lib/tasks/pgsequences.rake +++ /dev/null @@ -1,9 +0,0 @@ -namespace :pgsequences do - desc "Reset all Postgres sequences" - task :reset => :environment do - ActiveRecord::Base.connection.tables.each do |t| - ActiveRecord::Base.connection.reset_pk_sequence!(t) - puts "Reset pk sequence on table: #{t}" - end - end -end diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake new file mode 100644 index 00000000..d1e8ae8f --- /dev/null +++ b/lib/tasks/users.rake @@ -0,0 +1,55 @@ +namespace :users do + task :remove_null_emails => :environment do + team_user = User.find_by_email("team@pral2a.com") + count_team_devices = team_user.devices.length + count_users = User.count + count_moved_devices = 0 + count_deleted_users = 0 + User.where("email IS NULL or trim(email) = ''").each do |user| + puts "user #{user.id} (#{user.username}) has blank email, moving devices to team account:" + user.devices.each do |device| + device.owner = team_user + device.save(validate: false) + count_team_devices += 1 + count_moved_devices += 1 + puts "\t - Device #{device.id} moved to user: #{team_user.id}" + end + user.destroy! + count_users -= 1 + count_deleted_users += 1 + end + puts "Check moved devices: #{team_user.reload.devices.length} (should be #{count_team_devices}, #{count_moved_devices} moved)" + puts "Check remaining users: #{User.count} (should be #{count_users}, #{count_deleted_users} deleted)" + end + + task :deduplicate_emails => :environment do + count_deleted_users = 0 + count_users = User.count + count_devices = Device.count + count_moved_devices = 0 + ActiveRecord::Base.connection.execute( + "SELECT email FROM users GROUP BY email HAVING count(id) > 1" + ).each do |record| + email = record["email"] + users = User.where(email: email).to_a + puts "Found #{users.length} users for email: #{email}: (#{users.map(&:id).join(",")})" + move_to_user = users.shift + puts " - Moving devices from users (#{users.map(&:id).join(",")}) to user #{move_to_user.id}" + users.each do |user| + user.devices.each do |device| + device.owner = move_to_user + device.save(validate: false) + count_moved_devices += 1 + puts " - Moved device #{device.id} to user: #{move_to_user.id}" + end + end + puts " - Deleting users: #{users.map(&:id).join(",")}" + users.each do |user| + user.destroy! + count_deleted_users += 1 + end + end + puts "Check moved devices: #{Device.count} (should be #{count_devices}, #{count_moved_devices} moved)" + puts "Check deleted users: #{User.count} (should be #{count_users - count_deleted_users}, #{count_deleted_users} deleted)" + end +end \ No newline at end of file diff --git a/scripts/nginx-conf/api.smartcitizen.me.conf b/scripts/nginx-conf/api.smartcitizen.me.conf index a1371d0d..f46d6737 100644 --- a/scripts/nginx-conf/api.smartcitizen.me.conf +++ b/scripts/nginx-conf/api.smartcitizen.me.conf @@ -4,7 +4,7 @@ upstream docker-push { server { listen 80; - server_name ws.smartcitizen.me; + server_name staging-ws.smartcitizen.me ws.smartcitizen.me; location / { proxy_pass http://docker-push; diff --git a/spec/controllers/v0/devices_controller_spec.rb b/spec/controllers/v0/devices_controller_spec.rb index f2613fdd..ac3dc5da 100644 --- a/spec/controllers/v0/devices_controller_spec.rb +++ b/spec/controllers/v0/devices_controller_spec.rb @@ -1,7 +1,5 @@ require 'rails_helper' RSpec.describe V0::DevicesController do - skip { is_expected.to permit(:name,:description,:mac_address,:latitude,:longitude,:elevation,:exposure,:meta,:kit_id,:user_tags).for(:create) } - - + skip { is_expected.to permit(:name,:description,:mac_address,:latitude,:longitude,:elevation,:exposure,:meta,:user_tags).for(:create) } end diff --git a/spec/controllers/v0/kits_controller_spec.rb b/spec/controllers/v0/kits_controller_spec.rb deleted file mode 100644 index 0f682e8d..00000000 --- a/spec/controllers/v0/kits_controller_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe V0::KitsController do - skip { is_expected.to permit(:name,:description,:slug, :sensor_map).for(:update) } -end diff --git a/spec/controllers/v0/onboarding/orphan_devices_controller_spec.rb b/spec/controllers/v0/onboarding/orphan_devices_controller_spec.rb index de031aa2..eff31b44 100644 --- a/spec/controllers/v0/onboarding/orphan_devices_controller_spec.rb +++ b/spec/controllers/v0/onboarding/orphan_devices_controller_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe V0::Onboarding::OrphanDevicesController, type: :controller do - it { is_expected.to permit(:name, :description, :kit_id, :exposure, :latitude, :longitude, + it { is_expected.to permit(:name, :description, :exposure, :latitude, :longitude, :user_tags).for(:create) } describe "save_orphan_device" do diff --git a/spec/factories/components.rb b/spec/factories/components.rb index cb8d8845..fe29b672 100644 --- a/spec/factories/components.rb +++ b/spec/factories/components.rb @@ -1,7 +1,7 @@ FactoryBot.define do factory :component do uuid { SecureRandom.uuid } - association :board, factory: :kit + association :device association :sensor end diff --git a/spec/factories/kits.rb b/spec/factories/kits.rb deleted file mode 100644 index 0156f967..00000000 --- a/spec/factories/kits.rb +++ /dev/null @@ -1,7 +0,0 @@ -FactoryBot.define do - factory :kit do - name { "Testing kit" } - description { "A kit that was made for the test environment" } - end - -end diff --git a/spec/factories/orphan_devices.rb b/spec/factories/orphan_devices.rb index b76a65ad..d423151f 100644 --- a/spec/factories/orphan_devices.rb +++ b/spec/factories/orphan_devices.rb @@ -2,7 +2,6 @@ factory :orphan_device do name { "OrphanDeviceName" } description { "OrphanDeviceDescription" } - kit_id { 1 } exposure { "indoor" } # same coordinates used for testing Device latitude { 41.3966908 } diff --git a/spec/factories/sensors.rb b/spec/factories/sensors.rb index 24073787..2a72d612 100644 --- a/spec/factories/sensors.rb +++ b/spec/factories/sensors.rb @@ -3,5 +3,6 @@ name { "MiCS-2710" } description { "Metaloxide gas sensor" } unit { "KΩ" } + default_key { "key_#{SecureRandom.alphanumeric(4)}"} end end \ No newline at end of file diff --git a/spec/jobs/check_battery_level_below_job_spec.rb b/spec/jobs/check_battery_level_below_job_spec.rb index 0f5220b1..93427864 100644 --- a/spec/jobs/check_battery_level_below_job_spec.rb +++ b/spec/jobs/check_battery_level_below_job_spec.rb @@ -3,10 +3,10 @@ RSpec.describe CheckBatteryLevelBelowJob, type: :job do it 'should update notify_low_battery_timestamp and send email' do - device = create(:device, notify_low_battery: true) - + device = create(:device, notify_low_battery: true, updated_at: "2023-01-01 00:00:00") + updated_at_before = device.updated_at time_before = device.notify_low_battery_timestamp - device.update(data: { "10": '11'}) + device.update_columns(data: { "10": '11'}) expect(device.data["10"].to_i).to eq(11) @@ -14,5 +14,6 @@ device.reload expect(time_before).not_to eq(device.notify_low_battery_timestamp) + expect(device.updated_at).to eq(updated_at_before) end end diff --git a/spec/jobs/delete_archived_devices_job_spec.rb b/spec/jobs/delete_archived_devices_job_spec.rb index 481c5ded..77500c00 100644 --- a/spec/jobs/delete_archived_devices_job_spec.rb +++ b/spec/jobs/delete_archived_devices_job_spec.rb @@ -11,9 +11,9 @@ end it "should delete all archived devices, archived_at at least 24 hours ago" do - deviceNormal = create(:device, name: "dontDeleteMe", created_at: 6.weeks.ago) - deviceArchived = create(:device, name: "deleteMe", created_at: 1.month.ago) - deviceArchivedToday = create(:device, name: "dontDeleteMe", created_at: 2.months.ago) + deviceNormal = create(:device, name: "dontDeleteMe", created_at: 6.weeks.ago, components: [create(:component)]) + deviceArchived = create(:device, name: "deleteMe", created_at: 1.month.ago, components: [create(:component)]) + deviceArchivedToday = create(:device, name: "dontDeleteMe", created_at: 2.months.ago, components: [create(:component)]) deviceArchived.archive! deviceArchivedToday.archive! deviceArchived.update!({archived_at: 2.days.ago}) diff --git a/spec/lib/mqtt_messages_handler_spec.rb b/spec/lib/mqtt_messages_handler_spec.rb index c39bb414..a70ae86f 100644 --- a/spec/lib/mqtt_messages_handler_spec.rb +++ b/spec/lib/mqtt_messages_handler_spec.rb @@ -3,11 +3,14 @@ RSpec.describe MqttMessagesHandler do let(:device) { create(:device, device_token: 'aA1234') } let(:orphan_device) { create(:orphan_device, device_token: 'xX9876') } - let(:component) { build(:component, board: build(:kit), sensor: build(:sensor, id: 1)) } + let(:component) { build(:component, device: device, sensor: build(:sensor, id: 1, default_key: "key1")) } + let(:device_inventory) { create(:device_inventory, report: '{"random_property": "random_result"}') } before do device.components << component + create(:sensor, id: 13, default_key: "key13") + @data = [{ "recorded_at"=>"2016-06-08 10:30:00Z", @@ -78,7 +81,7 @@ #expect(Storer).to receive(:initialize).with('a', 'b') expect(Redis.current).to receive(:publish).with( 'telnet_queue', [{ - name: nil, + name: "key1", timestamp: 1465381800000, value: 21.0, tags: { @@ -90,6 +93,19 @@ MqttMessagesHandler.handle_topic(@packet.topic, @packet.payload) end + it 'handshakes the device if an orphan device exists' do + orphan_device = create(:orphan_device, device_token: device.device_token) + expect(orphan_device.device_handshake).to be false + allow(Redis.current).to receive(:publish) + expect(Redis.current).to receive(:publish).with( + 'token-received', { + onboarding_session: orphan_device.onboarding_session + }.to_json + ) + MqttMessagesHandler.handle_topic(@packet.topic, @packet.payload) + expect(orphan_device.reload.device_handshake).to be true + end + it 'does not queue when there is no data' do expect(Redis.current).not_to receive(:publish).with( 'telnet_queue', [{ @@ -117,18 +133,29 @@ end describe '#handle_raw' do - it 'processes raw data' do - the_data = "{ t:2017-03-24T13:35:14Z, 1:48.45, 13:66, 12:28, 10:4.45 }" + let(:the_data) { + "{ t:2017-03-24T13:35:14Z, 1:48.45, 13:66, 12:28, 10:4.45 }" + } + + it 'processes raw data' do expect(Redis.current).to receive(:publish).with( 'telnet_queue', [{ - name: nil, + name: "key1", timestamp: 1490362514000, value: 48.45, tags: { device_id: device.id, method: 'REST' } + },{ + name: "key13", + timestamp: 1490362514000, + value: 66.0, + tags: { + device_id: device.id, + method: 'REST' + } }].to_json ) @@ -137,10 +164,23 @@ # TODO: we should expect that a new Storer object should contain the correct, processed readings #expect(Storer).to receive(:new) end + + it 'handshakes the device if an orphan device exists' do + orphan_device = create(:orphan_device, device_token: device.device_token) + expect(orphan_device.device_handshake).to be false + allow(Redis.current).to receive(:publish) + expect(Redis.current).to receive(:publish).with( + 'token-received', { + onboarding_session: orphan_device.onboarding_session + }.to_json + ) + MqttMessagesHandler.handle_topic("device/sck/#{device.device_token}/readings/raw", the_data) + expect(orphan_device.reload.device_handshake).to be true + end end describe '#handle_hello' do - it 'logs device_token has been received' do + it 'handshakes the device if an orphan device exists' do expect(orphan_device.device_handshake).to be false expect(Redis.current).to receive(:publish).with( 'token-received', { @@ -172,6 +212,13 @@ MqttMessagesHandler.handle_topic(nil,'{"random_property":"random_result2"}') expect(DeviceInventory.count).to eq(0) end + + it 'does not handshake any device' do + expect(Redis.current).not_to receive(:publish) + MqttMessagesHandler.handle_topic( + @inventory_packet.topic, @inventory_packet.payload + ) + end end describe '#hardware_info' do @@ -183,6 +230,19 @@ expect(@hardware_info_packet.payload).to eq((device.hardware_info.to_json)) end + it 'handshakes the device if an orphan device exists' do + orphan_device = create(:orphan_device, device_token: device.device_token) + expect(orphan_device.device_handshake).to be false + allow(Redis.current).to receive(:publish) + expect(Redis.current).to receive(:publish).with( + 'token-received', { + onboarding_session: orphan_device.onboarding_session + }.to_json + ) + MqttMessagesHandler.handle_topic(@hardware_info_packet.topic, @hardware_info_packet.payload) + expect(orphan_device.reload.device_handshake).to be true + end + it 'does not handle bad topic' do expect(device.hardware_info["id"]).to eq(47) MqttMessagesHandler.handle_topic(@hardware_info_packet_bad.topic, @hardware_info_packet_bad.payload) diff --git a/spec/models/component_spec.rb b/spec/models/component_spec.rb index 07232d47..e6ea21ad 100644 --- a/spec/models/component_spec.rb +++ b/spec/models/component_spec.rb @@ -1,14 +1,34 @@ require 'rails_helper' RSpec.describe Component, :type => :model do - it { is_expected.to belong_to(:board) } + it { is_expected.to belong_to(:device) } it { is_expected.to belong_to(:sensor) } - it { is_expected.to validate_presence_of(:board) } + it { is_expected.to validate_presence_of(:device) } it { is_expected.to validate_presence_of(:sensor) } it "validates uniqueness of board to sensor" do - component = create(:component, board: create(:kit), sensor: create(:sensor)) - expect{ create(:component, board: component.board, sensor: component.sensor) }.to raise_error(ActiveRecord::RecordInvalid) + component = create(:component, device: create(:device), sensor: create(:sensor)) + expect{ create(:component, device: component.device, sensor: component.sensor) }.to raise_error(ActiveRecord::RecordInvalid) + end + + describe "creating a unique sensor key" do + let(:component) { + create(:component, device: create(:device), sensor: create(:sensor)) + } + + describe "when the given key is not in the list of existing keys" do + it "uses the key as is" do + generated_key = component.get_unique_key("key", ["other"]) + expect(generated_key).to eq("key") + end + end + + describe "when the given key is in the list of existing keys" do + it "adds an incremeting number to the key" do + generated_key = component.get_unique_key("key", ["key", "other", "key_1"]) + expect(generated_key).to eq("key_2") + end + end end end diff --git a/spec/models/device_spec.rb b/spec/models/device_spec.rb index d0988b13..eb79e12c 100644 --- a/spec/models/device_spec.rb +++ b/spec/models/device_spec.rb @@ -5,7 +5,6 @@ let(:mac_address) { "10:9a:dd:63:c0:10" } let(:device) { create(:device, mac_address: mac_address) } - it { is_expected.to belong_to(:kit).without_validating_presence } it { is_expected.to belong_to(:owner) } it { is_expected.to have_many(:devices_tags) } it { is_expected.to have_many(:tags).through(:devices_tags) } @@ -19,25 +18,27 @@ it "has last_reading_at" do Timecop.freeze do - device = create(:device, last_recorded_at: 1.minute.ago) + device = create(:device, last_reading_at: 1.minute.ago) expect(device.last_reading_at).to eq(1.minute.ago) end end - it "has added_at"do - Timecop.freeze do - expect(create(:device).added_at).to eq(Time.current.utc) - end - end - it "validates format of mac address, but allows nil" do expect{ create(:device, mac_address: '10:9A:DD:63:C0:10') }.to_not raise_error expect{ create(:device, mac_address: nil) }.to_not raise_error expect{ create(:device, mac_address: 123) }.to raise_error(ActiveRecord::RecordInvalid) end - describe "mac_address" do + describe "#sensor_map" do + it "maps the component key to the sensor id" do + device = create(:device) + sensor = create(:sensor, default_key: "sensor_key") + component = create(:component, device: device, sensor: sensor, key: "component_key") + expect(device.sensor_map).to eq({"component_key" => sensor.id}) + end + end + describe "mac_address" do it "takes mac_address from existing device on update" do device = FactoryBot.create(:device, mac_address: mac_address) new_device = FactoryBot.create(:device) @@ -202,24 +203,6 @@ end - describe "kit_version" do - it "has kit_version setter" do - device = build(:device, kit_version: "1.1") - expect(device.kit_id).to eq(3) - - device = build(:device, kit_version: "1.0") - expect(device.kit_id).to eq(2) - end - - it "has kit_version getter" do - device = build(:device, kit_id: 3) - expect(device.kit_version).to eq("1.1") - - device = build(:device, kit_id: 2) - expect(device.kit_version).to eq("1.0") - end - end - it "has to_s" do device = create(:device, name: 'cool device') expect(device.to_s).to eq('cool device') @@ -233,31 +216,15 @@ skip "has status" do expect(Device.new.status).to eq('new') - expect(create(:device, last_recorded_at: 1.minute.ago).status).to eq('online') - expect(create(:device, last_recorded_at: 10.minutes.ago).status).to eq('offline') + expect(create(:device, last_reading_at: 1.minute.ago).status).to eq('online') + expect(create(:device, last_reading_at: 10.minutes.ago).status).to eq('offline') end it "has firmware" do expect(create(:device, firmware_version: 'xyz').firmware).to eq('sck:xyz') end - context "with kit" do - - let(:kit) { build(:kit) } - let(:sensor) { build(:sensor) } - let(:device) { build(:device, kit: kit) } - - it "has the kit's sensors" do - expect(device.sensors).to eq(kit.sensors) - end - - it "has the kit's components" do - expect(device.components).to eq(kit.components) - end - - end - - context "without kit" do + context "creation" do let(:sensor) { create(:sensor) } let(:device) { create(:device, sensors: [sensor]) } @@ -267,7 +234,7 @@ end it "has its own components" do - expect(device.components).to eq([Component.find_by(board: device, sensor: sensor)]) + expect(device.components).to eq([Component.find_by(device: device, sensor: sensor)]) end end @@ -314,6 +281,33 @@ end end + describe "update_column_timestamps" do + + before do + @device = create(:device) + @component_1 = create(:component, device: @device, sensor: create(:sensor, id: 1), updated_at: "2023-01-01 12:00:00") + @component_2 = create(:component, device: @device, sensor: create(:sensor, id: 2)) + @device.reload + @timestamp = Time.parse("2023-10-06 06:00:00") + end + + it "updates the timestamp for components with the given sensor ids" do + @device.update_component_timestamps(@timestamp, [1]) + expect(@component_1.reload.last_reading_at).to eq(@timestamp) + end + + it "does not update the timesatamp for components without the given sensor ids" do + expect(@component_2).not_to receive(:update).with(last_reading_at: @timestamp) + @device.update_component_timestamps(@timestamp, [1]) + end + + it "does not update the component updated_at" do + updated_at = @component_1.updated_at + @device.update_component_timestamps(@timestamp, [1]) + expect(@component_1.reload.updated_at).to eq(updated_at) + end + end + context "notifications for low battery" do describe "do not get sent" do it 'when they are disabled' do @@ -378,7 +372,7 @@ context "notifications for stopped publishing" do describe "do not get sent" do it 'when they are disabled' do - device = create(:device, notify_stopped_publishing: false, last_recorded_at: 24.hours.ago) + device = create(:device, notify_stopped_publishing: false, last_reading_at: 24.hours.ago) expect(device).to have_attributes(notify_stopped_publishing: false) before_date = device.notify_stopped_publishing_timestamp CheckDeviceStoppedPublishingJob.perform_now @@ -389,7 +383,7 @@ it 'when they are enabled, but timestamp is too recent' do device = create(:device, notify_stopped_publishing: true, - last_recorded_at: 2.hours.ago, + last_reading_at: 2.hours.ago, notify_stopped_publishing_timestamp: 2.hours.ago) device.reload expect(device).to have_attributes(notify_stopped_publishing: true) @@ -403,9 +397,9 @@ end describe "do get sent" do - it 'when enabled, timestamp is more than 24 hours old AND last_recorded older than 10 minutes' do + it 'when enabled, timestamp is more than 24 hours old AND last_reading older than 10 minutes' do device = create(:device, notify_stopped_publishing: true, - last_recorded_at: 2.hours.ago, + last_reading_at: 2.hours.ago, notify_stopped_publishing_timestamp: 25.hours.ago) device.reload expect(device).to have_attributes(notify_stopped_publishing: true) @@ -431,4 +425,138 @@ end end + describe "hardware info" do + describe "hardware_name" do + context "when it has a name override" do + it "returns the overriden name" do + device = create(:device, hardware_name_override: "Overriden name") + expect(device.hardware_name).to eq("Overriden name") + end + end + + context "when it has a hardware_version" do + it "returns the string 'SmartCitizen Kit' concatenated with the version" do + device = create(:device) + expect(device).to receive(:hardware_version).at_least(:once).and_return("1.0.0") + expect(device.hardware_name).to eq("SmartCitizen Kit 1.0.0") + end + end + + context "otherwise" do + it "returns the string 'Unknown'" do + device = create(:device) + expect(device.hardware_name).to eq("Unknown") + end + end + end + + describe "hardware_type" do + context "when it has a type override" do + it "returns the overriden type" do + device = create(:device, hardware_type_override: "Overriden type") + expect(device.hardware_type).to eq("Overriden type") + end + end + + context "when it has a hardware_version" do + it "returns the string 'SCK'" do + expect(device).to receive(:hardware_version).and_return("1.0.0") + expect(device.hardware_type).to eq("SCK") + end + end + + context "otherwise" do + it "returns the string 'Unknown'" do + device = create(:device) + expect(device.hardware_type).to eq("Unknown") + end + end + end + + describe "hardware_version" do + context "when it has a version override" do + it "returns the overriden version" do + device = create(:device, hardware_version_override: "1.4.0+with+extra+sensors") + expect(device.hardware_version).to eq("1.4.0+with+extra+sensors") + end + end + + context "when it has hardware_info" do + it "returns the version from hardware_info" do + device = create(:device, hardware_info: { hw_ver: "1.5.0" }) + expect(device.hardware_version).to eq("1.5.0") + end + end + + context "otherwise" do + it "returns nil" do + device = create(:device) + expect(device.hardware_version).to be(nil) + end + end + end + + describe "hardware_slug" do + context "when it has a slug override" do + it "returns the overriden slug" do + device = create(:device, hardware_slug_override: "overriden_slug") + expect(device.hardware_slug).to eq("overriden_slug") + end + end + + context "when it has a hardware_version" do + it "returns the hardware type downcased, concatenated with the version number with periods translated to commas, seperated by a colon" do + device = create(:device) + expect(device).to receive(:hardware_type).and_return("SCK") + expect(device).to receive(:hardware_version).and_return("1.0.0") + expect(device.hardware_slug).to eq("sck:1,0,0") + end + end + + context "when it has no harware version" do + it "returns the hardware type downcased" do + device = create(:device) + expect(device.hardware_slug).to eq("unknown") + end + end + end + end + + describe "#find_or_create_component_by_sensor_id" do + context "when the sensor exists and a component already exists for this device" do + it "returns the existing component" do + sensor = create(:sensor) + component = create(:component, sensor: sensor, device: device) + expect(device.find_or_create_component_by_sensor_id(sensor.id)).to eq(component) + end + end + + context "when the sensor exists and a component does not already exist for this device" do + it "returns a new valid component with the correct sensor and device" do + sensor = create(:sensor) + component = device.find_or_create_component_by_sensor_id(sensor.id) + expect(component).not_to be_blank + expect(component).to be_a Component + expect(component.valid?).to be(true) + expect(component.persisted?).to be(true) + expect(component.device).to eq(device) + expect(component.sensor).to eq(sensor) + end + end + + context "when no sensor exists with this id" do + it "returns nil" do + create(:sensor, id: 12345) + expect(device.find_or_create_component_by_sensor_id(54321)).to be_blank + end + end + + context "when the id is nil" do + it "returns nil" do + expect(device.find_or_create_component_by_sensor_id(nil)).to be_blank + end + end + + end + end diff --git a/spec/models/kit_spec.rb b/spec/models/kit_spec.rb deleted file mode 100644 index e3d484b2..00000000 --- a/spec/models/kit_spec.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'rails_helper' - -RSpec.describe Kit, :type => :model do - it { is_expected.to have_many(:devices) } - it { is_expected.to have_many(:components) } - it { is_expected.to have_many(:sensors) } - - it { is_expected.to validate_presence_of(:name) } - it { is_expected.to validate_presence_of(:description) } -end diff --git a/spec/models/orphan_device_spec.rb b/spec/models/orphan_device_spec.rb index 60fb717d..cb01a7c3 100644 --- a/spec/models/orphan_device_spec.rb +++ b/spec/models/orphan_device_spec.rb @@ -57,9 +57,8 @@ it 'attributes hash without device_token and onboarding_session' do device_attributes_hash = orphan_device.device_attributes - expect(device_attributes_hash.length).to eq(8) + expect(device_attributes_hash.length).to eq(7) expect(device_attributes_hash.key?(:name)).to eq(true) - expect(device_attributes_hash.key?(:kit_id)).to eq(true) expect(device_attributes_hash.key?(:description)).to eq(true) expect(device_attributes_hash.key?(:user_tags)).to eq(true) expect(device_attributes_hash.key?(:longitude)).to eq(true) diff --git a/spec/models/raw_storer_spec.rb b/spec/models/raw_storer_spec.rb index b1450934..5a3debb3 100644 --- a/spec/models/raw_storer_spec.rb +++ b/spec/models/raw_storer_spec.rb @@ -1,87 +1,74 @@ -require 'rails_helper' +require "rails_helper" def to_ts(time) time.strftime("%Y-%m-%d %H:%M:%S") end RSpec.describe RawStorer, :type => :model do - before(:each) do - Kit.create!(id: 2, name: 'SCK 1.0 - Ambient Board Goteo Board', description: "Goteo Board", slug: 'sck:1,0', sensor_map: {"co": 9, "bat": 10, "hum": 5, "no2": 8, "nets": 21, "temp": 4, "light": 6, "noise": 7, "panel": 11}) - Kit.create!(id: 3, name: 'SCK 1.1 - Ambient Board Kickstarter Board', description: "Kickstarter Board", slug: 'sck:1,1', sensor_map: {"co": 16, "bat": 17, "hum": 13, "no2": 15, "nets": 21, "temp": 12, "light": 14, "noise": 7, "panel": 18}) - - Sensor.create!(id:3, name:'DHT22', description: 'test') - Sensor.create!(id:4, name:'DHT22 - Temperature', description: 'test') - Sensor.create!(id:5, name:'DHT22 - Humidity', description: 'test') - Sensor.create!(id:6, name:'PVD-P8001', description: 'test') - Sensor.create!(id:7, name:'POM-3044P-R', description: 'test') - Sensor.create!(id:8, name:'MICS-2710', description: 'test') - Sensor.create!(id:9, name:'MICS-5525', description: 'test') - Sensor.create!(id:10, name:'Battery', description: 'test') - Sensor.create!(id:11, name:'Solar Panel', description: 'test') - Sensor.create!(id:12, name:'HPP828E031', description: 'test') - Sensor.create!(id:13, name:'HPP828E031', description: 'test') - Sensor.create!(id:14, name:'BH1730FVC', description: 'test') - Sensor.create!(id:15, name:'MiCS-4514', description: 'test') - Sensor.create!(id:16, name:'MiCS-4514', description: 'test') - Sensor.create!(id:17, name:'Battery', description: 'test') - Sensor.create!(id:18, name:'Solar Panel', description: 'test') - Sensor.create!(id:19, name:'HPP828E031 (SHT21)', description: 'test') - Sensor.create!(id:20, name:'MiCS4514', description: 'test') - Sensor.create!(id:21, name:'Microchip RN-131', description: 'test') + Sensor.create!(id: 7, name: "POM-3044P-R", description: "test", default_key: "noise", equation: "Mathematician.table_calibration({0=>50,2=>55,3=>57,6=>58,20=>59,40=>60,60=>61,75=>62,115=>63,150=>64,180=>65,220=>66,260=>67,300=>68,375=>69,430=>70,500=>71,575=>72,660=>73,720=>74,820=>75,900=>76,975=>77,1050=>78,1125=>79,1200=>80,1275=>81,1320=>82,1375=>83,1400=>84,1430=>85,1450=>86,1480=>87,1500=>88,1525=>89,1540=>90,1560=>91,1580=>92,1600=>93,1620=>94,1640=>95,1660=>96,1680=>97,1690=>98,1700=>99,1710=>100,1720=>101,1745=>102,1770=>103,1785=>104,1800=>105,1815=>106,1830=>107,1845=>108,1860=>109,1875=>110},x)", reverse_equation: "x") + Sensor.create!(id: 12, name: "HPP828E031", description: "test", default_key: "temp", equation: "(175.72 / 65536.0 * x) - 53", reverse_equation: "x") + Sensor.create!(id: 13, name: "HPP828E031", description: "test", default_key: "hum", equation: "(125.0 / 65536.0 * x) + 7", reverse_equation: "x") + Sensor.create!(id: 14, name: "BH1730FVC", description: "test", default_key: "light", equation: "x", reverse_equation: "x/10.0") + Sensor.create!(id: 15, name: "MiCS-4514", description: "test", default_key: "no2", equation: "x", reverse_equation: "x/1000.0") + Sensor.create!(id: 16, name: "MiCS-4514", description: "test", default_key: "co", equation: "x", reverse_equation: "x/1000.0") + Sensor.create!(id: 17, name: "Battery", description: "test", default_key: "bat", equation: "x", reverse_equation: "x/10.0") + Sensor.create!(id: 18, name: "Solar Panel", description: "test", default_key: "panel", equation: "x", reverse_equation: "x/1000.0") + Sensor.create!(id: 21, name: "Microchip RN-131", description: "test", default_key: "nets", equation: "x", reverse_equation: "x") - Component.create!(id: 4, board: Kit.find(2), sensor: Sensor.find(4), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 5, board: Kit.find(2), sensor: Sensor.find(5), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 6, board: Kit.find(2), sensor: Sensor.find(6), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 7, board: Kit.find(2), sensor: Sensor.find(7), equation: 'Mathematician.table_calibration({0=>0,5=>45,10=>55,15=>63,20=>65,30=>67,40=>69,50=>70,60=>71,80=>72,90=>73,100=>74,130=>75,160=>76,190=>77,220=>78,260=>79,300=>80,350=>81,410=>82,450=>83,550=>84,600=>85,650=>86,750=>87,850=>88,950=>89,1100=>90,1250=>91,1375=>92,1500=>93,1650=>94,1800=>95,1900=>96,2000=>97,2125=>98,2250=>99,2300=>100,2400=>101,2525=>102,2650=>103},x)', reverse_equation: 'x') - Component.create!(id: 8, board: Kit.find(2), sensor: Sensor.find(8), equation: 'x', reverse_equation: 'x/1000.0') - Component.create!(id: 9, board: Kit.find(2), sensor: Sensor.find(9), equation: 'x', reverse_equation: 'x/1000.0') - Component.create!(id: 10, board: Kit.find(2), sensor: Sensor.find(10), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 11, board: Kit.find(2), sensor: Sensor.find(11), equation: 'x', reverse_equation: 'x/1000.0') - # - Component.create!(id: 12, board: Kit.find(3), sensor: Sensor.find(12), equation: '(175.72 / 65536.0 * x) - 53', reverse_equation: 'x') - Component.create!(id: 13, board: Kit.find(3), sensor: Sensor.find(13), equation: '(125.0 / 65536.0 * x) + 7', reverse_equation: 'x') - Component.create!(id: 14, board: Kit.find(3), sensor: Sensor.find(14), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 15, board: Kit.find(3), sensor: Sensor.find(7), equation: 'Mathematician.table_calibration({0=>50,2=>55,3=>57,6=>58,20=>59,40=>60,60=>61,75=>62,115=>63,150=>64,180=>65,220=>66,260=>67,300=>68,375=>69,430=>70,500=>71,575=>72,660=>73,720=>74,820=>75,900=>76,975=>77,1050=>78,1125=>79,1200=>80,1275=>81,1320=>82,1375=>83,1400=>84,1430=>85,1450=>86,1480=>87,1500=>88,1525=>89,1540=>90,1560=>91,1580=>92,1600=>93,1620=>94,1640=>95,1660=>96,1680=>97,1690=>98,1700=>99,1710=>100,1720=>101,1745=>102,1770=>103,1785=>104,1800=>105,1815=>106,1830=>107,1845=>108,1860=>109,1875=>110},x)', reverse_equation: 'x') - Component.create!(id: 16, board: Kit.find(3), sensor: Sensor.find(15), equation: 'x', reverse_equation: 'x/1000.0') - Component.create!(id: 17, board: Kit.find(3), sensor: Sensor.find(16), equation: 'x', reverse_equation: 'x/1000.0') - Component.create!(id: 18, board: Kit.find(3), sensor: Sensor.find(17), equation: 'x', reverse_equation: 'x/10.0') - Component.create!(id: 19, board: Kit.find(3), sensor: Sensor.find(18), equation: 'x', reverse_equation: 'x/1000.0') - Component.create!(id: 20, board: Kit.find(2), sensor: Sensor.find(21), equation: 'x', reverse_equation: 'x') - Component.create!(id: 21, board: Kit.find(3), sensor: Sensor.find(21), equation: 'x', reverse_equation: 'x') + Component.create!(id: 12, device: device, sensor: Sensor.find(12)) + Component.create!(id: 13, device: device, sensor: Sensor.find(13)) + Component.create!(id: 14, device: device, sensor: Sensor.find(14)) + Component.create!(id: 15, device: device, sensor: Sensor.find(7)) + Component.create!(id: 16, device: device, sensor: Sensor.find(15)) + Component.create!(id: 17, device: device, sensor: Sensor.find(16)) + Component.create!(id: 18, device: device, sensor: Sensor.find(17)) + Component.create!(id: 19, device: device, sensor: Sensor.find(18)) + Component.create!(id: 21, device: device, sensor: Sensor.find(21)) end let(:json) { - {"co": "118439", "bat": "1000", "hum": "21592", "no2": "260941", "nets": "17", "temp": "25768", "light": "509", "noise": "0", "panel": "0", "timestamp": to_ts(1.day.ago) } + { "co": "118439", "bat": "1000", "hum": "21592", "no2": "260941", "nets": "17", "temp": "25768", "light": "509", "noise": "0", "panel": "0", "timestamp": to_ts(1.day.ago) } } - let(:device) { create(:device, kit: Kit.last) } + let(:device) { create(:device) } # RawStorer.new data, mac, version, ip it "will not be created with invalid past timestamp" do ts = { timestamp: to_ts(5.years.ago) } - raw_storer = RawStorer.new( json.merge(ts), device.mac_address, "1.1-0.9.0-A", "127.0.0.1" ) + raw_storer = RawStorer.new(json.merge(ts), device.mac_address, "1.1-0.9.0-A", "127.0.0.1") + end + + it "updates component last_reading_at" do + includes_proxy = double({ where: double({last: device.reload})}) + allow(Device).to receive(:includes).and_return(includes_proxy) + + expect(device).to receive(:update_component_timestamps).with( + Time.parse(json[:timestamp]), + [16, 17, 13, 15, 21, 12, 14, 7, 18] + ) + + raw_storer = RawStorer.new(json, device.mac_address, "1.1-0.9.0-A", "127.0.0.1", true) end it "will not be created with invalid future timestamp" do ts = { timestamp: to_ts(2.days.from_now) } - raw_storer = RawStorer.new( json.merge(ts), device.mac_address, "1.1-0.9.0-A", "127.0.0.1" ) + raw_storer = RawStorer.new(json.merge(ts), device.mac_address, "1.1-0.9.0-A", "127.0.0.1") end it "will not be created with invalid data" do - expect(Kairos).to_not receive(:http_post_to) - raw_storer = RawStorer.new( {}, device.mac_address, "1.1-0.9.0-A", "127.0.0.1" ) + expect(Redis.current).not_to receive(:publish) + raw_storer = RawStorer.new({}, device.mac_address, "1.1-0.9.0-A", "127.0.0.1") end it "should return a correct sensor id number" do - expect(device.find_sensor_id_by_key(:co)).to eq(16) - expect(device.find_sensor_id_by_key(:bat)).to eq(17) + expect(device.reload.find_sensor_id_by_key(:co)).to eq(16) + expect(device.reload.find_sensor_id_by_key(:bat)).to eq(17) end - skip "will be created with valid data", :vcr do - expect(Kairos).to receive(:http_post_to) - raw_storer = RawStorer.new( json, device.mac_address, "1.1-0.9.0-A", "127.0.0.1" ) + it "will be created with valid data" do + expect(Redis.current).to receive(:publish) + raw_storer = RawStorer.new(json, device.mac_address, "1.1-0.9.0-A", "127.0.0.1", true) end - end diff --git a/spec/models/sensor_spec.rb b/spec/models/sensor_spec.rb index 6ea4783e..658d1b8b 100644 --- a/spec/models/sensor_spec.rb +++ b/spec/models/sensor_spec.rb @@ -8,8 +8,6 @@ # it { is_expected.to validate_presence_of(:unit) } it { is_expected.to have_many(:components) } - # it { is_expected.to have_many(:boards).through(:components) } - # it { is_expected.to have_many(:kits).through(:components) } it "has ancestry" diff --git a/spec/models/storer_spec.rb b/spec/models/storer_spec.rb index bbb84150..db3a784d 100644 --- a/spec/models/storer_spec.rb +++ b/spec/models/storer_spec.rb @@ -1,10 +1,10 @@ require 'rails_helper' RSpec.describe Storer, type: :model do - let(:kit){ build(:kit, id: 3, name: 'SCK', description: "Board", slug: 'sck', sensor_map: '{"temp": 12}')} - let(:sensor){ build(:sensor, id:12, name:'HPP828E031', description: 'test')} - let(:component){ create(:component, id: 12, board: kit, sensor: sensor, equation: '(175.72 / 65536.0 * x) - 53', reverse_equation: 'x')} - let(:device) { create(:device, device_token: 'aA1234', kit: kit) } + let(:sensor){ build(:sensor, id:12, name:'HPP828E031', description: 'test', equation: '(175.72 / 65536.0 * x) - 53', reverse_equation: 'x')} + let(:device) { create(:device, device_token: 'aA1234') } + let(:component){ create(:component, id: 12, device: device, sensor: sensor) } + context 'when receiving good data' do before do @@ -41,7 +41,16 @@ # model/storer.rb is not using Kairos, but Redis -> Telnet # expect(Kairos).to receive(:http_post_to).with("/datapoints", @karios_data) # expect_any_instance_of(Storer).to receive(:ws_publish) + expect do + Storer.new(device, @data) + end.not_to raise_error + end + it "updates the component last_reading_at timestamp for each of the provided sensors" do + expect(device).to receive(:update_component_timestamps).with( + Time.parse(@data['recorded_at']), + [sensor.id] + ) Storer.new(device, @data) end @@ -53,7 +62,7 @@ expect(device.reload.updated_at).to eq(updated_at) expect(device.reload.data).not_to eq(nil) - expect(device.reload.last_recorded_at).not_to eq(nil) + expect(device.reload.last_reading_at).not_to eq(nil) expect(device.reload.state).to eq('has_published') expect(Storer).to receive(:ws_publish) @@ -78,7 +87,7 @@ it 'does not update device' do expect{ Storer.new(device, @bad_data) }.to raise_error(ArgumentError) - expect(device.reload.last_recorded_at).to eq(nil) + expect(device.reload.last_reading_at).to eq(nil) expect(device.reload.data).to eq(nil) expect(device.reload.state).to eq('never_published') end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 74048c33..fafe6ae7 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -34,10 +34,17 @@ expect(build(:user, avatar: 'http://i.imgur.com/SZD8ADL.JPEG')).to be_valid end - it "has joined_at"do - Timecop.freeze do - expect(create(:user).joined_at).to eq(Time.current.utc) - end + it "is invalid with an email which isn't an email adddress" do + expect(build(:user, email: "not an email")).to be_invalid + end + + it "is invalid without an email" do + expect(build(:user, email: nil)).to be_invalid + end + + it "is invalid with an email which is already taken" do + create(:user, email: "taken@example.com") + expect(build(:user, email: "taken@example.com")).to be_invalid end it "has a location" do diff --git a/spec/policies/kit_policy_spec.rb b/spec/policies/kit_policy_spec.rb deleted file mode 100644 index 3cab0144..00000000 --- a/spec/policies/kit_policy_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -require 'rails_helper' - -describe KitPolicy do - subject { KitPolicy.new(user, kit) } - - let(:kit) { FactoryBot.build(:kit) } - - context "for a visitor" do - let(:user) { nil } - it { is_expected.to permitz(:show) } - it { is_expected.to_not permitz(:update) } - it { is_expected.to_not permitz(:create) } - end - - context "for a user" do - let(:user) { FactoryBot.create(:user) } - it { is_expected.to permitz(:show) } - it { is_expected.to_not permitz(:update) } - it { is_expected.to_not permitz(:create) } - end - - context "for an admin" do - let(:user) { FactoryBot.create(:admin) } - it { is_expected.to permitz(:show) } - it { is_expected.to permitz(:update) } - it { is_expected.to permitz(:create) } - end - -end diff --git a/spec/requests/v0/application_spec.rb b/spec/requests/v0/application_spec.rb index 44d20137..10482227 100644 --- a/spec/requests/v0/application_spec.rb +++ b/spec/requests/v0/application_spec.rb @@ -4,20 +4,20 @@ describe "format" do it "(JSON) returns ugly JSON, with JSON Mimetype" do - json = api_get '/kits' + json = api_get '/devices' #expect( response.body.to_s ).to_not eq( JSON.pretty_generate(json) ) expect(response.header['Content-Type']).to include('application/json') end skip "(JSON) returns pretty JSON, with JSON Mimetype if ?pretty=true" do - json = api_get '/v0/kits?pretty=true' + json = api_get '/v0/devices?pretty=true' expect( response.body.to_s ).to eq( JSON.pretty_generate(json) ) expect(response.header['Content-Type']).to include('application/json') end skip "(JSON-P) returns JS Mimetype if callback param present" do # rails now handles this - api_get '/v0/kits?callback=something' + api_get '/v0/devices?callback=something' expect(response.header['Content-Type']).to include('text/javascript') end end diff --git a/spec/requests/v0/components_spec.rb b/spec/requests/v0/components_spec.rb index c82925aa..b0550460 100644 --- a/spec/requests/v0/components_spec.rb +++ b/spec/requests/v0/components_spec.rb @@ -21,7 +21,7 @@ expect(json.length).to eq(2) expect(json[0]['uuid']).to eq(component1.uuid) expect(json[0].keys).to eq( - %w(id uuid board_id board_type sensor_id created_at updated_at) + %w(id uuid device_id sensor_id created_at updated_at) ) end end diff --git a/spec/requests/v0/devices_spec.rb b/spec/requests/v0/devices_spec.rb index 3c914beb..6b245500 100644 --- a/spec/requests/v0/devices_spec.rb +++ b/spec/requests/v0/devices_spec.rb @@ -24,8 +24,7 @@ expect(json.length).to eq(2) # expect(json[0]['name']).to eq(first.name) # expect(json[1]['name']).to eq(second.name) - expect(json[0].keys).to eq(%w(id uuid name description state postprocessing - hardware_info system_tags user_tags is_private notify_low_battery notify_stopped_publishing last_reading_at added_at updated_at mac_address device_token owner data kit)) + expect(json[0].keys).to eq(%w(id uuid name description state system_tags user_tags is_private last_reading_at created_at updated_at notify device_token postprocessing location hardware owner data)) end describe "when not logged in" do @@ -40,6 +39,13 @@ expect(j.count).to eq(1) expect(j[0]['id']).to eq(device.id) end + + it "does not show hardware_info" do + first = create(:device) + second = create(:device) + json = api_get 'devices' + expect(json[0]['hardware']['last_status_message']).to eq("[FILTERED]") + end end describe "when logged in as a normal user" do @@ -54,6 +60,13 @@ expect(j.count).to eq(2) expect(j[0]['id']).to be_in([device1.id, device2.id]) end + + it "does not show hardware_info" do + first = create(:device) + second = create(:device) + json = api_get 'devices', { access_token: token.token } + expect(json[0]['hardware']['last_status_message']).to eq("[FILTERED]") + end end describe "when logged in as an admin" do @@ -68,6 +81,13 @@ expect(j.count).to eq(3) expect(j[0]['id']).to be_in([device1.id, device2.id, device3.id]) end + + it "shows hardware_info" do + first = create(:device) + second = create(:device) + json = api_get 'devices', { access_token: admin_token.token} + expect(json[0]['hardware']['last_status_message']).not_to eq('[FILTERED]') + end end describe "world map" do @@ -139,8 +159,8 @@ expect(response.status).to eq(200) end - it "allows searching by last_recorded_at" do - json = api_get "devices?q[last_recorded_at_lt]=2023-09-26" + it "allows searching by last_reading_at" do + json = api_get "devices?q[last_reading_at_lt]=2023-09-26" expect(response.status).to eq(200) end @@ -234,25 +254,25 @@ describe "mac_address" do - it "filters mac address from guests" do + it "filters hardware info from guests" do j = api_get "devices/#{device.id}" - expect(j['mac_address']).to eq('[FILTERED]') + expect(j['hardware']['last_status_message']).to eq('[FILTERED]') end - it "filters mac address from users" do + it "filters hardware info from users" do j = api_get "devices/#{device.id}?access_token=#{token.token}" - expect(j['mac_address']).to eq('[FILTERED]') + expect(j['hardware']['last_status_message']).to eq('[FILTERED]') end - it "exposes mac address to device owner" do + it "exposes hardware info to device owner" do device = create(:device, owner: user) j = api_get "devices/#{device.id}?access_token=#{token.token}" - expect(j['mac_address']).to eq(device.mac_address) + expect(j['hardware']['last_status_message']).to eq(device.hardware_info) end - it "exposes mac address to admin" do + it "exposes hardware info address to admin" do j = api_get "devices/#{device.id}?access_token=#{admin_token.token}" - expect(j['mac_address']).to eq(device.mac_address) + expect(j['hardware']['last_status_message']).to eq(device.hardware_info) end end @@ -324,10 +344,15 @@ end it "will update a device with empty parameters access_token" do - api_put "devices/#{device.id}", { name: nil, access_token: token.token } + api_put "devices/#{device.id}", { access_token: token.token } expect(response.status).to eq(200) end + it "does not allow an empty device name" do + api_put "devices/#{device.id}", { name: nil, access_token: token.token } + expect(response.status).to eq(422) + end + it 'can read and update a jsonb' do expect(device.postprocessing).to be_nil j = api_put "devices/#{device.id}", { postprocessing_attributes: {"blueprint_url":"999"}, access_token: token.token, name: 'ABBA' } diff --git a/spec/requests/v0/kits_spec.rb b/spec/requests/v0/kits_spec.rb deleted file mode 100644 index f9e620f0..00000000 --- a/spec/requests/v0/kits_spec.rb +++ /dev/null @@ -1,90 +0,0 @@ -require 'rails_helper' - -describe V0::KitsController do - - let(:application) { build :application } - let(:user) { build :user } - let(:token) { build :access_token, application: application, resource_owner_id: user.id } - - let(:admin) { create :admin } - let(:admin_token) { create :access_token, application: application, resource_owner_id: admin.id } - - let(:kit) { build :kit } - - describe "GET /kits" do - it "returns all the kits" do - first = create(:kit) - second = create(:kit) - json = api_get 'kits' - - expect(response.status).to eq(200) - expect(json.length).to eq(2) - expect(json[0]['id']).to eq(first.id) - expect(json[0].keys).to eq(%w(id uuid slug name description created_at updated_at sensors)) - end - end - - describe "GET /kits/:id" do - it "returns a kit" do - kit = create(:kit) - j = api_get "kits/#{kit.id}" - expect(j['id']).to eq(kit.id) - expect(response.status).to eq(200) - end - - it "returns 404 if kit not found" do - j = api_get 'kits/100' - expect(j['id']).to eq('record_not_found') - expect(response.status).to eq(404) - end - end - - describe "POST /kits" do - - it "creates a kit" do - j = api_post 'kits', { - name: 'new kit', - description: 'blah blah blah', - access_token: admin_token.token - } - expect(j['name']).to eq('new kit') - expect(response.status).to eq(201) - end - - it "does not create a kit with missing parameters" do - j = api_post 'kits', { - access_token: admin_token.token - } - expect(j['id']).to eq('unprocessable_entity') - expect(response.status).to eq(422) - end - - end - - describe "PUT /kits/:id" do - - let!(:kit) { create :kit } - - it "updates a kit" do - api_put "kits/#{kit.id}", { name: 'new name', access_token: admin_token.token } - expect(response.status).to eq(200) - end - - it "does not update a kit with invalid access_token" do - api_put "kits/#{kit.id}", { name: 'new name', access_token: '123' } - expect(response.status).to eq(403) - end - - it "does not update a kit with missing access_token" do - api_put "kits/#{kit.id}", { name: 'new name', access_token: nil } - expect(response.status).to eq(403) - end - - it "does not update a kit with empty parameters access_token" do - api_put "kits/#{kit.id}", { name: nil, access_token: admin_token.token } - expect(response.status).to eq(422) - end - - end - -end diff --git a/spec/requests/v0/onboarding/device_registrations_spec.rb b/spec/requests/v0/onboarding/device_registrations_spec.rb index a0d9cca4..9e664750 100644 --- a/spec/requests/v0/onboarding/device_registrations_spec.rb +++ b/spec/requests/v0/onboarding/device_registrations_spec.rb @@ -33,9 +33,8 @@ end before do - create(:kit, id: 1) if Kit.where(id: 1).empty? - create(:tag, name: 'tag1') if Kit.where(name: 'tag1').empty? - create(:tag, name: 'tag2') if Kit.where(name: 'tag2').empty? + create(:tag, name: 'tag1') + create(:tag, name: 'tag2') end describe 'POST /onboarding/register' do @@ -63,7 +62,6 @@ expect(@device.device_token).to eq(orphan_device.device_token) expect(@device.exposure).to eq(orphan_device.exposure) expect(@device.description).to eq(orphan_device.description) - expect(@device.kit).to eq(Kit.first) expect(@device.tags.count).to eq(2) expect(@device.location['city']).to eq('Barcelona') end diff --git a/spec/requests/v0/onboarding/orphan_devices_spec.rb b/spec/requests/v0/onboarding/orphan_devices_spec.rb index 5848d815..435daa09 100644 --- a/spec/requests/v0/onboarding/orphan_devices_spec.rb +++ b/spec/requests/v0/onboarding/orphan_devices_spec.rb @@ -27,10 +27,10 @@ it 'creates an orphan_device with passed attributes' do j = api_post '/onboarding/device', { - kit_id: 3 + } - expect(OrphanDevice.where(kit_id: 3).count).to eq(1) + expect(OrphanDevice.count).to eq(1) end end diff --git a/spec/requests/v0/readings_spec.rb b/spec/requests/v0/readings_spec.rb index 0af8351d..48ae05e8 100644 --- a/spec/requests/v0/readings_spec.rb +++ b/spec/requests/v0/readings_spec.rb @@ -2,11 +2,10 @@ describe V0::ReadingsController do let(:user) { build(:user) } - let(:kit) { build(:kit, sensor_map: '{"noise": 7, "temp": 12, "light": 14, "no2": 15}' ) } - let(:device) { create(:device, owner: user, kit: kit) } + let(:device) { create(:device, owner: user) } let(:measurement) { build(:measurement) } let(:sensor) { build(:sensor, measurement: measurement) } - let(:component) { build(:component, board: kit, sensor: sensor) } + let(:component) { build(:component, device: device, sensor: sensor) } let(:application) { build :application } let(:token) { build :access_token, application: application, resource_owner_id: user.id } diff --git a/spec/requests/v0/static_spec.rb b/spec/requests/v0/static_spec.rb index cf75f917..45a435aa 100644 --- a/spec/requests/v0/static_spec.rb +++ b/spec/requests/v0/static_spec.rb @@ -13,7 +13,6 @@ current_user_url components_url devices_url - kits_url measurements_url sensors_url users_url diff --git a/spec/requests/v0/users_spec.rb b/spec/requests/v0/users_spec.rb index 65f0f576..55183971 100644 --- a/spec/requests/v0/users_spec.rb +++ b/spec/requests/v0/users_spec.rb @@ -58,8 +58,6 @@ it "does not include the device locations" do j = api_get "users/testguy" expect(j["devices"].map { |d| d["location"]}.compact).to be_empty - expect(j["devices"].map { |d| d["latitude"]}.compact).to be_empty - expect(j["devices"].map { |d| d["longitude"]}.compact).to be_empty end end @@ -73,8 +71,6 @@ it "includes the device locations" do j = api_get "users/testguy?access_token=#{token.token}" expect(j["devices"].map { |d| d["location"]}.compact).not_to be_empty - expect(j["devices"].map { |d| d["latitude"]}.compact).not_to be_empty - expect(j["devices"].map { |d| d["longitude"]}.compact).not_to be_empty end end @@ -95,8 +91,6 @@ it "does not include the device locations" do j = api_get "users/testguy?access_token=#{requesting_token.token}" expect(j["devices"].map { |d| d["location"]}.compact).to be_empty - expect(j["devices"].map { |d| d["latitude"]}.compact).to be_empty - expect(j["devices"].map { |d| d["longitude"]}.compact).to be_empty end end @@ -117,8 +111,6 @@ it "does not include the device locations" do j = api_get "users/testguy?access_token=#{requesting_token.token}" expect(j["devices"].map { |d| d["location"]}.compact).to be_empty - expect(j["devices"].map { |d| d["latitude"]}.compact).to be_empty - expect(j["devices"].map { |d| d["longitude"]}.compact).to be_empty end end @@ -139,8 +131,6 @@ it "includes the device locations" do j = api_get "users/testguy?access_token=#{requesting_token.token}" expect(j["devices"].map { |d| d["location"]}.compact).not_to be_empty - expect(j["devices"].map { |d| d["latitude"]}.compact).not_to be_empty - expect(j["devices"].map { |d| d["longitude"]}.compact).not_to be_empty end end end @@ -291,7 +281,7 @@ describe "PUT /users/|" do - let(:user) { create(:user, username: 'lisasimpson') } + let(:user) { create(:user, username: 'lisasimpson', country_code: "GB") } it "updates user" do j = api_put "users/#{[user.username,user.id].sample}", { @@ -301,6 +291,36 @@ expect(response.status).to eq(200) end + it "updates user country code" do + j = api_put "users/#{[user.username,user.id].sample}", { + country_code: 'ES', access_token: token.token + } + expect(j['location']['country_code']).to eq('ES') + expect(j['location']['country']).to eq('Spain') + expect(response.status).to eq(200) + expect(user.reload.country_code).to eq('ES') + end + + it "unsets user country code" do + j = api_put "users/#{[user.username,user.id].sample}", { + country_code: nil, access_token: token.token + } + expect(j['location']['country_code']).to be_nil + expect(j['location']['country']).to be_nil + expect(response.status).to eq(200) + expect(user.reload.country_code).to be_nil + end + + it "leaves country code as is when not explicitly passsed" do + j = api_put "users/#{[user.username,user.id].sample}", { + username: "new_username", access_token: token.token + } + expect(j['location']['country_code']).to eq("GB") + expect(j['location']['country']).to eq("United Kingdom of Great Britain and Northern Ireland") + expect(response.status).to eq(200) + expect(user.reload.country_code).to eq("GB") + end + it "does not update a user with invalid access_token" do j = api_put "users/#{[user.username,user.id].sample}", { username: 'bart', access_token: '123' diff --git a/spec/services/device_archive_spec.rb b/spec/services/device_archive_spec.rb index 183e63df..439f46c1 100644 --- a/spec/services/device_archive_spec.rb +++ b/spec/services/device_archive_spec.rb @@ -6,30 +6,34 @@ def kairos_query(key) end describe DeviceArchive do + + let(:device) { create(:device) } + before(:each) do create(:measurement, id: 1, name: 'temp') create(:measurement, id: 2, name: 'light') create(:measurement, id: 3, name: 'noise') create(:measurement, id: 4, name: 'NO2') - create(:kit, id: 3, name: 'SCK', description: "Board", slug: 'sck', sensor_map: {"noise": 7, "temp": 12, "light": 14, "no2": 15}) - create(:sensor, id:12, name:'HPP828E031', description: 'test', measurement_id: 1, unit: 'ºC') - create(:sensor, id:7, name:'POM-3044P-R', description: 'test', measurement_id: 2, unit: 'dB') - create(:sensor, id:14, name:'BH1730FVC', description: 'test', measurement_id: 2, unit: 'KΩ') - create(:sensor, id:15, name:'MiCS-4514', description: 'test', measurement_id: 4, unit: 'kOhm') - create(:component, id: 12, board: Kit.find(3), sensor: Sensor.find(12), equation: '(175.72 / 65536.0 * x) - 53', reverse_equation: 'x') - create(:component, id: 14, board: Kit.find(3), sensor: Sensor.find(14), equation: 'x', reverse_equation: 'x/10.0') - create(:component, id: 15, board: Kit.find(3), sensor: Sensor.find(7), equation: 'Mathematician.table_calibration({0=>50,2=>55,3=>57,6=>58,20=>59,40=>60,60=>61,75=>62,115=>63,150=>64,180=>65,220=>66,260=>67,300=>68,375=>69,430=>70,500=>71,575=>72,660=>73,720=>74,820=>75,900=>76,975=>77,1050=>78,1125=>79,1200=>80,1275=>81,1320=>82,1375=>83,1400=>84,1430=>85,1450=>86,1480=>87,1500=>88,1525=>89,1540=>90,1560=>91,1580=>92,1600=>93,1620=>94,1640=>95,1660=>96,1680=>97,1690=>98,1700=>99,1710=>100,1720=>101,1745=>102,1770=>103,1785=>104,1800=>105,1815=>106,1830=>107,1845=>108,1860=>109,1875=>110},x)', reverse_equation: 'x') - create(:component, id: 16, board: Kit.find(3), sensor: Sensor.find(15), equation: 'x', reverse_equation: 'x/1000.0') + + create(:sensor, id:12, name:'HPP828E031', description: 'test', measurement_id: 1, unit: 'ºC', default_key: "temp", equation: '(175.72 / 65536.0 * x) - 53', reverse_equation: 'x') + create(:sensor, id:14, name:'BH1730FVC', description: 'test', measurement_id: 2, unit: 'KΩ', default_key: "light", equation: 'x', reverse_equation: 'x/10.0') + create(:sensor, id:7, name:'POM-3044P-R', description: 'test', measurement_id: 2, unit: 'dB', default_key: "noise", equation: 'Mathematician.table_calibration({0=>50,2=>55,3=>57,6=>58,20=>59,40=>60,60=>61,75=>62,115=>63,150=>64,180=>65,220=>66,260=>67,300=>68,375=>69,430=>70,500=>71,575=>72,660=>73,720=>74,820=>75,900=>76,975=>77,1050=>78,1125=>79,1200=>80,1275=>81,1320=>82,1375=>83,1400=>84,1430=>85,1450=>86,1480=>87,1500=>88,1525=>89,1540=>90,1560=>91,1580=>92,1600=>93,1620=>94,1640=>95,1660=>96,1680=>97,1690=>98,1700=>99,1710=>100,1720=>101,1745=>102,1770=>103,1785=>104,1800=>105,1815=>106,1830=>107,1845=>108,1860=>109,1875=>110},x)', reverse_equation: 'x') + create(:sensor, id:15, name:'MiCS-4514', description: 'test', measurement_id: 4, unit: 'kOhm', default_key: "no2", equation: 'x', reverse_equation: 'x/1000.0') + + create(:component, id: 12, device: device, sensor: Sensor.find(12)) + create(:component, id: 14, device: device, sensor: Sensor.find(14)) + create(:component, id: 15, device: device, sensor: Sensor.find(7)) + create(:component, id: 16, device: device, sensor: Sensor.find(15)) end - let(:device) { create(:device, kit: Kit.find(3)) } + let(:csv) { - "timestamp,NO2 in kOhm (MiCS-4514),temp in ºC (HPP828E031),light in KΩ (BH1730FVC),light in dB (POM-3044P-R)\n"\ - "2013-04-03 06:00:00 UTC,1.0,-52.997318725585934,1.0,52.5\n"\ - "2013-04-19 06:00:00 UTC,2.0,-52.994637451171876,2.0,55.0\n"\ - "2013-04-23 06:00:00 UTC,3.0,-52.99195617675781,3.0,57.0\n"\ - "2013-04-30 06:00:00 UTC,4.0,-52.98927490234375,4.0,57.333333333333336\n" + "timestamp,temp in ºC (HPP828E031),light in KΩ (BH1730FVC),light in dB (POM-3044P-R),NO2 in kOhm (MiCS-4514)\n"\ + "2013-04-03 06:00:00 UTC,-52.997318725585934,1.0,52.5,1.0\n"\ + "2013-04-19 06:00:00 UTC,-52.994637451171876,2.0,55.0,2.0\n"\ + "2013-04-23 06:00:00 UTC,-52.99195617675781,3.0,57.0,3.0\n"\ + "2013-04-30 06:00:00 UTC,-52.98927490234375,4.0,57.333333333333336,4.0\n" } let(:http_response) { diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 629d244a..2d5c8cb4 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,6 +28,7 @@ # ...rather than: # # => "be bigger than 2" expectations.include_chain_clauses_in_custom_matcher_descriptions = true + expectations.max_formatted_output_length = nil end # rspec-mocks config goes here. You can use an alternate test double