From 74a3c60a689eb1ef253253983b2c259504117ec1 Mon Sep 17 00:00:00 2001 From: Kyle Aungst Date: Sat, 11 Aug 2018 21:32:38 -0700 Subject: [PATCH 1/4] Update .gitignore Will not include .idea files. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b844b14..d6f204d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ Gemfile.lock +.idea From d66f5a47f507a3bba22cd0b1b5ee1b55c9458f62 Mon Sep 17 00:00:00 2001 From: Kyle Aungst Date: Sat, 11 Aug 2018 21:36:48 -0700 Subject: [PATCH 2/4] Creates support for `scrub_type` Adds default scrub_types - scrub - anonymize field value - skip - keep field's value the same - wipe - set field's value to nil - sterilize - destroy all records --- README.md | 22 +++++++++++---- lib/acts_as_scrubbable.rb | 3 +-- lib/acts_as_scrubbable/scrub.rb | 14 +++++++--- lib/acts_as_scrubbable/scrubbable.rb | 25 +++++++++-------- lib/acts_as_scrubbable/tasks.rb | 26 ++++++++---------- spec/db/schema.rb | 6 ++++- spec/lib/acts_as_scrubbable/scrub_spec.rb | 27 +++++++++++++++++-- .../lib/acts_as_scrubbable/scrubbable_spec.rb | 24 ++++++++++++----- spec/support/database.rb | 14 +++++++--- 9 files changed, 109 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index d2f7639..9009389 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,22 @@ gem 'acts_as_scrubbable' ## Usage -Simple add the configuration for your fields that map directly to your columns +Add the configuration for your fields that map directly to your columns and a scrub_type +for those columns. +Default Scrub types include: +- `scrub` - scrub the field's value based on it's name or mapping (see mapping case below) +- `skip` - do not scrub the field's value +- `wipe` - set the field's value to nil on scrub +- `sterilize` - delete all records for this model on scrub ```ruby -class User < ActiveRecord::Base +class ScrubExample < ActiveRecord::Base ... - acts_as_scrubbable :first_name, :last_name + acts_as_scrubbable :scrub, :first_name # first_name will be random after `scrub!` + acts_as_scrubbable :skip, :middle_name # middle_name will be original value after `scrub!` + acts_as_scrubbable :wipe, :last_name # last_name will be `nil` after `scrub!` # optionally you can add a scope to limit the rows to update @@ -31,6 +39,11 @@ class User < ActiveRecord::Base ... end + +class SterilizeExample < ActiveRecord::Base + acts_as_scrubbable :sterilize # table will contain no records after `scrub!` +end + ``` @@ -38,11 +51,10 @@ Incase the mapping is not straight forward ```ruby class Address - acts_as_scrubbable :lng => :longitude, :lat => :latitude + acts_as_scrubbable :scrub, :lng => :longitude, :lat => :latitude end ``` - ### To run The confirmation message will be the db host diff --git a/lib/acts_as_scrubbable.rb b/lib/acts_as_scrubbable.rb index 918ef3d..ba88277 100644 --- a/lib/acts_as_scrubbable.rb +++ b/lib/acts_as_scrubbable.rb @@ -49,12 +49,11 @@ def self.scrub_map :longitude => -> { Faker::Address.longitude }, :username => -> { Faker::Internet.user_name }, :boolean => -> { [true, false ].sample }, - :school => -> { Faker::University.name } + :school => -> { Faker::University.name }, } end end - ActiveSupport.on_load(:active_record) do extend ActsAsScrubbable::Scrubbable end diff --git a/lib/acts_as_scrubbable/scrub.rb b/lib/acts_as_scrubbable/scrub.rb index 16dc14e..b869068 100644 --- a/lib/acts_as_scrubbable/scrub.rb +++ b/lib/acts_as_scrubbable/scrub.rb @@ -1,23 +1,29 @@ module ActsAsScrubbable module Scrub - def scrub! return unless self.class.scrubbable? run_callbacks(:scrub) do + if self.class.sterilizable? + self.destroy! + next + end + _updates = {} scrubbable_fields.each do |_field, value| unless self.respond_to?(_field) raise ArgumentError, "#{self.class} do not respond to #{_field}" end - next if self.send(_field).blank? + next if self.send(_field).blank? || value == :skip if ActsAsScrubbable.scrub_map.keys.include?(value) _updates[_field] = ActsAsScrubbable.scrub_map[value].call + elsif value == :wipe + _updates[_field] = nil else - puts "Undefined scrub: #{value} for #{self.class}#{_field}" - end + puts "Undefined scrub: #{value} for #{self.class}.#{_field}" + end end self.update_columns(_updates) unless _updates.empty? diff --git a/lib/acts_as_scrubbable/scrubbable.rb b/lib/acts_as_scrubbable/scrubbable.rb index 035f051..5c84b40 100644 --- a/lib/acts_as_scrubbable/scrubbable.rb +++ b/lib/acts_as_scrubbable/scrubbable.rb @@ -1,36 +1,35 @@ module ActsAsScrubbable module Scrubbable - - def scrubbable? false end + def acts_as_scrubbable(scrub_type=:scrub, *scrubbable_fields, **mapped_fields) + unless self.respond_to?(:scrubbable_fields) + class_attribute :scrubbable_fields + self.scrubbable_fields = {} + end - def acts_as_scrubbable(*scrubbable_fields, **mapped_fields) - - class_attribute :scrubbable_fields - - self.scrubbable_fields = {} - scrubbable_fields.each do |_field| - self.scrubbable_fields[_field] = _field + unless self.respond_to?(:sterilizable?) + class_attribute :sterilizable + self.sterilizable = scrub_type == :sterilize end - mapped_fields.each do |_field| - self.scrubbable_fields[_field.first] = _field.last + scrubbable_fields.each do |field_name| + self.scrubbable_fields[field_name] = scrub_type == :scrub ? field_name : scrub_type end + mapped_fields.each { |field_name, field_type| self.scrubbable_fields[field_name] = field_type } + class_eval do define_callbacks :scrub def self.scrubbable? true end - end include Scrub end - end end diff --git a/lib/acts_as_scrubbable/tasks.rb b/lib/acts_as_scrubbable/tasks.rb index 91a8a0b..9e91b50 100644 --- a/lib/acts_as_scrubbable/tasks.rb +++ b/lib/acts_as_scrubbable/tasks.rb @@ -2,21 +2,18 @@ require 'rake' namespace :scrub do - desc "scrub all" task all: :environment do - require 'highline/import' require 'term/ansicolor' require 'logger' require 'parallel' - include Term::ANSIColor @logger = Logger.new($stdout) @logger.formatter = proc do |severity, datetime, progname, msg| - "#{datetime}: [#{severity}] - #{msg}\n" + "#{datetime}: [#{severity}] - #{msg}\n" end db_host = ActiveRecord::Base.connection_config[:host] @@ -27,8 +24,8 @@ @logger.warn "Database: ".red + "#{db_name}".white unless ENV["SKIP_CONFIRM"] == "true" - answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white) + unless answer == db_host @logger.error "exiting ...".red exit @@ -41,21 +38,17 @@ @total_scrubbed = 0 - ar_classes = ActiveRecord::Base.descendants.select{|d| d.scrubbable? }.sort_by{|d| d.to_s } - + ar_classes = ActiveRecord::Base.descendants.select(&:scrubbable?).sort_by(&:to_s) # if the ENV variable is set - unless ENV["SCRUB_CLASSES"].blank? class_list = ENV["SCRUB_CLASSES"].split(",") - class_list = class_list.map {|_class_str| _class_str.constantize } - ar_classes = ar_classes & class_list + ar_classes &= class_list.map {|_class_str| _class_str.constantize } end @logger.info "Srubbable Classes: #{ar_classes.join(', ')}".white Parallel.each(ar_classes) do |ar_class| - # Removing any find or initialize callbacks from model ar_class.reset_callbacks(:initialize) ar_class.reset_callbacks(:find) @@ -65,10 +58,12 @@ scrubbed_count = 0 ActiveRecord::Base.connection_pool.with_connection do - if ar_class.respond_to?(:scrubbable_scope) - relation = ar_class.send(:scrubbable_scope) - else - relation = ar_class.all + relation = ar_class.respond_to?(:scrubbable_scope) ? ar_class.send(:scrubbable_scope) : ar_class.all + + if relation.sterilizable? + scrubbed_count += relation.count + relation.delete_all + next end relation.find_in_batches(batch_size: 1000) do |batch| @@ -83,6 +78,7 @@ @logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue end + ActiveRecord::Base.connection.verify! if ENV["SKIP_AFTERHOOK"].blank? diff --git a/spec/db/schema.rb b/spec/db/schema.rb index ec955f9..4186396 100644 --- a/spec/db/schema.rb +++ b/spec/db/schema.rb @@ -1,9 +1,13 @@ ActiveRecord::Schema.define(version: 20150421224501) do - create_table "scrubbable_models", force: true do |t| t.string "first_name" + t.string "middle_name" + t.string "last_name" t.string "address1" t.string "lat" end + create_table "sterilizable_models", force: true do |t| + t.string "irrelevant" + end end diff --git a/spec/lib/acts_as_scrubbable/scrub_spec.rb b/spec/lib/acts_as_scrubbable/scrub_spec.rb index f966fa1..17ee769 100644 --- a/spec/lib/acts_as_scrubbable/scrub_spec.rb +++ b/spec/lib/acts_as_scrubbable/scrub_spec.rb @@ -1,11 +1,10 @@ require 'spec_helper' RSpec.describe ActsAsScrubbable::Scrub do - describe '.scrub' do - # update_columns cannot be run on a new record subject{ ScrubbableModel.new } + before(:each) { subject.save } it 'changes the first_name attribute when scrub is run' do @@ -31,10 +30,34 @@ expect(subject.address1).to be_nil end + it "doesn't update the field if the scrub type is `:skip`" do + subject.middle_name = "Edward" + subject.save + subject.scrub! + expect(subject.middle_name).to eq "Edward" + end + + it "updates the field to nil if the scrub type is :wipe" do + subject.last_name = "Cortéz" + subject.save + subject.scrub! + expect(subject.last_name).to be_nil + end + it 'runs scrub callbacks' do subject.scrub! expect(subject.scrubbing_begun).to be(true) expect(subject.scrubbing_finished).to be(true) end + + context 'when sterilizable? is true' do + subject { SterilizableModel.new } + + it 'deletes all records' do + subject.save + subject.scrub! + expect(subject.class.all).to be_empty + end + end end end diff --git a/spec/lib/acts_as_scrubbable/scrubbable_spec.rb b/spec/lib/acts_as_scrubbable/scrubbable_spec.rb index a60e0df..f9c9a73 100644 --- a/spec/lib/acts_as_scrubbable/scrubbable_spec.rb +++ b/spec/lib/acts_as_scrubbable/scrubbable_spec.rb @@ -1,9 +1,8 @@ require 'spec_helper' RSpec.describe ActsAsScrubbable::Scrubbable do - context "not scrubbable" do - subject {NonScrubbableModel.new} + subject { NonScrubbableModel.new } describe "scrubbable?" do it 'returns false' do @@ -12,22 +11,33 @@ end end - context "scrubbable" do - subject {ScrubbableModel.new} + subject { ScrubbableModel.new } describe "scrubbable?" do - it 'returns false' do + it 'returns true' do expect(subject.class.scrubbable?).to eq true end end - describe "scrubbable_fields" do it 'returns the list of scrubbable fields' do - expect(subject.scrubbable_fields.keys.first).to eq :first_name + expect(subject.scrubbable_fields.keys).to match_array(%i[first_name middle_name last_name address1 lat]) end end + describe "sterilizable?" do + it 'returns false' do + expect(subject.class.sterilizable?).to eq false + end + + context "when :sterilize is passed as a scrub_type" do + subject { SterilizableModel.new } + + it 'returns true' do + expect(subject.class.sterilizable?).to eq true + end + end + end end end diff --git a/spec/support/database.rb b/spec/support/database.rb index 8a5b38e..6595282 100644 --- a/spec/support/database.rb +++ b/spec/support/database.rb @@ -1,7 +1,7 @@ require 'nulldb/rails' require 'nulldb_rspec' -ActiveRecord::Base.configurations.merge!("test" => {adapter: 'nulldb'}) +ActiveRecord::Base.configurations.merge!("test" => { adapter: 'nulldb' }) NullDB.configure do |c| c.project_root = './spec' @@ -11,16 +11,24 @@ config.include include NullDB::RSpec::NullifiedDatabase end - class NonScrubbableModel < ActiveRecord::Base; end class ScrubbableModel < ActiveRecord::Base - acts_as_scrubbable :first_name, :address1 => :street_address, :lat => :latitude attr_accessor :scrubbing_begun, :scrubbing_finished + + acts_as_scrubbable :scrub, :first_name, :address1 => :street_address, :lat => :latitude + acts_as_scrubbable :wipe, :last_name + acts_as_scrubbable :skip, :middle_name + set_callback :scrub, :before do self.scrubbing_begun = true end + set_callback :scrub, :after do self.scrubbing_finished = true end end + +class SterilizableModel < ActiveRecord::Base + acts_as_scrubbable :sterilize +end From df0d0e5b8bbc39ea077fd03113a24e7a00d312fe Mon Sep 17 00:00:00 2001 From: Kyle Aungst Date: Sat, 11 Aug 2018 21:40:29 -0700 Subject: [PATCH 3/4] Add support for `before_hook` --- README.md | 16 ++++++++++++---- lib/acts_as_scrubbable.rb | 9 ++++++++- lib/acts_as_scrubbable/tasks.rb | 8 ++++++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9009389..a4a7d16 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ end ``` - Incase the mapping is not straight forward ```ruby @@ -87,22 +86,31 @@ If you want to limit the classes you to be scrubbed you can set the `SCRUB_CLASS rake scrub SCRUB_CLASSES=Blog,Post ``` -If you want to skip the afterhook +If you want to skip the beforehook ``` -rake scrub SKIP_AFTERHOOK=true +rake scrub SKIP_BEFOREHOOK=true ``` +If you want to skip the afterhook +``` +rake scrub SKIP_AFTERHOOK=true +``` ### Extending -You may find the need to extend or add additional generators or an after_hook +You may find the need to extend or add additional generators or an before_hook/after_hook ```ruby ActsAsScrubbable.configure do |c| c.add :email_with_prefix, -> { "prefix-#{Faker::Internet.email}" } + c.before_hook do + puts "Running before scrub" + raise "Don't run in production" if Rails.env.production? + end + c.after_hook do puts "Running after commit" ActiveRecord::Base.connection.execute("TRUNCATE some_table") diff --git a/lib/acts_as_scrubbable.rb b/lib/acts_as_scrubbable.rb index ba88277..e98089f 100644 --- a/lib/acts_as_scrubbable.rb +++ b/lib/acts_as_scrubbable.rb @@ -11,11 +11,18 @@ module ActsAsScrubbable autoload :Scrub autoload :VERSION - def self.configure(&block) yield self end + def self.before_hook(&block) + @before_hook = block + end + + def self.execute_before_hook + @before_hook.call if @before_hook + end + def self.after_hook(&block) @after_hook = block end diff --git a/lib/acts_as_scrubbable/tasks.rb b/lib/acts_as_scrubbable/tasks.rb index 9e91b50..b581aef 100644 --- a/lib/acts_as_scrubbable/tasks.rb +++ b/lib/acts_as_scrubbable/tasks.rb @@ -16,8 +16,12 @@ "#{datetime}: [#{severity}] - #{msg}\n" end - db_host = ActiveRecord::Base.connection_config[:host] - db_name = ActiveRecord::Base.connection_config[:database] + if ENV["SKIP_BEFOREHOOK"].blank? + @logger.info "Running before hook".red + ActsAsScrubbable.execute_before_hook + end + + db_host, db_name = ActiveRecord::Base.connection_config.values_at(:host, :database) @logger.warn "Please verify the information below to continue".red @logger.warn "Host: ".red + " #{db_host}".white From 108525e93ab09fca484c984806a9eef364698e11 Mon Sep 17 00:00:00 2001 From: Kyle Aungst Date: Mon, 13 Aug 2018 09:57:12 -0700 Subject: [PATCH 4/4] Add bs, phone_number, and full_address scrub types. --- lib/acts_as_scrubbable.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/acts_as_scrubbable.rb b/lib/acts_as_scrubbable.rb index e98089f..8ca828e 100644 --- a/lib/acts_as_scrubbable.rb +++ b/lib/acts_as_scrubbable.rb @@ -52,11 +52,14 @@ def self.scrub_map :state_abbr => -> { Faker::Address.state_abbr }, :state => -> { Faker::Address.state }, :city => -> { Faker::Address.city }, + :full_address => -> { Faker::Address.full_address }, :latitude => -> { Faker::Address.latitude }, :longitude => -> { Faker::Address.longitude }, :username => -> { Faker::Internet.user_name }, :boolean => -> { [true, false ].sample }, :school => -> { Faker::University.name }, + :bs => -> { Faker::Company.bs }, + :phone_number => -> { Faker::PhoneNumber.phone_number } } end end