Merge branch 'LEF-24-repl-fixes' into 'master'

LEF-24 - Use preserve event for repl job

See merge request cdr/longleaf!21

lib/longleaf/candidates/service_candidate_filesystem_iterator.rb

@@ -70,6 +70,7 @@ def needs_run?(file_rec)
 
       # File is not a valid candidate for services if it is deregistered, unless performing cleanup
       if @event != EventNames::CLEANUP && md_rec.deregistered?
+        logger.debug("Skipping deregistered file: #{file_rec.path}")
         return false
       end
 
@@ -79,15 +80,18 @@ def needs_run?(file_rec)
 
       # When in force mode, candidate needs a run as long as there are any services configured for it.
       if @force && expected_services.length > 0
+        logger.debug("Forcing run needed for file: #{file_rec.path}")
         return true
       end
 
       expected_services.each do |service_name|
         if service_manager.service_needed?(service_name, md_rec)
+          logger.debug("Service #{service_name} needed for file: #{file_rec.path}")
           return true
         end
       end
 
+      logger.debug("Run not needed for file: #{file_rec.path}")
       # No services needed to be run for this file
       false
     end

lib/longleaf/events/event_names.rb

@@ -3,7 +3,6 @@ module Longleaf
   class EventNames
     REGISTER = 'register'
     PRESERVE = 'preserve'
-    REPLICATE = 'replicate'
     CLEANUP = 'cleanup'
     DEREGISTER = 'deregister'
   end

lib/longleaf/preservation_services/rsync_replication_service.rb

@@ -131,7 +131,7 @@ def perform(file_rec, event)
     # @return [Boolean] returns true if this service is applicable for the provided event
     def is_applicable?(event)
       case event
-      when EventNames::REPLICATE
+      when EventNames::PRESERVE
         true
       else
         false

spec/features/deregister_command_spec.rb

@@ -20,7 +20,7 @@
   end
 
   context 'with valid configuration' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1')
         .map_services('loc1', 'serv1')

spec/features/preserve_command_spec.rb

@@ -166,7 +166,7 @@
     let!(:fail_script) { create_work_class(lib_dir, 'PresService', 'pres_service.rb',
         perform: "raise Longleaf::PreservationServiceError.new") }
 
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1', work_script: fail_script, frequency: '1 year')
         .map_services('loc1', 'serv1')
@@ -187,7 +187,7 @@
   context 'storage location with multiple files' do
     let!(:work_script_file) { create_work_class(lib_dir, 'PresService', 'pres_service.rb') }
 
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1', work_script: work_script_file, frequency: '1 hour')
         .map_services('loc1', 'serv1')
@@ -284,7 +284,7 @@
     let!(:work_script_file2) { create_work_class(lib_dir, 'PresService2', 'pres_service2.rb') }
     let!(:work_script_file3) { create_work_class(lib_dir, 'PresService3', 'pres_service3.rb') }
 
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1', work_script: work_script_file1, frequency: '1 hour')
         .with_service(name: 'serv2', work_script: work_script_file2, frequency: '1 hour')
@@ -369,7 +369,7 @@
         update_timestamp(file_path, config_path, 'serv2')
         update_timestamp(file_path, config_path, 'serv3')
 
-        run_simple("longleaf preserve -c #{config_path} -I #{lib_dir} -f #{file_path}", fail_on_error: false)
+        run_simple("longleaf preserve -c #{config_path} -I #{lib_dir} -f #{file_path} --log_level 'DEBUG'", fail_on_error: false)
       end
 
       it 'indicates that no services ran' do

spec/features/register_command_spec.rb

@@ -36,7 +36,7 @@
   end
 
   context 'invalid storage location' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: nil, md_path: md_dir)
         .with_services
         .with_mappings
@@ -55,7 +55,7 @@
   end
 
   context 'with valid configuration' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1')
         .map_services('loc1', 'serv1')

spec/features/validate_application_config_spec.rb

@@ -31,7 +31,7 @@
 
   context 'invalid storage location' do
     let(:md_dir) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: nil, md_path: md_dir)
         .with_services
         .with_mappings
@@ -56,7 +56,7 @@
   context 'unavailable storage location' do
     let(:path_dir) { FileUtils.rmdir(Dir.mktmpdir('path'))[0] }
     let(:md_dir) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_services
         .with_mappings
@@ -80,7 +80,7 @@
   context 'valid storage location' do
     let(:path_dir) { Dir.mktmpdir('path') }
     let(:md_dir) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_services
         .with_mappings
@@ -104,7 +104,7 @@
     let(:path_dir1) { Dir.mktmpdir('path') }
     let(:md_dir1) { Dir.mktmpdir('metadata') }
     let(:md_dir2) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir1, md_path: md_dir1)
         .with_location(name: 'loc2', path: path_dir1, md_path: md_dir2)
         .with_services
@@ -129,7 +129,7 @@
   context 'invalid service definition' do
     let(:path_dir) { Dir.mktmpdir('path') }
     let(:md_dir) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1', work_script: nil)
         .with_mappings
@@ -151,7 +151,7 @@
   end
 
   context 'valid service definition' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_locations
         .with_service(name: 'serv1', work_script: 'preserve.rb')
         .with_mappings
@@ -167,7 +167,7 @@
   context 'valid service mapping' do
     let(:path_dir) { Dir.mktmpdir('path') }
     let(:md_dir) { Dir.mktmpdir('metadata') }
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1', work_script: 'preserve.rb')
         .map_services('loc1', 'serv1')

spec/features/validate_metadata_command_spec.rb

@@ -19,7 +19,7 @@
   end
 
   context 'location with one digest configured' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir, md_digests: ['sha1'])
         .with_service(name: 'serv1')
         .map_services('loc1', 'serv1')
@@ -112,7 +112,7 @@
   end
 
   context 'location with no digests configured' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir)
         .with_service(name: 'serv1')
         .map_services('loc1', 'serv1')
@@ -158,7 +158,7 @@
   end
 
   context 'location with multiple digests configured' do
-    let(:config_path) { ConfigBuilder.new
+    let!(:config_path) { ConfigBuilder.new
         .with_location(name: 'loc1', path: path_dir, md_path: md_dir, md_digests: ['sha1', 'sha512'])
         .with_service(name: 'serv1')
         .map_services('loc1', 'serv1')

spec/longleaf/preservation_services/rsync_replication_service_spec.rb

@@ -11,7 +11,7 @@
 
   RsyncService ||= Longleaf::RsyncReplicationService
   ConfigBuilder ||= Longleaf::ConfigBuilder
-  REPLICATE_EVENT ||= Longleaf::EventNames::REPLICATE
+  PRESERVE_EVENT ||= Longleaf::EventNames::PRESERVE
 
   let(:md_src_dir) { Dir.mktmpdir('metadata') }
   let(:path_src_dir) { Dir.mktmpdir('path') }
@@ -106,7 +106,7 @@
     let(:service) { RsyncService.new(service_def, app_manager) }
 
     it "returns true for replicate event" do
-      expect(service.is_applicable?(Longleaf::EventNames::REPLICATE)).to be true
+      expect(service.is_applicable?(Longleaf::EventNames::PRESERVE)).to be true
     end
 
     it "returns false for non-verify event" do
@@ -131,7 +131,7 @@
         original_file = create_test_file(dir: path_src_dir)
         file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-        service.perform(file_rec, REPLICATE_EVENT)
+        service.perform(file_rec, PRESERVE_EVENT)
 
         replica_path = File.join(path_dest_dir, File.basename(original_file))
 
@@ -151,7 +151,7 @@
         original_file = create_test_file(dir: original_path)
         file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-        service.perform(file_rec, REPLICATE_EVENT)
+        service.perform(file_rec, PRESERVE_EVENT)
 
         replica_path = File.join(path_dest_dir, "nested/path/to/", File.basename(original_file))
 
@@ -170,7 +170,7 @@
         # Remove the destination so that is is "unavailable"
         FileUtils.rmdir(path_dest_dir)
 
-        expect { service.perform(file_rec, REPLICATE_EVENT) }.to raise_error(Longleaf::StorageLocationUnavailableError)
+        expect { service.perform(file_rec, PRESERVE_EVENT) }.to raise_error(Longleaf::StorageLocationUnavailableError)
       end
 
       context 'with additional rsync options' do
@@ -181,7 +181,7 @@
           original_file = create_test_file(dir: path_src_dir)
           file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-          service.perform(file_rec, REPLICATE_EVENT)
+          service.perform(file_rec, PRESERVE_EVENT)
 
           replica_path = File.join(path_dest_dir, File.basename(original_file))
 
@@ -198,7 +198,7 @@
           original_file = create_test_file(dir: path_src_dir)
           file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-          expect { service.perform(file_rec, REPLICATE_EVENT) }.to raise_error(Longleaf::PreservationServiceError)
+          expect { service.perform(file_rec, PRESERVE_EVENT) }.to raise_error(Longleaf::PreservationServiceError)
         end
       end
     end
@@ -213,7 +213,7 @@
         original_file = create_test_file(dir: path_src_dir)
         file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-        service.perform(file_rec, REPLICATE_EVENT)
+        service.perform(file_rec, PRESERVE_EVENT)
 
         replica_path = File.join(dest_dir, File.basename(original_file))
 
@@ -227,7 +227,7 @@
         original_file = create_test_file(dir: original_path)
         file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-        service.perform(file_rec, REPLICATE_EVENT)
+        service.perform(file_rec, PRESERVE_EVENT)
 
         replica_path = File.join(dest_dir, "nested/path/to/", File.basename(original_file))
 
@@ -241,7 +241,7 @@
 
         FileUtils.rmdir(dest_dir)
 
-        expect { service.perform(file_rec, REPLICATE_EVENT) }.to raise_error(Longleaf::StorageLocationUnavailableError)
+        expect { service.perform(file_rec, PRESERVE_EVENT) }.to raise_error(Longleaf::StorageLocationUnavailableError)
       end
     end
 
@@ -255,7 +255,7 @@
         original_file = create_test_file(dir: path_src_dir)
         file_rec = make_file_record(original_file, md_rec, "source_loc", app_manager)
 
-        service.perform(file_rec, REPLICATE_EVENT)
+        service.perform(file_rec, PRESERVE_EVENT)
 
         replica_path = File.join(path_dest_dir, File.basename(original_file))