diff --git a/lib/Ravada.pm b/lib/Ravada.pm index 044195766..886fbe84a 100644 --- a/lib/Ravada.pm +++ b/lib/Ravada.pm @@ -1473,6 +1473,7 @@ sub _upgrade_tables { $self->_upgrade_table('requests','start_time','int(11) DEFAULT NULL'); $self->_upgrade_table('requests','output','text DEFAULT NULL'); $self->_upgrade_table('requests','after_request','int(11) DEFAULT NULL'); + $self->_upgrade_table('requests','after_request_ok','int(11) DEFAULT NULL'); $self->_upgrade_table('requests','at_time','int(11) DEFAULT NULL'); $self->_upgrade_table('requests','run_time','float DEFAULT NULL'); @@ -3524,6 +3525,56 @@ sub _cmd_refresh_storage($self, $request=undef) { $vm->refresh_storage(); } +sub _list_mnt($vm, $type) { + my ($out, $err) = $vm->run_command("findmnt","-$type"); + my %tab; + for my $line ( split /\n/,$out ) { + my ($target) = $line =~ /^(.*?) /; + next if $target eq 'TARGET'; + $tab{$target} = $line; + } + return %tab; +} + +sub _search_partition($path, $tab) { + confess if ref($path); + my $curr_path = ""; + my $found=''; + for my $dir (split /\//,$path ) { + $dir = "" if !defined $dir; + $curr_path .= "/$dir"; + $curr_path =~ s{\/\/+}{/}g; + next if !exists $tab->{$curr_path}; + my $curr_found = $tab->{$curr_path}; + $found = $curr_path if $curr_found && length($curr_found) > length($found); + } + return $found; +} + +sub _check_mounted($path, $fstab, $mtab) { + my $partition = _search_partition($path, $fstab); + return 1 if exists $mtab->{$partition} && $mtab->{$partition}; + + die "Error: partition $partition not mounted. Retry.\n"; +} + +sub _cmd_check_storage($self, $request) { + my $contents = "a" x 160; + for my $vm ( $self->list_vms ) { + next if !$vm->is_local; + my %fstab = _list_mnt($vm,"s"); + my %mtab = _list_mnt($vm,"m"); + + for my $storage ( $vm->list_storage_pools ) { + next if $storage !~ /tst/; + my $path = ''.$vm->_storage_path($storage); + _check_mounted($path,\%fstab,\%mtab); + my ($ok,$err) = $vm->write_file("$path/check_storage",$contents); + die "Error on starage pool $storage : $err. Retry.\n" if $err; + } + } +} + sub _cmd_refresh_machine($self, $request) { my $id_domain = $request->args('id_domain'); @@ -3939,6 +3990,7 @@ sub _req_method { ,rebase => \&_cmd_rebase ,refresh_storage => \&_cmd_refresh_storage +,check_storage => \&_cmd_check_storage ,refresh_machine => \&_cmd_refresh_machine ,domain_autostart=> \&_cmd_domain_autostart ,change_owner => \&_cmd_change_owner diff --git a/lib/Ravada/Request.pm b/lib/Ravada/Request.pm index 7614ae471..a33c69d96 100644 --- a/lib/Ravada/Request.pm +++ b/lib/Ravada/Request.pm @@ -78,6 +78,7 @@ our %VALID_ARG = ( ,hybernate=> {uid => 1, id_domain => 1} ,download => {uid => 2, id_iso => 1, id_vm => 2, verbose => 2, delay => 2, test => 2} ,refresh_storage => { id_vm => 2 } + ,check_storage => { uid => 1 } ,set_base_vm=> {uid => 1, id_vm=> 1, id_domain => 1, value => 2 } ,cleanup => { } ,clone => { uid => 1, id_domain => 1, name => 2, memory => 2, number => 2 @@ -425,7 +426,7 @@ sub _check_args { my $args = { @_ }; my $valid_args = $VALID_ARG{$sub}; - for (qw(at after_request retry _no_duplicate)) { + for (qw(at after_request after_request_ok retry _no_duplicate)) { $valid_args->{$_}=2 if !exists $valid_args->{$_}; } @@ -576,12 +577,12 @@ sub _new_request { confess "ERROR: Different id_domain: ".Dumper(\%args) if $args{id_domain} && $args{id_domain} ne $id_domain_args; $args{id_domain} = $id_domain_args; - $args{after_request} = delete $args{args}->{after_request} - if exists $args{args}->{after_request}; - $args{retry} = delete $args{args}->{retry} - if exists $args{args}->{retry}; - } + for (qw(after_request after_request_ok retry)) { + $args{$_} = delete $args{args}->{$_} + if exists $args{args}->{$_}; + } + $args{args} = encode_json($args{args}); } _init_connector() if !$CONNECTOR || !$$CONNECTOR; @@ -1284,11 +1285,25 @@ sub priority($self) { sub requirements_done($self) { my $after_request = $self->after_request(); - return 1 if !defined $after_request; - - my $req = Ravada::Request->open($self->after_request); - return 1 if $req->status eq 'done'; - return 0; + my $after_request_ok = $self->after_request_ok(); + return 1 if !defined $after_request && !defined $after_request_ok; + + my $ok = 0; + if ($after_request) { + $ok = 0; + my $req = Ravada::Request->open($self->after_request); + $ok = 1 if $req->status eq 'done'; + } + if ($after_request_ok) { + $ok = 0; + my $req = Ravada::Request->open($self->after_request_ok); + if ($req->status eq 'done' && $req->error ) { + $self->status('done'); + $self->error($req->error); + } + $ok = 1 if $req->status eq 'done' && $req->error eq ''; + } + return $ok; } sub AUTOLOAD { diff --git a/lib/Ravada/VM/Void.pm b/lib/Ravada/VM/Void.pm index f122e6cdd..3e3b7449b 100644 --- a/lib/Ravada/VM/Void.pm +++ b/lib/Ravada/VM/Void.pm @@ -190,6 +190,11 @@ sub dir_img { sub dir_base { return dir_img } sub dir_clone { return dir_img } +sub _storage_path($self, $storage) { + confess "Error: unknown storage '$storage'" if $storage ne 'default'; + return dir_img; +} + sub _list_domains_local($self, %args) { my $active = delete $args{active}; diff --git a/script/rvd_back b/script/rvd_back index 01f884719..0f9a5d5c8 100755 --- a/script/rvd_back +++ b/script/rvd_back @@ -208,6 +208,10 @@ sub clean_old_requests { sub autostart_machines { my $ravada = shift; + my $req = Ravada::Request->check_storage( + uid => Ravada::Utils::user_daemon->id + ,retry => 10 + ); for my $domain ( $ravada->list_domains_data ) { next unless $domain->{autostart} && ! $domain->{is_base} && $domain->{status} !~ /active/i; @@ -217,6 +221,7 @@ sub autostart_machines { Ravada::Request->start_domain( id_domain => $domain->{id} ,uid => $domain->{id_owner} + ,after_request_ok => $req->id ); } } diff --git a/t/lib/Test/Ravada.pm b/t/lib/Test/Ravada.pm index b418d4509..31f99a591 100644 --- a/t/lib/Test/Ravada.pm +++ b/t/lib/Test/Ravada.pm @@ -828,6 +828,7 @@ sub wait_request { if ( $req->status ne 'done' ) { diag("Waiting for request ".$req->id." ".$req->command." ".$req->status ." ".($req->error or '')) if $debug && (time%5 == 0); + sleep 1; $done_all = 0; } elsif (!$done{$req->id}) { $t0 = time; @@ -965,6 +966,10 @@ sub _delete_qemu_pool($pool) { my $xml = XML::LibXML->load_xml(string => $pool->get_xml_description()); my ($path) = $xml->findnodes('/pool/target/path'); my $dir = $path->textContent(); + for my $file ( qw(check_storage) ) { + my $path ="$dir/$file"; + unlink $path or die "$! $path" if -e $path; + } rmdir($dir) or die "$! $dir"; } @@ -1765,7 +1770,7 @@ sub shutdown_nodes { } } -sub create_storage_pool($vm) { +sub create_storage_pool($vm, $dir=undef) { if (!ref($vm)) { $vm = rvd_back->search_vm($vm); } @@ -1775,7 +1780,9 @@ sub create_storage_pool($vm) { my $capacity = 1 * 1024 * 1024; my $pool_name = new_pool_name(); - my $dir = "/var/tmp/$pool_name"; + if (!$dir) { + $dir = "/var/tmp/$pool_name"; + } mkdir $dir if ! -e $dir; diff --git a/t/vm/s30_storage.t b/t/vm/s30_storage.t new file mode 100644 index 000000000..014b2a188 --- /dev/null +++ b/t/vm/s30_storage.t @@ -0,0 +1,108 @@ +use warnings; +use strict; + +use Carp qw(confess); +use Data::Dumper; +use File::Copy; +use Test::More; + +no warnings "experimental::signatures"; +use feature qw(signatures); + +use lib 't/lib'; +use Test::Ravada; + + +########################################################### + +sub test_storage_pools($vm) { + my $req = Ravada::Request->check_storage( + uid => user_admin->id + ); + my $req_cleanup = Ravada::Request->cleanup( + after_request_ok => $req->id + ); + wait_request( debug => 1); + is($req->status,'done'); + is($req->error, ''); + + is($req_cleanup->status,'done'); + is($req_cleanup->error, ''); +} + +sub _add_fstab($vm) { + my $dir = "/mnt/".base_domain_name(); + if (!-e $dir) { + mkdir $dir; + } + my %fstab = Ravada::_list_mnt($vm,"s"); + return $dir if $fstab{$dir}; + + copy("/etc/fstab","/etc/fstab.tst_rvd_backup") or die $!; + + open my $fstab,">>","/etc/fstab" or die $!; + print $fstab "bogus.ravada:$dir $dir nfs rw,_netdev 0 0\n"; + close $fstab; + + return $dir; +} + +sub remove_fstab($vm, $dir) { + my $file = "$dir/check_storage";# or die "$!"; + unlink $file or die "$! $file" if -e $file; + copy("/etc/fstab.tst_rvd_backup","/etc/fstab") +} + +sub test_storage_pools_fail($vm) { + return if $vm->type ne 'KVM'; + my $dir = _add_fstab($vm); + + create_storage_pool($vm, $dir); + + my $req = Ravada::Request->check_storage( + uid => user_admin->id + ,retry => 2 + ); + my $req_cleanup = Ravada::Request->cleanup( + after_request_ok => $req->id + ); + is($req_cleanup->after_request_ok,$req->id); + wait_request( debug => 1, check_error => 0); + is($req->status,'done'); + like($req->error, qr/not mounted/); + + is($req_cleanup->status,'done'); + like($req_cleanup->error, qr/not mounted/); + + remove_fstab($vm, $dir); + $vm->refresh_storage_pools(); +} + +sub _clean_local { + my $dir = "/mnt/".base_domain_name(); + my $file = "$dir/check_storage";# or die "$!"; + unlink $file or die "$! $file" if -e $file; +} +########################################################### + +_clean_local(); +clean(); + +for my $vm_name (vm_names() ) { + ok($vm_name); + SKIP: { + my $vm = rvd_back->search_vm($vm_name); + my $msg = "SKIPPED: No virtual managers found"; + if ($vm && $vm_name =~ /kvm/i && $>) { + $msg = "SKIPPED: Test must run as root"; + $vm = undef; + } + + skip($msg,10) if !$vm; + test_storage_pools($vm); + test_storage_pools_fail($vm); + } +} + +end(); +done_testing();