#!/usr/bin/perl
# 
# This program shuts downs a server (or servers). It can be called as either a job from the webui or directly
# from another program or a terminal.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 
# TODO: 
# - We need to support shutdown ordering (inverese of boot ordering)
# 

use strict;
use warnings;
use Anvil::Tools;
use Sys::Virt;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->Get->switches({list => [
	"confirm",
	"immediate",
	"no-db", 
	"no-wait", 
	"reset", 
	"server", 
	"wait"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to DBs.
if ($anvil->data->{switches}{'no-db'})
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "log_0743"});
	
	# If there was a job-uuid, clear it.
	$anvil->data->{sys}{database}{connections} = 0;
	$anvil->data->{switches}{'job-uuid'}       = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'sys::database::connections' => $anvil->data->{sys}{database}{connections},
		'switches::job-uuid'         => $anvil->data->{switches}{'job-uuid'},
	}});
}
else
{
	$anvil->Database->connect();
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
	if (not $anvil->data->{sys}{database}{connections})
	{
		# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
		# again after we exit.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
		sleep 10;
		$anvil->nice_exit({exit_code => 1});
	}
}

$anvil->data->{sys}{host_type} = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	'sys::host_type' => $anvil->data->{sys}{host_type},
}});

if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->get_job_details({
		debug    => 2,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	$anvil->Job->clear();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "job_0283", 
	});
	
	# Pull out the job data.
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		if ($line =~ /server=(.*?)$/)
		{
			$anvil->data->{switches}{server} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'switches::server' => $anvil->data->{switches}{server},
			}});
		}
		if ($line =~ /server-uuid=(.*?)$/)
		{
			# This is not used anymore. If it is for some reason, make it 'server'
			my $server_uuid = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
			if ($server_uuid)
			{
				$anvil->data->{switches}{server} = $1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::server' => $anvil->data->{switches}{server},
				}});
			}
		}
		if ($line =~ /task=(.*?)$/)
		{
			my $task = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { task => $task }});
			
			if ($task eq "reset")
			{
				$anvil->data->{switches}{'reset'} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::reset' => $anvil->data->{switches}{'reset'},
				}});
			}
		}
	}
	
	# If the job is too old, abort.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_age" => $anvil->data->{jobs}{job_age}, 
	}});
	if (($anvil->data->{jobs}{job_age}) && ($anvil->data->{jobs}{job_age} > 120))
	{
		# Too old.
		$anvil->Job->update_progress({
			progress   => 100, 
			job_status => "aborted", 
			message    => "job_0479",
			'print'    => 1, 
			log_level  => 1, 
			variables  => {
				age    => $anvil->data->{jobs}{job_age}, 
				server => $anvil->Get->server_from_switch({server => $anvil->data->{switches}{server}}),
			},
		});
		$anvil->nice_exit({exit_code => 0});
	}
}

# Get the server name and UUID if not 'all'
if (not $anvil->data->{switches}{server})
{
	# Um...  Unable to proceed.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0263"});
	$anvil->Job->update_progress({progress => 100, message => "error_0263"});
	$anvil->nice_exit({exit_code => 1});
}
elsif ($anvil->data->{switches}{server} eq "all")
{
	if ($anvil->data->{switches}{immediate})
	{
		# Not allowed to force-kill all servers.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0462"});
		$anvil->Job->update_progress({progress => 100, message => "error_0462"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Stopping all servers.
	$anvil->data->{switches}{server_name} = "";
	$anvil->data->{switches}{server_uuid} = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"switches::server_name" => $anvil->data->{switches}{server_name},
		"switches::server_uuid" => $anvil->data->{switches}{server_uuid},
	}});
}
else
{
	# Get the server name and UUID from the switch.
	$anvil->Get->server_from_switch({server => $anvil->data->{switches}{server}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"switches::server_name" => $anvil->data->{switches}{server_name},
		"switches::server_uuid" => $anvil->data->{switches}{server_uuid},
	}});
	
	if ((not $anvil->data->{switches}{server_name}) or (not $anvil->data->{switches}{server_uuid}))
	{
		$anvil->Job->update_progress({
			progress  => 100, 
			'print'   => 1, 
			log_level => 1,
			message   => "error_0080",
			variables => {
				server => $anvil->data->{switches}{server}, 
			},
		});
		$anvil->nice_exit({exit_code => 1});
	}
}

# If the server isn't all, check to see if the server is running elsewhere. If it is, reassign the job to that host.
if (($anvil->data->{switches}{server} !~ /all/i) && ($anvil->data->{sys}{database}{connections}))
{
	# What's the server's host?
	$anvil->Database->get_servers({debug => 2});
	my $server_name      = $anvil->data->{switches}{server_name};
	my $server_uuid      = $anvil->data->{switches}{server_uuid};
	my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
	my $server_host_name = $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid});
	my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:server_name'      => $server_name,
		's2:server_uuid'      => $server_uuid, 
		's3:server_host_uuid' => $server_host_uuid, 
		's4:server_host_name' => $server_host_name, 
		's5:server_state'     => $server_state, 
	}});
	
	if ((not $server_host_uuid) or ($server_host_uuid eq "NULL"))
	{
		$anvil->Job->update_progress({
			progress  => 100, 
			'print'   => 1, 
			log_level => 1,
			message   => "warning_0017",
			variables => {
				server => $anvil->data->{switches}{server_name}, 
			},
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	if (($server_state eq "running") && ($server_host_uuid ne $anvil->Get->host_uuid))
	{
		# Re-assign the job.
		$anvil->Job->update_progress({
			progress   => 99,
			message    => "job_0357", 
			job_status => "reassigned", 
			log_level  => 1, 
			'print'    => 1, 
			variables  => {
				server    => $server_name, 
				host_name => $server_host_name, 
			},
		});
		
		my ($job_uuid) = reassign_job($anvil, $server_uuid, $server_host_uuid);
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "job_0137", 
			log_level  => 1, 
			'print'    => 1, 
			variables  => {
				host_name => $server_host_name,
				job_uuid  => $job_uuid, 
			},
		});
		
		$anvil->nice_exit({exit_code => 0});
	}
}

# Are we a node or DR host?
if (($anvil->data->{sys}{host_type} ne "node") && ($anvil->data->{sys}{host_type} ne "dr"))
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0264"});
	$anvil->Job->update_progress({progress => 100, message => "error_0264"});
	$anvil->nice_exit({exit_code => 1});
}

# If we're an Anvil!, read out Anvil! uuid
$anvil->data->{sys}{anvil_uuid} = "";
if (($anvil->data->{sys}{host_type} eq "node") && ($anvil->data->{sys}{database}{connections}))
{
	$anvil->data->{sys}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'sys::anvil_uuid' => $anvil->data->{sys}{anvil_uuid},
	}});
}

# Is the server valid and running here?
my $short_host_name = $anvil->Get->short_host_name;
$anvil->Server->connect_to_libvirt({debug => 2});
if ($anvil->data->{switches}{server} !~ /all/i)
{
	# Make sure the server name is real.
	my $found   = 0;
	my @domains = $anvil->data->{libvirtd}{$short_host_name}{connection}->list_all_domains();
	foreach my $domain_handle (@domains)
	{
		my $server_name = $domain_handle->get_name;
		my $server_uuid = $domain_handle->get_uuid_string();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			domain_handle => $domain_handle, 
			server_name   => $server_name, 
			server_uuid   => $server_uuid, 
		}});
		if (($server_name eq $anvil->data->{switches}{server_name}) or
		    ($server_uuid eq $anvil->data->{switches}{server_uuid}) or 
		    (($anvil->data->{switches}{server_uuid}) && ($server_uuid eq $anvil->data->{switches}{server_uuid})))
		{
			$found = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
			last;
		}
	}
	
	if (not $found)
	{
		# Do we know where the server is running, if anywhere?
		$anvil->Database->get_servers({debug => 2});
		my $server_uuid      = $anvil->data->{switches}{server_uuid};
		my $server_name      = $anvil->data->{switches}{server_name};
		my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
		my $server_state     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_uuid'      => $server_uuid,
			's2:server_name'      => $server_name, 
			's3:server_host_uuid' => $server_host_uuid, 
			's4:server_state'     => $server_state, 
		}});
		
		# Is the server deleted?
		if ($server_state eq "DELETED")
		{
			$anvil->Job->update_progress({
				progress  => 100, 
				'print'   => 1, 
				log_level => 1,
				message   => "warning_0016",
				variables => {
					server => $anvil->data->{switches}{server_name}, 
				},
			});
			$anvil->nice_exit({exit_code => 1});
		}
		if ($server_state eq "shut off")
		{
			$anvil->Job->update_progress({
				progress  => 100, 
				'print'   => 1, 
				log_level => 1,
				message   => "warning_0018",
				variables => {
					server => $anvil->data->{switches}{server_name}, 
				},
			});
			$anvil->nice_exit({exit_code => 1});
		}
		if ((not $server_host_uuid) or ($server_host_uuid eq "NULL"))
		{
			$anvil->Job->update_progress({
				progress  => 100, 
				'print'   => 1, 
				log_level => 1,
				message   => "warning_0017",
				variables => {
					server => $anvil->data->{switches}{server_name}, 
				},
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		if (($server_host_uuid) && ($server_state))
		{
			# Re-assign the job.
			my $server_host_name = $anvil->Get->host_name_from_uuid({host_uuid => $server_host_uuid});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_host_name => $server_host_name }});
			$anvil->Job->update_progress({
				progress   => 99,
				message    => "job_0357", 
				job_status => "reassigned", 
				log_level  => 1, 
				'print'    => 1, 
				variables  => {
					server    => $server_name, 
					host_name => $server_host_name, 
				},
			});
			
			my ($job_uuid) = reassign_job($anvil, $server_uuid, $server_host_uuid);
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0137", 
				log_level  => 1, 
				'print'    => 1, 
				variables  => {
					host_name => $server_host_name,
					job_uuid  => $job_uuid, 
				},
			});
			
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			# It doesn't seem to be running anywhere.
			$anvil->Job->update_progress({
				progress  => 100, 
				'print'   => 1, 
				log_level => 1,
				message   => "warning_0009",
				variables => {
					server => $anvil->data->{switches}{server_name}, 
				},
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}
}

# If we don't have a job_uuid and the user didn't pass 'confirm', ask to proceed.
if ((not $anvil->data->{switches}{'job-uuid'}) && (not $anvil->data->{switches}{'confirm'}))
{
	if ($anvil->data->{switches}{server} =~ /all/i)
	{
		print $anvil->Words->string({key => "message_0092"})." ";
	}
	elsif ($anvil->data->{switches}{'reset'})
	{
		print $anvil->Words->string({key => "message_0151", variables => { server_name => $anvil->data->{switches}{server_name} }})." ";
	}
	else
	{
		print $anvil->Words->string({key => "message_0093", variables => { server_name => $anvil->data->{switches}{server_name} }})." ";
	}
	my $answer = <STDIN>;
	chomp($answer);
	if (($answer ne "y") && ($answer ne "Y"))
	{
		# Abort and exit.
		print $anvil->Words->string({key => "message_0061"})."\n";
		$anvil->nice_exit({exit_code => 0});
	}
}

# This is copied from anvil-boot-server, but it works here as well. We can't use 'pcs' without pacemaker 
# being up.
if ($anvil->data->{sys}{host_type} eq "node")
{
	wait_for_pacemaker($anvil);
}

# If 'server' is 'all', shut down all servers.
if (lc($anvil->data->{switches}{server}) eq "all")
{
	shutdown_all_servers($anvil);
}
else
{
	my $wait = $anvil->data->{switches}{'no-wait'} ? 0 : 1;
	shutdown_server($anvil, $anvil->data->{switches}{server_name}, $wait, 50);
}

$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0281"});
$anvil->Job->update_progress({progress => 100, message => "job_0281"}) if $anvil->data->{switches}{'job-uuid'};

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub wait_for_pacemaker
{
	my ($anvil) = @_;

	# Shutdown the server using pcs, but of course, wait for the node to be up.
	my $waiting = 1;
	while($waiting)
	{
		my $problem = $anvil->Cluster->parse_cib({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if (not $problem)
		{
			my $node_name = $anvil->data->{cib}{parsed}{'local'}{name};
			my $ready     = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{ready};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ready => $ready }});
			if ($ready)
			{
				# We're good. 
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0279"});
				$anvil->Job->update_progress({progress => 15, message => "job_0279"}) if $anvil->data->{switches}{'job-uuid'};
			}
			else
			{
				# Node isn't ready yet.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0278"});
				$anvil->Job->update_progress({progress => 10, message => "job_0278"}) if $anvil->data->{switches}{'job-uuid'};
			}
		}
		else
		{
			# Cluster hasn't started.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0277"});
			$anvil->Job->update_progress({progress => 5, message => "job_0277"}) if $anvil->data->{switches}{'job-uuid'};
		}
		if ($waiting)
		{
			sleep 10;
		}
	}
	
	return(0);
}

sub shutdown_server
{
	my ($anvil, $server, $wait, $progress) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server   => $server,
		'wait'   => $wait, 
		progress => $progress, 
	}});
	
	if (($wait) && (($anvil->data->{switches}{immediate}) or ($anvil->data->{switches}{'reset'})))
	{
		$wait = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'wait' => $wait }});
	}
	
	if ($anvil->data->{sys}{host_type} eq "node")
	{
		# Is the server in the cluster?
		if (not exists $anvil->data->{cib}{parsed}{data}{server}{$server})
		{
			# Nope.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0548", variables => { server => $server }});
			$anvil->Job->update_progress({progress => 100, message => "log_0548,!!server!".$server."!!"}) if $anvil->data->{switches}{'job-uuid'};
			$anvil->nice_exit({exit_code => 1});
		}
		
		my $status = $anvil->data->{cib}{parsed}{data}{server}{$server}{status};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { status => $status }});
		if ($status eq "off")
		{
			# It's off already
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0284", variables => { server => $server }});
			$anvil->Job->update_progress({progress => $progress, message => "job_0284,!!server!".$server."!!"}) if $anvil->data->{switches}{'job-uuid'};
			return(0);
		}
	}
	
	# Now shut down. Forcibly?
	if ($anvil->data->{switches}{immediate})
	{
		# Kill it!
		$anvil->Job->update_progress({log_level => 1, 'print' => 1, file => $THIS_FILE, line => __LINE__, progress => $progress, message => "job_0474", variables => { server => $server }});
	}
	elsif ($anvil->data->{switches}{'reset'})
	{
		# Reset it (don't destroy it)
		$anvil->Job->update_progress({log_level => 1, 'print' => 1, file => $THIS_FILE, line => __LINE__, progress => $progress, message => "job_0463", variables => { server => $server }});
	}
	else
	{
		# Lets be gentle
		$anvil->Job->update_progress({log_level => 1, 'print' => 1, file => $THIS_FILE, line => __LINE__, progress => $progress, message => "job_0289", variables => { server => $server }});
	}
	
	my $problem = 0;
	if ($anvil->data->{sys}{host_type} eq "dr")
	{
		# Shut down using virsh. Invert the return.
		if ($anvil->data->{switches}{'reset'})
		{
			my $success = $anvil->Server->shutdown_virsh({
				debug   => 2,
				server  => $server, 
				'reset' => 1,
			});
			$problem = $success ? 0 : 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				success => $success,
				problem => $problem, 
			}});
		}
		else
		{
			my $force   = $anvil->data->{switches}{immediate} ? 1 : 0;
			my $do_wait = $wait ? 0 : 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				force   => $force, 
				do_wait => $do_wait, 
			}});
			my $success = $anvil->Server->shutdown_virsh({
				debug     => 2,
				server    => $server, 
				force     => $force, 
				wait_time => $do_wait,
			});
			$problem = $success ? 0 : 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				success => $success,
				problem => $problem, 
			}});
		}
	}
	else
	{
		if ($anvil->data->{switches}{'reset'})
		{
			# We're not powering it off, so don't ask the cluster to do anything.
			my $success = $anvil->Server->shutdown_virsh({
				debug   => 2,
				server  => $server, 
				'reset' => 1,
			});
			$problem = $success ? 0 : 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				success => $success,
				problem => $problem, 
			}});
		}
		else
		{
			$problem = $anvil->Cluster->shutdown_server({
				debug  => 2, 
				server => $server, 
				'wait' => $wait,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			
			# If we're being asked to force the server off, do it now regardless of what pacemaker returned.
			if ($anvil->data->{switches}{immediate})
			{
				my $success = $anvil->Server->shutdown_virsh({
					debug     => 2,
					server    => $server, 
					force     => 1, 
					wait_time => 0,
				});
				$problem = $success ? 0 : 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					success => $success,
					problem => $problem, 
				}});
			}
		}
	}
	
	if ($problem)
	{
		# Failed, abort.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0268", variables => { server => $server }});
		$anvil->Job->update_progress({progress => 100, message => "error_0268,!!server!".$server."!!"}) if $anvil->data->{switches}{'job-uuid'};
		$anvil->nice_exit({exit_code => 1});
	}
	else
	{
		if ($wait)
		{
			# Stopped!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0285", variables => { server => $server }});
			$anvil->Job->update_progress({progress => $progress, message => "job_0285,!!server!".$server."!!"}) if $anvil->data->{switches}{'job-uuid'};
		}
		else
		{
			# Stop requested.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0286", variables => { server => $server }});
			$anvil->Job->update_progress({progress => $progress, message => "job_0286,!!server!".$server."!!"}) if $anvil->data->{switches}{'job-uuid'};
		}
	}
	
	return(0);
}

sub shutdown_all_servers
{
	my ($anvil) = @_;
	
	# Get the server data so that we know the stop order.
	$anvil->Database->get_servers({debug => 2});
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({debug => 2});
	my $target     = $anvil->Get->short_host_name();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid => $anvil_uuid,
		target     => $target, 
	}});
	
	if ($anvil->data->{sys}{host_type} eq "dr")
	{
		# Get a list of servers and their states from libvirt.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"libvirtd::${target}::connection" => $anvil->data->{libvirtd}{$target}{connection}, 
		}});
		my @domains = $anvil->data->{libvirtd}{$target}{connection}->list_all_domains();
		foreach my $domain (@domains)
		{
			my $server_name             = $domain->get_name;
			my $server_uuid             = $domain->get_uuid_string;
			my ($state, $reason)        = $domain->get_state();
			my $is_running              = (($state) && ($state != 5)) ? 1 : 0;
			my $start_after_server_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid};
			   $start_after_server_uuid = "" if $start_after_server_uuid eq "NULL";
			my $start_after_server_name = $start_after_server_uuid ? $anvil->data->{servers}{server_uuid}{$start_after_server_uuid}{server_name} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s1:server_name"             => $server_name,
				"s2:server_uuid"             => $server_uuid, 
				"s3:state"                   => $state, 
				"s4:reason"                  => $reason,
				"s5:is_running"              => $is_running, 
				"s6:start_after_server_uuid" => $start_after_server_uuid, 
				"s7:start_after_server_name" => $start_after_server_name, 
			}});
			
			### Reasons are dependent on the state. 
			### See: https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainShutdownReason
			my $server_state = "unknown";
			if ($state == 1)    { $server_state = "running"; }	# Server is running.
			elsif ($state == 2) { $server_state = "blocked"; }	# Server is blocked (IO contention?).
			elsif ($state == 3) { $server_state = "paused"; }	# Server is paused (migration target?).
			elsif ($state == 4) { $server_state = "in shutdown"; }	# Server is shutting down.
			elsif ($state == 5) { $server_state = "shut off"; }	# Server is shut off.
			elsif ($state == 6) { $server_state = "crashed"; }	# Server is crashed!
			elsif ($state == 7) { $server_state = "pmsuspended"; }	# Server is suspended.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state  => $server_state }});
			
			# If we boot after another server, make sure that the other server know to stop before us.
			if ($start_after_server_name)
			{
				$anvil->data->{server_stop}{$start_after_server_name}{wait_for}{$server_name} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"server_stop::${start_after_server_name}::wait_for::${server_name}" => $anvil->data->{server_stop}{$start_after_server_name}{wait_for}{$server_name},
				}});
			}
			
			# This makes it quicker to track the state of other servers.
			$anvil->data->{server_stop}{$server_name}{is_off}       = $is_running ? 0 : 1;
			$anvil->data->{server_stop}{$server_name}{stop_called}  = 0;
			$anvil->data->{server_stop}{$server_name}{server_uuid}  = $server_uuid;
			$anvil->data->{server_stop}{$server_name}{server_state} = $server_state;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"server_stop::${server_name}::is_off"       => $anvil->data->{server_stop}{$server_name}{is_off},
				"server_stop::${server_name}::stop_called"  => $anvil->data->{server_stop}{$server_name}{stop_called},
				"server_stop::${server_name}::server_uuid"  => $anvil->data->{server_stop}{$server_name}{server_uuid},
				"server_stop::${server_name}::server_state" => $anvil->data->{server_stop}{$server_name}{server_state},
			}});
		}
	}
	else
	{
		# We need to know which servers shut down before others, and this is the reverse of the boot
		# order. If a server boots after another, the other must stop before we shut the first down.
		# The delay doesn't matter, as the boot_after server has to be off, and once off, waiting 
		# makes no sense.
		$anvil->Database->get_servers({debug => 3});
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{server_name}})
		{
			my $server_uuid  = $anvil->data->{servers}{server_name}{$server_name}{server_uuid};
			my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:server_name'  => $server_name,
				's2:server_uuid'  => $server_uuid,
				's3:server_state' => $server_state,
			}});
			next if $server_state eq "DELETED";
			
			# If this server is set to boot after another, record that the other can't stop until
			# we're down.
			next if not $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid};
			next if $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid} eq "NULL";
			my $start_after_server_uuid       = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid};
			my $start_after_server_name       = $anvil->data->{servers}{server_uuid}{$start_after_server_uuid}{server_name};
			my $start_after_server_state      = $anvil->data->{servers}{server_uuid}{$start_after_server_uuid}{server_state};
			my $start_after_server_anvil_uuid = $anvil->data->{servers}{server_uuid}{$start_after_server_uuid}{server_anvil_uuid};
			my $start_after_server_anvil_name = $anvil->data->{anvils}{anvil_uuid}{$start_after_server_anvil_uuid}{anvil_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:start_after_server_uuid'       => $start_after_server_uuid,
				's2:start_after_server_name'       => $start_after_server_name,
				's3:start_after_server_state'      => $start_after_server_state, 
				's4:start_after_server_anvil_uuid' => $start_after_server_anvil_uuid, 
				's5:start_after_server_anvil_name' => $start_after_server_anvil_name, 
			}});
			
			$anvil->data->{server_stop}{$start_after_server_name}{wait_for}{$server_name} = $server_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"server_stop::${start_after_server_name}::wait_for::${server_name}" => $anvil->data->{server_stop}{$start_after_server_name}{wait_for}{$server_name},
			}});
			
			$anvil->data->{server_stop}{$server_name}{is_off}       = $server_state !~ /off/ ? 0 : 1;
			$anvil->data->{server_stop}{$server_name}{stop_called}  = 0;
			$anvil->data->{server_stop}{$server_name}{server_uuid}  = $server_uuid;
			$anvil->data->{server_stop}{$server_name}{server_state} = $server_state;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s1:server_stop::${server_name}::is_off"       => $anvil->data->{server_stop}{$server_name}{is_off},
				"s2:server_stop::${server_name}::stop_called"  => $anvil->data->{server_stop}{$server_name}{stop_called},
				"s3:server_stop::${server_name}::server_uuid"  => $anvil->data->{server_stop}{$server_name}{server_uuid},
				"s4:server_stop::${server_name}::server_state" => $anvil->data->{server_stop}{$server_name}{server_state},
			}});
		}
	}
	
	# Loop until all servers are off (or asked to power off).
	my $all_processed = 0;
	until($all_processed)
	{
		# We top out at 90, bottom is 20. 
		$all_processed = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_processed => $all_processed }});
		
		# Update our view of the servers.
		$anvil->Database->get_servers({debug => 3});
		
		# If we're a node, make sure pacemaker is ready
		if ($anvil->data->{sys}{host_type} eq "node")
		{
			my $problem = $anvil->Cluster->parse_cib({debug => 2});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				$all_processed = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_processed => $all_processed }});
				
				# Tell the user we can't shut down yet.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0004"});
				sleep 5; 
				next;
			}
		}
		my $server_count = keys %{$anvil->data->{server_stop}};
		my $increment    = $server_count ? int(70 / $server_count) : 70;
		my $percent      = 15;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_count => $server_count, 
			increment    => $increment,
		}});
		
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{server_stop}})
		{
			my $server_uuid       = $anvil->data->{servers}{server_name}{$server_name}{server_uuid};
			my $server_anvil_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
			my $server_anvil_name = $anvil->data->{anvils}{anvil_uuid}{$server_anvil_uuid}{anvil_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:server_name'       => $server_name,
				's2:server_uuid'       => $server_uuid, 
				's3:server_anvil_uuid' => $server_anvil_uuid, 
				's4:server_anvil_name' => $server_anvil_name, 
			}});
			next if $anvil->data->{server_stop}{$server_name}{is_off};
			if ($anvil->data->{sys}{host_type} eq "node")
			{
				# Not for us to deal with
				next if $server_anvil_uuid ne $anvil_uuid;
			}
			
			# If this server needs to wait, check that the others are off.
			my $stop = 1;
			if ($anvil->data->{server_stop}{$server_name}{wait_for})
			{
				foreach my $wait_for_server_name (sort {$a cmp $b} keys %{$anvil->data->{server_stop}{$server_name}{wait_for}})
				{
					my $wait_for_server_uuid       = $anvil->data->{server_stop}{$server_name}{wait_for}{$wait_for_server_name};
					my $wait_for_server_state      = $anvil->data->{servers}{server_uuid}{$wait_for_server_uuid}{server_state};
					my $wait_for_server_anvil_uuid = $anvil->data->{servers}{server_uuid}{$wait_for_server_uuid}{server_anvil_uuid};
					my $wait_for_server_anvil_name = $anvil->data->{anvils}{anvil_uuid}{$wait_for_server_anvil_uuid}{anvil_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"s1:wait_for_server_name"       => $wait_for_server_name,
						's2:wait_for_server_uuid'       => $wait_for_server_uuid, 
						's3:wait_for_server_state'      => $wait_for_server_state, 
						's4:wait_for_server_anvil_uuid' => $wait_for_server_anvil_uuid, 
						's5:wait_for_server_anvil_name' => $wait_for_server_anvil_name, 
					}});
					if ($wait_for_server_state ne "shut off")
					{
						# We need to keep waiting.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0005", variables => {
							server_name            => $server_name, 
							wait_for_server_name   => $wait_for_server_name, 
							node                   => $wait_for_server_anvil_name, 
							wait_for_server_status => $wait_for_server_state, 
						}});
						
						$all_processed = 0;
						$stop          = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							all_processed => $all_processed,
							stop          => $stop,
						}});
					}
				}
			}
			next if not $stop;
			
			# Still here? Ask the server to stop
			my $state = $anvil->data->{server_stop}{$server_name}{server_state};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'state' => $state }});
			
			if ($state =~ /off/i)
			{
				# It's off, have we recorded that?
				if (not $anvil->data->{server_stop}{$server_name}{is_off})
				{
					# Nope, do so now
					$anvil->data->{server_stop}{$server_name}{is_off} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server_stop::${server_name}::is_off" => $anvil->data->{server_stop}{$server_name}{is_off},
					}});
				}
			}
			elsif (not $anvil->data->{server_stop}{$server_name}{stop_called})
			{
				# Shut it down (don't wait).
				my $wait    =  $anvil->data->{switches}{'wait'} ? 1 : 0;
				   $percent += $increment;
				shutdown_server($anvil, $server_name, $wait, $percent);
				
				$anvil->data->{server_stop}{$server_name}{stop_called} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"server_stop::${server_name}::stop_called" => $anvil->data->{server_stop}{$server_name}{stop_called},
				}});
			}
		}
		if (not $all_processed)
		{
			# Sleep a bit before checking again.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "warning_0006"});
			sleep 5;
		}
	}
	
	return(0);
}

sub reassign_job
{
	my ($anvil, $server_uuid, $host_uuid) = @_;

	my $say_immediate =  $anvil->data->{switches}{immediate} ? "true" : "";
	my $say_no_db     =  $anvil->data->{switches}{'no-db'}   ? "true" : "";
	my $say_no_wait   =  $anvil->data->{switches}{'no-wait'} ? "true" : "";
	my $say_wait      =  $anvil->data->{switches}{'wait'}    ? "true" : "";
	my $job_data      =  "confirm=true\n";
	   $job_data      .= "immediate=".$say_immediate."\n";
	   $job_data      .= "no-db=".$say_no_db."\n"; 
	   $job_data      .= "no-wait=".$say_no_wait."\n"; 
	   $job_data      .= "server=".$anvil->data->{switches}{server_name}."\n";
	   $job_data      .= "server-uuid=".$anvil->data->{switches}{server_uuid}."\n";
	   $job_data      .= "wait=".$say_wait;
	if ($anvil->data->{switches}{'reset'})
	{
		$job_data .= "\ntask=reset";
	}
	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
		debug           => 2,
		job_command     => $anvil->data->{path}{exe}{$THIS_FILE}.$anvil->Log->switches,
		job_data        => $job_data, 
		job_host_uuid   => $host_uuid, 
		job_progress    => 0, 
		job_name        => "set_power::server::off", 
		job_title       => "job_0515", 
		job_description => "job_0516", 
	});

	return($job_uuid);
}
