#!/usr/bin/perl
# 
# This program will disable our daemons on all machines, then update each striker. It then walks through all 
# DR hosts and Anvil! nodes. With nodes, it migrates servers to the peer, takes the node out of the cluster, 
# updates it, reboots if the kernel was updated, and then rejoins the cluster, migrates the VMs and the does
# the same process on the peer sub-node.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 
# TODO: 
# 
# USAGE:
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Text::Diff;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->Get->switches({list => [
	"clear-cache", 
	"force", 
	"no-reboot", 
	"reboot", 
	"reboot-self", 
	"timeout", 
	"y", 
	"yes"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0305"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# Make sure we're running as 'root'
# $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0))
{
	# Not root
	print $anvil->Words->string({key => "error_0005"})."\n";
	$anvil->nice_exit({exit_code => 1});
}

# Make sure we're a striker.
if ($anvil->Get->host_type ne "striker")
{
	# This has to be run on a Striker dashboard.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0422", variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}});
	$anvil->nice_exit({exit_code => 1});
}

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Job->update_progress({
		progress         => get_progress($anvil),
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		'print'          => 1,
		message          => "message_0319", 
	});
}

# Update beginning. Verifying all known machines are accessible...
$anvil->Job->update_progress({
	'print'  => 1,
	progress => get_progress($anvil),
	message  => "job_0469", 
});
my $all_access = verify_access($anvil);
if ((not $all_access) && (not $anvil->data->{switches}{force}))
{
	# Not all systems are accessible. Update aborted!
	$anvil->Job->update_progress({
		'print'   => 1,
		progress  => 100,
		message   => "message_0423", 
		log_level => 1,
		variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}
	});
	$anvil->nice_exit({exit_code => 1});
}

# Success!
$anvil->Job->update_progress({
	'print'   => 1,
	progress  => get_progress($anvil),
	message   => "message_0424", 
	log_level => 1,
	variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}
});

if (($anvil->data->{switches}{y}) or ($anvil->data->{switches}{yes}))
{
	# Proceeding without confirmation, '-y' or '--yes' used.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0425", variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}});
}
else
{
	# Make sure the user is ready to proceed.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0426", variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}});
	print $anvil->Words->string({key => "message_0021"})."\n";
	my $answer = <STDIN>;
	chomp $answer;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});

	if ($answer =~ /^y/i)
	{
		print $anvil->Words->string({key => "message_0175"})."\n";
	}
	else
	{
		print $anvil->Words->string({key => "message_0022"})."\n";
		$anvil->nice_exit({exit_code => 0});
	}
}

manage_daemons($anvil, "stop");

# Update systems
update_strikers_and_dr($anvil);

# Update DR Host
update_nodes($anvil);

manage_daemons($anvil, "start");

# Updates complete!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0427", variables => {
	timestamp => $anvil->Get->date_and_time({time_only => 1}),
}});

my $host_uuid       = $anvil->Get->host_uuid;
my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
	's1:host_uuid'       => $host_uuid,
	's2:short_host_name' => $short_host_name, 
}});
if ($anvil->data->{sys}{reboot_needed})
{
	if ($anvil->data->{switches}{'reboot-self'})
	{
		# The local system needs to be rebooted, and '--reboot-self' was used. Rebooting in 60 seconds!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0428", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
		my $waiting = 60;
		while ($waiting)
		{
			print $waiting.", ";
			sleep 5;
			$waiting -= 5;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $waiting }});
		}
		print "\n";
		# Rebooting now!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0429", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
		
		my $shell_call = $anvil->data->{path}{exe}{systemctl}." reboot";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
		
		# Reboot requested, exiting.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0430", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
	}
	else
	{
		# This host needs to be rebooted to activate the new kernel. Tell the user to reboot asap
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0431", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
	}
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This returns a number between 1 and 99, incrementing by '1'
sub get_progress
{
	my ($anvil) = @_;
	
	if (not exists $anvil->data->{sys}{progress})
	{
		$anvil->data->{sys}{progress} = 0;
	}
	
	if ($anvil->data->{sys}{progress} < 99)
	{
		$anvil->data->{sys}{progress}++
	}
	return($anvil->data->{sys}{progress});
}

# For some unknown reason, 'sys::timeout' gets periodically blanked and 'switches::timeout' resets. This
# function ensures a sensible timeout is returned.
sub return_timeout
{
	my ($anvil) = @_;
	
	if ($anvil->data->{sys}{timeout})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::timeout" => $anvil->data->{sys}{timeout} }});
		return($anvil->data->{sys}{timeout})
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::timeout" => $anvil->data->{switches}{timeout} }});
	if ($anvil->data->{switches}{timeout})
	{
		$anvil->data->{sys}{timeout} = $anvil->Convert->to_seconds({
			debug  => 2,
			string => $anvil->data->{switches}{timeout},
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::timeout" => $anvil->data->{sys}{timeout} }});
		
		if ((not $anvil->data->{sys}{timeout}) or ($anvil->data->{sys}{timeout} =~ /\D/))
		{
			# The --timeout switch was used, but the value isn't valid.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0432", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				timeout   => $anvil->data->{switches}{timeout},
			}});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	else
	{
		$anvil->data->{sys}{timeout} = 86400;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::timeout" => $anvil->data->{sys}{timeout} }});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::timeout" => $anvil->data->{sys}{timeout} }});
	return($anvil->data->{sys}{timeout});
}

sub update_nodes
{
	my ($anvil) = @_;
	
	# Here, we loop through anvil systems, and find which sub nodes will be updated first, and which will
	# be updated second.
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid            = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		my $anvil_description     = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_description};
		my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $primary_host_uuid     = $anvil->Cluster->get_primary_host_uuid({anvil_uuid => $anvil_uuid});
		   $primary_host_uuid     = $anvil_node1_host_uuid if not $primary_host_uuid;
		my $secondary_host_uuid   = $primary_host_uuid eq $anvil_node1_host_uuid ? $anvil_node2_host_uuid : $anvil_node1_host_uuid;
		my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_node1_host_uuid}{short_host_name};
		my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_node2_host_uuid}{short_host_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:anvil_name'            => $anvil_name, 
			's2:anvil_uuid'            => $anvil_uuid,
			's3:anvil_description'     => $anvil_description, 
			's4:anvil_node1_host_uuid' => $anvil_node1_host_uuid, 
			's5:anvil_node2_host_uuid' => $anvil_node2_host_uuid, 
			's6:primary_host_uuid'     => $primary_host_uuid, 
			's7:secondary_host_uuid'   => $secondary_host_uuid, 
			's8:node1_short_host_name' => $node1_short_host_name, 
			's9:node2_short_host_name' => $node2_short_host_name, 
		}});
		
		# Before we proceed, are both nodes online? If so, great. If not, are both offline? If only 
		# one is online, abort. Check now in case things have changed since our first scan
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0433", variables => {
			timestamp  => $anvil->Get->date_and_time({time_only => 1}),
			anvil_name => $anvil_name,
		}});
		foreach my $host_uuid ($secondary_host_uuid, $primary_host_uuid)
		{
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_uuid'       => $host_uuid,
				's2:short_host_name' => $short_host_name, 
			}});
			# Verifying access to the subnode
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0434", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name,
			}});
			my $matches = $anvil->Network->find_access({
				debug  => 2,
				target => $host_uuid, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
			$anvil->data->{peer}{$short_host_name}{access}{ip}      = "";
			$anvil->data->{peer}{$short_host_name}{access}{network} = "";
			foreach my $preferred_network ("bcn", "mn", "ifn", "sn", "any")
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { preferred_network => $preferred_network }});
				foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_name => $network_name }});
					if (($network_name !~ /^$preferred_network/) && ($preferred_network ne "any"))
					{
						next;
					}
					my $target_ip   = $anvil->data->{network_access}{$network_name}{target_ip_address};
					my $test_access = $anvil->Remote->test_access({target => $target_ip});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:target_ip'   => $target_ip, 
						's2:test_access' => $test_access, 
					}});
					
					if ($test_access)
					{
						# We're good.
						$anvil->data->{peer}{$short_host_name}{access}{ip}      = $target_ip;
						$anvil->data->{peer}{$short_host_name}{access}{network} = $network_name;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							"s1:peer::${short_host_name}::access::ip"      => $anvil->data->{peer}{$short_host_name}{access}{ip}, 
							"s2:peer::${short_host_name}::access::network" => $anvil->data->{peer}{$short_host_name}{access}{network}, 
						}});
						# Access found 
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0435", variables => {
							timestamp    => $anvil->Get->date_and_time({time_only => 1}),
							network_name => $network_name,
							target_ip    => $target_ip, 
						}});
					}
					last if $anvil->data->{peer}{$short_host_name}{access}{ip};
				}
				last if $anvil->data->{peer}{$short_host_name}{access}{ip};
			}
			if (not $anvil->data->{peer}{$short_host_name}{access}{ip})
			{
				# Access not found!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0436", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
			}
		}
		
		if ((($anvil->data->{peer}{$node1_short_host_name}{access}{ip}) && (not $anvil->data->{peer}{$node2_short_host_name}{access}{ip})) or 
		    ((not $anvil->data->{peer}{$node1_short_host_name}{access}{ip}) && ($anvil->data->{peer}{$node2_short_host_name}{access}{ip})))
		{
			# Only one node online, skip this Anvil node.
			if ($anvil->data->{switches}{force})
			{
				# Skip this Anvil! system
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0437", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				next;
			}
			else
			{
				# Exiting update!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0438", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		# Update anvil-node on both machines before we do the main OS update. This is needed because,
		# depending on what has changed, migations could fail if the changes in version (ie: db calls to added/removed columns) breaks live migration.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0103", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
		foreach my $host_uuid ($secondary_host_uuid, $primary_host_uuid)
		{
			# Update anvil-node (which will update anvil-core also)
			my $short_host_name      = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $peer_host_uuid       = $host_uuid eq $primary_host_uuid ? $secondary_host_uuid : $primary_host_uuid;
			my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_uuid'            => $host_uuid, 
				's2:short_host_name'      => $short_host_name, 
				's3:peer_host_uuid'       => $peer_host_uuid, 
				's4:peer_short_host_name' => $peer_short_host_name,
			}});
			
			# Preparing to update the target host
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0104", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name, 
			}});
			
			my $shell_call = $anvil->data->{path}{exe}{dnf}." clean expire-cache && ".$anvil->data->{path}{exe}{dnf}." -y update anvil-node";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			my ($output, $error, $return_code) = $anvil->Remote->call({
				debug      => 2,
				shell_call => $shell_call, 
				target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error,
				return_code => $return_code,
			}});
		}
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0105", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
		
		# Update the secondary first, as it should have no VMs on it.
		foreach my $host_uuid ($secondary_host_uuid, $primary_host_uuid)
		{
			# Withdraw the node from the cluster.
			my $short_host_name      = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $peer_host_uuid       = $host_uuid eq $primary_host_uuid ? $secondary_host_uuid : $primary_host_uuid;
			my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_uuid'            => $host_uuid, 
				's2:short_host_name'      => $short_host_name, 
				's3:peer_host_uuid'       => $peer_host_uuid, 
				's4:peer_short_host_name' => $peer_short_host_name,
			}});
			
			# Preparing to update the target host
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0439", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name, 
			}});
			
			# Make sure VMs are off, DRBD is down and the node is out of the cluster. Call this 
			# with nohup so it doesn't get killed by the loss of the SSH connection.
			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --no-db".$anvil->Log->switches()." >/dev/null 2>&1 &";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $error, $return_code) = $anvil->Remote->call({
				debug      => 2,
				shell_call => $shell_call, 
				target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error,
				return_code => $return_code,
			}});
			
			# Now wait for DRBD resources to stop (which requires VMs be off).
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0440", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			my $wait_until = time + return_timeout($anvil);
			my $next_log   = time + 60;
			my $waiting    = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				wait_until => $wait_until, 
				next_log   => $next_log,
				waiting    => $waiting,
			}});
			while ($waiting)
			{
				my $drbd_up      = 0;
				my $pacemaker_up = 0;
				$anvil->DRBD->get_status({
					debug  => 2,
					host   => $short_host_name, 
					target => $anvil->data->{peer}{$short_host_name}{access}{ip},
				});
				
				# How may resources are up?
				my $resource_count = keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_count => $resource_count }});
				if ($resource_count)
				{
					# DRBD is still up.
					$drbd_up = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up => $drbd_up }});
				}
				
				# Is pacemaker down?
				my $problem = $anvil->Cluster->parse_cib({debug => 2, target => $anvil->data->{peer}{$short_host_name}{access}{ip}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				
				if (not $problem)
				{
					# Node is still in the cluster.
					$pacemaker_up = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pacemaker_up => $pacemaker_up }});
				}
				
				if ((not $pacemaker_up) && (not $drbd_up))
				{
					$waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
				
				if ($waiting)
				{
					# Log which resources are still up
					if (time > $next_log)
					{
						if ($pacemaker_up)
						{
							# The subnode is still in the cluster.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0441", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
							}});
						}
						else
						{
							# The subnode is no longer in the cluster, good.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0442", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
							}});
						}
						
						foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}})
						{
							# The resource is still up.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0443", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
								resource  => $resource,
							}});
						}
						   $next_log      = time + 60;
						my $time_left     = $wait_until - time;
						   $time_left     = 0 if $time_left < 0;
						my $say_time_left = $anvil->Convert->time({
							'time'    => $time_left,
							translate => 1,
							long      => 0,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							next_log      => $next_log,
							time_left     => $time_left, 
							say_time_left => $say_time_left, 
						}});
						# Keep waiting
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0444", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							time_left => $say_time_left,
						}});
					}
					if (time > $wait_until)
					{
						# Timeout.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0445", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							host_name => $short_host_name,
						}});
						$anvil->nice_exit({exit_code => 1});
					}
					
					sleep 10;
				}
			}
			
			my $update_switches = "";
			if ($anvil->data->{switches}{'no-reboot'})
			{
				$update_switches .= " --no-reboot";
			}
			if ($anvil->data->{switches}{reboot})
			{
				$update_switches .= " --reboot";
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_switches => $update_switches }});
			
			# We register a job, even though anvil-daemon isn't running. This will get picked up 
			# by 'anvil-update-systems --no-db' towards the end of it's run.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0446", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			$shell_call = $anvil->data->{path}{exe}{'anvil-update-system'}.$update_switches.$anvil->Log->switches();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			my $job_uuid = $anvil->Database->insert_or_update_jobs({
				debug           => 2,
				job_command     => $shell_call,
				job_description => "job_0468",
				job_host_uuid   => $host_uuid,
				job_name        => "system::update-system",
				job_progress    => 0,
				job_title       => "job_0467"
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
			# Job registered, tell the user we're waiting
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0447", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				job_uuid  => $job_uuid, 
			}});
			
			# Now call anvil-update-system with --no-db and background it so we can close
			# the DB connection without killing the process.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0448", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name, 
			}});
			$shell_call = $anvil->data->{path}{exe}{nohup}." ".$anvil->data->{path}{exe}{'anvil-update-system'}." --no-db".$update_switches;
			if ($anvil->data->{switches}{'clear-cache'})
			{
				# We'll only call clear-cache on this one.
				$shell_call .= " --clear-cache";
			}
			$shell_call .= $anvil->Log->switches()." >/dev/null 2>&1 &";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			($output, $error, $return_code) = $anvil->Remote->call({
				debug      => 2,
				shell_call => $shell_call, 
				target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				error       => $error,
				return_code => $return_code,
			}});
			
			# Record the start time so that we can be sure the subnode has rebooted (uptime is 
			# less than the current time minus this start time), if the host reboots as part of 
			# the update.
			my $rebooted    = 0;
			my $reboot_time = time;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				rebooted        => $rebooted, 
				reboot_time     => $reboot_time,
				short_host_name => $short_host_name, 
			}});
			
			# Verify that the node is no longer in the cluster.
			$wait_until = time + return_timeout($anvil);
			$waiting    = 1;
			$next_log   = time + 60;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				wait_until => $wait_until, 
				next_log   => $next_log,
			}});
			while ($waiting)
			{
				$anvil->Job->get_job_details({job_uuid => $job_uuid});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"jobs::job_progress" => $anvil->data->{jobs}{job_progress},
					"jobs::job_data"     => $anvil->data->{jobs}{job_data},
				}});
				if ($anvil->data->{jobs}{job_progress} == 100)
				{
					# Done! The subnode has been updated.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0449", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						host_name => $short_host_name, 
					}});
					$waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
					
					# Did it reboot?
					if ($anvil->data->{jobs}{job_data} eq "rebooted")
					{
						$rebooted = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { rebooted => $rebooted }});
					}
					
					# Did it fail?
					if ($anvil->data->{jobs}{job_data} eq "failed")
					{
						# Abort! There was a problem updating the subnode! Anvil! 
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0450", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
						}});
						$anvil->nice_exit({exit_code => 1});
					}
				}
				else
				{
					if (time > $next_log)
					{
						# Continuing to wait.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0451", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							progress  => $anvil->data->{jobs}{job_progress},
						}});
						if ($anvil->data->{jobs}{job_progress} eq "0")
						{
							# It is expected for the job to stay at '0' for a while.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0452", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
							}});
						}
						   $next_log      = time + 60;
						my $time_left     = $wait_until - time;
						my $say_time_left = $anvil->Convert->time({
							'time'    => $time_left,
							translate => 1,
							long      => 0,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							next_log      => $next_log,
							time_left     => $time_left, 
							say_time_left => $say_time_left, 
						}});
						# Waiting, will check again shortly.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0453", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							time_left => $say_time_left, 
						}});
					}
					if (time > $wait_until)
					{
						# Timed out while waiting for the subnode to update. Aborting the update.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0454", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							host_name => $short_host_name, 
						}});
						$anvil->nice_exit({exit_code => 1});
					}
					sleep 5;
				}
			}
			
			# Update completed successfully! Checking if a reboot is needed.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0455", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			my $run_anvil_safe_start = 0;
			if ($rebooted)
			{
				# Rebooted! Will wait for it to come back up.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0456", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				wait_for_reboot($anvil, $host_uuid, $reboot_time);
			}
			else
			{
				# Reboot not needed, kernel appears to be up to date.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0457", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				$run_anvil_safe_start = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { run_anvil_safe_start => $run_anvil_safe_start }});
			}
			
			# Wait for the node to rejoin the cluster. As before, this is a time 
			# unrestricted wait loop.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0458", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			   $wait_until   = time + return_timeout($anvil);
			   $waiting      = 1;
			my $start_called = 0; 
			   $next_log     = time + 60;
			my $manual_start = time + 60; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:wait_until'   => $wait_until, 
				's2:next_log'     => $next_log,
				's3:manual_start' => $manual_start, 
			}});
			while($waiting)
			{
				# Should we call a start to the cluster?
				if ((not $start_called) && ($run_anvil_safe_start))
				{
					# Calling 'anvil-safe-start' to rejoin the subnode to the node.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0459", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
					}});
					   $start_called = 1;
					my $shell_call   = $anvil->data->{path}{exe}{'anvil-safe-start'}.$anvil->Log->switches()." >/dev/null 2>&1 &";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						start_called => $start_called,
						shell_call   => $shell_call,
					}});
					
					my ($output, $error, $return_code) = $anvil->Remote->call({
						debug      => 2,
						shell_call => $shell_call, 
						target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output, 
						error       => $error,
						return_code => $return_code,
					}});
				}
				
				# Pull the CIB and make sure both nodes are ready, and that DRBD resources 
				# are all UpToDate if this is the reboot from the first node.
				my ($problem) = $anvil->Cluster->parse_cib({target => $anvil->data->{peer}{$short_host_name}{access}{ip}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				
				# Are both nodes ready?
				if (not $problem)
				{
					# Both nodes are in the cluster, but are they full members yet?
					my $both_ready = 1;
					my $node_count = 0;
					foreach my $node_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}})
					{
						my $ready = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{ready};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							node_name => $node_name, 
							ready     => $ready,
						}});
						if (not $ready)
						{
							$both_ready = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_ready => $both_ready }});
						}
						$node_count++;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_count => $node_count }});
					}
					
					# Did we see two nodes and are both ready?
					if (($node_count == 2) && ($both_ready))
					{
						# Yes! If this is the first subnode, we need to wait for DRBD
						# to be UpToDate. If it's the second, we just wait for the 
						# connections to be up.
						# NOTE: We call the peer to get the DRBD data as it's got a 
						#       better view of the storage 
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0460", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
						}});
						$anvil->DRBD->get_status({
							host   => $peer_short_host_name, 
							target => $anvil->data->{peer}{$peer_short_host_name}{access}{ip},
						});
						
						if ($host_uuid eq $primary_host_uuid)
						{
							### NOTE: Should we wait for all connections 
							###       to be up?
							# This is the second node, we don't have to wait.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0461", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
							}});
							$waiting = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						}
						else
						{
							# This is the first node. Wait for all volumes to be 
							# UpToDate.
							if (time > $next_log)
							{
								# Waiting for all volumes to be UpToDate before updating the other subnode.
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0462", variables => {
									timestamp => $anvil->Get->date_and_time({time_only => 1}),
								}});
							}
							my $all_uptodate = 1;
							my $resources    = 0;
							foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$peer_short_host_name}{resource}})
							{
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
								foreach my $peer_name (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$peer_short_host_name}{resource}{$resource}{connection}})
								{
									# We don't care about DR hosts for this upgrade
									my $peer_uuid = $anvil->Get->host_uuid_from_name({host_name => $peer_name}); 
									my $peer_type = $anvil->data->{hosts}{host_uuid}{$peer_uuid}{host_type};
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										's1:peer_name' => $peer_name,
										's2:peer_uuid' => $peer_uuid,
										's3:peer_type' => $peer_type,
									}});
									next if $peer_type ne "node";
									foreach my $volume (sort {$a <=> $b} keys %{$anvil->data->{drbd}{status}{$peer_short_host_name}{resource}{$resource}{connection}{$peer_name}{volume}})
									{
										# This is this subnode's disk state, 
										# as the DRBD data was collected 
										# from the peer.
										my $disk_state = $anvil->data->{drbd}{status}{$peer_short_host_name}{resource}{$resource}{connection}{$peer_name}{volume}{$volume}{'peer-disk-state'};
										$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
											's1:volume'     => $volume,
											's2:disk_state' => $disk_state,
										}});
										
										if (lc($disk_state) ne "uptodate")
										{
											$all_uptodate   = 0;
											my $eta_in_seconds = $anvil->data->{drbd}{status}{$peer_short_host_name}{resource}{$resource}{connection}{$peer_name}{volume}{$volume}{'estimated-seconds-to-finish'};
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
												all_uptodate   => $all_uptodate,
												eta_in_seconds => $eta_in_seconds, 
											}});
											if (time > $next_log)
											{
												if ($eta_in_seconds)
												{
													# The resource is not synced yet
													$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0463", variables => {
														timestamp      => $anvil->Get->date_and_time({time_only => 1}),
														resource       => $$resource, 
														volume         => $volume, 
														eta_in_seconds => $eta_in_seconds, 
													}});
												}
												else
												{
													# The resource is not yet UpToDate
													$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0464", variables => {
														timestamp => $anvil->Get->date_and_time({time_only => 1}),
														resource  => $resource, 
														volume    => $volume, 
													}});
												}
											}
										}
									} # End foreach volume
								} # End foreach peer
							} # End foreach resource
							
							if ($all_uptodate)
							{
								# All resources appear to be ready,
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0465", variables => {
									timestamp => $anvil->Get->date_and_time({time_only => 1}),
								}});
								$waiting = 0;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
							}
						} # End if host is first or second subnode
					} # End if both ready
					elsif (time > $next_log)
					{
						# Both subnodes are not online yet, still waiting.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0466", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
						}});
					}
				} # End if CIB was parsed 
				elsif (time > $next_log)
				{
					# Unable to parse the node's cluster information base, will try again soon.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0467", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
					}});
				}
				
				if (time > $next_log)
				{
					   $next_log      = time + 60;
					my $time_left     = $wait_until - time;
					my $say_time_left = $anvil->Convert->time({
						'time'    => $time_left,
						translate => 1,
						long      => 0,
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:next_log'      => $next_log,
						's2:time_left'     => $time_left, 
						's3:say_time_left' => $say_time_left, 
					}});
					
					# Tell the user we're still waiting.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0468", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						host_name => $short_host_name, 
						time_left => $say_time_left, 
					}});
				}
				if (time > $wait_until)
				{
					# Timed out while waiting for the subnode to join the subcluster. Aborting the update.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0469", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						host_name => $short_host_name, 
					}});
					$anvil->nice_exit({exit_code => 1});
				}
				
				if ($waiting)
				{
					sleep 5;
				}
			} # End while waiting for subnode to return
			
			# Run anvil-version-change
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0470", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			$output      = "";
			$error       = "";
			$return_code = "";
			$shell_call  = $anvil->data->{path}{exe}{'anvil-version-changes'};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			if ($host_uuid eq $anvil->Get->host_uuid)
			{
				($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code, 
				}});
			}
			else
			{
				($output, $error, $return_code) = $anvil->Remote->call({
					debug      => 2,
					shell_call => $shell_call, 
					target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error,
					return_code => $return_code,
				}});
			}
			# Done!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0471", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
		}
	}
	
	return(0);
}

sub update_strikers_and_dr
{
	my ($anvil) = @_;

	foreach my $host_type ("striker", "dr")
	{
		$anvil->Database->get_hosts({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			my $host_uuid       = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $this_host_type  = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'       => $host_name, 
				's2:host_uuid'       => $host_uuid,
				's3:short_host_name' => $short_host_name, 
				's4:this_host_type'  => $this_host_type, 
			}});
			next if $this_host_type ne $host_type;
			
			if ($host_type eq "striker")
			{
				# Starting the update of the Striker dashboard
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0472", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
			}
			else
			{
				# Starting the update of the DR host
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0473", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
			}
			
			# If this is the local system, set the variable to track if we need to reboot. 
			# Otherwise, see if we have access to the peer.
			if ($host_uuid eq $anvil->Get->host_uuid)
			{
				$anvil->data->{sys}{reboot_needed} = 0;
			}
			elsif(not $anvil->data->{peer}{$short_host_name}{access}{ip})
			{
				if ($host_type eq "striker")
				{
					# No access to the Striker dashboard, skipping.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0474", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						host_name => $short_host_name, 
					}});
				}
				else
				{
					# No access to the DR host, skipping.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0475", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						host_name => $short_host_name, 
					}});
				}
				next;
			}
			
			# Record the start time so that we can be sure the subnode has rebooted (uptime is 
			# less than the current time minus this start time), if the host reboots as part of 
			# the update.
			my $reboot_time = time;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_time => $reboot_time }});
			
			# Beginning OS update
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0476", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name, 
			}});
			my $rebooted = 0;
			if (($anvil->data->{switches}{'clear-cache'}) && ($host_uuid eq $anvil->Get->host_uuid))
			{
				my $shell_call = $anvil->data->{path}{exe}{dnf}." clean all";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code, 
				}});
				
				# Cache cleared.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0478", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
			}
			# Tell the user to watch dnf in ps.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0477", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			if ($host_uuid eq $anvil->Get->host_uuid)
			{
				my $shell_call = $anvil->data->{path}{exe}{dnf}." -y update";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code, 
				}});
				if ($return_code)
				{
					# Report the error and output
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0479", variables => {
						timestamp   => $anvil->Get->date_and_time({time_only => 1}),
						return_code => $return_code, 
						output      => $output, 
					}});
				}
				
				# Loop through the output.
				my $package_changes = 0;
				foreach my $line (split/\n/, $output)
				{
					$line = $anvil->Words->clean_spaces({string => $line});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					
					if ($line =~ / (\d+) Packages$/i)
					{
						$package_changes += $1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { package_changes => $package_changes }});
					}
				}
				
				# Did the user want to reboot on any update?
				if (($package_changes) && ($anvil->data->{switches}{reboot}) && ($anvil->data->{switches}{'reboot-self'}))
				{
					# Reboot needed
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0480", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
						packages  => $package_changes, 
					}});
					$anvil->data->{sys}{reboot_needed} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"sys::reboot_needed" => $anvil->data->{sys}{reboot_needed},
					}});
				}
				
				# Get the newest installed kernel
				$shell_call = $anvil->data->{path}{exe}{rpm}." -q kernel | ".$anvil->data->{path}{exe}{'sort'}." | ".$anvil->data->{path}{exe}{tail}." -n 1";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				(my $installed_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					installed_kernel => $installed_kernel, 
					return_code      => $return_code, 
				}});
				$installed_kernel =~ s/^kernel-(\d+.\d+\.\d+-\d+)\..*$/$1/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { installed_kernel => $installed_kernel }});
				
				# Get the running kernel
				$shell_call = $anvil->data->{path}{exe}{uname}." -r";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				(my $active_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					active_kernel => $active_kernel, 
					return_code   => $return_code, 
				}});
				$active_kernel =~ s/(\d+.\d+\.\d+-\d+)\..*$/$1/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_kernel => $active_kernel }});
				
				if ($installed_kernel eq $active_kernel)
				{
					# The kernel has not been updated.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0481", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
					}});
				}
				else
				{
					# The kernel appears to have been upgraded, reboot needed!
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0482", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
					}});
					$anvil->data->{sys}{reboot_needed} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"sys::reboot_needed" => $anvil->data->{sys}{reboot_needed},
					}});
				}
			}
			else
			{
				# Call anvil-update-system and then wait.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0483", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
				if ($host_type eq "dr")
				{
					# Make sure VMs are off and DRBD is down. Call this with nohup so it 
					# doesn't get killed by the loss of the SSH connection.
					my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'}." --no-db".$anvil->Log->switches()." >/dev/null 2>&1 &";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $error, $return_code) = $anvil->Remote->call({
						debug      => 2,
						shell_call => $shell_call, 
						target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output, 
						error       => $error,
						return_code => $return_code,
					}});
					
					# Now wait for DRBD resources to stop (which requires VMs be off).
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0484", variables => {
						timestamp => $anvil->Get->date_and_time({time_only => 1}),
					}});
					my $wait_until = time + return_timeout($anvil);
					my $next_log   = time + 60;
					my $waiting    = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						wait_until => $wait_until, 
						next_log   => $next_log,
						waiting    => $waiting,
					}});
					while ($waiting)
					{
						my $drbd_up = 0;
						$anvil->DRBD->get_status({
							host   => $short_host_name, 
							target => $anvil->data->{peer}{$short_host_name}{access}{ip},
						});
						
						# How may resources are up?
						my $resource_count = keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_count => $resource_count }});
						
						if (not $resource_count)
						{
							# Done!
							$waiting = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						}
						
						if ($waiting)
						{
							# Log which resources are still up
							if (time > $next_log)
							{
								foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$short_host_name}{resource}})
								{
									# The resource is still up
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0485", variables => {
										timestamp => $anvil->Get->date_and_time({time_only => 1}),
										resource  => $resource, 
									}});
								}
								   $next_log      = time + 60;
								my $time_left     = $wait_until - time;
								my $say_time_left = $anvil->Convert->time({
									'time'    => $time_left,
									translate => 1,
						  			long      => 0,
								});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									next_log      => $next_log,
									time_left     => $time_left, 
									say_time_left => $say_time_left, 
								}});
								# Waiting for bit, will check again shortly
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0486", variables => {
									timestamp => $anvil->Get->date_and_time({time_only => 1}),
									time_left => $say_time_left, 
								}});
							}
							if (time > $wait_until)
							{
								# Timeout.
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0487", variables => {
									timestamp => $anvil->Get->date_and_time({time_only => 1}),
									host_name => $short_host_name, 
								}});
								$anvil->nice_exit({exit_code => 1});
							}
							
							sleep 10;
						}
					}
				}
				
				my $update_switches = "";
				if ($anvil->data->{switches}{'no-reboot'})
				{
					$update_switches .= " --no-reboot";
				}
				if ($anvil->data->{switches}{reboot})
				{
					$update_switches .= " --reboot";
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_switches => $update_switches }});
				
				# We register a job, even though anvil-daemon isn't running. This will get 
				# picked up by 'anvil-update-systems --no-db' towards the end of it's run.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0488", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				my $shell_call = $anvil->data->{path}{exe}{'anvil-update-system'}.$update_switches.$anvil->Log->switches();
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				my $job_uuid = $anvil->Database->insert_or_update_jobs({
					debug           => 2,
					job_command     => $shell_call,
					job_description => "job_0468",
					job_host_uuid   => $host_uuid,
					job_name        => "system::update-system",
					job_progress    => 0,
					job_title       => "job_0467"
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
				
				# Job registered, waiting for it to complete.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0489", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					job_uuid  => $job_uuid, 
				}});
				
				# Now call anvil-update-system with --no-db and background it so we can close
				# the DB connection without killing the process.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0490", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
				$shell_call = $anvil->data->{path}{exe}{nohup}." ".$anvil->data->{path}{exe}{'anvil-update-system'}." --no-db".$update_switches;
				if ($anvil->data->{switches}{'clear-cache'})
				{
					# We'll only call clear-cache on this one.
					$shell_call .= " --clear-cache";
				}
				$shell_call .= $anvil->Log->switches()." >/dev/null 2>&1 &";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my ($output, $error, $return_code) = $anvil->Remote->call({
					debug      => 2,
					shell_call => $shell_call, 
					target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error,
					return_code => $return_code,
				}});
				
				# Verify / wait until the update is done.
				my $wait_until = time + return_timeout($anvil);
				my $waiting    = 1;
				my $next_log   = time + 60;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { next_log => $next_log }});
				while ($waiting)
				{
					$anvil->Job->get_job_details({job_uuid => $job_uuid});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"jobs::job_progress" => $anvil->data->{jobs}{job_progress},
						"jobs::job_data"     => $anvil->data->{jobs}{job_data},
					}});
					if ($anvil->data->{jobs}{job_progress} == 100)
					{
						# Done!
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0491", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							host_name => $short_host_name, 
						}});
						$waiting = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						
						# Did it reboot?
						if ($anvil->data->{jobs}{job_data} eq "rebooted")
						{
							$rebooted = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { rebooted => $rebooted }});
						}
						
						# Did it fail?
						if ($anvil->data->{jobs}{job_data} eq "failed")
						{
							# Abort!
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0492", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
							}});
							$anvil->nice_exit({exit_code => 1});
						}
					}
					else
					{
						if (time > $next_log)
						{
							# continuing to wait.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0493", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
								progress  => $anvil->data->{jobs}{job_progress}, 
							}});
							if ($anvil->data->{jobs}{job_progress} == 0)
							{
								# It is normal for the job to show '0' progress until the database access is restored.
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0494", variables => {
									timestamp => $anvil->Get->date_and_time({time_only => 1}),
								}});
							}
							   $next_log      = time + 60;
							my $time_left     = $wait_until - time;
							my $say_time_left = $anvil->Convert->time({
								'time'    => $time_left,
								translate => 1,
								long      => 0,
							});
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								next_log      => $next_log,
								time_left     => $time_left, 
								say_time_left => $say_time_left, 
							}});
							print $anvil->Get->date_and_time({time_only => 1})."; - Waiting for another: [".$say_time_left."], will check again shortly.\n";
						}
						if (time > $wait_until)
						{
							# Timed out while waiting to update the OS. Aborting the update.\n";
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0495", variables => {
								timestamp => $anvil->Get->date_and_time({time_only => 1}),
								host_name => $short_host_name, 
							}});
							$anvil->nice_exit({exit_code => 1});
						}
						sleep 5;
					}
				}
				
			}
			
			if ($rebooted)
			{
				# Rebooted! Will wait for it to come back up.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0496", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				wait_for_reboot($anvil, $host_uuid, $reboot_time);
			}
			else
			{
				# Reboot not needed, kernel appears to be up to date.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0497", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
			}
			
			# Run anvil-version-change
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0498", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
			my $output      = "";
			my $error       = "";
			my $return_code = "";
			my $shell_call  = $anvil->data->{path}{exe}{'anvil-version-changes'};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			if ($host_uuid eq $anvil->Get->host_uuid)
			{
				($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code, 
				}});
			}
			else
			{
				($output, $error, $return_code) = $anvil->Remote->call({
					debug      => 2,
					'close'    => 1,
					no_cache   => 1,
					shell_call => $shell_call, 
					target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error,
					return_code => $return_code,
				}});
			}
		}
	}
	
	# Run anvil-version-change to make sure any DB schema updates are sorted out
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0470", variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}});
	my $shell_call = $anvil->data->{path}{exe}{'anvil-version-changes'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	return(0);
}

sub wait_for_reboot
{
	my ($anvil, $host_uuid, $reboot_time) = @_;
	my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:host_uuid'       => $host_uuid,
		's2:short_host_name' => $short_host_name, 
		's3:reboot_time'     => $reboot_time, 
	}});
	
	my $matches = $anvil->Network->find_access({
		debug  => 2,
		target => $host_uuid, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
	
	# Wait until the node comes back up.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0499", variables => {
		timestamp => $anvil->Get->date_and_time({time_only => 1}),
	}});
	
	# This is an infinite loop, there is no timeout for this.
	my $wait_until = time + return_timeout($anvil);
	my $waiting    = 1;
	my $next_log   = time + 60;
	my $delay      = 30;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { next_log => $next_log }});
	while($waiting)
	{
		# We wait initially because we're going to be called right after the target is rebooted.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "log_0816", variables => { delay => $delay }});
		sleep $delay;
		
		# Test access
		my $target = $anvil->data->{peer}{$short_host_name}{access}{ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			target          => $target,
			short_host_name => $short_host_name, 
		}});
		my $test_access = $anvil->Remote->test_access({
			debug => 2, 
			target => $target,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_access => $test_access }});
		
		if ($test_access)
		{
			# What's the machine's uptime?
			my $uptime            = $anvil->Get->uptime({debug => 2, target => $anvil->data->{peer}{$short_host_name}{access}{ip}});
			my $time_since_reboot = time - $reboot_time; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				uptime            => $uptime,
				time_since_reboot => $time_since_reboot, 
				short_host_name   => $short_host_name, 
			}});
			
			if (($uptime) && ($uptime < $time_since_reboot))
			{
				# Rebooted!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0500", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		if ($waiting)
		{
			if (time > $next_log)
			{
				   $next_log      = time + 60;
				my $time_left     = $wait_until - time;
				my $say_time_left = $anvil->Convert->time({
					'time'    => $time_left,
					translate => 1,
					long      => 0,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:next_log'      => $next_log,
					's2:time_left'     => $time_left, 
					's3:say_time_left' => $say_time_left, 
				}});
				
				# Tell the user we're still waiting.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0501", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
					time_left => $say_time_left, 
				}});
			}
			if (time > $wait_until)
			{
				# Timeout.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0502", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}
	
	return(0);
}

sub manage_daemons
{
	my ($anvil, $task) = @_;
	
	$task = "start" if not $task;
	
	my $do_task = $task eq "start" ? "enable --now" : "stop";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { do_task => $do_task }});
	
	if ($task eq "stop")
	{
		# Disabling Anvil! daemons on all hosts...
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0503", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
	}
	else
	{
		# Enabling Anvil! daemons on all hosts...
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0504", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
	}
	my $daemons = ["anvil-daemon", "scancore"];
	foreach my $host_type ("dr", "node", "striker")
	{
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			my $host_uuid       = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $this_host_type  = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'       => $host_name, 
				's2:host_uuid'       => $host_uuid,
				's3:short_host_name' => $short_host_name, 
				's4:this_host_type'  => $this_host_type, 
			}});
			next if $host_type ne $this_host_type;
			
			if ($task eq "stop")
			{
				# Disabling dameons
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0505", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
			}
			else
			{
				# Enabling dameons
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0506", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
					host_name => $short_host_name, 
				}});
			}
			if (not $anvil->data->{peer}{$short_host_name}{access}{ip})
			{
				# Offline! Skipping.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0507", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				next;
			}
			
			# Local
			foreach my $daemon (@{$daemons})
			{
				my $shell_call = $anvil->data->{path}{exe}{systemctl}." ".$do_task." ".$daemon;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my $output      = "";
				my $error       = "";
				my $return_code = 999;
				if ($host_uuid eq $anvil->Get->host_uuid)
				{
					# Local
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output, 
						return_code => $return_code, 
					}});
				}
				else
				{
					# Remote, it'll be a while before we hit some clients, so close this 
					# connection so later access to the machines don't fail with ssh 
					# connection timeouts.
					($output, $error, $return_code) = $anvil->Remote->call({
						debug      => 2,
						'close'    => 1,
						no_cache   => 1,
						shell_call => $shell_call, 
						target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output, 
						error       => $error,
						return_code => $return_code,
					}});
				}
				if (not $return_code)
				{
					if ($task eq "stop")
					{
						# Stopped
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0508", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							daemon    => $daemon, 
						}});
					}
					else
					{
						# started
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0509", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							daemon    => $daemon, 
						}});
					}
				}
				else
				{
					if ($task eq "stop")
					{
						# didn't stop!
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0510", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							daemon    => $daemon, 
						}});
					}
					else
					{
						# didn't start!
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0511", variables => {
							timestamp => $anvil->Get->date_and_time({time_only => 1}),
							daemon    => $daemon, 
						}});
					}
				}
			}
			# Done!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0471", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
			}});
		}
	}
	
	# If we're in the stop stage, we need to ensure that anvil-safe-start is enabled on the nodes, or else we'll hang when the node reboots.
	if ($task eq "stop")
	{
		# Enabling 'anvil-safe-start' on nodes to prevent hangs on reboot.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0512", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			my $host_uuid       = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $host_type       = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'       => $host_name, 
				's2:host_uuid'       => $host_uuid,
				's3:short_host_name' => $short_host_name, 
				's4:host_type'       => $host_type, 
			}});
			next if $host_type ne "node";
			
			if (not $anvil->data->{peer}{$short_host_name}{access}{ip})
			{
				# Offline! Skipping.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0513", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				next;
			}
			
			# Local
			my $shell_call = $anvil->data->{path}{exe}{systemctl}." enable anvil-safe-start.service";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my $output      = "";
			my $error       = "";
			my $return_code = 999;
			if ($host_uuid eq $anvil->Get->host_uuid)
			{
				# Local
				($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code, 
				}});
			}
			else
			{
				# Remote, it'll be a while before we hit some clients, so close this 
				# connection so later access to the machines don't fail with ssh 
				# connection timeouts.
				($output, $error, $return_code) = $anvil->Remote->call({
					debug      => 2,
					'close'    => 1,
					no_cache   => 1,
					shell_call => $shell_call, 
					target     => $anvil->data->{peer}{$short_host_name}{access}{ip},
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					error       => $error,
					return_code => $return_code,
				}});
			}
		}
		# Done!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0471", variables => {
			timestamp => $anvil->Get->date_and_time({time_only => 1}),
		}});
	}
	
	return(0);
}

sub verify_access
{
	my ($anvil) = @_;
	
	# Load host and Anvil! data.
	$anvil->Database->get_hosts();
	
	# Make sure all are available before we start.
	my $all_access = 1;
	foreach my $host_type ("dr", "node", "striker")
	{
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			my $host_uuid       = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
			my $this_host_type  = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'       => $host_name, 
				's2:host_uuid'       => $host_uuid,
				's3:short_host_name' => $short_host_name, 
				's4:this_host_type'  => $this_host_type, 
			}});
			next if $host_type ne $this_host_type;
			
			# Verifying access
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0514", variables => {
				timestamp => $anvil->Get->date_and_time({time_only => 1}),
				host_name => $short_host_name, 
			}});
			my $matches = $anvil->Network->find_access({
				debug  => 2,
				target => $host_uuid, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
			
			$anvil->data->{peer}{$short_host_name}{access}{ip}      = "";
			$anvil->data->{peer}{$short_host_name}{access}{network} = "";
			foreach my $preferred_network ("bcn", "mn", "ifn", "sn", "any")
			{
				next if $anvil->data->{peer}{$short_host_name}{access}{ip};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { preferred_network => $preferred_network }});
				foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_name => $network_name }});
					if (($network_name !~ /^$preferred_network/) && ($preferred_network ne "any"))
					{
						next;
					}
					
					my $target_ip   = $anvil->data->{network_access}{$network_name}{target_ip_address};
					my $test_access = $anvil->Remote->test_access({
						'close' => 1,
						target  => $target_ip,
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's2:target_ip'    => $target_ip, 
						's3:test_access'  => $test_access, 
					}});
					
					if ($test_access)
					{
						# We're good.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0515", variables => {
							timestamp    => $anvil->Get->date_and_time({time_only => 1}),
							target_ip    => $target_ip, 
							network_name => $network_name, 
						}});
						$anvil->data->{peer}{$short_host_name}{access}{ip}      = $target_ip;
						$anvil->data->{peer}{$short_host_name}{access}{network} = $network_name;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							"s1:peer::${short_host_name}::access::ip"      => $anvil->data->{peer}{$short_host_name}{access}{ip}, 
							"s2:peer::${short_host_name}::access::network" => $anvil->data->{peer}{$short_host_name}{access}{network}, 
						}});
					}
				}
			}
			
			if (not $anvil->data->{peer}{$short_host_name}{access}{ip})
			{
				# No access! Skipping.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0516", variables => {
					timestamp => $anvil->Get->date_and_time({time_only => 1}),
				}});
				$all_access = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_access => $all_access }});
			}
		}
	}
	
	return($all_access);
}
