#!/usr/bin/perl
# 
# This adds a node (or gets a node to join to a new/rebuilt peer or added DR) to an Anvil!.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to connect to any database.
# 2 = Failed to load/parse the manifest.
# 3 = Failed to change the host name.
# 4 = Failed to reconnect to any database after the network was reconfigured
# 5 = Problem parsing job data or loading manifest or anvil data using job data.
# 6 = A pcs call failed.
# 7 = No job was found to run.
# 
# TODO: 
# - Check to see if this is a cluster node and/or running VMs, and if so, refuse to run.
# 

use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
use String::ShellQuote;
use Text::Diff;
use NetAddr::IP;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches({list => [
	"as-machine", 
	"join", 
	"manifest", 
	"no-auto-grow-pv", 
	"rejoin", 
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0129"});
	sleep 2;
	$anvil->nice_exit({exit_code => 1});
}

# If there isn't a job UUID, see if the user is asking to rejoin this subnode to a cluster.
if (($anvil->data->{switches}{rejoin}) && (not $anvil->data->{switches}{'job-uuid'}))
{
	process_rejoin($anvil);
	$anvil->nice_exit({exit_code => 0});
}

# Get the job details
load_job($anvil);

# Did the user ask to check for free space?
if (not $anvil->data->{switches}{'no-auto-grow-pv'})
{
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0200", variables => { daemon => "anvil-safe-start.service" }});
	$anvil->Storage->auto_grow_pv({debug => 2});
}

# Make sure the firewall is setup and ready.
$anvil->Network->manage_firewall({debug => 2, task => "check"});

# Update the user passwords
update_passwords($anvil);

# Hold until both subnodes are marked as configured and not in maintenance mode.
wait_for_subnodes($anvil);

# Check if we need to change any IPs or our hostname.
check_local_network($anvil);

# Make sure the hosts file has entries for all nets for both subnodes
wait_for_etc_hosts($anvil);

# Wait until we can ping our peer on all networks.
wait_for_access($anvil);

# See if we can assemble any storage groups.
$anvil->Cluster->assemble_storage_groups({debug => 2});

# Check for replacement VGs
check_for_replaced_vgs($anvil);

# (wait for out peer and) Configure pacemaker
configure_pacemaker($anvil);

# Configure DRBD
configure_drbd($anvil);

# If we're rejoining, add missing servers.
configure_servers($anvil);

# Enable anvil-safe-start
if (1)
{
	my ($return_code) = $anvil->System->enable_daemon({daemon => "anvil-safe-start.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	update_progress($anvil, 99, "job_0094,!!daemon!anvil-safe-start.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }});
}

# Update NTP, if needed.
my $ntp_updated = $anvil->System->check_ntp({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ntp_updated => $ntp_updated }});

update_progress($anvil, 100, "job_0129");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0129"});

# Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({
	variable_name         => "system::all::configured", 
	variable_value        => 1, 
	variable_default      => "", 
	variable_description  => "striker_0297", 
	variable_section      => "system", 
	variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
	variable_source_table => "hosts", 
});

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# Check for replacement VGs
sub check_for_replaced_vgs
{
	my ($anvil) = @_;
	# If there's only one, it might be a replacement during a node rebuild. Is there a 
	# group with a matching size but missing a VG?
	$anvil->Database->get_storage_group_data({debug => 3});
	$anvil->Cluster->get_peers({debug => 3});
	my $anvil_uuid      = $anvil->Cluster->get_anvil_uuid();
	my $host_uuid       = $anvil->Get->host_uuid();
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $peer_host_uuid  = $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:anvil_uuid'      => $anvil_uuid,
		's2:host_uuid'       => $host_uuid, 
		's3:peer_host_uuid'  => $peer_host_uuid, 
		's4:node1_host_uuid' => $node1_host_uuid, 
		's5:node2_host_uuid' => $node2_host_uuid, 
	}});
	foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}})
	{
		my $storage_group_uuid = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:storage_group_name' => $storage_group_name, 
			's2:storage_group_uuid' => $storage_group_uuid,
		}});
		
		if (not exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid})
		{
			# This will be the minimum size of the replacement VG.
			my $needed_size                   = 0;
			my $old_storage_group_member_uuid = "";
			
			# We're not a member in this group, did we used to?
			my $query = "
SELECT 
    a.storage_group_member_uuid, 
    a.storage_group_member_host_uuid, 
    a.storage_group_member_vg_uuid, 
    b.scan_lvm_vg_name, 
    b.scan_lvm_vg_size 
FROM 
    storage_group_members a, 
    scan_lvm_vgs b 
WHERE 
    a.storage_group_member_storage_group_uuid = ".$anvil->Database->quote($storage_group_uuid)." 
AND 
    a.storage_group_member_vg_uuid = b.scan_lvm_vg_internal_uuid
;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			
			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				results => $results, 
				count   => $count, 
			}});
			foreach my $row (@{$results})
			{
				my $storage_group_member_uuid      = $row->[0];
				my $storage_group_member_host_uuid = $row->[1];
				my $storage_group_member_vg_uuid   = $row->[2];
				my $scan_lvm_vg_name               = $row->[3];
				my $scan_lvm_vg_size               = $row->[4];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:storage_group_member_uuid'      => $storage_group_member_uuid, 
					's2:storage_group_member_host_uuid' => $storage_group_member_host_uuid, 
					's3:storage_group_member_vg_uuid'   => $storage_group_member_vg_uuid, 
					's4:scan_lvm_vg_name'               => $scan_lvm_vg_name, 
					's5:scan_lvm_vg_size'               => $scan_lvm_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_lvm_vg_size}).")", 
				}});
				
				if ($storage_group_member_host_uuid eq $peer_host_uuid)
				{
					$needed_size = $scan_lvm_vg_size;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						needed_size => $needed_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}).")", 
					}});
				}
				elsif (($storage_group_member_host_uuid eq $host_uuid) && ($scan_lvm_vg_name eq "DELETED"))
				{
					$old_storage_group_member_uuid = $storage_group_member_uuid;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						old_storage_group_member_uuid => $old_storage_group_member_uuid, 
					}});
				}
			}
			
			# If we have a needed size, see if we can find a new VG that's close.
			if (($needed_size) && ($old_storage_group_member_uuid))
			{
				my $query = "
SELECT 
    scan_lvm_vg_internal_uuid, 
    scan_lvm_vg_name, 
    scan_lvm_vg_size 
FROM 
    scan_lvm_vgs 
WHERE 
    scan_lvm_vg_name != 'DELETED' 
AND 
    scan_lvm_vg_host_uuid = ".$anvil->Database->quote($host_uuid)." 
ORDER BY 
    scan_lvm_vg_size ASC
;";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				
				my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
				my $count   = @{$results};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					results => $results, 
					count   => $count, 
				}});
				foreach my $row (@{$results})
				{
					my $scan_lvm_vg_internal_uuid = $row->[0];
					my $scan_lvm_vg_name          = $row->[1];
					my $scan_lvm_vg_size          = $row->[2];
					my $this_storage_group_uuid   = exists $anvil->data->{storage_groups}{vg_uuid}{$scan_lvm_vg_internal_uuid} ? $anvil->data->{storage_groups}{vg_uuid}{$scan_lvm_vg_internal_uuid}{storage_group_uuid} : "";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						scan_lvm_vg_internal_uuid => $scan_lvm_vg_internal_uuid, 
						scan_lvm_vg_name          => $scan_lvm_vg_name, 
						scan_lvm_vg_size          => $scan_lvm_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_lvm_vg_size}).")", 
						this_storage_group_uuid   => $this_storage_group_uuid, 
					}});
					
					# Is this VG in another SG?
					next if $needed_size > $scan_lvm_vg_size;
					next if $this_storage_group_uuid;
					
					# Still here? Use this one.
					my $difference = $scan_lvm_vg_size - $needed_size;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						difference => $difference." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $difference}).")", 
					}});
					
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0351", variables => { 
						storage_group_name => $storage_group_name,
						vg_name            => $scan_lvm_vg_name, 
					}});
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "log_0351,!!storage_group_name!".$storage_group_name."!!,!!vg_name!".$scan_lvm_vg_name."!!");
					
					# Update the SG. 
					my $query = "
UPDATE 
    storage_group_members 
SET 
    storage_group_member_vg_uuid = ".$anvil->Database->quote($scan_lvm_vg_internal_uuid).",
    modified_date                = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
    storage_group_member_uuid    = ".$anvil->Database->quote($old_storage_group_member_uuid)."
;";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
					$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
					
					# Reload the Storage Group data.
					$anvil->Database->get_storage_group_data({debug => 3});
					last;
				}
			}
		}
	}
	
	return(0);
}

# This looks for servers to add.
sub configure_servers
{
	my ($anvil) = @_;
	
	# Are we in the cluster? Can't resync a server until we and our peer are.
	my $short_host_name = $anvil->Get->short_host_name;
	my $host_uuid       = $anvil->Get->host_uuid;
	my $problem         = $anvil->Cluster->parse_cib({debug => 3});
	my $anvil_uuid      = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:short_host_name' => $short_host_name, 
		's2:host_uuid'       => $host_uuid, 
		's3:problem'         => $problem,
		's4:anvil_uuid'      => $anvil_uuid,
	}});
	
	# Is our peer offline? If it's online, do nothing
	if (($problem) or (not $anvil->data->{cib}{parsed}{'local'}{ready}))
	{
		print __LINE__.": We can't sync missing servers until we're in the cluster.\n";
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not $anvil->data->{cib}{parsed}{peer}{ready})
	{
		print __LINE__.": The peer is not (fully) in the cluster yet. Please wait until it is.\n";
		$anvil->nice_exit({exit_code => 1});
	}
	
	# How shall I connect to my peer?
	my $peer_name                              = $anvil->data->{cib}{parsed}{peer}{name};
	my $peer_host_uuid                         = $anvil->Get->host_uuid_from_name({host_name => $peer_name});
	my ($peer_target_ip, $peer_target_network) = $anvil->Network->find_target_ip({host_uuid => $peer_host_uuid});
	my $peer_short_host_name                   = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:peer_name'            => $peer_name, 
		's2:peer_host_uuid'       => $peer_host_uuid, 
		's3:peer_target_ip'       => $peer_target_ip, 
		's4:peer_target_network'  => $peer_target_network, 
		's5:peer_short_host_name' => $peer_short_host_name, 
	}});
	
	# Make sure LVM data is up to date.
	print __LINE__.": Running scan-lvm prior to checking Storage Groups.\n";
	$anvil->ScanCore->call_scan_agents({debug => 2, agent => "scan-lvm"});
	
	# Look to see if our VG(s) have changed from what's in the storage groups.
	$anvil->Database->get_lvm_data({debug => 3});
	$anvil->Database->get_storage_group_data({debug => 3});
	foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}})
	{
		my $storage_group_uuid        = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}{$storage_group_name}{storage_group_uuid};
		my $storage_group_member_uuid = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{storage_group_member_uuid};
		my $vg_internal_uuid          = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_internal_uuid};
		my $vg_size                   = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_size};
		my $peer_vg_size              = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$peer_host_uuid}{vg_size};
		my $storage_group_member_note = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{storage_group_member_note};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:storage_group_name'        => $storage_group_name, 
			's2:storage_group_uuid'        => $storage_group_uuid, 
			's3:storage_group_member_uuid' => $storage_group_member_uuid, 
			's4:vg_internal_uuid'          => $vg_internal_uuid, 
			's5:vg_size'                   => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")",
			's6:peer_vg_size'              => $peer_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $peer_vg_size}).")", 
			's7:storage_group_member_note' => $storage_group_member_note, 
		}});
		
		my $vg_found = 0;
		foreach my $scan_lvm_vg_name (sort {$a cmp $b} keys %{$anvil->data->{lvm}{host_name}{$short_host_name}{vg}})
		{
			next if $scan_lvm_vg_name eq "DELETED";
			my $scan_lvm_vg_internal_uuid = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{scan_lvm_vg_internal_uuid};
			my $scan_lvm_vg_size          = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{scan_lvm_vg_size};
			my $this_storage_group_uuid   = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{storage_group_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:scan_lvm_vg_name'          => $scan_lvm_vg_name, 
				's2:scan_lvm_vg_internal_uuid' => $scan_lvm_vg_internal_uuid, 
				's3:scan_lvm_vg_size'          => $scan_lvm_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_lvm_vg_size}).")", 
				's4:this_storage_group_uuid'   => $this_storage_group_uuid, 
			}});
			
			if ($vg_internal_uuid eq $scan_lvm_vg_internal_uuid)
			{
				# It's probably been marked as DELETED, so we need to find a replacement. 
				if ($scan_lvm_vg_name =~ /DELETED:vg_uuid=(.*)$/)
				{
					my $vg_uuid = $1;
					print __LINE__.": The VG with the ScanCore UUID: [".$vg_uuid."] was in the SG: [".$storage_group_name."], but is now deleted.\n";
					print __LINE__.": Will now try to find a replacement...\n";
				}
				else
				{
					print __LINE__.": The VG: [".$scan_lvm_vg_name."] is in the SG: [".$storage_group_name."], good.\n";
					$vg_found = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { vg_found => $vg_found }});
					last;
				}
			}
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { vg_found => $vg_found }});
		if (not $vg_found)
		{
			print __LINE__.": The volume group: [".$vg_internal_uuid."] that was in the storage group: [".$storage_group_name."] no longer exists.\n";
			print __LINE__.": [ Note ] - Checking for a replacement VG now.\n";
			print __LINE__.": [ Suggestion ] - You may need to run 'anvil-manage-host --auto-grow-pv' if this is a rebuilt sub node!\n";
			my $closest_size             = 0;
			my $closest_vg_uuid          = "";
			my $closest_vg_name          = "";
			my $closest_vg_internal_uuid = "";
			foreach my $scan_lvm_vg_name (sort {$a cmp $b} keys %{$anvil->data->{lvm}{host_name}{$short_host_name}{vg}})
			{
				next if $scan_lvm_vg_name eq "DELETED";
				my $scan_lvm_vg_uuid          = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{scan_lvm_vg_uuid};
				my $scan_lvm_vg_internal_uuid = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{scan_lvm_vg_internal_uuid};
				my $scan_lvm_vg_size          = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{scan_lvm_vg_size};
				my $this_storage_group_uuid   = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$scan_lvm_vg_name}{storage_group_uuid};
				next if not $this_storage_group_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:scan_lvm_vg_name'          => $scan_lvm_vg_name, 
					's2:scan_lvm_vg_uuid'          => $scan_lvm_vg_uuid, 
					's3:scan_lvm_vg_internal_uuid' => $scan_lvm_vg_internal_uuid, 
					's4:scan_lvm_vg_size'          => $scan_lvm_vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_lvm_vg_size}).")", 
					's5:this_storage_group_uuid'   => $this_storage_group_uuid, 
				}});
				
				if (($scan_lvm_vg_size >= $peer_vg_size) && ((not $closest_size) or ($scan_lvm_vg_size < $closest_size)))
				{
					$closest_size             = $scan_lvm_vg_size;
					$closest_vg_uuid          = $scan_lvm_vg_uuid;
					$closest_vg_name          = $scan_lvm_vg_name;
					$closest_vg_internal_uuid = $scan_lvm_vg_internal_uuid;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:closest_size'             => $closest_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $closest_size}).")", 
						's2:closest_vg_uuid'          => $closest_vg_uuid, 
						's3:closest_vg_name'          => $closest_vg_name, 
						's4:closest_vg_internal_uuid' => $closest_vg_internal_uuid, 
					}});
				}
			}
			
			if ($closest_vg_name)
			{
				# Found the closest match.
				my $difference = $peer_vg_size - $closest_size;
				if (not $difference)
				{
					print __LINE__.": The VG: [".$closest_vg_name."] is: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $closest_size})."], exactly matching our peer's VG size.\n";
				}
				else
				{
					print __LINE__.": The VG: [".$closest_vg_name."] is: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $closest_size})."], and the peer's VG size is: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $peer_vg_size})."] (we're: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $difference})."] smaller)\n";
				}
				print __LINE__.": Updating the storage group: [".$storage_group_name."] to use this VG instead.\n";
				$anvil->Database->insert_or_update_storage_group_members({
					debug                                   => 2,
					storage_group_member_uuid               => $storage_group_member_uuid,
					storage_group_member_note               => "auto-replaced",
					storage_group_member_storage_group_uuid => $storage_group_uuid,
					storage_group_member_host_uuid          => $host_uuid, 
					storage_group_member_vg_uuid            => $closest_vg_internal_uuid, 
				});
				
				# Refesh
				$anvil->Database->get_storage_group_data({debug => 3});
			}
			else
			{
				print __LINE__.": [ Error ] - No local VGs exist to use in this Storage Group!\n";
				print __LINE__.": [ Error ]   Unable to proceed.\n";
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}
	
	# Walk through the resources and any we find, see if they need to be sync'ed.
	$anvil->Database->get_servers({debug => 2});
	foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}})
	{
		my $status  = $anvil->data->{cib}{parsed}{data}{server}{$server}{status};
		my $active  = $anvil->data->{cib}{parsed}{data}{server}{$server}{active};
		my $blocked = $anvil->data->{cib}{parsed}{data}{server}{$server}{blocked};
		my $failed  = $anvil->data->{cib}{parsed}{data}{server}{$server}{failed};
		my $role    = $anvil->data->{cib}{parsed}{data}{server}{$server}{role};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server'  => $server, 
			's2:status'  => $status, 
			's3:active'  => $active, 
			's4:blocked' => $blocked,
			's5:failed'  => $failed, 
			's6:role'    => $role, 
		}});
		
		my $server_uuid = $anvil->Get->server_uuid_from_name({
			anvil_uuid  => $anvil_uuid,
			server_name => $server, 
		});
		my $server_definition = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:server_uuid'       => $server_uuid,
			's2:server_definition' => $server_definition, 
		}});
		$anvil->Server->parse_definition({
			debug      => 3,
			host       => $short_host_name,
			server     => $server, 
			source     => "from_db",
			definition => $server_definition, 
		});
		
		# Do we have the DRBD resource file?
		my $drbd_res_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server.".res";
		   $drbd_res_file =~ s/\/\//\//g;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_file => $drbd_res_file }});
		
		my $drbd_res_body = "";
		if (not -f $drbd_res_file)
		{
			# Try copying it from the peer.
			print __LINE__.": The DRBD resource file: [".$drbd_res_file."] was not found, copying it from the peer: [".$peer_name."]\n";
			$drbd_res_body = $anvil->Storage->read_file({
				file   => $drbd_res_file, 
				target => $peer_target_ip, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_body => $drbd_res_body }});
			
			if (($drbd_res_body) && ($drbd_res_body ne "!!error!!"))
			{
				# Write it out.
				print __LINE__.": The DRBD resource file was found, saving it locally.\n";
				$anvil->Storage->write_file({
					file  => $drbd_res_file, 
					body  => $drbd_res_body, 
					user  => "root", 
					group => "root",
					mode  => "644",
				});
				
				# Run scan-drbd.
				print __LINE__.": Running scan-drbd to pick up the new resource config.\n";
				my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code,
				}});
			}
			else
			{
				# Can't proceed.
				print __LINE__.": The DRBD resource file was NOT found on the peer, skipping this server for now.\n";
				next;
			}
		}
		
		# Read in the DRBD resource file
		$anvil->Database->get_drbd_data({debug => 2});
		
		my $drbd_res_xml = $anvil->data->{drbd}{host_name}{$short_host_name}{resource_name}{$server}{xml};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_xml => $drbd_res_xml }});
		print __LINE__.": Checking the resource file: [".$drbd_res_file."]\n";
		
		$anvil->DRBD->parse_resource({
			debug => 2, 
			xml   => $drbd_res_xml, 
		});
		
		$anvil->Database->get_lvm_data({debug => 2});
		foreach my $volume (sort {$a <=> $b} keys %{$anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}})
		{
			print __LINE__.": - Checking volume: [".$volume."]\n";
			my $volume_path        = $anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}{$volume}{device_path};
			my $local_lv_path      = $anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}{$volume}{backing_disk};
			my $peer_lv_path       = $anvil->data->{new}{resource}{$server}{host_uuid}{$peer_host_uuid}{volume_number}{$volume}{backing_disk};
			my $meta_disk          = $anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}{$volume}{meta_disk};
			my $volume_size        = $anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}{$volume}{size};	# Will be '0' if the resource is down.
			my $storage_group_uuid = $anvil->data->{new}{resource}{$server}{host_uuid}{$host_uuid}{volume_number}{$volume}{storage_group_uuid};
			my $lv_name            = ($local_lv_path =~ /^.*\/(.*)$/)[0];
			my $local_lv_size      = $anvil->data->{lvm}{host_name}{$short_host_name}{lv}{$lv_name}{scan_lvm_lv_size} // 0;
			my $peer_lv_size       = $anvil->data->{lvm}{host_name}{$peer_short_host_name}{lv}{$lv_name}{scan_lvm_lv_size} // 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's01:volume'             => $volume, 
				's02:volume_path'        => $volume_path, 
				's03:local_lv_path'      => $local_lv_path, 
				's04:peer_lv_path'       => $peer_lv_path, 
				's05:meta_disk'          => $meta_disk, 
				's06:volume_size'        => $volume_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size}).")", 
				's07:storage_group_uuid' => $storage_group_uuid, 
				's08:lv_name'            => $lv_name, 
				's09:local_lv_size'      => $local_lv_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $local_lv_size}).")", 
				's10:peer_lv_size'       => $peer_lv_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $peer_lv_size}).")", 
			}});
			
			if (not $storage_group_uuid)
			{
				# The storage group is found by the LV. That's not likely to work, so if it 
				# didn't, look it up by the volume group.
				my $query = "
SELECT 
    d.storage_group_uuid 
FROM 
    scan_lvm_lvs a, 
    scan_lvm_vgs b, 
    storage_group_members c, 
    storage_groups d 
WHERE 
    a.scan_lvm_lv_path = ".$anvil->Database->quote($peer_lv_path)." 
AND 
    a.scan_lvm_lv_on_vg = b.scan_lvm_vg_name 
AND 
    b.scan_lvm_vg_internal_uuid = c.storage_group_member_vg_uuid 
AND 
    c.storage_group_member_storage_group_uuid = d.storage_group_uuid 
AND 
    a.scan_lvm_lv_name != 'DELETED'
AND 
    b.scan_lvm_vg_name != 'DELETED' 
AND 
    d.storage_group_name != 'DELETED'
;";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				
				my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
				my $count   = @{$results};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					results => $results, 
					count   => $count, 
				}});
				
				if ($count == 1)
				{
					$storage_group_uuid = $results->[0]->[0];
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { storage_group_uuid => $storage_group_uuid }});
				}
				else
				{
					print __LINE__.": [ Error ] - There is storage group found for this volume!\n";
					print __LINE__.": [ Suggestion ] - You may need to run 'anvil-manage-host --auto-grow-pv' if this is a rebuilt sub node!\n";
					$anvil->nice_exit({exit_code => 1});
				}
			}
			
			if ($peer_lv_size > $local_lv_size)
			{
				$local_lv_size = $peer_lv_size;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's01:local_lv_size' => $local_lv_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $local_lv_size}).")", 
				}});
			}
			
			my $new_lv_path = "";
			if (not -e $local_lv_path)
			{
				print __LINE__.": - The backing LV: [".$local_lv_path."] doesn't exist.\n";
				my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
				my $vg_internal_uuid   = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_internal_uuid} // "";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:storage_group_name' => $storage_group_name, 
					's2:vg_internal_uuid'   => $vg_internal_uuid, 
				}});
				
				if (not $vg_internal_uuid)
				{
					print __LINE__.": [ Error ] - There is no volume group for the storage group: [".$storage_group_name."]!\n";
					print __LINE__.": [ Suggestion ] - You may need to run 'anvil-manage-host --auto-grow-pv' if this is a rebuilt sub node!\n";
					$anvil->nice_exit({exit_code => 1});
					next;
				}
				
				my $vg_name        = $anvil->data->{lvm}{vg_internal_uuid}{$vg_internal_uuid}{scan_lvm_vg_name};
				my $vg_free        = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$vg_name}{scan_lvm_vg_free};
				my $vg_extent_size = $anvil->data->{lvm}{host_name}{$short_host_name}{vg}{$vg_name}{scan_lvm_vg_extent_size};
				my $extent_count   = $local_lv_size / $vg_extent_size;
				   $new_lv_path    = "/dev/".$vg_name."/".$lv_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:vg_name'        => $vg_name, 
					's2:vg_free'        => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", 
					's3:vg_extent_size' => $vg_extent_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_extent_size}).")", 
					's4:extent_count'   => $extent_count." (".$anvil->Convert->add_commas({number => $extent_count}).")", 
					's5:new_lv_path'    => $new_lv_path, 
				}});
				
				if (-e $new_lv_path)
				{
					# The new LV already exists, a previous rebuild might have been 
					# interrupted.
					print __LINE__.": The backing LV: [".$local_lv_path."] doesn't exist, and the new LV: [".$new_lv_path."] does exist.\n";
					print __LINE__.": - Was a previous rebuild interrupted? Proceeding with the new LV path.\n";
				}
				else
				{
					# Round up if needed.
					if ($extent_count =~ /^(\d+)\./)
					{
						$extent_count = $1 + 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							'extent_count' => $extent_count." (".$anvil->Convert->add_commas({number => $extent_count}).")", 
						}});
					}
					
					# Can we create the LV?
					print __LINE__.": The local backing LV: [".$local_lv_path."] is missing, we'll recreate it.\n";
					print __LINE__.": - The new LV: [".$new_lv_path."] will be: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $local_lv_size})."] from the storage group: [".$storage_group_name."].\n";
					print __LINE__.": - We'll use the VG: [".$vg_name."] which has: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free})."] free.\n";
					
					### NOTE: We use '--yes' because on rebuilds, if the geometry lines up to 
					###       previous LVs, we could get a prompt to remove the old drbd 
					###       signature. There's no way to determine if it's good or useful, so 
					###       we want a full resync.
					my $shell_call = $anvil->data->{path}{exe}{lvcreate}." --yes -l ".$extent_count." -n ".$lv_name." ".$vg_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code,
					}});
					
					# Run scan-lvm
					if ($return_code)
					{
						print __LINE__.": [ Warning ] - The call to create the LV returned the return code: [".$return_code."] (0 expected).\n";
						print __LINE__.": [ Warning ]   The output, if any, was:\n";
						print __LINE__.": =========================\n";
						print $output;
						print __LINE__.": =========================\n";
					}
					else
					{
						print __LINE__.": [ Success ] - The LV apprears to have been created!\n";
					}
					print __LINE__.": - Rescanning the LVM data to pickup the new status, please be patient.\n";
					$anvil->ScanCore->call_scan_agents({debug => 2, agent => "scan-lvm"});
					
					# refresh
					$anvil->Database->get_lvm_data({debug => 3});
					$anvil->Database->get_storage_group_data({debug => 3});
				}
			}
			
			# If the LV name has changed, update DRBD's resource file.
			if (($new_lv_path) && ($new_lv_path ne $local_lv_path))
			{
				print __LINE__.": [ Note ] - The LV path has changed! Updating the DRBD resource file.\n";
				### TODO: This needs to be verified!
				my $old_body = $anvil->Storage->read_file({
					file   => $drbd_res_file, 
					target => $peer_target_ip, 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_body => $old_body }});
				
				if (not $drbd_res_body)
				{
					$drbd_res_body = $anvil->Storage->read_file({
						file   => $drbd_res_file, 
						target => $peer_target_ip, 
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_res_body => $drbd_res_body }});
				}
				
				if ((not $drbd_res_body) && ($drbd_res_body ne "!!error!!"))
				{
					# Failed to read the res file
					print __LINE__.": [ Error ] - Failed to read the DRBD resource file: [".$drbd_res_file."] locally or from the peer using: [".$peer_target_ip."].\n";
					$anvil->nice_exit({exit_code => 1});
				}
				
				# loop through, replacing the old LV with the new one.
				my $hosts    = [];
				my $new_body = "";
				foreach my $line (split/\n/, $drbd_res_body)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					
					if ($line =~ /$local_lv_path/)
					{
						$line =~ s/$local_lv_path/$new_lv_path/g;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					}
					
					if ($line =~ /\s.*?on (.*) \{/)
					{
						my $host = $1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host }});
						
						push @{$hosts}, $host;
					}
					
					$new_body .= $line."\n";
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_body => $new_body }});
				
				my $difference = diff \$drbd_res_body, \$new_body, { STYLE => 'Unified' };
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
				
				if ($difference)
				{
					### TODO: We need to have a job to update the peers 
					###       if the peer isn't online when this runs.
					# Write it out on the peer(s);
					foreach my $host (@{$hosts})
					{
						my $problem = $anvil->Storage->write_file({
							file      => $drbd_res_file, 
							body      => $new_body, 
							user      => "root", 
							group     => "root",
							mode      => "644",
							target    => $host,
							overwrite => 1, 
							backup    => 1,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
						
						if ($problem)
						{
							print __LINE__.": [ Warning ] - Failed to update the DRBD resource file: [".$drbd_res_file."] on the host: [".$host."]\n";
							print __LINE__.": [ Warning ] - Please copy the version from this machine as soon as possible!\n";
						}
						else
						{
							print __LINE__.": [ Success ] - Successfully updated: [".$drbd_res_file."] to update the backing disk.\n";
						}
					}
					
					# Now adjust on all hosts.
					foreach my $host (@{$hosts})
					{
						print __LINE__.": Asking the host: [".$host."] to update the configuration.\n";
						my $return_code = $anvil->DRBD->manage_resource({
							debug    => 2,
							task     => "adjust",
							resource => $server, 
							target   => $host,
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
					}
				}
				
				# Now update the LV path.
				$local_lv_path = $new_lv_path;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_lv_path => $local_lv_path }});
			}
			
			# Check if there is metadata on the backing disk.
			my $shell_call = $anvil->data->{path}{exe}{drbdadm}." get-gi ".$server."/".$volume;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code,
			}});
			
			if (not $output)
			{
				print __LINE__.": It appears that there is no metadata on the backing disk, creating it now.\n";
				my $shell_call = $anvil->data->{path}{exe}{drbdadm}." create-md --force ".$server."/".$volume;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code,
				}});
				
				if ($return_code)
				{
					print __LINE__.": [ Error ] - There appears to have been a problem creating the metadata!\n";
					print __LINE__.": [ Error ]   There was a non-zero return code: [".$return_code."]. The output, if any, was:\n";
					print __LINE__.": =========================\n";
					print $output;
					print __LINE__.": =========================\n";
				}
				else
				{
					print __LINE__.": [ Success ] - The metadata was created!\n";
				}
			}
			elsif ($output =~ /^(.*?):/)
			{
				print __LINE__.": Metadata appears to exist already.\n";
			}
			
			# Call adjust to try to connect the disk.
			print __LINE__.": Calling 'drbdadm adjust ".$server."' to make sure we connect to our peer, if possible.\b";
			$shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code,
			}});
			
			print __LINE__.": - Done!\n";
		}
	}
	
	return(0);
}

# Rejoin this host to an existing subnode.
sub process_rejoin
{
	my ($anvil) = @_;
	
	print __LINE__.": -=] Rejoin this host: [".$anvil->Get->short_host_name."] as a subnode in an existing Anvil! node.\n";
	my $host_uuid     = $anvil->Get->host_uuid();
	my $as_machine    = $anvil->data->{switches}{'as-machines'};
	my $manifest      = $anvil->data->{switches}{manifest};
	my $manifest_name = "";
	my $manifest_uuid = "";
	my $anvil_uuid    = "";
	my $rebuild       = 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:host_uuid'  => $host_uuid, 
		's2:as_machine' => $as_machine,
		's3:manifest'   => $manifest, 
	}});
	
	# If ScanCore isn't running, run it once.
	print __LINE__.": Checking if ScanCore is running.\n";
	my $scancore_running = $anvil->System->check_daemon({daemon => "scancore.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scancore_running => $scancore_running }});
	if ($scancore_running)
	{
		print __LINE__.": - ScanCore is running, proceeding.\n";
	}
	else
	{
		print __LINE__.": - [ Note ] - ScanCore is NOT running!\n";
		print __LINE__.":   [ Note ] - Running once to make sure the data in the database is up to date.\n";
		print __LINE__.":   [ Note ] - If the install has problems, verify that ScanCore is running on the peer, also.\n";
		print __LINE__.":   [ Note ] - Please wait, this can take a minute...\n";
		my $shell_call = $anvil->data->{path}{exe}{scancore}.$anvil->Log->switches()." --run-once";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		print __LINE__.": - ScanCore run complete!\n";
	}
	
	# Load data
	$anvil->Database->get_hosts();
	$anvil->Database->get_anvils();
	$anvil->Database->get_manifests();
	
	# In case we're being re-installed, see if we can find our own data.
	my $old_manifest_uuid = "";
	if (exists $anvil->data->{anvils}{host_uuid}{$host_uuid})
	{
		# Found it.
		my $anvil_name = $anvil->data->{anvils}{host_uuid}{$host_uuid}{anvil_name};
		   $anvil_uuid = $anvil->data->{anvils}{host_uuid}{$host_uuid}{anvil_uuid};
		my $old_role   = $anvil->data->{anvils}{host_uuid}{$host_uuid}{role};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:anvil_name' => $anvil_name,
			's2:anvil_uuid' => $anvil_uuid, 
		}});
		print __LINE__.": This host used to be in the Anvil! [".$anvil_name."] as: [".$old_role."]\n";
		if (exists $anvil->data->{manifests}{manifest_name}{$anvil_name})
		{
			# Found the manifest.
			$old_manifest_uuid = $anvil->data->{manifests}{manifest_name}{$anvil_name}{manifest_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_manifest_uuid => $old_manifest_uuid }});
			print __LINE__.": The manifest was found with the UUID: [".$old_manifest_uuid."]\n";
		}
		else
		{
			# Didn't find the manifest, something is wrong.
			print __LINE__.": The manifest was NOT found (was it deleted?).\n";
			print __LINE__.": - The manifes must be created for this host to rejoin the Anvil! node.\n";
			print __LINE__.": - Alternatively, this host must be removed for the Anvil!\n";
			print __LINE__.": [ Error ] - Unable to proceed at this time.\n";
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Does it match? 
		if (($manifest) && (($manifest ne $anvil_name) && ($manifest ne $old_manifest_uuid)))
		{
			# They asked for this machine to be joined to a different Anvil!.
			print __LINE__.": [ Error ] - You asked to join the Anvil! node: [".$manifest."].\n";
			print __LINE__.": [ Error ] - This host must be removed from: [".$anvil_name."] first.\n";
			print __LINE__.": [ Error ] - Unable to proceed at this time.\n";
			$anvil->nice_exit({exit_code => 1});
		}
		
		if (($as_machine) && ($as_machine ne $old_role))
		{
			print __LINE__.": [ Error ] - You asked to join the Anvil! node: [".$manifest."] as: [".$as_machine."]\n";
			print __LINE__.": [ Error ] - This host was previously: [".$old_role."], so it needs to be that role again.\n";
			print __LINE__.": [ Error ] - Alternatively, remove this host from that Anvil! and try again.\n";
			print __LINE__.": [ Error ] - Unable to proceed at this time.\n";
			$anvil->nice_exit({exit_code => 1});
		}
		
		$manifest   = $anvil_name;
		$as_machine = $old_role;
		$rebuild    = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:manifest'   => $manifest,
			's2:as_machine' => $as_machine, 
			's3:rebuild'    => $rebuild, 
		}});
	}
	
	# Does this host need to be reconfigured? If so, register a job, if one doesn't exist yet, and exit.
	my $configured = $anvil->System->check_if_configured({
		debug    => 2,
		thorough => 1,
	});
	if ($configured)
	{
		print __LINE__.": The host appears to already be configured, good.\n";
	}
	else
	{
		print __LINE__.": [ Note ] - The host appears to NOT be configured yet!\n";
		if (exists $anvil->data->{anvils}{host_uuid}{$host_uuid})
		{
			# Can we find the data needed to reconfigure this subnode?
			my $job_uuid = create_anvil_configure_host_job($anvil, $host_uuid);
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
			if ($job_uuid)
			{
				print __LINE__.": [ Success ] - Found the data needed to configure the network on this subnode! The anvil-join-anvil job has\n";
				print __LINE__.": [ Success ]   been created with the job UUID: [".$job_uuid."]\n";
				print __LINE__.": [ Note ] - This subnode will reboot one minute after the network configuration completes. Once rebooted,\n";
				print __LINE__.": [ Note ]   please run this again.\n";
				$anvil->nice_exit({exit_code => 0});
			}
			else
			{
				print __LINE__.": [ Warning ] - Failed to automatically create the anvil-join-anvil job.\n";
				print __LINE__.": [ Warning ] - Please use Striker to initialize this host. Once the network has been configured, try rejoining\n";
				print __LINE__.": [ Warning ]   this subnode again.\n";
				$anvil->nice_exit({exit_code => 1});
			}
		}
		else
		{
			# We can't configure it automatically.
			print __LINE__.": [ Warning ] - This is a new host, so the information needed to configure the network does not exist.\n";
			print __LINE__.": [ Warning ] - Please use Striker to initialize this host. Once the network has been configured, try rejoining\n";
			print __LINE__.": [ Warning ]   this subnode again.\n";
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	if ($manifest)
	{
		# Did we get a valid manifest?
		if (exists $anvil->data->{manifests}{manifest_name}{$manifest})
		{
			$manifest_name = $manifest;
			$manifest_uuid = $anvil->data->{manifests}{manifest_name}{$manifest}{manifest_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:manifest_name' => $manifest_name,
				's2:manifest_uuid' => $manifest_uuid, 
			}});
		}
		elsif (exists $anvil->data->{manifests}{manifest_uuid}{$manifest})
		{
			$manifest_name = $anvil->data->{manifests}{manifest_uuid}{$manifest}{manifest_name};
			$manifest_uuid = $manifest;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:manifest_name' => $manifest_name,
				's2:manifest_uuid' => $manifest_uuid, 
			}});
		}
		else
		{
			print __LINE__.": [ Error ] - The manifest: [".$manifest."] was not found in the database.\n";
			print __LINE__.": [ Error ] - Try again without this switch to see the available manifests.\n"; 
			$anvil->nice_exit({exit_code => 1});
		}
	}
	else
	{
		# Show the existing manifests.
		my $count = keys %{$anvil->data->{manifests}{manifest_name}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
		foreach my $this_manifest_name (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_name}})
		{
			my $this_manifest_uuid = $anvil->data->{manifests}{manifest_name}{$this_manifest_name}{manifest_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:this_manifest_name' => $this_manifest_name,
				's2:this_manifest_uuid' => $this_manifest_uuid, 
			}});
			if ($count == 1)
			{
				$manifest_name = $this_manifest_name;
				$manifest_uuid = $this_manifest_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:manifest_name' => $manifest_name,
					's2:manifest_uuid' => $manifest_uuid, 
				}});
			}
			else
			{
				print __LINE__.": - Manifest: [".$manifest_name."] (uuid: [".$manifest_uuid."])\n";
			}
		}
		print __LINE__.": - Which manifest do you want to join this host to?\n";
		my $answer = <STDIN>;
		chomp($answer);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		# Did we get a valid manifest?
		if (exists $anvil->data->{manifests}{manifest_name}{$answer})
		{
			$manifest_name = $answer;
			$manifest_uuid = $anvil->data->{manifests}{manifest_name}{$answer}{manifest_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:manifest_name' => $manifest_name,
				's2:manifest_uuid' => $manifest_uuid, 
			}});
		}
		elsif (exists $anvil->data->{manifests}{manifest_uuid}{$answer})
		{
			$manifest_name = $anvil->data->{manifests}{manifest_uuid}{$answer}{manifest_name};
			$manifest_uuid = $answer;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:manifest_name' => $manifest_name,
				's2:manifest_uuid' => $manifest_uuid, 
			}});
		}
		else
		{
			if ($answer)
			{
				print __LINE__.": [ Error ] - Your answer: [".$answer."] doesn't match a valid manifest.\n"; 
			}
			print __LINE__.": [ Error ] - Please try again (you can use --manifest <name or UUID> to avoind this prompt).\n"; 
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	if (not $as_machine)
	{
		print __LINE__.": - Will this node be 'node1' or 'node2'?\n";
		my $answer = <STDIN>;
		chomp($answer);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		if ($answer eq "node1")
		{
			$as_machine = "node1";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { as_machine => $as_machine }});
		}
		elsif ($answer eq "node2")
		{
			$as_machine = "node2";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { as_machine => $as_machine }});
		}
		else
		{
			print __LINE__.": [ Error ] - Please enter 'node1' or 'node2'.\n";
			print __LINE__.": [ Error ] - Please try again (you can use '--as-manchine node{1,2}' to avoind this prompt).\n"; 
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	if (not $anvil_uuid)
	{
		$anvil_uuid = $anvil->data->{anvils}{anvil_name}{$manifest_name}{anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		if (not $anvil_uuid)
		{
			print __LINE__.": [ Error ] - Failed to find an Anvil! UUID for the node: [".$manifest_name."].\n";
			print __LINE__.": [ Error ] - Has the Anvil! been deleted from the database?\n"; 
			print __LINE__.": [ Error ] - Unable to proceed.\n"; 
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	print __LINE__.": \n";
	print __LINE__.": -=] Joining: [".$manifest_name."] as: [".$as_machine."] (Anvil! UUID: [".$anvil_uuid."]\n";
	if ($rebuild)
	{
		print __LINE__.": [ Note ] - This is a rebuild, the previous data recorded by this host will be\n";
		print __LINE__.":            preserved.\n";
	}
	else
	{
		print __LINE__.": [ Warning ] - This will replace the old subnode, and all the previous data\n";
		print __LINE__.":               associated with it!\n";
		print __LINE__.": [ Warning ] - Be certain the old host will NOT come back! If it does, it can\n";
		print __LINE__.":               cause confusion with the Anvil! node!\n";
	}
	if (($anvil->data->{switches}{y}) or ($anvil->data->{switches}{yes}))
	{
		print __LINE__.": [ Note ] - Confirmed by switch, proceeding.\b";
	}
	else
	{
		print $anvil->Words->string({key => "message_0021"})." ";
		my $answer = <STDIN>;
		chomp($answer);
		if ($answer !~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	
	# If this isn't a rebuild, purge the old host.
	if (not $rebuild)
	{
		my $node_key      = $as_machine eq "node1" ? "anvil_node1_host_uuid" : "anvil_node2_host_uuid";
		my $old_host_uuid = $anvil->data->{anvils}{anvil_name}{$manifest_name}{$node_key} ? $anvil->data->{anvils}{anvil_name}{$manifest_name}{$node_key} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node_key'      => $node_key,
			's2:old_host_uuid' => $old_host_uuid,
		}});
		
		if ($old_host_uuid)
		{
			print __LINE__.": [ Note ] - Purging the old host: [".$old_host_uuid."] from the database.\n";
			print __LINE__.": [ Note ] - Please be patient!\n";
			my $shell_call = $anvil->data->{path}{exe}{'striker-purge-target'}."  ".$anvil->Log->switches;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
		}
	}
	
	# Register a job.
	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
		job_host_uuid   => $host_uuid, 
		job_command     => $anvil->data->{path}{exe}{'anvil-join-anvil'}.$anvil->Log->switches, 
		job_data        => "as_machine=".$as_machine.",manifest_uuid=".$manifest_uuid.",anvil_uuid=".$anvil_uuid.",rejoin=true", 
		job_name        => "join_anvil::".$as_machine, 
		job_title       => "job_0072", 
		job_description => "job_0073", 
		job_progress    => 0,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});

	print __LINE__.": Registering a job. This host should (re)join the Anvil! node shortly.\n";
	
	return(0);
}

sub create_anvil_configure_host_job
{
	my ($anvil, $host_uuid) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
	
	# This is just to make the job_data cleaner.
	if (exists $anvil->data->{job_vars})
	{
		delete $anvil->data->{job_vars};
	}
	if (exists $anvil->data->{iface_map})
	{
		delete $anvil->data->{iface_map};
	}
	
	# Load the network interface data to validate/update the MACs to interface names. If the subnode was
	# rebuilt, the device name could have reverted. So we walk through the history to see if the same MAC
	# used to have a name we recognize.
	my $query = "
SELECT
    network_interface_uuid, 
    network_interface_mac_address, 
    network_interface_name, 
    network_interface_device, 
    round(extract(epoch from modified_date))  
FROM 
    history.network_interfaces 
WHERE 
    network_interface_operational != 'DELETED' 
AND 
    network_interface_host_uuid = ".$anvil->Database->quote($host_uuid)."
ORDER BY 
    modified_date DESC
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $network_interface_uuid = $row->[0];
		my $mac_address            = $row->[1];
		my $name                   = $row->[2];
		my $device                 = $row->[3];
		my $modified_date_unix     = $row->[4];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:network_interface_uuid" => $network_interface_uuid, 
			"s2:mac_address"            => $mac_address, 
			"s3:name"                   => $name, 
			"s4:device"                 => $device, 
			"s5:modified_date_unix"     => $modified_date_unix, 
		}});
		
		$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address} = $mac_address;
		$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name}	= $name;
		$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device}      = $device;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::mac_address" => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address},
			"s2:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::name"        => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name},
			"s3:iface_map::${network_interface_uuid}::timestamp::${modified_date_unix}::device"      => $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device},
		}});
	}
	
	# Load the variables, if not already done so.
	if (not exists $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid})
	{
		my $short_host_name = $anvil->Get->short_host_name();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { short_host_name => $short_host_name }});
		
		$anvil->Network->get_ips({debug => 3, target => $short_host_name});
		$anvil->Network->collect_data({debug => 3});
		$anvil->Database->get_variables({debug => 2});
	}
	
	# The variables were already read in, so it's easy to recreate the job.
	my $form_data_found = 0;
	foreach my $variable_uuid (sort {$a cmp $b} keys %{$anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}})
	{
		my $variable_name  = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_name};
		my $variable_value = $anvil->data->{variables}{source_table}{hosts}{source_uuid}{$host_uuid}{variable_uuid}{$variable_uuid}{variable_value};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:variable_uuid"  => $variable_uuid,
			"s2:variable_name"  => $variable_name, 
			"s3:variable_value" => $variable_value, 
		}});
		next if $variable_name !~ /^form::config_step\d::/;
		
		if (not $form_data_found)
		{
			$form_data_found = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { form_data_found => $form_data_found }});
		}
		
		# If the config value is the MAC address, check to see if there was a change since the
		# original variables were recorded.
		if ($variable_name =~ /form::config_step2::(.*?)_mac_to_set::value/)
		{
			my $interface = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface => $interface }});
			
			my $iface_found = "";
			my $mac_found   = "";
			foreach my $network_interface_uuid (sort {$a cmp $b} keys %{$anvil->data->{iface_map}})
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_interface_uuid => $network_interface_uuid }});
				foreach my $modified_date_unix (sort {$b <=> $a} keys %{$anvil->data->{iface_map}{$network_interface_uuid}{timestamp}})
				{
					my $mac_address = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{mac_address};
					my $name        = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{name};
					my $device      = $anvil->data->{iface_map}{$network_interface_uuid}{timestamp}{$modified_date_unix}{device};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						#"s1:modified_date_unix" => $modified_date_unix, 
						"s2:mac_address"        => $mac_address, 
						"s3:name"               => $name, 
						"s4:device"             => $device, 
					}});
					
					if ((not $iface_found) && ($device eq $interface))
					{
						$iface_found = $mac_address;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { iface_found => $iface_found }});
					}
					if ((not $iface_found) && ($name eq $interface))
					{
						$iface_found = $mac_address;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { iface_found => $iface_found }});
					}
					if ((not $mac_found) && ($mac_found eq $variable_value))
					{
						$mac_found = $interface;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mac_found => $mac_found }});
					}
				}
			}
			
			if ((not $iface_found) && (not $mac_found))
			{
				# Abort, we can't find this interface by name or MAC address.
				print __LINE__.": [ Error ] - The interface: [".$interface."] which was originally assigned to the device with the MAC\n";
				print __LINE__.": [ Error ]   address: [".$variable_value."] could not be found. We checked for the interface name in the\n";
				print __LINE__.": [ Error ]   'network_interfaces' historical schema, in case the interface was moved to a new NIC in the\n";
				print __LINE__.": [ Error ]   past. You will need to use the Striker UI (Striker logo -> Anvil -> Prepare Network) to \n";
				print __LINE__.": [ Error ]   initialize the network for this node before it can rejoin the cluster.\n";
				return("");
			}
			# If I found the mac address, we're done. If not, but we found the interface, update
			# the variable_value to the new MAC address.
			if (not $iface_found)
			{
				$variable_value = $mac_found;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_value => $variable_value }});
			}
		}
		
		$anvil->data->{job_vars}{$variable_name} = $variable_value;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"job_vars::${variable_name}" => $anvil->data->{job_vars}{$variable_name}, 
		}});
	}
	
	if (not $form_data_found)
	{
		print __LINE__.": [ Error ] - It appears that the data used to configure this node is no longer stored in the database.\n";
		print __LINE__.": [ Error ]   You will need to use the Striker UI (Striker logo -> Anvil -> Prepare Network) to initialize\n";
		print __LINE__.": [ Error ]   the network for this node before it can rejoin the cluster.\n";
		return("");
	}
	
	my $job_lines = 0;
	my $job_data  = "";
	foreach my $variable_name (sort {$a cmp $b} keys %{$anvil->data->{job_vars}})
	{
		$job_data .= $variable_name."=".$anvil->data->{job_vars}{$variable_name}."\n";
		$job_lines++;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:job_lines" => $job_lines, 
			"s2:job_data"  => $job_data,
		}});
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_data => $job_data }});
	my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
		job_command     => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, 
		job_data        => $job_data, 
		job_name        => "configure::network", 
		job_title       => "job_0001", 
		job_description => "job_0071", 
		job_progress    => 0,
		job_host_uuid   => $host_uuid,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});	
	
	return($job_uuid)
}

# Make sure the hosts file has entries for all nets for both subnodes
sub wait_for_etc_hosts
{
	my ($anvil) = @_;
	
	my $anvil_uuid           = $anvil->data->{sys}{anvil_uuid};
	my $node1_host_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $manifest_uuid        = $anvil->data->{sys}{manifest_uuid};
	my $i_am                 = $anvil->data->{sys}{machine};
	my $peer_is              = $i_am eq "node1" ? "node2" : "node1";
	my $peer_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$peer_is."_host_uuid"};
	my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:anvil_uuid'           => $anvil_uuid, 
		's2:manifest_uuid'        => $manifest_uuid, 
		's3:node1_host_uuid'      => $node1_host_uuid, 
		's4:node2_host_uuid'      => $node2_host_uuid, 
		's5:i_am'                 => $i_am,
		's6:peer_is'              => $peer_is,
		's7:peer_host_uuid'       => $peer_host_uuid, 
		's8:peer_short_host_name' => $peer_short_host_name, 
	}});
	
	my $problem = $anvil->Striker->load_manifest({debug => 2, manifest_uuid => $manifest_uuid});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
		problem       => $problem, 
	}});
	if ($problem)
	{
		# Something went wrong, fatally. Abort the job.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Create the list of host names we need to see in /etc/hosts and the hostnames to use in corosync.
	foreach my $machine (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}})
	{
		my $this_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$machine."_host_uuid"};
		my $this_short_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $bcn_name             = "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:machine'              => $machine,
			's2:this_host_uuid'       => $this_host_uuid, 
			's3:this_short_host_name' => $this_short_host_name, 
		}});
		foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
		{
			my $host_name  = $this_short_host_name.".".$network_name;
			my $ip_address = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network_name}{ip};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:network_name' => $network_name,
				's2:host_name'    => $host_name, 
				's3:ip_address'   => $ip_address,
			}});
			
			$anvil->data->{networks}{$host_name}{ip_address} = $ip_address;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s1:networks::${host_name}::ip_address" => $anvil->data->{networks}{$host_name}{ip_address},
			}});
			
			# If this is the first BCN, record it's name (in case BCN1 isn't used here) and IP 
			# address, and add the short and full host names.
			if (($network_name =~ /^bcn/) && (not $bcn_name))
			{
				   $bcn_name       = $network_name;
				my $this_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{host_name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bcn_name => $bcn_name }});
				
				$anvil->data->{networks}{$this_short_host_name}{ip_address} = $ip_address;
				$anvil->data->{networks}{$this_host_name}{ip_address}       = $ip_address;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:networks::${this_short_host_name}::ip_address" => $anvil->data->{networks}{$this_short_host_name}{ip_address},
					"s2:networks::${this_host_name}::ip_address"       => $anvil->data->{networks}{$this_host_name}{ip_address},
				}});
			}
		}
	}
	
	my $waiting = 1;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0477");
	while($waiting)
	{
		# Update the /etc/hosts file. Note that this doesn't add hosts that are not yet trusted, so 
		# anything not found will be inserted here.
		$anvil->System->update_hosts({debug => 2});
		
		# Now lets see if all expected hosts names are in the /etc/hosts file yet.
		my $ready      = 1;
		my $hosts_file = $anvil->Storage->read_file({
			file       => $anvil->data->{path}{configs}{hosts},
			force_read => 1,
			cache      => 0,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts_file => $hosts_file }});
		
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{networks}})
		{
			my $found      = 0;
			my $ip_address = $anvil->data->{networks}{$host_name}{ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name'  => $host_name,
				's2:ip_address' => $ip_address, 
			}});
			
			foreach my $line (split/\n/, $hosts_file)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				
				$line =~ s/#.*$//;
				if ($line =~ /^(\d.*?)\s+(.*)$/)
				{
					my $this_ip = $1;
					my $hosts   = $2;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						this_ip => $this_ip,
						hosts   => $hosts, 
					}});
					
					if ($anvil->Validate->ip({ip => $this_ip}))
					{
						foreach my $this_host (split/\s+/, $hosts)
						{
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_host => $this_host }});
							
							if ($host_name eq $this_host)
							{
								my $variables = {
									host_name  => $host_name, 
									ip_address => $ip_address,
									found_ip   => $this_ip,
								};
								if ($ip_address eq $this_ip)
								{
									# Found it
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0830", variables => $variables });
									$found = 1;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
								}
								else
								{
									# Found the host, but the IP doesn't match.
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0831", variables => $variables });
								}
							}
							last if $found;
						}
					}
				}
				last if $found;
			}
			
			# Is the network missing?
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
			if (not $found)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
				if ($host_name =~ /^(.*)\.(.*?)$/)
				{
					my $short_host_name = $1;
					my $network         = $2;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"s1:short_host_name"            => $short_host_name, 
						"s2:network"                    => $network,
					}});
					
					$anvil->data->{unused_network}{$network} = 0 if not defined $anvil->data->{unused_network}{$network};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"unused_network::${network}" => $anvil->data->{unused_network}{$network},
					}});
					
					if ($anvil->data->{unused_network}{$network})
					{
						# This network doesn't exist, don't wait for it.
						$found = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
						
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0122", variables => { 
							host_name => $host_name,
							network   => $network,
						}});
					}
				}
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
			if (not $found)
			{
				$ready = 0;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0832", variables => { host_name => $host_name }});
				
				# Add the IP to be added to /etc/hosts.
				$anvil->data->{to_add}{$host_name} = $ip_address;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "to_add::${host_name}" => $anvil->data->{to_add}{$host_name} }});
			}
		}
		
		if ($ready)
		{
			# Ready!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0833"});
			
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
		else
		{
			# Not ready, wait a bit.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0834"});
			sleep 5;
			
			# Try to update the /etc/hosts file
			$anvil->System->update_hosts({debug => 2});
			sleep 1;
		}
	}
	
	return(0);
}

sub wait_for_access
{
	my ($anvil) = @_;
	
	# NOTE: This logic is a copy of anvil-safe-start.
	$anvil->Database->get_hosts();
	$anvil->Database->get_anvils();
	my $anvil_uuid           = $anvil->Cluster->get_anvil_uuid();
	my $host_uuid            = $anvil->Get->host_uuid();
	my $node1_host_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid      = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $peer_host_uuid       = $host_uuid eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $short_host_name      = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name}      // "";
	my $peer_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name} // "";
	my $peer_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password}    // "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:anvil_uuid'           => $anvil_uuid, 
		's2:host_uuid'            => $host_uuid,
		's3:node1_host_uuid'      => $node1_host_uuid,
		's4:node2_host_uuid'      => $node2_host_uuid, 
		's5:short_host_name'      => $short_host_name, 
		's6:peer_host_uuid'       => $peer_host_uuid, 
		's7:peer_short_host_name' => $peer_short_host_name, 
		's8:peer_password'        => $anvil->Log->is_secure($peer_password), 
	}});
	
	if (not $peer_short_host_name)
	{
		die "Peer not found!\n";
	}
	
	my $waiting = 1;
	while ($waiting)
	{
		# This will get set back to '1' if 
		$waiting = 0;
		
		# Load IPs (again, to catch changes that might be delaying startup)
		$anvil->Network->load_ips({
			clear     => 1,
			host      => $short_host_name,
			host_uuid => $host_uuid, 
			
		});
		$anvil->Network->load_ips({
			clear     => 1,
			host      => $peer_short_host_name,
			host_uuid => $peer_host_uuid, 
			
		});
		
		# Loop through our interfaces and then loop our peers. Test access over them and set 
		# 'waiting' back to '1' if the connection fails.
		foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$short_host_name}{interface}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				interface => $interface,
				waiting   => $waiting, 
			}});
			
			# Only care about our networks.
			next if $waiting;
			if (not $anvil->Network->is_our_interface({interface => $interface}))
			{
				# Not an interface we care about
				next;
			}
			
			my $this_network = ($interface =~ /^(.*?)_/)[0];
			my $ip_address   = $anvil->data->{network}{$short_host_name}{interface}{$interface}{ip};
			my $subnet_mask  = $anvil->data->{network}{$short_host_name}{interface}{$interface}{subnet_mask};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:this_network' => $this_network, 
				's2:ip_address'   => $ip_address, 
				's3:subnet_mask'  => $subnet_mask, 
			}});
			
			### NOTE: I know I could match interface names, but that's not certain enough. It's 
			###       possible (if unlikely) that the network name+number differs on our peer. So
			###       this is safer.
			# Loop through my peer's interfaces and see if we're sharing this one.
			my $local_network    = NetAddr::IP->new($ip_address."/".$subnet_mask);
			my $peer_match_found = 0;
			foreach my $peer_interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$peer_short_host_name}{interface}})
			{
				last if $peer_match_found;
				my $peer_ip_address  = $anvil->data->{network}{$peer_short_host_name}{interface}{$peer_interface}{ip};
				my $peer_subnet_mask = $anvil->data->{network}{$peer_short_host_name}{interface}{$peer_interface}{subnet_mask};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer_interface   => $peer_interface, 
					peer_ip_address  => $peer_ip_address, 
					peer_subnet_mask => $peer_subnet_mask, 
				}});
				
				# This the matching network?
				next if $subnet_mask ne $peer_subnet_mask;
				
				my $peer_network = NetAddr::IP->new($peer_ip_address."/".$peer_subnet_mask);
				if ($peer_network->within($local_network))
				{
					# Match, test access.
					   $peer_match_found = 1;
					my $access           = $anvil->Remote->test_access({
						target   => $peer_ip_address,
						password => $peer_password,
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
					if ($access)
					{
						# This network is good.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0604", variables => {
							peer    => $peer_short_host_name, 
							network => $this_network, 
							peer_ip => $peer_ip_address,
						}});
						
						$anvil->data->{sys}{peer_target_ip} = $peer_ip_address;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							"sys::peer_target_ip" => $anvil->data->{sys}{peer_target_ip},
						}});
					}
					else
					{
						# No access, wait and try it again.
						$waiting = 1;
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0605", variables => {
							peer    => $peer_short_host_name, 
							network => $this_network, 
							peer_ip => $peer_ip_address,
						}});
						
						my $job_created = check_for_bad_key($anvil, $peer_ip_address);
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_created => $job_created }});
						
						if ($job_created)
						{
							# Sleep for ten seconds to give the job time to get 
							# picked up and run.
							sleep 10;
						}
					}
				} 
				
			}
		}
		
		if ($waiting)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0606", variables => { peer => $peer_short_host_name }});
			sleep 5;
			
			# Run anvil-manage-keys to see if we're failing to connect because of key changes.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0586"});
			
			### TODO: This should be safe, but it's worth revisiting with fresh eyes to be sure.
			my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-keys'}." --test --confirm".$anvil->Log->switches();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
		}
	}
	
	# All networks are up.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0607", variables => { peer => $peer_short_host_name }});
	
	return(0);
}

# This takes an IP or short host name and checks to see if a bad key exists in states. If so, it creates a 
# job to remove it.
sub check_for_bad_key
{
	my ($anvil, $peer) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { peer => $peer }});
	
	my $job_created = 0;
	my $state_name  = "host_key_changed::".$peer;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { state_name => $state_name }});
	if ((exists $anvil->data->{key_jobs}{$state_name}) && ($anvil->data->{key_jobs}{$state_name}))
	{
		# We've already processed this key, don't do it again.
		my $job_uuid = $anvil->data->{key_jobs}{$state_name};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0627", variables => { 
			peer     => $peer, 
			job_uuid => $job_uuid, 
		}});
		return($job_created);
	}
	
	my $query = "
SELECT 
    state_uuid, 
    state_note 
FROM 
    states 
WHERE 
    state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." 
AND 
    state_name      = ".$anvil->Database->quote($state_name)."
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		results => $results, 
		count   => $count, 
	}});
	
	if ($count == 1)
	{
		my $state_uuid = $results->[0]->[0];
		my $state_note = $results->[0]->[1];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			state_uuid => $state_uuid,
			state_note => $state_note, 
		}});
		
		my $bad_key = ($state_note =~ /key=(.*)$/)[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_key => $bad_key }});
		
		next if not $bad_key;
		
		### NOTE: Rationale for automating this; If a malicious actor can run this program, they can
		###       just as easily register a job to clear the keys directly. So this favours 
		###       usability.
		# Register a job to clean up the bad key.
		$anvil->data->{key_jobs}{$state_name} = $anvil->Database->insert_or_update_jobs({
			debug           => 2,
			file            => $THIS_FILE, 
			line            => __LINE__, 
			job_title       => "job_0056",
			job_description => "job_0057",
			job_host_uuid   => $anvil->Get->host_uuid, 
			job_data        => "bad_key=".$bad_key,
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-keys'}.$anvil->Log->switches, 
			job_name        => "manage::broken_keys", 
			job_progress    => 0,
		});
		$job_created = $anvil->data->{key_jobs}{$state_name} ? 1 : 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:key_jobs::${state_name}" => $anvil->data->{key_jobs}{$state_name},
			"s2:job_created"             => $job_created, 
		}});
		
		# Tell the user we've created a new job.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "log_0625", variables => {
			job_uuid => $anvil->data->{key_jobs}{$state_name}, 
			peer     => $peer, 
		}});
	}
	
	return($job_created);
}

# Configure DRBD
sub configure_drbd
{
	my ($anvil) = @_;
	
	### TODO: See if there is a hardware RAID controller and, if so, auto-enable 
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0126");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0126"});
	my $updated = $anvil->DRBD->update_global_common({
		usage_count => $anvil->data->{sys}{privacy}{strong} ? 0 : 1,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }});
	if ($updated)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0127");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0127"});
	}
	else
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0128");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});
	}
	
	# Record that we're done configuring DRBD.
	$anvil->Database->insert_or_update_variables({
		variable_name         => "system::drbd::configured", 
		variable_value        => 1, 
		variable_default      => "", 
		variable_description  => "striker_0296", 
		variable_section      => "system", 
		variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
		variable_source_table => "hosts", 
	});
	
	return(0);
}

# Update the passwords for user accounts.
sub update_passwords
{
	my ($anvil) = @_;
	
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	my $anvil_name    = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $anvil_uuid    = $anvil->data->{sys}{anvil_uuid};
	my $new_password  = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $users         = $machine =~ /dr\d+/ ? ["root", "admin"] : ["root", "admin", "hacluster"];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		machine          => $machine, 
		manifest_uuid    => $manifest_uuid,
		anvil_name       => $anvil_name, 
		anvil_uuid       => $anvil_uuid, 
		new_password     => $anvil->Log->is_secure($new_password), 
	}});
	
	foreach my $user (@{$users})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { user => $user }});
		
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0093,!!user!".$user."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0093", variables => { user => $user }});
		my ($return_code) = $anvil->System->change_shell_user_password({
			debug        => 3,
			user         => $user,
			new_password => $new_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	}
	
	return(0);
}

sub check_for_reauth
{
	my ($anvil, $node1_host_name, $node2_host_name, $escaped_password) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		node1_host_name  => $node1_host_name,
		node2_host_name  => $node2_host_name,  
		escaped_password => $anvil->Log->is_secure($escaped_password),
	}});
	
	my $auth       = 0;
	my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster status";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	
	my $in_pcsd = 0;
	foreach my $line (split/\n/, $output)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($in_pcsd)
		{
			if ($line =~ /Unable to authenticate/i)
			{
				$auth = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { auth => $auth }});
			}
		}
		if ($line =~ /PCSD Status/i)
		{
			$in_pcsd = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_pcsd => $in_pcsd }});
		}
		last if $auth;
	}
	
	# If we're rebuilding, the cluster could be trying to start, but auth is still needed. So check to 
	# see if '/var/lib/pcsd/known-hosts' exists. If not, try to auth.
	if ((not $auth) && (not -f '/var/lib/pcsd/known-hosts'))
	{
		$auth = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { auth => $auth }});
	}
	
	if ($auth)
	{
		# Also, re-auth. We need to run this on both hosts.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0487"});
		my $shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({secure => 1, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	return(0);
}

# (wait for our peer and) Configure pacemaker. If this is a DR host, this is skipped.
sub configure_pacemaker
{
	my ($anvil) = @_;
	
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	
	### TODO: Move these to variables in the 'sys' hash
	my $anvil_name       =  $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{name};
	my $anvil_uuid       =  $anvil->data->{sys}{anvil_uuid};
	my $host_name        =  $anvil->Get->host_name;
	my $new_password     =  $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid  =  $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name  =  $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	   $node1_host_name  =~ s/\..*$//;
	my $node2_host_uuid  =  $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name  =  $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	   $node2_host_name  =~ s/\..*$//;
	my $peer_host_name   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name;
	my $peer_host_uuid   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
	my $escaped_password =  shell_quote($new_password);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		machine          => $machine, 
		anvil_uuid       => $anvil_uuid, 
		anvil_name       => $anvil_name, 
		host_name        => $host_name, 
		manifest_uuid    => $manifest_uuid,
		node1_host_uuid  => $node1_host_uuid, 
		node1_host_name  => $node1_host_name, 
		node2_host_uuid  => $node2_host_uuid, 
		node2_host_name  => $node2_host_name, 
		peer_host_name   => $peer_host_name, 
		peer_host_uuid   => $peer_host_uuid, 
		new_password     => $anvil->Log->is_secure($new_password), 
		escaped_password => $anvil->Log->is_secure($escaped_password),
	}});
	
	# If this is a DR box, we don't use pacemaker.
	if ($machine =~ /dr\d+/)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0096"});
		return(0);
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 'path::configs::corosync-authkey' => $anvil->data->{path}{configs}{'corosync-authkey'} }});
	if (not -f $anvil->data->{path}{configs}{'corosync-authkey'})
	{
		# In case we're rebuilding, see if the peer already has the '/etc/corosync/authkey' file.
		my $corosync_authkey = $anvil->Storage->read_file({
			file   => $anvil->data->{path}{configs}{'corosync-authkey'},
			target => $peer_host_name,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { corosync_authkey => $corosync_authkey }});
		if ($corosync_authkey ne "!!error!!")
		{
			# Write the file out.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0485"});
			$anvil->Storage->write_file({
				debug => 2,
				body  => $corosync_authkey,
				file  => $anvil->data->{path}{configs}{'corosync-authkey'}, 
				user  => "root",
				group => "root",
				mode  => "0400",
			});
		}
	}
	
	### Run on both nodes.
	# Enable pcsd and start the pcsd daemon.
	my ($return_code) = $anvil->System->enable_daemon({daemon => "pcsd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->start_daemon({daemon => "pcsd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "pcsd.service" }});
	
	# Enable libvirtd and start the libvirtd daemon.
	($return_code) = $anvil->System->enable_daemon({daemon => "libvirtd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->start_daemon({daemon => "libvirtd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!libvirtd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "libvirtd.service" }});
	
	# Disabled and stop the drbd daemon.
	($return_code) = $anvil->System->disable_daemon({daemon => "drbd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	($return_code) = $anvil->System->stop_daemon({daemon => "drbd.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!drbd.service!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "drbd.service" }});
	
	# Disabled and stop the ksm and ksmtuned daemon.
	($return_code) = $anvil->System->disable_daemon({daemon => "ksm.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksm.service" }});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksm.service!!");
	
	($return_code) = $anvil->System->stop_daemon({daemon => "ksmtuned.service"});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
	$return_code = undef;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "ksmtuned.service" }});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!ksmtuned.service!!");
	
	# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, we'll initialize the
	# cluster shortly.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 'path::configs::corosync.conf' => $anvil->data->{path}{configs}{'corosync.conf'} }});
	if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
	{
		my $corosync_conf = $anvil->Storage->read_file({
			file   => $anvil->data->{path}{configs}{'corosync.conf'},
			target => $peer_host_name,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }});
		if ($corosync_conf ne "!!error!!")
		{
			# Write the file out.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0100"});
			$anvil->Storage->write_file({
				debug => 2,
				body  => $corosync_conf,
				file  => $anvil->data->{path}{configs}{'corosync.conf'}, 
				user  => "root",
				group => "root",
				mode  => "0644",
			});
		}
		
		# Restart corosync
		my ($return_code) = $anvil->System->restart_daemon({daemon => "corosync.service"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
		$return_code = undef;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0095", variables => { daemon => "corosync.service" }});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0095,!!daemon!corosync.service!!");
	}
	
	# Node 1 initializes, node 2 waits.
	if ($machine eq "node2")
	{
		my $start_time = 0;
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0103"});
		
		# If we're rebuilding, we may need to re-auth. So check right off the bat.
		if (not -f '/var/lib/pcsd/known-hosts')
		{
			check_for_reauth($anvil, $node1_host_name, $node2_host_name, $escaped_password);
		}
		
		# We loop until the peer finishes or the peer's job hit's 100.
		my $tried_starting = 0;
		my $both_online    = 0;
		my $try_starting   = time + 30;
		my $delay          = 5;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			delay        => $delay, 
			try_starting => $try_starting,
		}});
		until($both_online)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'path::configs::corosync.conf' => $anvil->data->{path}{configs}{'corosync.conf'},
			}});
			if (-e $anvil->data->{path}{configs}{'corosync.conf'})
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { start_time => $start_time }});
				if (not $start_time)
				{
					# Corosync is configured, we'll wait up to two minutes and then try
					# joining the cluster ourselves.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0108"});
					$start_time = time + 60;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { start_time => $start_time }});
				}
				elsif ((time > $start_time) && (not $tried_starting))
				{
					# We've waited a minute, try (re)authenticating, and then try starting the cluster.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0486"});
					check_for_reauth($anvil, $node1_host_name, $node2_host_name, $escaped_password);
					
					# Now try starting the cluster.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0106"});
					my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
					
					# Mark that weve tried to start.
					$tried_starting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { tried_starting => $tried_starting }});
				}
				
				# Can I access my peer?
				my $access = $anvil->Remote->test_access({
					debug    => 2,
					target   => $peer_host_name,
					password => $new_password,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
				if ($access)
				{
					# I can, can I parse the CIB?
					my $problem = $anvil->Cluster->parse_cib({debug => 2});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
					if (not $problem)
					{
						# See if both nodes are online.
						my $node1_ready = $anvil->Cluster->check_node_status({debug => 2, node_name => $node1_host_name});
						my $node2_ready = $anvil->Cluster->check_node_status({debug => 2, node_name => $node2_host_name});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							node1_ready => $node1_ready,
							node2_ready => $node2_ready, 
						}});
						if (($node1_ready) && ($node2_ready))
						{
							$both_online = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
						}
						else
						{
							# Not online yet, wait a bit.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
								node1_name   => $node1_host_name,
								node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
								node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
								node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
								node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
								node2_name   => $node2_host_name,
								node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
								node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
								node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
								node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
							}});
						}
					}
				}
			}
			else
			{
				# corosync.conf doesn't exist yet.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0107"});
			}
			if (not $both_online)
			{
				if (time > $try_starting)
				{
					# Try starting pacemaker, in case we're rebuilding.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0164");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0164"});
					
					$try_starting += 60;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { try_starting => $try_starting }});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
				}
				else
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0816", variables => { delay => $delay }});
					sleep $delay;
				}
			}
		}
		
		# Check if we need to re-auth with pcsd
		check_for_reauth($anvil, $node1_host_name, $node2_host_name, $escaped_password);
	}
	else
	{
		# We're node 1, proceed with cluster setup.
		my $waiting         = 1;
		my $warning_printed = 0;
		while($waiting)
		{
			# Try to authenticate against the peer.
			my $auth_shell_call = $anvil->data->{path}{exe}{pcs}." host auth ".$node1_host_name." ".$node2_host_name." -u hacluster -p ".$escaped_password;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { auth_shell_call => $auth_shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, secure => 1, shell_call => $auth_shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong. 
				if (not $warning_printed)
				{
					# Update the job
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097");
					$warning_printed = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }});
				}
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0097"});
				sleep 5;
			}
			else
			{
				# We're good.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0098"});
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		# If there is no corosync.conf, see if the peer has it. If so, copy it. If not, initialize 
		# the cluster.
		if (not -e $anvil->data->{path}{configs}{'corosync.conf'})
		{
			# There's no cluster yet, initialize it.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0099", variables => { anvil_name => $anvil_name }});
			
			my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			if ($return_code)
			{
				# Something went wrong
				update_progress($anvil, 100, "job_0101,!!error!".$output."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0101", variables => { error => $output }});
				$anvil->nice_exit({exit_code => 5});
			}
		}
		
		# If we can parse the CIB, then pcsd is running. 
		my $problem = $anvil->Cluster->parse_cib({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($problem)
		{
			# Start the cluster.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0102"});
			
			my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
		}
		
		# Now wait for both nodes to come online.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0109");
		my $both_online = 0;
		my $start_again = time + 60;
		until ($both_online)
		{
			### TODO: If we're waiting more that five minutes, call 'pcs cluster start --all' again.
			my $problem = $anvil->Cluster->parse_cib({debug => 2});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if (not $problem)
			{
				# See if both nodes are online.
				my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
				my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node1_ready => $node1_ready,
					node2_ready => $node2_ready, 
				}});
				if (($node1_ready) && ($node2_ready))
				{
					$both_online = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
					
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
				}
				else
				{
					# Not online yet, wait a bit.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
						node1_name   => $node1_host_name,
						node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
						node1_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
						node1_join   => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{'join'},
						node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{ready},
						node2_name   => $node2_host_name,
						node2_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{in_ccm},
						node2_crmd   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{crmd},
						node2_join   => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{'join'},
						node2_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node2_host_name}{node_state}{ready},
					}});
				}
			}
			if (time > $start_again)
			{
				### NOTE: We can't just call 'start --all' again anymore. Now we need to 
				###       stop -> start. Before we do this, make sure there are no servers
				###       running.
				   $start_again  = time + 60;
				my $restart      = 1;
				my $server_count = keys %{$anvil->data->{cib}{parsed}{data}{server}};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					start_again  => $start_again, 
					server_count => $server_count,
				}});
				foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"cib::parsed::data::server::${server}::active" => $anvil->data->{cib}{parsed}{data}{server}{$server}{active},
					}});
					if ($anvil->data->{cib}{parsed}{data}{server}{$server}{active})
					{
						$restart = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart => $restart }});
					}
				}
				
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart => $restart }});
				if ($restart)
				{
					# Call cluster start again.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0272"});
					my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster stop --all";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						start_again => $start_again, 
						shell_call  => $shell_call,
					}});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					
					$shell_call  = $anvil->data->{path}{exe}{pcs}." cluster start --all";
					($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
				}
			}
			sleep 5 if not $both_online;
		}
		
		### Setup fencing! 
		$anvil->Striker->get_fence_data();
		
		# IPMI first, if applicable.
		my $something_changed = {};
		my $fence_order       = {};
		my $node1_use_delay   = 0;
		my $node2_use_delay   = 0;
		my $fence_devices     = {};
		foreach my $node ("node1", "node2")
		{
			my $node_name         = $node eq "node1" ? $node1_host_name : $node2_host_name;
			my $host_uuid         = $node eq "node1" ? $node1_host_uuid : $node2_host_uuid;
			my $host_ipmi         = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
			my $ipmi_stonith_name = "ipmilan_".$node; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				node_name         => $node_name, 
				host_uuid         => $host_uuid, 
				host_ipmi         => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
				ipmi_stonith_name => $ipmi_stonith_name, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0115,!!node!".$node_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0115", variables => { node => $node_name }});
			
			# This will store the fence level order. If something changes
			$fence_order->{$node_name} = [];
			
			# This will switch to '1' if something changed, triggering a reconfig of the fencing levels.
			$something_changed->{$node_name} = 0;
			
			# Does this stonith method already exist?
			my $create_entry    = 0;
			my $delete_old      = 0;
			my $pcs_add_command = "";
			if ($host_ipmi)
			{
				push @{$fence_order->{$node_name}}, "fence_ipmilan";
				$fence_devices->{$node_name}{fence_ipmilan} = [$ipmi_stonith_name];
				
				# The --action switch needs to be 'pcmk_off_action' in pcs, so we convert it here.
				$host_ipmi =~ s/--action status//;
				$host_ipmi =~ s/--action/--pcmk_off_action/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
				}});
				
				# We have IPMI, so we also want fence_delay for this node.
				if ($node eq "node1")
				{
					$node1_use_delay = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_use_delay => $node1_use_delay }});
				}
				else
				{
					$node2_use_delay = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_use_delay => $node2_use_delay }});
				}
				
				# If we're here, break up the command and turn it into the pcs call.
				my $old_switches              = {};
				my ($fence_agent, $arguments) = ($host_ipmi =~ /^\/.*\/(.*?)\s+(.*)$/);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					fence_agent  => $fence_agent,
					arguments    => $arguments =~ /passw/ ? $anvil->Log->is_secure($arguments) : $arguments,
				}});
				
				   $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$ipmi_stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ";
				my $switches = $anvil->System->parse_arguments({debug => 2, arguments => $arguments});
				foreach my $switch (sort {$a cmp $b} keys %{$switches})
				{
					# Ignore 'delay', we handle that in Cluster->set_delay(); Also, 
					# convert '#!SET!#' to 'true'.
					my $value =  $switches->{$switch};
					   $value =~ s/"/\\"/g;
					   $value =~ s/#!SET!#/true/g;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						switch => $switch,
						value  => $value,
					}});
					next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "delay";
					next if $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name} eq "action";
					
					# Find the argument=value version.
					my $argument        =  $anvil->data->{fence_data}{$fence_agent}{switch}{$switch}{name};
					   $pcs_add_command .= $argument."=\"".$value."\" ";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						argument        => $argument,
						value           => $argument        =~ /passw/ ? $anvil->Log->is_secure($value)           : $value,
						pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
					}});
					
					# Store this to see if it's different from what's already in the CIB.
					$old_switches->{$argument} = $value;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"old_switches->{$argument}" => $old_switches->{$argument},
					}});
				}
				$pcs_add_command .= "op monitor interval=\"60\"";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
				}});
				
				# If there's an entry in the CIB, see if it's different somehow
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
				{
					foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}})
					{
						next if $argument eq "delay";
						next if $argument eq "action";
						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}{$argument}{value};
						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:argument'  => $argument, 
							's2:old_entry' => $old_entry,
							's3:new_entry' => $new_entry,
						}});
						
						if ($old_entry ne $new_entry)
						{
							# Changed, delete and recreate.
							$delete_old   = 1;
							$create_entry = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								delete_old   => $delete_old,
								create_entry => $create_entry,
							}});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
							last;
						}
						
						# Delete the old switch.
						delete $old_switches->{$argument};
					}
					
					# Are there any old switches left?
					my $old_switch_count = keys %{$old_switches};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						delete_old       => $delete_old, 
						old_switch_count => $old_switch_count,
					}});
					if ((not $delete_old) && ($old_switch_count))
					{
						# Delete and recreate. 
						$delete_old   = 1;
						$create_entry = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							delete_old   => $delete_old,
							create_entry => $create_entry,
						}});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
					}
				}
				else
				{
					# No existing entry, add a new one.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0116");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0116"});
			
					$create_entry = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
				}
			}
			elsif (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
			{
				# There was an existing fence config, but there's no entry in 'host_ipmi'. 
				# Remove the stonith entry.
				$delete_old = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { delete_old => $delete_old }});
				
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0118");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0118"});
			}
			
			# Process the IPMI entry.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				delete_old   => $delete_old,
				create_entry => $create_entry, 
			}});
			if ($delete_old)
			{
				# Delete
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$ipmi_stonith_name."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $ipmi_stonith_name }});
				
				my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$ipmi_stonith_name;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
				if ($return_code)
				{
					# Something went wrong.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
						shell_call  => $shell_call, 
						output      => $output, 
						return_code => $return_code, 
					}});
					update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
					sleep 2;
					$anvil->nice_exit({exit_code => 6});
				}
				
				$something_changed->{$node_name} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			}
			if ($create_entry)
			{
				# Create.
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$ipmi_stonith_name."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $ipmi_stonith_name }});
				
				my $shell_call = $pcs_add_command;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
				if ($return_code)
				{
					# If this is because the method already exists, ignore the error.
					if ($output =~ /already exists/i)
					{
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "warning_0174", variables => {
							method => $ipmi_stonith_name, 
						}});
					}
					else
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
				}
				
				$something_changed->{$node_name} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			}
			
			
			### Now any other fence devices.
			foreach my $device (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}})
			{
				my $delete_old      = 0;
				my $create_entry    = 0;
				my $old_switches    = {};
				my $fence_uuid      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{fences}{$device}{uuid};
				my $fence_name      = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_name};
				my $fence_arguments = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_arguments};
				my $fence_agent     = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_agent};
				my $stonith_name    = ($fence_agent =~ /^fence_(.*)$/)[0]."_".$node."_".$fence_name; 
				my $port            = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}{$device}{port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					device          => $device, 
					fence_uuid      => $fence_uuid, 
					fence_name      => $fence_name, 
					fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments,
					stonith_name    => $stonith_name, 
					port            => $port, 
				}});
				
				# We use this to tell if there are two or more entries per agent. If there
				# are, we link them later when setting up the fence levels.
				if (not exists $fence_devices->{$node_name}{$fence_agent})
				{
					push @{$fence_order->{$node_name}}, $fence_agent;
					$fence_devices->{$node_name}{$fence_agent} = [];
				}
				push @{$fence_devices->{$node_name}{$fence_agent}}, $stonith_name;
				
				# Fence arguments use 'action', but pcs deprecated it in favour of 'pcmk_off_action', so rename it.
				$fence_arguments =~ s/action=/pcmk_off_action=/;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments, 
				}});
				
				# Build the pcs command
				my $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ".$fence_arguments." ";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
				}});
				while ($fence_arguments =~ /=/)
				{
					# Ignore 'delay', we handle that in Cluster->set_delay();
					my $pair               =  ($fence_arguments =~ /(\S*?=".*?")/)[0];
					   $fence_arguments    =~ s/$pair//;
					   $fence_arguments    =~ s/^\s+//;
					   $fence_arguments    =~ s/\s+$//;
					my ($argument, $value) =  ($pair =~ /(.*)="(.*)"/);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:fence_arguments' => $fence_arguments, 
						's2:pair'            => $pair =~ /passw/ ? $anvil->Log->is_secure($pair) : $pair,
						's3:argument'        => $argument,
						's4:value'           => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value,
					}});
					
					# Ignore 'delay', we handle that in Cluster->set_delay();
					if (($argument ne "pcmk_off_action")                                           && 
					    (exists $anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name}) && 
					    ($anvil->data->{fence_data}{$fence_agent}{switch}{$argument}{name} eq "delay"))
					{
						next;
					}
					
					# Store this to see if it's different from what's already in the CIB.
					$old_switches->{$argument} = $value;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"old_switches->{$argument}" => $old_switches->{$argument},
					}});
				}
				if ($port)
				{
					$port                 =~ s/"/\\"/g;
					$pcs_add_command      .= "port=\"".$port."\" ";
					$old_switches->{port} =  $port;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
						"old_switches->{port}" => $old_switches->{port},
					}});
				}
				$pcs_add_command .= "op monitor interval=\"60\"";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
				}});
				
				# Does this device exist already?
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
				{
					foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}})
					{
						next if $argument eq "delay";
						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}{$argument}{value};
						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:argument'  => $argument, 
							's2:old_entry' => $old_entry,
							's3:new_entry' => $new_entry,
						}});
						
						if ($old_entry ne $new_entry)
						{
							# Changed, delete and recreate.
							$delete_old   = 1;
							$create_entry = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								delete_old   => $delete_old,
								create_entry => $create_entry,
							}});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
							last;
						}
						
						# Delete the old switch.
						delete $old_switches->{$argument};
					}
					
					# Are there any old switches left?
					my $old_switch_count = keys %{$old_switches};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }});
					if ((not $delete_old) && ($old_switch_count))
					{
						# Delete and recreate. 
						$delete_old   = 1;
						$create_entry = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							delete_old   => $delete_old,
							create_entry => $create_entry,
						}});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
					}
				}
				elsif (exists $anvil->data->{cib}{parsed}{cib}{resource}{primitive}{$stonith_name})
				{
					# If this isn't connected to a node, like PDUs, we need to look for it elsewhere.
					foreach my $argument_name (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{cib}{resource}{primitive}{$stonith_name}{instance_attributes}})
					{
						my $argument  = $anvil->data->{cib}{parsed}{cib}{resource}{primitive}{$stonith_name}{instance_attributes}{$argument_name}{name};
						my $old_entry = $anvil->data->{cib}{parsed}{cib}{resource}{primitive}{$stonith_name}{instance_attributes}{$argument_name}{value};
						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							's1:argument_name' => $argument_name, 
							's2:argument'      => $argument,
							's3:old_entry'     => $old_entry, 
							's4:new_entry'     => $new_entry,
						}});
						next if $argument eq "pcmk_host_list";
						
						if ($old_entry ne $new_entry)
						{
							# Changed, delete and recreate.
							$delete_old   = 1;
							$create_entry = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								delete_old   => $delete_old,
								create_entry => $create_entry,
							}});
							
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
							last;
						}
						
						# Delete the old switch.
						delete $old_switches->{$argument};
					}
					
					# Are there any old switches left?
					my $old_switch_count = keys %{$old_switches};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }});
					if ((not $delete_old) && ($old_switch_count))
					{
						# Delete and recreate. 
						$delete_old   = 1;
						$create_entry = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							delete_old   => $delete_old,
							create_entry => $create_entry,
						}});
						
						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
					}
				}
				else
				{
					# No existing entry, add a new one.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0122,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0122", variables => { device => $stonith_name }});
					
					$create_entry = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
				}
				
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					delete_old   => $delete_old,
					create_entry => $create_entry, 
				}});
				if ($delete_old)
				{
					# Delete
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $stonith_name }});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$stonith_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
					
					$something_changed->{$node_name} = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
				}
				if ($create_entry)
				{
					# Create.
					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$stonith_name."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }});
					
					my $shell_call = $pcs_add_command;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# See if this was because the fence method already existed.
						if ($output =~ /already exists/)
						{
							# It already existed, we're fine.
							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0481,!!device!".$stonith_name."!!");
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0481", variables => { device => $stonith_name }});
						}
						else
						{
							# Something went wrong.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
								shell_call  => $shell_call, 
								output      => $output, 
								return_code => $return_code, 
							}});
							update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
							sleep 2;
							$anvil->nice_exit({exit_code => 6});
						}
					}
					else
					{
						$something_changed->{$node_name} = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
					}
				}
			}
			
			
			### If we had a fence_ipmilan entry, add a 'fence_delay' entry, if needed.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				node            => $node, 
				node1_use_delay => $node1_use_delay,
				node2_use_delay => $node2_use_delay, 
			}});
			if ((($node eq "node1") && ($node1_use_delay)) or 
			    (($node eq "node2") && ($node2_use_delay)))
			{
				my $stonith_name = "delay_".$node; 
				push @{$fence_order->{$node_name}}, "fence_delay";
				$fence_devices->{$node_name}{fence_delay} = [$stonith_name];
				
				# Add the fence delay if it doesn't exist yet.
				if (not exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
				{
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." fence_delay pcmk_host_list=\"".$node_name."\" wait=\"60\" op monitor interval=\"60\"";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
				}
			}
		}
		
		# Setup fence levels.
		foreach my $node_name (sort {$a cmp $b} keys %{$fence_order})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
			if ($something_changed->{$node_name})
			{
				# Update our view of the cluster.
				my $problem = $anvil->Cluster->parse_cib({debug => 2});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				
				# Delete any existing fence levels
				if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name})
				{
					foreach my $index (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{order}})
					{
						# pcs stonith level delete <index> <target>
						my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level delete ".$index." ".$node_name;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
						
						my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							output      => $output,
							return_code => $return_code, 
						}});
						if ($return_code)
						{
							# Something went wrong.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
								shell_call  => $shell_call, 
								output      => $output, 
								return_code => $return_code, 
							}});
							update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
							sleep 2;
							$anvil->nice_exit({exit_code => 6});
						}
					}
				}
				
				# Create the new fence levels
				my $i = 1;
				foreach my $fence_agent (@{$fence_order->{$node_name}})
				{
					my $devices = "";
					foreach my $device (sort {$a cmp $b} @{$fence_devices->{$node_name}{$fence_agent}})
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
						$devices .= $device.",";
					}
					$devices =~ s/,$//;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { devices => $devices }});
					
					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level add ".$i." ".$node_name." ".$devices;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					if ($return_code)
					{
						# Something went wrong.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
							shell_call  => $shell_call, 
							output      => $output, 
							return_code => $return_code, 
						}});
						update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
						sleep 2;
						$anvil->nice_exit({exit_code => 6});
					}
					
					$i++;
				}
			}
		}
	}

	# Make sure logind is update to handle fencing properly
	# see - https://access.redhat.com/solutions/1578823
	$anvil->Cluster->configure_logind({debug => 2});

	# Enable fencing and set the retry to INFINITY, if needed.
	$anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} = "" if not defined $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::data::stonith::max-attempts" => $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}, 
	}});
	if ($anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} ne "INFINITY")
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0124");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0124"});
		
		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-max-attempts=INFINITY";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
				shell_call  => $shell_call, 
				output      => $output, 
				return_code => $return_code, 
			}});
			update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 6});
		}
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"cib::parsed::data::stonith::enabled" => $anvil->data->{cib}{parsed}{data}{stonith}{enabled}, 
	}});
	if (not $anvil->data->{cib}{parsed}{data}{stonith}{enabled})
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0125");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0125"});
		
		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-enabled=true";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
				shell_call  => $shell_call, 
				output      => $output, 
				return_code => $return_code, 
			}});
			update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 6});
		}
	}
	
	# Update (if needed) corosync.conf to use the BCN1, MN1 and SN1 as knet networks.
	if ($machine eq "node1")
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0344");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0344"});
		
		check_corosync($anvil);
	}
	
	# Record that we're done configuring pacemaker.
	$anvil->Database->insert_or_update_variables({
		variable_name         => "system::pacemaker::configured", 
		variable_value        => 1, 
		variable_default      => "", 
		variable_description  => "striker_0295", 
		variable_section      => "system", 
		variable_source_uuid  => $anvil->data->{sys}{host_uuid}, 
		variable_source_table => "hosts", 
	});
	
	# Make sure ScanCore's DB is updated.
	print __LINE__.": Running scan-cluster prior to checking Storage Groups.\n";
	$anvil->ScanCore->call_scan_agents({debug => 2, agent => "scan-cluster"});
	
	return(0);
}
	
sub check_corosync
{
	my ($anvil) = @_;
	
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0345");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0345"});
	my $waiting      = 1;
	my $anvil_uuid   = $anvil->data->{sys}{anvil_uuid};
	my $new_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $host_uuid    = $anvil->Get->host_uuid();
	while ($waiting)
	{
		### TODO: These shouldn't be statically <net>1, but instead based on the actual nets on the 
		###       system.
		my $problem          =  $anvil->Cluster->parse_cib({debug => 3});
		my $peer_ready       =  $anvil->data->{cib}{parsed}{peer}{ready};
		my $peer_name        =  $anvil->data->{cib}{parsed}{peer}{name};
		my $peer_short_name  =  $peer_name;
		   $peer_short_name  =~ s/\..*$//;
		my $peer_bcn_name    =  $peer_short_name.".bcn1";
		my $peer_sn_name     =  $peer_short_name.".sn1";
		my $peer_mn_name     =  $peer_short_name.".mn1";
		my $peer_ifn_name    =  $peer_short_name.".ifn1";
		my $local_ready      =  $anvil->data->{cib}{parsed}{'local'}{ready};
		my $local_name       =  $anvil->data->{cib}{parsed}{'local'}{name};
		my $local_short_name =  $anvil->Get->short_host_name;
		my $local_bcn_name   =  $local_short_name.".bcn1";
		my $local_sn_name    =  $local_short_name.".sn1";
		my $local_mn_name    =  $local_short_name.".mn1";
		my $local_ifn_name   =  $local_short_name.".ifn1";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			problem          => $problem, 
			peer_ready       => $peer_ready,
			peer_name        => $peer_name, 
			peer_short_name  => $peer_short_name, 
			peer_bcn_name    => $peer_bcn_name, 
			peer_ifn_name    => $peer_ifn_name, 
			peer_sn_name     => $peer_sn_name, 
			peer_mn_name     => $peer_mn_name, 
			local_ready      => $local_ready,
			local_name       => $local_name, 
			local_short_name => $local_short_name, 
			local_bcn_name   => $local_bcn_name, 
			local_ifn_name   => $local_ifn_name,
			local_sn_name    => $local_sn_name, 
			local_mn_name    => $local_mn_name, 
		}});
		if ((not $problem) && ($peer_ready) && ($local_ready))
		{
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0346");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0346"});
			
			# Verify we can ping the peer on the BCN and SN.
			$anvil->Database->get_ip_addresses({debug => 2});
			
			# Some builds failed on this logic below, so this logging helps us debug when theat 
			# happens. 
			foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{hosts}{host_uuid}{$host_uuid}{network}})
			{
				my $ip = $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{$network}{ip_address};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:network' => $network,
					's2:ip'      => $ip,
				}});
			}
			
			my $bcn_access = 0;
			my $ifn_access = 0;
			my $sn_access  = 0;
			my $mn_access  = 0;
			my $problem    = 0;
			if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{bcn1}) && 
			    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{bcn1}{ip_address}))
			{
				$bcn_access = $anvil->Remote->test_access({
					target   => $peer_bcn_name,
					password => $new_password,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bcn_access => $bcn_access }});
				if (not $bcn_access)
				{
					$problem = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				}
			}
			if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{ifn1}) && 
			    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{ifn1}{ip_address}))
			{
				$ifn_access = $anvil->Remote->test_access({
					target   => $peer_ifn_name,
					password => $new_password,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ifn_access => $ifn_access }});
				if (not $ifn_access)
				{
					$problem = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				}
			}
			if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{sn1}) && 
			    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{sn1}{ip_address}))
			{
				$sn_access = $anvil->Remote->test_access({
					target   => $peer_sn_name,
					password => $new_password,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sn_access => $sn_access }});
				if (not $sn_access)
				{
					$problem = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				}
			}
			if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{mn1}) && 
			    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{mn1}{ip_address}))
			{
				$mn_access = $anvil->Remote->test_access({
					target   => $peer_mn_name,
					password => $new_password,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mn_access => $mn_access }});
				if (not $mn_access)
				{
					$problem = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				}
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				bcn_access => $bcn_access, 
				ifn_access => $ifn_access, 
				sn_access  => $sn_access, 
				mn_access  => $mn_access, 
			}});
			if (not $problem)
			{
				# We're ready!
				update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0347");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0347"});
				
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				
				my $in_totem          = 0;
				my $token_seen        = 0;
				my $in_nodelist       = 0;
				my $in_node           = 0;
				my $ring0_addr        = "";
				my $ring1_addr        = "";
				my $in_node_name      = "";
				my $nodelist_body     = "";
				my $old_corosync_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'corosync.conf'}});
				my $new_corosync_conf = "";
				foreach my $line (split/\n/, $old_corosync_conf)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					if ($line =~ /totem \{/)
					{
						$in_totem = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_totem => $in_totem }});
						
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
						next;
					}
					if ($line =~ /nodelist \{/)
					{
						$in_nodelist = 1;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
						next;
					}
					
					if ($in_nodelist)
					{
						if ($line =~ /node \{/)
						{
							$in_node = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
							
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							next;
						}
						if ($in_node)
						{
							if ($line =~ /name:(.*)$/)
							{
								$in_node_name = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node_name => $in_node_name }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /ring0_addr:(.*)$/)
							{
								$ring0_addr = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring0_addr => $ring0_addr }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /ring1_addr:(.*)$/)
							{
								$ring1_addr = $anvil->Words->clean_spaces({string => $1});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ring1_addr => $ring1_addr }});
								
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
							elsif ($line =~ /}/)
							{
								$in_node = 0;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_node => $in_node }});
								
								# Which networks are we using? Ideally BCN 
								# and MN, but it depends if the MN, or even
								# SN, exists.
								# First, is this the local or peer node?
								my $this_is = "peer";
								if (($in_node_name eq $anvil->Get->host_name) or ($in_node_name eq $anvil->Get->short_host_name))
								{
									# It's us
									$this_is = "local";
								}
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_is => $this_is }});
								
								my $test_ring0_addr = "";
								my $test_ring1_addr = "";
								if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{bcn1}) && 
								    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{bcn1}{ip_address}))
								{
									# BCN is the primary ring, when available
									if ($this_is eq "local")
									{
										$test_ring0_addr = $local_bcn_name;
									}
									else
									{
										$test_ring0_addr = $peer_bcn_name;
									}
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
								}
								# The migration network is the second choice.
								if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{mn1}) && 
								    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{mn1}{ip_address}))
								{
									if ($this_is eq "local")
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $local_mn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $local_mn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
									else
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $peer_mn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $peer_mn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
								}
								# storage network is the third choice
								if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{sn1}) && 
								    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{sn1}{ip_address}))
								{
									if ($this_is eq "local")
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $local_sn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $local_sn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
									else
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $peer_sn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $peer_sn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
								}
								# The internet facing network is the network 
								# of last resort
								if ((exists $anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{ifn1}) && 
								    ($anvil->data->{hosts}{host_uuid}{$host_uuid}{network}{ifn1}{ip_address}))
								{
									if ($this_is eq "local")
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $local_ifn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $local_ifn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
									else
									{
										# Have we found ring0 yet?
										if (not $test_ring0_addr)
										{
											# No, use this for ring0.
											$test_ring0_addr = $peer_ifn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring0_addr => $test_ring0_addr }});
										}
										elsif (not $test_ring1_addr)
										{
											# Yes, use this for ring1
											$test_ring1_addr = $peer_ifn_name;
											$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_ring1_addr => $test_ring1_addr }});
										}
									}
								}
								
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									's1:ring0_addr'      => $ring0_addr, 
									's2:test_ring0_addr' => $test_ring0_addr,
									's3:ring1_addr'      => $ring1_addr, 
									's4:test_ring1_addr' => $test_ring1_addr, 
								}});
								
								if ((not $ring1_addr) or ($ring0_addr ne $test_ring0_addr) or ($ring1_addr ne $test_ring1_addr))
								{
									# Rewrite the nodelist body.
									my $new_nodelist_body  = "        ring0_addr: ".$test_ring0_addr."\n";
									if ($test_ring1_addr)
									{
										$new_nodelist_body .= "        ring1_addr: ".$test_ring1_addr."\n";
									}
									foreach my $nodelist_line (split/\n/, $nodelist_body)
									{
										$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_line => $nodelist_line }});
										next if $nodelist_line =~ /ring\d_addr/;
										
										$new_nodelist_body .= $nodelist_line."\n";
										$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
									}
									
									$nodelist_body = $new_nodelist_body;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_nodelist_body => $new_nodelist_body }});
								}
								
								$new_corosync_conf .= $nodelist_body;
								$new_corosync_conf .= $line."\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
								
								$ring0_addr         = "";
								$ring1_addr         = "";
								$in_node_name       = "";
								$nodelist_body      = "";
								next;
							}
							else
							{
								# Normal line, stash it.
								$nodelist_body .= $line."\n"; 
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodelist_body => $nodelist_body }});
								next;
							}
						}
						elsif ($line =~ /}/)
						{
							$in_nodelist = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_nodelist => $in_nodelist }});
							
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							next;
						}
					}
					if ($in_totem)
					{
						if ($line =~ /}/)
						{
							if (not $token_seen)
							{
								$new_corosync_conf .= "    token: 10000\n";
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							}
							$new_corosync_conf .= $line."\n";
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
							
							$in_totem   = 0;
							$token_seen = 0;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								in_totem   => $in_totem,
								token_seen => $token_seen,
							}});
							next;
						}
						if ($line =~ /token:/)
						{
							$token_seen = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
						}
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
					}
					else
					{
						# Normal line 
						$new_corosync_conf .= $line."\n";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_corosync_conf => $new_corosync_conf }});
					}
				}
				
				# Take the last new lines of the file bodies.
				$old_corosync_conf =~ s/\n$//g;
				$new_corosync_conf =~ s/\n$//g;
				
				my $difference = diff \$old_corosync_conf, \$new_corosync_conf, { STYLE => 'Unified' };
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
				
				if ($difference)
				{
					# Update the corosync.conf, sync it and reload corosync.
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "log_0643,!!difference!".$difference."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0643", variables => { difference => $difference }});
					$anvil->Storage->write_file({
						file      => $anvil->data->{path}{configs}{'corosync.conf'}, 
						body      => $new_corosync_conf,
						overwrite => 1,
						backup    => 1,
					});
					
					# Sync
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0644"});
					my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster sync";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					# RC 0 is OK
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0348,!!return_code!".$return_code."!!,!!output!".$output."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0348", variables => { 
						output      => $output,
						return_code => $return_code, 
					}});
					
					# Reload
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0645"});
					$shell_call = $anvil->data->{path}{exe}{pcs}." cluster reload corosync";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					
					# RC 0 is OK
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					update_progress($anvil, ($anvil->data->{job}{progress}+1), "job_0349,!!return_code!".$return_code."!!,!!output!".$output."!!");
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0349", variables => { 
						output      => $output,
						return_code => $return_code, 
					}});
				}
			}
		}
		
		if ($waiting)
		{
			sleep 5;
		}
	}
	
	return(0);
}

sub do_reboot
{
	my ($anvil) = @_;
	
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0488");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0488"});
	
	my $time_left = 60;
	while ($time_left)
	{
		# Give them the countdown.
		$anvil->Job->update_progress({
			progress  => $anvil->data->{job}{progress} += 1, 
			message   => "log_0626", 
			job_uuid  => $anvil->data->{job}{uuid},
			'print'   => 1,
			log_level => 1,
			file      => $THIS_FILE, 
			line      => __LINE__,
			variables => { seconds => $time_left },
		});
		sleep 10;
		$time_left -= 10;
	}
	
	# Last, log that we're going down now.
	$anvil->Job->update_progress({
		progress  => $anvil->data->{job}{progress} += 1, 
		message   => "message_0389", 
		job_uuid  => $anvil->data->{job}{uuid},
		'print'   => 1,
		log_level => 1,
		file      => $THIS_FILE, 
		line      => __LINE__,
	});
	
	my $shell_call = $anvil->data->{path}{exe}{systemctl}." reboot";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
	
	return(0);
}


# Check if we need to change any IPs or our hostname.
sub check_local_network
{
	my ($anvil) = @_;
	
	# What host name and IP(s) should I have?
	my $local_host    = $anvil->Get->short_host_name();
	my $machine       = $anvil->data->{sys}{machine};
	my $manifest_uuid = $anvil->data->{sys}{manifest_uuid};
	my $domain        = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{domain};
	my $old_host_name = $anvil->Get->host_name;
	my $new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name};
	if ($domain)
	{
		$new_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{name}.".".$domain;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's2:domain'        => $domain, 
		's3:old_host_name' => $old_host_name, 
		's4:new_host_name' => $new_host_name, 
	}});
	
	$anvil->data->{sys}{host_name} = $new_host_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'sys::host_name' => $anvil->data->{sys}{host_name}, 
	}});
	
	# If the hostname isn't the same, change it.
	if ($old_host_name ne $new_host_name)
	{
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0061,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0061", variables => { host_name => $new_host_name }});
		my ($now_host_name) = $anvil->System->host_name({
			debug => 2,
			set   => $new_host_name,
		});
		if ($now_host_name eq $new_host_name)
		{
			# Success!
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0045");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0045"});
			
			# Reboot.
			do_reboot($anvil);
		}
		else
		{
			# Failed
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0063", variables => {
				host_name         => $new_host_name, 
				current_host_name => $now_host_name, 
			}});
			update_progress($anvil, 0, "job_0063,!!host_name!".$new_host_name."!!,!!current_host_name!".$now_host_name."!!");
			sleep 2;
			$anvil->nice_exit({exit_code => 3});
		}
	}
	else
	{
		# No need to change
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0077", variables => { host_name => $new_host_name }});
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'sys::host_name' => $anvil->data->{sys}{host_name}, 
	}});
	
	# Read the local network manager data.
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0080"});
	$anvil->Network->read_nmcli({debug => 2});
	
	# Now check IP addresses.
	$anvil->Network->get_ips({debug => 2});
	
	# This will be set when the first IFN with a gateway is set.
	my $default_gateway_interface = "";
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
	{
		if ($anvil->data->{network}{$local_host}{interface}{$in_iface}{default_gateway})
		{
			$default_gateway_interface = $in_iface;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			last;
		}
	}
	
	my $dns = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{dns};
	my $mtu = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{mtu};
	my $ntp = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{ntp};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dns => $dns,
		mtu => $mtu,
		ntp => $ntp,
	}});
	
	if (not $dns)
	{
		$dns = "8.8.8.8,8.8.4.4";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dns => $dns }});
	}
	if (not $mtu)
	{
		$mtu = "1500";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mtu => $mtu }});
	}
	
	### TODO: sorting the array seems inconsistent, so sorting in a hash
	# The DNS are comma-separated lists, that may or may not have spaces and may or may not be in 
	# alphabetical order. To properly compare, we'll rebuild the CSV string of the current and desired 
	# DNS settings.
	my @dns_array       = split/,/, $dns;
	my $wanted_dns_hash = {};
	foreach my $this_dns (sort {$a cmp $b} @dns_array)
	{
		$this_dns                     = $anvil->Words->clean_spaces({string => $this_dns});
		$wanted_dns_hash->{$this_dns} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"wanted_dns_hash->${this_dns}" => $wanted_dns_hash->{$this_dns}, 
		}});
	}
	my $cleaned_wanted_dns = "";
	foreach my $wanted_dns (sort {$a cmp $b} keys %{$wanted_dns_hash})
	{
		$cleaned_wanted_dns .= $wanted_dns.",";
	}
	$cleaned_wanted_dns =~ s/,$//;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cleaned_wanted_dns => $cleaned_wanted_dns }});
	
	# If any network interface is changed, we'll write out the config file then, when done, disconnect 
	# from the database, restart networking and then reconnect before moving on.
	my $restart_interfaces = [];
	
	foreach my $network (sort {$a cmp $b} keys %{$anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}})
	{
		my $ip      = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$machine}{network}{$network}{ip};
		my $subnet  = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{subnet};
		my $cidr    = $anvil->Convert->cidr({subnet_mask => $subnet});
		my $gateway = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{networks}{name}{$network}{gateway};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:network' => $network, 
			's2:ip'      => $ip,
			's3:subnet'  => $subnet, 
			's4:cidr'    => $cidr, 
			's5:gateway' => $gateway, 
		}});
		
		# If a network in the manifest isn't found locally, note it so we don't wait for it in 
		# /etc/hosts. This gets set to '0' if found.
		$anvil->data->{unused_network}{$network} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"unused_network::${network}" => $anvil->data->{unused_network}{$network}, 
		}});
		
		foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
		{
			# Only one interface will start with the network name and have an IP address.
			next if $in_iface !~ /^${network}_/;
			next if not $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
			my $current_ip      = $anvil->data->{network}{$local_host}{interface}{$in_iface}{ip};
			my $current_subnet  = $anvil->data->{network}{$local_host}{interface}{$in_iface}{subnet_mask};
			my $current_gateway = $anvil->data->{network}{$local_host}{interface}{$in_iface}{gateway};
			my $current_dns     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{dns};
			my $current_mtu     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
			my $mac_address     = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mac_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:in_iface'        => $in_iface,
				's2:current_ip'      => $current_ip,
				's4:current_subnet'  => $current_subnet, 
				's5:current_gateway' => $current_gateway, 
				's6:current_dns'     => $current_dns, 
				's7:current_mtu'     => $current_mtu, 
			}});
			
			# Mark that there's an interface for this network so we wait for it in /etc/hosts
			$anvil->data->{unused_network}{$network} = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"unused_network::${network}" => $anvil->data->{unused_network}{$network}, 
			}});
			
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0081,!!name!".$in_iface."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0081", variables => { name => $in_iface }});
			
			if ((not $default_gateway_interface) && ($in_iface =~ /^ifn/) && ($gateway))
			{
				# No existing default gateway, but this is the first IFN we've seen with a 
				# gateway defined, so we'll use this one.
				$default_gateway_interface = $in_iface;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { default_gateway_interface => $default_gateway_interface }});
			}
			
			my $change            = 0;
			my $current_dns_hash  = {};
			my @current_dns_array = split/,/, $current_dns;
			foreach my $this_current_dns (sort {$a cmp $b} @current_dns_array)
			{
				$this_current_dns                      = $anvil->Words->clean_spaces({ string => $this_current_dns });
				$current_dns_hash->{$this_current_dns} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"current_dns_hash->::${this_current_dns}" => $current_dns_hash->{$this_current_dns}, 
				}});
			}
			my $cleaned_current_dns = "";
			foreach my $current_dns (sort {$a cmp $b} keys %{$current_dns_hash})
			{
				$cleaned_current_dns .= $current_dns.",";
			}
			$cleaned_current_dns =~ s/,$//;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				cleaned_wanted_dns  => $cleaned_wanted_dns, 
				cleaned_current_dns => $cleaned_current_dns,
			}});
			
			if (($current_ip ne $ip) or ($current_subnet ne $subnet))
			{
				# IP / subnet changed.
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# gateway?
			if ($current_gateway ne $gateway)
			{
				#print __LINE__.": Gateway: [".$current_gateway."] -> [".$gateway."]\n";
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			# Check DNS only if this is the default gateway interface.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				in_iface                  => $in_iface,
				default_gateway_interface => $default_gateway_interface, 
				cleaned_current_dns       => $cleaned_current_dns, 
			}});
			if ($in_iface eq $default_gateway_interface)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					cleaned_current_dns => $cleaned_current_dns,
					cleaned_wanted_dns  => $cleaned_wanted_dns, 
				}});
				if ($cleaned_wanted_dns ne $cleaned_current_dns)
				{
					$change = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
				}
			}
			elsif ($cleaned_current_dns)
			{
				# Remove the old DNS entries.
				$change = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { change => $change }});
			}
			
			if (not $change)
			{
				# No change
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0082");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0082"});
			}
			else
			{
				# Update the config. 
				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0078,!!interface!".$in_iface."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0078", variables => { interface => $in_iface }});
				
				my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
				my $filename       = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
				my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:in_iface'       => $in_iface, 
					's2:interface_name' => $interface_name, 
					's3:filename'       => $filename, 
					's4:interface_uuid' => $interface_uuid, 
				}});
				
				# Record the name to restart
				push @{$restart_interfaces}, $interface_name;
				
				# Query each value to see if it needs to be updated.
				my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection show ".$interface_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					return_code => $return_code, 
				}});
				foreach my $line (split/\n/, $output)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
					next if $line !~ /^ipv4/;
					
					my ($variable, $old_value) = ($line =~ /^(ipv4.*?):\s+(.*?)/);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						variable  => $variable,
						old_value => $old_value, 
					}});
					
					if ($variable eq "ipv4.addresses")
					{
						my $new_value = $ip."/".$cidr;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_value => $new_value }});
						if ($new_value ne $old_value)
						{
							update_nmcli($anvil, $interface_uuid, $variable, $new_value);
						}
					}
					if ($variable eq "ipv4.gateway")
					{
						# Is this the default gateway? 
						my $new_value = "";
						if ($default_gateway_interface eq $in_iface)
						{
							# Yes.
							$new_value = $gateway;
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_value => $new_value }});
						if ($new_value ne $old_value)
						{
							update_nmcli($anvil, $interface_uuid, $variable, $new_value);
						}
					}
					if ($variable eq "ipv4.dns")
					{
						# Is this the default gateway? 
						my $new_value = "";
						if ($default_gateway_interface eq $in_iface)
						{
							# Yes.
							$new_value = $cleaned_wanted_dns;
						}
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_value => $new_value }});
						if ($new_value ne $old_value)
						{
							update_nmcli($anvil, $interface_uuid, $variable, $new_value);
						}
					}
					if ($variable eq "ipv4.method")
					{
						# Is this the default gateway? 
						if ($old_value eq "auto")
						{
							update_nmcli($anvil, $interface_uuid, $variable, "manual");
						}
					}
				}
			}
			last;
		}
	}
	
	# If there are any entries in '$restart_interfaces', restart
	my $restart_interface_count = @{$restart_interfaces};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { restart_interface_count => $restart_interface_count }});
	if ($restart_interface_count)
	{
		# Disconnect from the database, as we're about to tear down our connection.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0079"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0079");
		$anvil->Database->disconnect();
		
		# Reload
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0463"});
		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection reload"});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		# NM seems to have a race issue, so we sleep a second after nmcli calls.
		sleep 1;
		
		# Restart
		$anvil->System->stop_daemon({debug => 2, daemon => "NetowkrManager.service"});
		sleep 5;
		$anvil->System->start_daemon({debug => 2, daemon => "NetowkrManager.service"});
		
		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
		my $wait_until = time + 130;
		until ($anvil->data->{sys}{database}{connections})
		{
			$anvil->refresh();
			$anvil->Database->connect();
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0132"});
			if (not $anvil->data->{sys}{database}{connections})
			{
				if (time > $wait_until)
				{
					# Failed to reconnect, exit. 
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0107"});
					$anvil->nice_exit({exit_code => 4});
				}
				
				# No databases, sleep and then try again.
				sleep 2;
			}
			
			# reload the job data.
			load_job($anvil);
		}
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0084"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0084");
	}
	
	# Remove virbr0 if it exists.
	if (exists $anvil->data->{network}{$local_host}{interface}{virbr0})
	{
		# Remove the NAT'ed bridge
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
		
		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default"});
		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default "});
		
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
	}
	
	# Update network view
	$anvil->Network->read_nmcli({debug => 3});
	$anvil->Network->get_ips({debug => 3});
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
	
	### NOTE: MTUs are dynamic on EL9, so we don't configure this anymore.
	# Update MTUs (running interface and config) if needed.
=cut
	foreach my $in_iface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
	{
		# Only one interface will start with the network name and have an IP address.
		my $current_mtu = $anvil->data->{network}{$local_host}{interface}{$in_iface}{mtu};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:in_iface'    => $in_iface, 
			's2:current_mtu' => $current_mtu,
		}});
		
		if (($mtu) && ($current_mtu eq $mtu))
		{
			# It's fine
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0087,!!interface!".$in_iface."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0087", variables => { 
				interface => $in_iface,
				mtu       => $mtu,
			}});
		}
		else
		{
			# Change the MTU both on the running interface and in the config file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0088,!!interface!".$in_iface."!!,!!old_mtu!".$current_mtu."!!,!!mtu!".$mtu."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0088", variables => { 
				interface => $in_iface, 
				old_mtu   => $current_mtu, 
				mtu       => $mtu, 
			}});
			
			# Change the live MTU.
			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{ip}." link set ".$in_iface." mtu ".$mtu});
			
			# Now update the config file.
			my $interface_uuid = $anvil->data->{nmcli}{$local_host}{device_to_uuid}{$in_iface};
			my $filename       = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{filename};
			my $interface_name = $anvil->data->{nmcli}{$local_host}{uuid}{$interface_uuid}{name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:interface_name' => $interface_name, 
				's2:filename'       => $filename, 
				's3:interface_uuid' => $interface_uuid, 
			}});
			
			my $mtu_seen   = 0;
			my $new_config = "";
			my $old_config = $anvil->Storage->read_file({file => $filename});
			foreach my $line (split/\n/, $old_config)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /^MTU=".*?"/)
				{
					$mtu_seen   =  1;
					$new_config .= "MTU=\"".$mtu."\"\n";
				}
				else
				{
					$new_config .= $line."\n";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_config => $new_config }});
				}
			}
			if (not $mtu_seen)
			{
				# Inject the MTU variable
				$new_config .= "MTU=\"".$mtu."\"\n";
			}
			
			# Write out the new file.
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0083,!!file!".$filename."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0083", variables => { file => $filename }});
			$anvil->Storage->write_file({
				debug     => 2,
				file      => $filename, 
				body      => $new_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644", 
				overwrite => 1,
			});
			
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");
		}
	}
=cut
	
	# Update NTP if set and needed.
	if ($ntp)
	{
		# Break up the NTP servers into a list, we'll set to '1' the ones we find.
		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0089");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0089"});
		my $updated = $anvil->System->check_ntp({debug => 2});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { updated => $updated }});
	}
	
	# Configure SSH by adding ours and our peer's SSH keys to ~/.ssh/known_hosts
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0113"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0113");
	$anvil->System->check_ssh_keys({debug => 2});
	
	# Setup IPMI, if needed.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0114"});
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0114");
	$anvil->System->configure_ipmi({debug => 2, manifest_uuid => $manifest_uuid});
	
	# Wait now until our peer's host name matches what's in the manifest.
	my $anvil_uuid     = $anvil->data->{sys}{anvil_uuid};
	my $peer_machine   = $anvil->data->{sys}{peer_machine};
	my $peer_host_name = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$peer_machine}{name};
	my $peer_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{"anvil_".$peer_machine."_host_uuid"};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"s1:anvil_uuid"     => $anvil_uuid, 
		"s2:peer_machine"   => $peer_machine,
		"s3:peer_host_name" => $peer_host_name,
		"s4:peer_host_uuid" => $peer_host_uuid,
	}});
	
	my $waiting = 1;
	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0482,!!peer_name!".$peer_host_name."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0482", variables => { peer_name => $peer_host_name }});
	while ($waiting)
	{
		$anvil->Database->get_hosts();
		my $current_host_name       = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{host_name};
		my $current_short_host_name = $anvil->data->{hosts}{host_uuid}{$peer_host_uuid}{short_host_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:current_host_name"       => $current_host_name,
			"s2:current_short_host_name" => $current_short_host_name,
		}});
		if (($peer_host_name eq $current_host_name) or 
		    ($peer_host_name eq $current_short_host_name))
		{
			# Done!
			$waiting = 0;
			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0483,!!peer_name!".$peer_host_name."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0483", variables => { peer_name => $peer_host_name }});
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0484", variables => { 
				old_peer_name => $current_short_host_name." / ".$current_host_name, 
				peer_name     => $peer_host_name,
			}});
			sleep 5;
		}
	}
	
	return(0);
}

sub update_nmcli
{
	my ($anvil, $interface_uuid, $variable, $new_value) = @_;
	
	my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection modify ".$interface_uuid." ".$variable." \"".$new_value."\"";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	
	return(0);
}

# Load the job details.
sub load_job
{
	my ($anvil) = @_;
	
	# See if we can find the job details. This method checks for the 'job-uuid' switch if it was used.
	$anvil->data->{switches}{'job-uuid'} = "" if not exists $anvil->data->{switches}{'job-uuid'};
	$anvil->data->{jobs}{job_uuid}       = "" if not exists $anvil->data->{jobs}{job_uuid};
	
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
	}});
	
	if (not $anvil->data->{jobs}{job_uuid})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0130"});
		sleep 1;
		$anvil->nice_exit({exit_code => 7});
	}
	
	$anvil->data->{job}{progress} = 0;
	update_progress($anvil, $anvil->data->{job}{progress}, "clear");
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0074,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0074", variables => { 'job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
	
	# Pull out the job data.
	$anvil->data->{job}{as_machine}    = "";
	$anvil->data->{job}{manifest_uuid} = "";
	$anvil->data->{job}{anvil_uuid}    = "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 
		'job_data' => $anvil->data->{jobs}{job_data},
	}});
	if ($anvil->data->{jobs}{job_data})
	{
		foreach my $pair (split/,/, $anvil->data->{jobs}{job_data})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pair => $pair }});
			if ($pair =~ /^(.*?)=(.*)$/)
			{
				my $variable = $1;
				my $value    = $2;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					variable => $variable,
					value    => $value, 
				}});
				
				$anvil->data->{job}{$variable} = $value;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"job::${variable}" => $anvil->data->{job}{$variable}, 
				}});
			}
		}
	}
	
	# Set the rejoin switch if needed.
	if ((not $anvil->data->{switches}{rejoin}) && ($anvil->data->{job}{rejoin}))
	{
		$anvil->data->{switches}{rejoin} = $anvil->data->{job}{rejoin};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"switches::rejoin" => $anvil->data->{switches}{rejoin}, 
		}});
	}
	
	# Verify we got what we needed.
	if ((not $anvil->data->{job}{as_machine})    or 
	    (not $anvil->data->{job}{manifest_uuid}) or 
	    (not $anvil->data->{job}{anvil_uuid}))
	{
		# This occassionally is hit, but then works when tried again. 
		update_progress($anvil, 100, "error_0308,!!job-uuid!".$anvil->data->{switches}{'job-uuid'}."!!,!!raw!".$anvil->data->{jobs}{job_data}."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0308", variables => { 
			'job-uuid' => $anvil->data->{switches}{'job-uuid'},
			raw        => $anvil->data->{jobs}{job_data},
		}});	
		sleep 2;
		$anvil->nice_exit({exit_code => 5});
	}
	
	$anvil->data->{sys}{machine}       = $anvil->data->{job}{as_machine};
	$anvil->data->{sys}{peer_machine}  = $anvil->data->{job}{as_machine} eq "node1" ? "node2" : "node1";
	$anvil->data->{sys}{manifest_uuid} = $anvil->data->{job}{manifest_uuid};
	$anvil->data->{sys}{anvil_uuid}    = $anvil->data->{job}{anvil_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"s1:sys::machine"       => $anvil->data->{sys}{machine},
		"s2:sys::peer_machine"  => $anvil->data->{sys}{peer_machine},
		"s3:sys::manifest_uuid" => $anvil->data->{sys}{manifest_uuid},
		"s4:sys::anvil_uuid"    => $anvil->data->{sys}{anvil_uuid},
	}});
	
	# Load in the host, manifest and anvil data.
	$anvil->Database->get_hosts();
	$anvil->Database->get_manifests();
	$anvil->Database->get_anvils();
	
	# Parse the manifest
	my $manifest_uuid = $anvil->data->{job}{manifest_uuid};
	if (not exists $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0046", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 0, "warning_0046,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
		sleep 10;
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Parse the manifest
	my $problem = $anvil->Striker->load_manifest({manifest_uuid => $anvil->data->{sys}{manifest_uuid}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
		problem       => $problem, 
	}});
	if ($problem)
	{
		# Something went wrong, fatally. Abort the job.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "job_0076", variables => { uuid => $anvil->data->{sys}{manifest_uuid} }});
		update_progress($anvil, 100, "job_0076,!!uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
		$anvil->nice_exit({exit_code => 2});
	}
	
	# Make sure we have a valid Anvil! 
	my $anvil_uuid = $anvil->data->{job}{anvil_uuid};
	if (not exists $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid})
	{
		# Odd. Error out, the Anvil! might not be loaded yet.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "warning_0121", variables => { uuid => $anvil->data->{sys}{anvil_uuid} }});
		update_progress($anvil, 0, "warning_0121,!!uuid!".$anvil->data->{sys}{anvil_uuid}."!!");
		sleep 10;
		$anvil->nice_exit({exit_code => 5});
	}
	
	# Load the anvil
	$anvil->data->{sys}{node1_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	$anvil->data->{sys}{node2_host_uuid} = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"sys::node1_host_uuid" => $anvil->data->{sys}{node1_host_uuid},
		"sys::node2_host_uuid" => $anvil->data->{sys}{node2_host_uuid},
	}});
	update_progress($anvil, ($anvil->data->{job}{progress} += 1), "job_0075,!!machine!".$anvil->data->{sys}{machine}."!!,!!manifest_uuid!".$anvil->data->{sys}{manifest_uuid}."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0075", variables => { 
		machine       => $anvil->data->{sys}{machine},
		manifest_uuid => $anvil->data->{sys}{manifest_uuid},
	}});	
	
	return(0);
}

# If this is being called as a job, this will allow the progress to be updated.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:progress' => $progress,
		's2:message'  => $message, 
	}});

	$progress = 98 if $progress > 100;
	if (not $anvil->data->{switches}{'job-uuid'})
	{
		return(0);
	}
	
	$anvil->Job->update_progress({
		debug    => 3,
		progress => $progress, 
		message  => $message,
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}

sub wait_for_subnodes
{
	my ($anvil) = @_;
	
	my $anvil_uuid      = $anvil->data->{sys}{anvil_uuid};
	my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid      => $anvil_uuid, 
		node1_host_uuid => $node1_host_uuid, 
		node2_host_uuid => $node2_host_uuid, 
	}});
	
	my $waiting = 1;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0477");
	while($waiting)
	{
		my $ready = 1;
		foreach my $host_uuid ($node1_host_uuid, $node2_host_uuid)
		{
			my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				host_uuid => $host_uuid, 
				host_name => $host_name, 
			}});
			
			my ($maintenance_mode, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
				variable_name         => "maintenance_mode",
				variable_source_table => "hosts",
				variable_source_uuid  => $host_uuid,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { maintenance_mode => $maintenance_mode }});
			(my $configured, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
				variable_name         => "system::configured", 
				variable_source_uuid  => $host_uuid, 
				variable_source_table => "hosts", 
			});
			$maintenance_mode = 1 if not defined $maintenance_mode;
			$configured       = 0 if not defined $configured;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configured => $configured }});
			
			# Is anvil-configure-host running?
			my $job_running = 0;
			$anvil->Database->get_jobs({job_host_uuid => $host_uuid});
			foreach my $job_uuid (sort {$a cmp $b} keys %{$anvil->data->{jobs}{running}})
			{
				my $job_command  = $anvil->data->{jobs}{running}{$job_uuid}{job_command};
				my $job_data     = $anvil->data->{jobs}{running}{$job_uuid}{job_data};
				my $job_progress = $anvil->data->{jobs}{running}{$job_uuid}{job_progress};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:job_uuid'     => $job_uuid, 
					's2:job_command'  => $job_command, 
					's3:job_data'     => $job_data, 
					's4:job_progress' => $job_progress, 
				}});
				next if $job_progress == 100;
				if ($job_command =~ /anvil-configure-host/)
				{
					$job_running = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_running => $job_running }});
				}
			}
			
			if (($maintenance_mode) or ($job_running) or (not $configured))
			{
				$ready = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ready => $ready }});
				
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0475", variables => { 
					subnode          => $host_name,
					configured       => $configured,
					maintenance_mode => $maintenance_mode, 
					job_running      => $job_running, 
				}});
			}
		}
		
		if ($ready)
		{
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0476"});
			sleep 5;
		}
	}
	
	return(0);
}
