#!/usr/bin/perl
# 
# This manages if a server is backed up to a DR host or not. When enabled, it can start or stop replication.
# 
# NOTE: Unlike most jobs, this one will directly work on the peer node and the DR host using SSH connections.
#       This behaviour is likely to change later as it's not ideal.
# 
# TODO: --remove is not yet implemented, this needs to be done. Use anvil-delete-server for methods to delete.
# EXAMPLES:
# - anvil-manage-dr --link --dr-host vm-a01dr02 --anvil vm-anvil-01
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Any problem that causes an early exit.
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Text::Diff;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

$anvil->Get->switches({list => [
	"anvil",
	"confirm",
	"connect",
	"disconnect",
	"dr-host",
	"job-uuid",
	"license-file",
	"link",
	"protect",
	"protocol",
	"remove",
	"restore",
	"server",
	"show", 
	"unlink",
	"update",
	"Yes"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect({debug => 2});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0306"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

# Backwards support.
if ($anvil->data->{switches}{Yes})
{
	$anvil->data->{switches}{confirm} = 1;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"switches::confirm" => $anvil->data->{switches}{confirm}, 
	}});
}

# Are we linking or unlinking?
if (($anvil->data->{switches}{'link'}) or ($anvil->data->{switches}{'unlink'}))
{
	handle_links($anvil);
}

# If we've got a job UUID, load the job details.
if ($anvil->data->{switches}{'job-uuid'})
{
	load_job($anvil);
}
elsif ($anvil->data->{switches}{show})
{
	# If '--show' was used, show all Nodes, DR hosts and servers.
	show_list($anvil);
}

# Are we looking for servers that should be protected by us, but aren't.
if ($anvil->data->{switches}{restore})
{
	handle_restore($anvil);
}

my $terminal = "";
if (not $anvil->data->{switches}{'job-uuid'})
{
	my $term    = $ENV{TERM} ? $ENV{TERM} : "xterm-256color";
	my $termios = new POSIX::Termios;
	$termios->getattr;
	my $ospeed   = $termios->getospeed;
	   $terminal = Tgetent Term::Cap { TERM => $term, OSPEED => $ospeed };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { terminal => $terminal }});
	
	$terminal->Trequire(qw/ce ku kd/);
	print $terminal->Tputs('cl');
}

sanity_check($anvil, $terminal);


$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub handle_restore
{
	my ($anvil) = @_;
	
	# If we're not a DR host, there's nothing to do.
	my $host_type = $anvil->Get->host_type();
	my $host_uuid = $anvil->Get->host_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_type => $host_type,
		host_uuid => $host_uuid, 
	}});
	
	if ($host_type ne "dr")
	{
		print $anvil->Words->string({key => "message_0218"})."\n";
		$anvil->nice_exit({exit_code => 1});
	}

	# If we're a DR host and we're asked to restore, see if we're connected to any anvils.
	$anvil->Database->get_anvils({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});
	my $link_count = 0;
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		my $anvil_description = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_description};
		my $dr_link_uuid      = exists $anvil->data->{dr_links}{by_host_uuid}{$host_uuid}{dr_link_anvil_uuid}{$anvil_uuid} ? $anvil->data->{dr_links}{by_host_uuid}{$host_uuid}{dr_link_anvil_uuid}{$anvil_uuid}{dr_link_uuid} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			's1:anvil_name'        => $anvil_name,
			's2:anvil_uuid'        => $anvil_uuid, 
			's3:anvil_description' => $anvil_description, 
			's4:dr_link_uuid'      => $dr_link_uuid, 
		}});
		if ($dr_link_uuid)
		{
			# We'll restore this one.
			$link_count++;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { link_count => $link_count }});
			
			# Tell the user we'll look for servers to restore
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0358", variables => {
				node_name   => $anvil_name, 
				description => $anvil_description, 
			}});
		}
		else
		{
			# Tell the user we will NOT look for servers to restore.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0359", variables => {
				node_name   => $anvil_name, 
				description => $anvil_description, 
			}});
		}
	}
	if (not $link_count)
	{
		# We're not linked to any nodes, nothing to do.
		$anvil->Job->update_progress({
			progress   => 100,
			log_level  => 1,
			'print'    => 1, 
			message    => "error_0496", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 5}),
		log_level => 1,
		'print'   => 1, 
		message   => "message_0222", 
	});
	
	# Load data
	$anvil->Database->get_hosts();
	$anvil->Database->get_storage_group_data({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});
	$anvil->Database->get_servers();
	$anvil->Database->get_lvm_data({debug => 2});
	
	# Which Anvil! nodes are we linked to?
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_host_uuid}{$host_uuid}{dr_link_anvil_name}})
	{
		my $dr_link_uuid = $anvil->data->{dr_links}{by_host_uuid}{$host_uuid}{dr_link_anvil_name}{$anvil_name}{dr_link_uuid};
		my $anvil_uuid   = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_anvil_uuid};
		my $dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:anvil_name"   => $anvil_name,
			"s2:dr_link_uuid" => $dr_link_uuid, 
			"s3:anvil_uuid"   => $anvil_uuid, 
			"s4:dr_link_note" => $dr_link_note, 
		}});
		next if $dr_link_note eq "DELETED";
		
		my $anvil_description     = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_description};
		my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
		my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
		my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
		my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:anvil_description"     => $anvil_description,
			"s2:anvil_password"        => $anvil->Log->is_secure($anvil_password), 
			"s3:node1_host_uuid"       => $node1_host_uuid, 
			"s4:node1_short_host_name" => $node1_short_host_name, 
			"s5:node2_host_uuid"       => $node2_host_uuid, 
			"s6:node2_short_host_name" => $node2_short_host_name, 
		}});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 1}),
			log_level => 1,
			'print'   => 1, 
			message   => "message_0223", 
			variables => {
				anvil_name        => $anvil_name, 
				anvil_description => $anvil_description, 
			},
		});
		
		# Can I access both subnodes?
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 1}),
			log_level => 1,
			'print'   => 1, 
			message   => "message_0224", 
			variables => {
				subnode_name => $node1_short_host_name, 
			},
		});
		my ($node1_target_ip, $node1_target_network) = $anvil->Network->find_target_ip({
			debug       => 3, 
			host_uuid   => $node1_host_uuid,
			ping        => 1, 
			test_access => 1,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:node1_target_ip"      => $node1_target_ip,
			"s2:node1_target_network" => $node1_target_network, 
		}});
		if ($node1_target_ip)
		{
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0225", 
				variables => {
					ip_address => $node1_target_ip, 
					network    => $node1_target_network, 
				},
			});
		}
		else
		{
			# No access
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0226", 
			});
		}
		
		 my ($node2_target_ip, $node2_target_network) = $anvil->Network->find_target_ip({
			debug       => 3, 
			host_uuid   => $node2_host_uuid,
			ping        => 1, 
			test_access => 1,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"s1:node2_target_ip"      => $node2_target_ip,
			"s2:node2_target_network" => $node2_target_network, 
		}});
		if ($node2_target_ip)
		{
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0225", 
				variables => {
					ip_address => $node2_target_ip, 
					network    => $node2_target_network, 
				},
			});
		}
		else
		{
			# No access
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0226", 
			});
		}
		
		# Can we proceed?
		if ((not $node1_target_ip) && (not $node2_target_ip))
		{
			# Nope. 
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0247", 
			});
			next;
		}
		
		my $use_ip        = $node1_target_ip ? $node1_target_ip       : $node2_target_ip;
		my $use_host_name = $node1_target_ip ? $node1_short_host_name : $node2_short_host_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			use_ip        => $use_ip,
			use_host_name => $use_host_name, 
		}});
		
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 1}),
			log_level => 1,
			'print'   => 1, 
			message   => "message_0249", 
			variables => {
				subnode_name => $use_host_name, 
			},
		});
		
		# Read in all of the DRBD resource files on the host.
		my $shell_call = $anvil->data->{path}{exe}{ls}." ".$anvil->data->{path}{directories}{drbd_resources};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_ip => $use_ip }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $use_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		# This will be updated, if needed, and pushed back to the source nodes if it changes.
		foreach my $file (split/\n/, $output)
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
			next if $file !~ /\.res$/;
			
			# Read the file:
			my $server_name =  $file;
			   $server_name =~ s/\.res$//;
			my $full_path   =  $anvil->data->{path}{directories}{drbd_resources}."/".$file;
			   $full_path   =~ s/\/\//\//g;
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0252", 
				variables => {
					file        => $full_path, 
					server_name => $server_name,
				},
			});
			
			if (-e $full_path)
			{
				# It's already here, skip it.
				$anvil->Job->update_progress({
					progress  => $anvil->Job->bump_progress({steps => 1}),
					log_level => 1,
					'print'   => 1, 
					message   => "message_0255", 
				});
				next;
			}
			
			# Read it from the peer
			my $new_resource_body = "";
			my $resource_body = $anvil->Storage->read_file({
				file     => $full_path,
				target   => $use_ip, 
				password => $anvil_password,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_body => $resource_body }});
			
			my $in_on          = 0;
			my $in_volume      = 0;
			my $this_volume    = "";
			my $this_disk      = "";
			my $this_host_name = "";
			my $this_host_uuid = "";
			foreach my $line (split/\n/, $resource_body)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /\}/)
				{
					if ($in_volume)
					{
						$in_volume = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_volume => $in_volume }});
					}
					elsif ($in_on)
					{
						# Record the data.
						if ($this_volume eq "")
						{
							die "Reached end of section without a volume.\n";
						}
						
						# How big is this LV?
						my $this_short_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_short_host_name => $this_short_host_name }});
						
						my $this_lv_size  = 0;
						my $this_lv_on_vg = "";
						if (exists $anvil->data->{lvm}{host_name}{$this_short_host_name}{lv_path}{$this_disk})
						{
							my $scan_lvm_lv_name     = $anvil->data->{lvm}{host_name}{$this_short_host_name}{lv_path}{$this_disk}{scan_lvm_lv_name};
							   $this_lv_size         = $anvil->data->{lvm}{host_name}{$this_short_host_name}{lv}{$scan_lvm_lv_name}{scan_lvm_lv_size};
							   $this_lv_on_vg        = $anvil->data->{lvm}{host_name}{$this_short_host_name}{lv}{$scan_lvm_lv_name}{scan_lvm_lv_on_vg};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:scan_lvm_lv_name' => $scan_lvm_lv_name, 
								's2:this_lv_size'     => $this_lv_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_lv_size}).")",
								's3:this_lv_on_vg'    => $this_lv_on_vg, 
							}});
						}
						
						$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{host_uuid}                      = $this_host_uuid;
						$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_path}  = $this_disk;
						$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_size}  = $this_lv_size;
						$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_on_vg} = $this_lv_on_vg;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							"s1:drbd_resource::${file}::host_name::${this_host_name}::host_uuid"                        => $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{host_uuid},
							"s2:drbd_resource::${file}::host_name::${this_host_name}::volume::${this_volume}::lv_path"  => $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_path},
							"s3:drbd_resource::${file}::host_name::${this_host_name}::volume::${this_volume}::lv_size"  => $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_size}}).")",
							"s4:drbd_resource::${file}::host_name::${this_host_name}::volume::${this_volume}::lv_on_vg" => $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_on_vg}, 
						}});
						
						$in_on          = 0;
						$this_host_name = "";
						$this_host_uuid = "";
						$this_disk      = "";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							in_on          => $in_on, 
							this_host_name => $this_host_name,
							this_host_uuid => $this_host_uuid, 
							this_disk      => $this_disk, 
						}});
					}
				}
				if ($line =~ /on (.*?) \{/)
				{
					$this_host_name = $1;
					$this_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $this_host_name});
					$in_on          = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						this_host_name => $this_host_name,
						this_host_uuid => $this_host_uuid, 
						in_on          => $in_on, 
					}});
				}
				if ($line =~ /volume (\d+) \{/)
				{
					$this_volume = $1;
					$in_volume   = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						this_volume => $this_volume,
						in_volume   => $in_volume, 
					}});
				}
				if ($line =~ /\sdisk\s+(.*?);/)
				{
					$this_disk = $1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_disk => $this_disk }});
				}
			}
			
			# Get the LV sizes we'll need to create.
			foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$file}{host_name}})
			{
				my $this_host_uuid = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{host_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_host_name => $this_host_name,
					this_host_uuid => $this_host_uuid, 
				}});
				foreach my $this_volume (sort {$a <=> $b} keys %{$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}})
				{
					my $lv_path       = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_path};
					my $lv_size       = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_size};
					my $this_lv_on_vg = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_on_vg};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:this_volume'   => $this_volume, 
						's2:lv_path'       => $lv_path, 
						's3:lv_size'       => $lv_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}).")",
						's4:this_lv_on_vg' => $this_lv_on_vg, 
					}});
					
					if ((not exists $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size}) or 
					    ($lv_size > $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size}))
					{
						$anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size} = $lv_size;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							"drbd_resource::${file}::volume::${this_volume}::needed_size" => $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size}}).")",
						}});
					}
					
					foreach my $this_storage_group_uuid (sort {$a cmp $b} keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}})
					{
						next if $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{storage_group_uuid};
						if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$this_storage_group_uuid}{host_uuid}{$this_host_uuid})
						{
							my $this_vg_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$this_storage_group_uuid}{host_uuid}{$this_host_uuid}{vg_name};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_vg_name => $this_vg_name }});
							
							if ($this_lv_on_vg eq $this_vg_name)
							{
								$anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{storage_group_uuid} = $this_storage_group_uuid;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									"drbd_resource::${file}::volume::${this_volume}::storage_group_uuid" => $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{storage_group_uuid},
								}});
							}
						}
					}
				}
			}
			
			# Do I need to make this LV?
			foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$file}{host_name}})
			{
				my $this_host_uuid = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{host_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_host_name => $this_host_name,
					this_host_uuid => $this_host_uuid, 
				}});
				next if $this_host_uuid ne $host_uuid;
				
				foreach my $this_volume (sort {$a <=> $b} keys %{$anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}})
				{
					my $drbd_device        = "/dev/drbd/by-res/".$server_name."/".$this_volume;
					my $original_lv_path   = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_path};
					my $this_lv_on_vg      = $anvil->data->{drbd_resource}{$file}{host_name}{$this_host_name}{volume}{$this_volume}{lv_on_vg};
					my $needed_size        = $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{needed_size};
					my $storage_group_uuid = $anvil->data->{drbd_resource}{$file}{volume}{$this_volume}{storage_group_uuid};
					my $storage_group_name = $storage_group_uuid ? $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{storage_group_name} : "";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:this_volume'        => $this_volume,
						's2:drbd_device'        => $drbd_device, 
						's3:original_lv_path'   => $original_lv_path, 
						's4:this_lv_on_vg'      => $this_lv_on_vg, 
						's5:needed_size'        => $needed_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}).")", 
						's6:storage_group_uuid' => $storage_group_uuid, 
						's7:storage_group_name' => $storage_group_name, 
					}});
					
					# Do I have a storage group?
					if (not $storage_group_uuid)
					{
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "warning_0023", 
						});
						next;
					}
					
					if (not $needed_size)
					{
						# Something went wrong with calculating the space we needed.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "warning_0025", 
						});
						next;
					}
					
					$anvil->Job->update_progress({
						progress  => $anvil->Job->bump_progress({steps => 1}),
						log_level => 1,
						'print'   => 1, 
						message   => "message_0256", 
						variables => {
							drbd_device => $drbd_device,
						},
					});
					if (-e $drbd_device)
					{
						# The DRBD device exists, nothing to do.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "message_0257", 
						});
						next;
					}
					elsif (-e $original_lv_path)
					{
						# The LV exists, nothing to do.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "message_0258", 
							variables => {
								lv_path => $original_lv_path,
							},
						});
						next;
					}
					
					# If we're still here, we need to create the LV. What VG though? It 
					# might have changed names.
					if (not exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid})
					{
						# We're not in this storage group yet.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "message_0259", 
							variables => {
								volume        => $this_volume,
								storage_group => $storage_group_name,
								'bytes'       => $anvil->Convert->add_commas({number => $needed_size}), 
								hr_size       => $anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}),
							},
						});
						next;
					}
					
					# Do I have enough space?
					my $vg_name      = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_name};
					my $vg_size      = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_size};
					my $vg_free      = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_free};
					my $extent_size  = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$host_uuid}{vg_extent_size};
					my $extent_count = int($needed_size / $extent_size);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:vg_name'      => $vg_name,
						's2:vg_size'      => $vg_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_size}).")", 
						's3:vg_free'      => $vg_free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}).")", 
						's4:needed_size'  => $needed_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}).")", 
						's5:extent_size'  => $extent_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
						's6:extent_count' => $extent_count, 
					}});
					
					if ($needed_size > $vg_free)
					{
						# Not enough space.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "warning_0024", 
							variables => {
								vg_name            => $vg_name, 
								needed_space_bytes => $anvil->Convert->add_commas({number => $needed_size}), 
								needed_space_hr    => $anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}), 
								free_space_bytes   => $anvil->Convert->add_commas({number => $vg_free}), 
								free_space_hr      => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}), 
							},
						});
						next;
					}
					
					# Has the new LV path changed?
					my $new_lv_name =  $server_name."_".$this_volume;
					my $new_lv_path = "/dev/".$vg_name."/".$new_lv_name;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:server_name' => $server_name,
						's2:new_lv_name' => $new_lv_name, 
						's3:new_lv_path' => $new_lv_path, 
					}});
					$anvil->Job->update_progress({
						progress  => $anvil->Job->bump_progress({steps => 1}),
						log_level => 1,
						'print'   => 1, 
						message   => "message_0260", 
						variables => {
							vg_name            => $vg_name, 
							new_lv_name        => $new_lv_name, 
							needed_space_bytes => $anvil->Convert->add_commas({number => $needed_size}), 
							needed_space_hr    => $anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}), 
							free_space_bytes   => $anvil->Convert->add_commas({number => $vg_free}), 
							free_space_hr      => $anvil->Convert->bytes_to_human_readable({'bytes' => $vg_free}), 
						},
					});
					
					# If the LV path is a different name, warn the user we'll be updating
					# the DRBD resource config and updating the nodes. Confirm before 
					# proceeding.
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:new_lv_path'      => $new_lv_path, 
						's2:original_lv_path' => $original_lv_path, 
					}});
					if ($new_lv_path ne $original_lv_path)
					{
						# Ask for confirmation.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0261", variables => {
							new_lv_name => $new_lv_path, 
							old_lv_name => $original_lv_path, 
						}});
						if ($anvil->data->{switches}{confirm})
						{
							# Don't prompt.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0150"});
						}
						else
						{
							my $answer = <STDIN>;
							chomp $answer;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 0, list => { answer => $answer }});
							if ($answer =~ /^y/i)
							{
								# Proceed
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0262"});
							}
							elsif ($answer =~ /^a/i)
							{
								# Proceed with all.
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0520"});
								$anvil->data->{switches}{confirm} = 1;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									'switches::confirm' => $anvil->data->{switches}{confirm}, 
								}});
							}
							else
							{
								# Skip
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0268"});
								next;
							}
						}
					}
					else
					{
						# The LV path is the same, so we can safely just proceed.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0269", variables => { 
							lv_path      => $new_lv_path,
							lv_size      => $anvil->Convert->bytes_to_human_readable({'bytes' => $needed_size}), 
							extent_count => $anvil->Convert->add_commas({number => $extent_count}), 
							extent_size  => $anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}), 
						}});
					}
					
					# Create the LV
					my $shell_call = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$new_lv_name." ".$vg_name." -y";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call });
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					
					# Update our LVM view
					run_scanagent($anvil, "scan-lvm");
					
					sleep 1;
					# Does it exist now?
					if (not -e $new_lv_path)
					{
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "warning_0026", 
						});
						next;
					}
					
					# Tell the user we create the LV
					$anvil->Job->update_progress({
						progress  => $anvil->Job->bump_progress({steps => 1}),
						log_level => 1,
						'print'   => 1, 
						message   => "message_0521", 
						variables => {
							lv_path => $new_lv_name, 
						},
					});
					
					# Update the DRBD resource file.
					foreach my $line (split/\n/, $resource_body)
					{
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { '>>line' => $line }});
						$line =~ s/$original_lv_path/$new_lv_path/;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { '<<line' => $line }});
						
						$new_resource_body .= $line."\n";
					}
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_body => $new_resource_body }});
					
					my $difference = diff \$resource_body, \$new_resource_body, { STYLE => 'Unified' };
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
					
					if ($difference)
					{
						$resource_body = $new_resource_body;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_body => $resource_body }});
						
						# Verify that the new config is valid
						test_config($anvil, $server_name, $full_path, $resource_body);
						
						# Tell the user we're updating the peers'
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "message_0522", 
							variables => {
								file       => $full_path, 
								difference => $difference, 
							},
						});
					}
					
					# Write out the resource file (if we didn't in a previous volume or it changed).
					if ((not -e $full_path) or ($difference))
					{
						# Write out a test file.
						$anvil->Job->update_progress({
							progress  => $anvil->Job->bump_progress({steps => 1}),
							log_level => 1,
							'print'   => 1, 
							message   => "message_0322", 
							variables => {
								file => $full_path, 
							},
						});
						my ($problem) = $anvil->Storage->write_file({
							debug     => 2,
							backup    => 1,
							overwrite => 1,
							file      => $full_path, 
							body      => $resource_body,
							user      => "root", 
							group     => "root", 
							mode      => "0644",
						});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
						run_scanagent($anvil, "scan-drbd");
					}
					
					# If there's a difference, update the peers, to.
					if ($difference)
					{
						foreach my $target_ip ($node1_target_ip, $node2_target_ip)
						{
							# Tell the user we're updating the peers'
							$anvil->Job->update_progress({
								progress  => $anvil->Job->bump_progress({steps => 1}),
								log_level => 1,
								'print'   => 1, 
								message   => "message_0523", 
								variables => {
									file      => $full_path, 
									target_ip => $target_ip, 
									resource  => $server_name, 
								},
							});
							
							# Write the new config.
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }});
							my ($problem) = $anvil->Storage->write_file({
								backup    => 1,
								overwrite => 1,
								file      => $full_path, 
								body      => $resource_body,
								user      => "root", 
								group     => "root", 
								mode      => "0644",
								target    => $target_ip, 
								password  => $anvil_password,
							});
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
							
							my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
							my ($output, $error, $return_code) = $anvil->Remote->call({
								target     => $target_ip, 
								password   => $anvil_password,
								shell_call => $shell_call,
							});
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								error       => $error,
								output      => $output,
								return_code => $return_code, 
							}});
							
							# Call 'scan-drbd' on the peer.
							$shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
							
							($output, $error, $return_code) = $anvil->Remote->call({
								target     => $target_ip, 
								password   => $anvil_password,
								shell_call => $shell_call,
							});
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								error       => $error,
								output      => $output,
								return_code => $return_code, 
							}});
						}
					}
					
					# Created, now create the MD. 
					$anvil->Job->update_progress({
						progress  => $anvil->Job->bump_progress({steps => 1}),
						log_level => 1,
						'print'   => 1, 
						message   => "message_0323", 
						variables => {
							server_name => $server_name, 
							volume      => $this_volume, 
						},
					});
					$shell_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$server_name."/".$this_volume;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
					($output, $return_code) = $anvil->System->call({shell_call => $shell_call });
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output,
						return_code => $return_code, 
					}});
					
					# Update our DRBD view
					run_scanagent($anvil, "scan-drbd");
				}
			}
			
			# Adjust the DRBD resource.
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				log_level => 1,
				'print'   => 1, 
				message   => "message_0327", 
				variables => {
					server_name => $server_name, 
				},
			});
			my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call });
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output,
				return_code => $return_code, 
			}});
			
			sleep 1;
			
			# Call scan-drbd
			run_scanagent($anvil, "scan-drbd");
		}
		# All servers on this Anvil! node processed.
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 1}),
			log_level => 1,
			'print'   => 1, 
			message   => "message_0355", 
			variables => {
				node_name => $anvil_name, 
			},
		});
	}
	
	# Make sure the firewall is updated.
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 5}),
		log_level => 1,
		'print'   => 1, 
		message   => "message_0519", 
	});
	my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-firewall'}.$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	
	# All nodes processed, we're done.
	$anvil->Job->update_progress({
		progress  => 100,
		log_level => 1,
		'print'   => 1, 
		message   => "message_0356", 
	});
	
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}

sub run_scanagent
{
	my ($anvil, $agent) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { agent => $agent }});
	
	# Call scan-lvm
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0518", variables => { scan_agent => $agent }});
	$anvil->ScanCore->call_scan_agents({debug => 2, agent => $agent});
	
	return(0);
}

sub show_list
{
	my ($anvil) = @_;
	
	# Show Anvil! nodes
	$anvil->Database->get_hosts();
	$anvil->Database->get_storage_group_data({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});
	$anvil->Database->get_servers();
	
	# Show Anvil! nodes.
	print $anvil->Words->string({key => "header_0081"})."\n";
	my $anvil_count = keys %{$anvil->data->{anvils}{anvil_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_count => $anvil_count }});
	if ($anvil_count)
	{
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			my $dr_link_count     = keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				anvil_name        => $anvil_name,
				anvil_uuid        => $anvil_uuid, 
				anvil_description => $anvil_description, 
				dr_link_count     => $dr_link_count, 
			}});
			print $anvil->Words->string({key => "message_0364", variables => { 
				anvil_name  => $anvil_name,
				description => $anvil_description,
			}})."\n";
			if ($dr_link_count)
			{
				foreach my $dr_link_host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
				{
					my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$dr_link_host_name}{dr_link_uuid};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						dr_link_host_name => $dr_link_host_name,
						dr_link_uuid      => $dr_link_uuid, 
					}});
					print $anvil->Words->string({key => "message_0365", variables => { 
						host_name => $dr_link_host_name,
						link_uuid => $dr_link_uuid,
					}})."\n";
				}
			}
			else
			{
				# No DR hosts yet
				print $anvil->Words->string({key => "message_0366"})."\n";
			}
		}
	}
	else
	{
		# No nodes yet
		print $anvil->Words->string({key => "message_0367"})."\n";
	}
	
	# DR Hosts
	print "\n".$anvil->Words->string({key => "header_0115"})."\n";
	my $dr_host_count = keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_count => $dr_host_count }});
	if ($dr_host_count)
	{
		# TODO: Show free space of VGs, available RAM and CPU core count and type/speed
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_type}{dr}{host_name}})
		{
			print $anvil->Words->string({key => "message_0368", variables => { host_name => $host_name }})."\n";
		}
	}
	else
	{
		# No DR hosts yet
		print $anvil->Words->string({key => "message_0366"})."\n";
	}
	
	# DR Hosts
	print "\n".$anvil->Words->string({key => "header_0116"})."\n";
	my $server_count = keys %{$anvil->data->{servers}{server_uuid}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
	if ($server_count)
	{
		# If there's servers, there has to be Anvil! nodes.
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			# TODO: Show each server's disk(s) size, RAM and CPU core count
			my $anvil_uuid  = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
			{
				my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
				my $server_name = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
				print $anvil->Words->string({key => "message_0370", variables => { 
					server_name => $server_name,
					anvil_name  => $anvil_name,
				}})."\n";
			}
		}
	}
	else
	{
		# No servers yet exist
		print $anvil->Words->string({key => "message_0371"})."\n";
	}
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}

sub handle_links
{
	my ($anvil) = @_;

	# Get a list of DR hosts.
	$anvil->Database->get_dr_links();

	# Do we habe an Anvil! and a DR host?
	my $anvil_uuid = $anvil->Database->get_anvil_uuid_from_string({string => $anvil->data->{switches}{anvil}});
	my $host_uuid  = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{'dr-host'}});
	my $host_type  = $anvil->Get->host_type();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		's1:anvil_uuid' => $anvil_uuid, 
		's2:host_uuid'  => $host_uuid, 
		's3:host_type'  => $host_type, 
	}});
	
	if ((not $host_uuid) && ($host_type eq "dr"))
	{
		# Use ourself.
		$host_uuid = $anvil->Get->host_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
	}
	
	if ((not $anvil_uuid) && ($host_type eq "node"))
	{
		# Use this node.
		$anvil_uuid = $anvil->Cluster->get_anvil_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
	}

	my $problem = 0;
	if (not $anvil_uuid)
	{
		$problem = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		if ($anvil->data->{switches}{anvil})
		{
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				job_status => "failed",
				message    => "error_0405",
				variables  => {
					anvil  => $anvil->data->{switches}{anvil},
				},
			});
		}
		else
		{
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				job_status => "failed",
				message    => "error_0406",
			});
		}
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $anvil_uuid        = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			print "- ".$anvil_name." - ".$anvil_description."\n";
		}
	}

	my $show_hosts = 0;
	if ($host_uuid)
	{
		# Is it a DR host.
		my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
		if ($host_type ne "dr")
		{
			$show_hosts = 1;
			$problem    = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				show_hosts => $show_hosts,
				problem    => $problem,
			}});
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				job_status => "failed",
				message    => "error_0408",
				variables  => {
					host   => $anvil->data->{switches}{'dr-host'},
				},
			});
		}
	}
	else
	{
		$show_hosts = 1;
		$problem    = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			show_hosts => $show_hosts,
			problem    => $problem,
		}});
		if ($anvil->data->{switches}{anvil})
		{
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				job_status => "failed",
				message    => "error_0407",
				variables  => {
					host   => $anvil->data->{switches}{'dr-host'},
				},
			});
		}
		else
		{
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				job_status => "failed",
				message    => "error_0409",
			});
		}
	}

	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { show_hosts => $show_hosts }});
	if ($show_hosts)
	{
		$anvil->Database->get_hosts();
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
			my $this_host_uuid  = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $short_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
			my $host_type       = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{host_type};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				this_host_uuid  => $this_host_uuid,
				host_name       => $host_name,
				short_host_name => $short_host_name,
				host_type       => $host_type,
			}});
			next if $host_type ne "dr";
			print "- ".$short_host_name."\n";
		}
	}
	
	if ($problem)
	{
		$anvil->Job->update_progress({
			progress   => $anvil->Job->bump_progress({steps => 1}),
			log_level  => 1,
			'print'    => 1,
			job_status => "failed",
			message    => "error_0410",
		});
		foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
		{
			my $this_anvil_uuid   = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
			my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
			print "- ".$anvil_name." - ".$anvil_description."\n";

			foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$this_anvil_uuid}{dr_link_host_name}})
			{
				my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$this_anvil_uuid}{dr_link_host_name}{$host_name}{dr_link_uuid};
				print " ^-> ".$host_name."\n";
			}
		}
		print "\n";
		$anvil->Job->update_progress({progress => 100});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Are there any servers using this link?
	$anvil->Database->get_servers({debug => 2});
	$anvil->Database->get_drbd_data({debug => 2});
	my $protected_server_count = 0;
	my $dr_short_host_name     = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_short_host_name => $dr_short_host_name }});
	foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
	{
		my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			server_name => $server_name, 
			server_uuid => $server_uuid, 
		}});
		
		if (exists $anvil->data->{drbd}{host_name}{$dr_short_host_name}{resource_name}{$server_name})
		{
			my $resource_uuid = $anvil->data->{drbd}{host_name}{$dr_short_host_name}{resource_name}{$server_name}{resource_uuid}; 
			my $resource_up   = $anvil->data->{drbd}{host_name}{$dr_short_host_name}{resource_name}{$server_name}{up}; 
			my $resource_xml  = $anvil->data->{drbd}{host_name}{$dr_short_host_name}{resource_name}{$server_name}{xml}; 
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				resource_uuid => $resource_uuid, 
				resource_up   => $resource_up, 
				resource_xml  => $resource_xml, 
			}});
			next if $resource_xml eq "DELETED";
			
			# We can't unlink
			$protected_server_count++;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { protected_server_count => $protected_server_count }});
			print $anvil->Words->string({key => "warning_0027", variables => { 
				server  => $server_name, 
				dr_name => $dr_short_host_name, 
			}})."\n";
		}
	}
	
	if ($protected_server_count)
	{
		# Abort.
		my $anvil_name = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
		$anvil->Job->update_progress({
			progress   => 100,
			log_level  => 1,
			'print'    => 1, 
			job_status => "failed",
			message    => "error_0198",
			variables  => {
				node_name => $anvil_name, 
				dr_host   => $dr_short_host_name, 
			}, 
		});
		$anvil->nice_exit({exit_code => 1});
	}

	# Still alivee? Update!
	my $dr_link_uuid = "";
	if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$host_uuid})
	{
		$dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$host_uuid}{dr_link_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_link_uuid => $dr_link_uuid }});
	}

	if ($anvil->data->{switches}{'link'})
	{
		my $returned_dr_link_uuid = $anvil->Database->insert_or_update_dr_links({
			debug => 2,
			dr_link_host_uuid  => $host_uuid,
			dr_link_anvil_uuid => $anvil_uuid,
			dr_link_note       => "user-created",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { returned_dr_link_uuid => $returned_dr_link_uuid }});
		
		$anvil->Job->update_progress({
			progress   => $anvil->Job->bump_progress({steps => 1}),
			log_level  => 1,
			'print'    => 1,
			message    => "log_0734",
			variables  => {
				host   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name},
				anvil  => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
			},
		});
	}
	elsif ($anvil->data->{switches}{'unlink'})
	{
		if ($dr_link_uuid)
		{
			# Delete
			$anvil->Database->insert_or_update_dr_links({
				debug        => 2,
				dr_link_uuid => $dr_link_uuid,
				'delete'     => 1,
			});
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				message    => "log_0735",
				variables  => {
					host   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name},
					anvil  => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
				},
			});
		}
		else
		{
			# Link didn't exist, nothing to unlink.
			$anvil->Job->update_progress({
				progress   => $anvil->Job->bump_progress({steps => 1}),
				log_level  => 1,
				'print'    => 1,
				message    => "log_0736",
				variables  => {
					host   => $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name},
					anvil  => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
				},
			});
		}
	}
	$anvil->Job->update_progress({progress => 100});
	$anvil->nice_exit({exit_code => 0});

	return(0);
}

sub sanity_check
{
	my ($anvil, $terminal) = @_;
	
	# Begin sanity checks
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 5}),
		log_level => 1,
		'print'   => 1, 
		message   => "job_0358", 
	});

	# Somehow, duplicate entries are being created in dr_links. Look for and purge them.
	check_for_duplicate_dr_links($anvil);

	# Are we a node or DR?
	my $host_type  = $anvil->Get->host_type();
	my $host_uuid  = $anvil->Get->host_uuid();
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_type  => $host_type,
		host_uuid  => $host_uuid, 
		anvil_uuid => $anvil_uuid, 
	}});

	if ($host_type eq "striker")
	{
		# This has to be run on an Anvil! sub-node. (For now at least. We need to make a menu to let
		# the user select a VM interactively).
		$anvil->Job->update_progress({
			progress   => 100,
			log_level  => 1,
			'print'    => 1, 
			message    => "error_0404",
			job_status => "failed",
		});
		$anvil->nice_exit({exit_code => 1});
	}

	# If we're protecting, removing, connecting, disconnecting, or update, we need to have a server.
	$anvil->Database->get_servers();
	my $server_name  = "";
	my $server_uuid  = "";
	if ($anvil->data->{switches}{server})
	{
		$server_uuid = $anvil->Database->get_server_uuid_from_string({
			debug  => 2,
			string => $anvil->data->{switches}{server},
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_uuid => $server_uuid }});
	}
	
	my $host_anvil_uuid = "";
	if ($server_uuid)
	{
		# Lastly, is this server on this Anvil! node?
		$server_name     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
		$host_anvil_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			server_name     => $server_name,
			host_anvil_uuid => $host_anvil_uuid." (".$anvil->data->{anvils}{anvil_uuid}{$host_anvil_uuid}{anvil_name}.")",
		}});
	}

	my $dr_link_uuid = "";
	my $dr_link_note = "";
	my $dr_host_uuid = "";
	my $dr_linked    = 0;
	if (($anvil->data->{switches}{'connect'})    or
	    ($anvil->data->{switches}{'disconnect'}) or
	    ($anvil->data->{switches}{protect})      or
	    ($anvil->data->{switches}{remove})       or
	    ($anvil->data->{switches}{update}))
	{
		# Were we given a server?
		if (not $anvil->data->{switches}{server})
		{
			# Please specify the server to manager using '--server <name or uuid>'.
			$anvil->Job->update_progress({
				progress   => 100,
				log_level  => 1,
				'print'    => 1, 
				message    => "error_0340",
				job_status => "failed",
			});

			# Show the servers on this anvil and exit.
			show_servers($anvil, $anvil_uuid);
			$anvil->nice_exit({exit_code => 1});
		}
		if (not $server_uuid)
		{
			$anvil->Job->update_progress({
				progress   => 100,
				log_level  => 1,
				'print'    => 1, 
				job_status => "failed",
				message    => "error_0411",
				variables  => { 
					server => $anvil->data->{switches}{server},
				},
			});

			# Show the servers on this anvil and exit.
			show_servers($anvil, $anvil_uuid);
			$anvil->nice_exit({exit_code => 1});
		}
		else
		{
			if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$host_anvil_uuid}{dr_link_host_uuid}{$host_uuid})
			{
				$dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$host_anvil_uuid}{dr_link_host_uuid}{$host_uuid}{dr_link_uuid};
				$dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note};
				$dr_host_uuid = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					dr_link_uuid => $dr_link_uuid,
					dr_link_note => $dr_link_note, 
					dr_host_uuid => $dr_host_uuid, 
				}});
				
				if ($dr_link_note ne "DELETED")
				{
					$dr_linked = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_linked => $dr_linked }});
				}
				else
				{
					# Clear the link UUID
					$dr_link_uuid = "";
					$dr_link_note = "";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						dr_link_uuid => $dr_link_uuid,
						dr_link_note => $dr_link_note, 
					}});
				}
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				'switches::connect'    => $anvil->data->{switches}{'connect'},
				'switches::disconnect' => $anvil->data->{switches}{'disconnect'},
				'switches::protect'    => $anvil->data->{switches}{protect},
				'switches::remove'     => $anvil->data->{switches}{remove},
				'switches::update'     => $anvil->data->{switches}{update},
			}});
		}

		if ($anvil_uuid ne $host_anvil_uuid)
		{
			if ($host_type eq "dr")
			{
				# DR hosts don't belong to a single anvil, so see if this DR host is peered 
				# with the Anvil!.
				if ($dr_linked)
				{
					# We're good, set the Anvil! UUID to the host UUID.
					$anvil_uuid                     = $host_anvil_uuid;
					$anvil->data->{sys}{anvil_uuid} = $anvil_uuid;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						anvil_uuid        => $anvil_uuid,
						"sys::anvil_uuid" => $anvil->data->{sys}{anvil_uuid}, 
					}});
				}
				else
				{
					# We're not linked to this Anvil! node.
					$anvil->Job->update_progress({
						progress   => 100,
						log_level  => 1,
						'print'    => 1, 
						job_status => "failed",
						message    => "error_0012",
						variables  => {
							server => $server_name,
							anvil  => $anvil->data->{anvils}{anvil_uuid}{$host_anvil_uuid}{anvil_name},
						},
					});

					# Show the servers on this anvil and exit.
					show_servers($anvil, $anvil_uuid);
					$anvil->nice_exit({exit_code => 1});
				}
			}
			else
			{
				# Server isn't here.
				$anvil->Job->update_progress({
					progress   => 100,
					log_level  => 1,
					'print'    => 1, 
					job_status => "failed",
					message    => "error_0412",
					variables  => {
						server     => $server_name,
						this_anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
						on_anvil   => $anvil->data->{anvils}{anvil_uuid}{$host_anvil_uuid}{anvil_name},
					},
				});

				# Show the servers on this anvil and exit.
				show_servers($anvil, $anvil_uuid);
				$anvil->nice_exit({exit_code => 1});
			}
		}
	}

	# If we're (dis}connecting, is the server being protected in the first place?
	if (($anvil->data->{switches}{'connect'}) or ($anvil->data->{switches}{'disconnect'}))
	{
		# Is this server configured to be protected?
		my $config_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
		   $config_file =~ s/\/\//\//g;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
		if (not -e $config_file)
		{
			# Can't connect an unprotected server.
			$anvil->Job->update_progress({
				progress   => 100,
				log_level  => 1,
				'print'    => 1, 
				job_status => "failed", 
				message    => "error_0371", 
				variables  => {
					server      => $server_name, 
					config_file => $config_file, 
				}, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}

	# If we're doing a --protect or --remove, make sure we're a node, the cluster is up, and both nodes 
	# are ready.
	if (($anvil->data->{switches}{protect}) or ($anvil->data->{switches}{remove}))
	{
		# Make sure we're in an Anvil! if we're a node (DR link will be checked later).
		if (($host_type eq "node") && (not $anvil_uuid))
		{
			# This must be run on a node active in the cluster hosting the server being managed.
			$anvil->Job->update_progress({
				progress   => 100,
				log_level  => 1,
				'print'    => 1, 
				message    => "error_0332", 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# If we're a node, make sure we're in a cluster.
		if ($host_type eq "node")
		{
			# Can we parse the CIB?
			my ($problem) = $anvil->Cluster->parse_cib();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
			if ($problem)
			{
				# No. Clear the job,
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0336"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0336", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			
			# Both nodes need to be in the cluster, are they?
			if (not $anvil->data->{cib}{parsed}{'local'}{ready})
			{
				# We're not a full member of the cluster yet. Please try again once we're fully in. Exiting.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0337"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0337", 
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
		}
		
		if ($anvil->data->{switches}{protect})
		{
			if (not $anvil->data->{switches}{protocol})
			{
				$anvil->data->{switches}{protocol} = "short-throw";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::protocol' => $anvil->data->{switches}{protocol},
				}});
			}
			elsif (($anvil->data->{switches}{protocol} ne "sync")        && 
			       ($anvil->data->{switches}{protocol} ne "short-throw") && 
			       ($anvil->data->{switches}{protocol} ne "long-throw"))
			{
				# The protocol is invalid. Please use '--help' for more information.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0342", variables => { protocol => $anvil->data->{switches}{protocol} }});
				$anvil->Job->update_progress({
					progress   => 100,
					message    => "error_0342,!!protocol!".$anvil->data->{switches}{protocol}."!!",
					job_status => "failed", 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			
			if ($anvil->data->{switches}{protocol} eq "long-throw")
			{
				# If there isn't a license file, make sure it's being provided.
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'path::config::drbd-proxy.license' => $anvil->data->{path}{configs}{'drbd-proxy.license'},
				}});
				if (not -e $anvil->data->{path}{configs}{'drbd-proxy.license'})
				{
					if (not $anvil->data->{switches}{'license-file'})
					{
						# Proxy wouldn't work.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0373"});
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0373", 
							job_status => "failed", 
						});
						$anvil->nice_exit({exit_code => 1});
					}
					elsif (not -f $anvil->data->{switches}{'license-file'})
					{
						# It was passed, but doesn't point to the file.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0374", variables => { file => $anvil->data->{switches}{'license-file'} }});
						$anvil->Job->update_progress({
							progress   => 100,
							message    => "error_0374,!!file!".$anvil->data->{switches}{'license-file'}."!!", 
							job_status => "failed", 
						});
						$anvil->nice_exit({exit_code => 1});
					}
					
					# Still here? Copy the license file.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0429", variables => { file => $anvil->data->{switches}{'license-file'} }});
					my $failed = $anvil->Storage->copy_file({
						source_file => $anvil->data->{switches}{'license-file'}, 
						target_file => $anvil->data->{path}{configs}{'drbd-proxy.license'},
					});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
				}
				
				# Lastly, read in the license file to make sure it _looks_ ok.
				my $problem = $anvil->DRBD->check_proxy_license({debug => 2});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
				if ($problem)
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0375"});
					$anvil->Job->update_progress({
						progress   => 100,
						message    => "error_0375", 
						job_status => "failed", 
					});
					$anvil->nice_exit({exit_code => 1});
				}
			}
		}
	}
	
	# Get the Anvil! details.
	$anvil->Database->get_hosts();
	$anvil->Database->get_storage_group_data({debug => 2});
	$anvil->Database->get_dr_links({debug => 2});

	# Does this Anvil! have at least one DR node? If there's only one, use it. If more than one, we need
	# a '--dr-host' switch.
	if ($anvil->data->{switches}{'dr-host'})
	{
		$dr_host_uuid = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{'dr-host'}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_uuid => $dr_host_uuid }});
		if (not $dr_host_uuid)
		{
			# Bad host.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0400", variables => {
				dr_host => $anvil->data->{switches}{'dr-host'},
			}});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0400,!!dr_host!".$anvil->data->{switches}{'dr-host'}."!!",
				job_status => "failed",
			});
			$anvil->nice_exit({exit_code => 1});
		}
	}

	# If I don't have a dr_host_uuid yet, see which are available. If only one, use it. If two or more, 
	# and if the server is already being protected, determine which to use from it's config. Otherwise,
	# tell the user they need to specify which.
	if (not $dr_host_uuid)
	{
		my $dr_count = keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			anvil_uuid => $anvil_uuid,
			dr_count   => $dr_count,
		}});

		if (not $dr_count)
		{
			print $anvil->Words->string({key => "error_0333"})."\n";
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0333",
				job_status => "failed",
			});
			$anvil->nice_exit({exit_code => 1});
		}
		if ($dr_count eq "1")
		{
			# Only one, use it.
			foreach my $host_uuid (keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}})
			{
				$dr_host_uuid = $host_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_host_uuid => $dr_host_uuid }});
			}
		}
		else
		{
			# Two or more. Is the server already protected? If so, try to find which DR it's 
			# using.
			if (($anvil->data->{switches}{'connect'}) or ($anvil->data->{switches}{'disconnect'}))
			{
				# Read the config.
				my $config_file =  $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
				   $config_file =~ s/\/\//\//g;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
				
				my $resource_config = $anvil->Storage->read_file({file => $config_file});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource_config => $resource_config }});
				foreach my $line (split/\n/, $resource_config)
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lineg => $line }});
					
				}
			}
		}
	}

	# If I still don't have a DR host, fail out.
	if (not $dr_host_uuid)
	{
		print $anvil->Words->string({key => "error_0402"})."\n";
		foreach my $dr_link_host_name (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}})
		{
			my $dr_link_uuid      = $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_name}{$dr_link_host_name}{dr_link_uuid};
			my $dr_link_host_uuid = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				dr_link_host_name => $dr_link_host_name,
				dr_link_uuid      => $dr_link_uuid,
				dr_link_host_uuid => $dr_link_host_uuid,
			}});
			print $anvil->Words->string({key => "error_0403", variables => {
				host_name => $dr_link_host_name,
				host_uuid => $dr_link_host_uuid,
			}})."\n";
		}
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0402",
			job_status => "failed",
		});
		$anvil->nice_exit({exit_code => 1});
	}

	my $dr_host_name       = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{host_name};
	my $dr_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		dr_host_name       => $dr_host_name,
		dr_short_host_name => $dr_short_host_name,
	}});

	$anvil->data->{sys}{dr_host_uuid}       = $dr_host_uuid;
	$anvil->data->{sys}{dr_host_name}       = $dr_host_name;
	$anvil->data->{sys}{dr_short_host_name} = $dr_short_host_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		"sys::dr_host_uuid"       => $anvil->data->{sys}{dr_host_uuid},
		"sys::dr_host_name"       => $anvil->data->{sys}{dr_host_name},
		"sys::dr_short_host_name" => $anvil->data->{sys}{dr_short_host_name},
	}});

	### TODO: Switch to: my $target_ip = $anvil->Network->find_target_ip({});
	# Can we access DR, if we're not the DR host?
	$anvil->Database->get_ip_addresses({debug => 2});
	my $password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password} // "";
	my $dr_ip    = $anvil->System->find_matching_ip({
		debug => 2,
		host  => $dr_host_name,
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		password => $anvil->Log->is_secure($password),
		dr_ip    => $dr_ip,
	}});
	if ($dr_host_uuid ne $anvil->Get->host_uuid)
	{
		if ((not $dr_ip) or ($dr_ip eq "!!error!!"))
		{
			# Failed to find an IP we can access the DR host. Has it been configured? Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0334", variables => { host_name => $dr_host_name }});
			$anvil->Job->update_progress({
				progress   => 0,
				message    => "error_0334,!!host_name!".$dr_host_name."!!",
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Test access.
		my $access = $anvil->Remote->test_access({
			target   => $dr_ip,
			password => $password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
		if (not $access)
		{
			# Failed to access the DR host. Is it running? Exiting.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0335", variables => { 
				host_name  => $dr_host_name,
				ip_address => $dr_ip, 
			}});
			$anvil->Job->update_progress({
				progress   => 0,
				message    => "error_0335,!!host_name!".$dr_host_name."!!,!!ip_address!".$dr_ip."!!",
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		### TODO: We can queue a job to update the peer later, there's no real need, in the long run,
		###       for the peer to be online.
		# If we're protecting or removing a server from DR, the peer needs to be up.
		if ((($anvil->data->{switches}{protect})   or 
		     ($anvil->data->{switches}{remove})    or 
		     ($anvil->data->{switches}{protocol})) && 
		    (not $anvil->data->{cib}{parsed}{peer}{ready}))
		{
			if ($anvil->data->{switches}{protect})
			{
				# We can't setup a server to be protected unless both nodes are up, and the 
				# peer isn't at this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0338"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0338", 
					job_status => "failed", 
				});
			}
			else
			{
				# We can't remove a server from DR unless both nodes are up, and the peer 
				# isn't at this time.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0339"});
				$anvil->Job->update_progress({
					progress   => 0,
					message    => "error_0339", 
					job_status => "failed", 
				});
			}
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Verify we found the server.
	$anvil->data->{server}{'server-name'} = $server_name;
	$anvil->data->{server}{'server-uuid'} = $server_uuid;
	$anvil->data->{server}{'anvil-uuid'}  = $anvil_uuid;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'server::server-uuid' => $anvil->data->{server}{'server-uuid'},
		'server::server-name' => $anvil->data->{server}{'server-name'},
	}});
	
	# Get and parse the server's definition to find the DRBD devices.
	if ((not $anvil->data->{server}{'server-uuid'}) or (not $anvil->data->{server}{'server-name'}))
	{
		# Failed to find the server by name or UUID.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0341", variables => { server => $anvil->data->{switches}{server} }});
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0341,!!server!".$anvil->data->{switches}{server}."!!", 
			job_status => "failed", 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Are we being asked to actuall do something?
	if (((not $anvil->data->{switches}{'connect'})  && 
	     (not $anvil->data->{switches}{disconnect}) && 
	     (not $anvil->data->{switches}{protect})    && 
	     (not $anvil->data->{switches}{remove})     && 
	     (not $anvil->data->{switches}{update}))    or 
	     ($anvil->data->{switches}{help})           or 
	     ($anvil->data->{switches}{h}))
	{
		# Show the man page.
		system($anvil->data->{path}{exe}{man}." ".$THIS_FILE);
		if (($anvil->data->{switches}{help}) or ($anvil->data->{switches}{h}))
		{
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0359"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0359", 
	});
	
	# If we've got a got job-uuid, wait for / make sure all machines are online.
	if ($anvil->data->{switches}{'job-uuid'})
	{
		# Loop until we have access to both the peer machines.
		my $waiting  = 1;
		my $wait_for = 10;
		while ($waiting)
		{
			# This will get set to 1 if we have to keep waiting.
			$waiting = 0;
			my $password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
			my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
			my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
			my $dr_host_uuid    = $anvil->data->{sys}{dr_host_uuid};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				password        => $anvil->Log->is_secure($password),
				node1_host_uuid => $node1_host_uuid,
				node2_host_uuid => $node2_host_uuid, 
				dr_host_uuid    => $dr_host_uuid,
			}});
			foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
			{
				next if $this_host_uuid eq $anvil->Get->host_uuid();
				my $this_host_name = $anvil->Get->host_name_from_uuid({host_uuid => $this_host_uuid});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_host_uuid => $this_host_uuid,
					this_host_name => $this_host_name,
				}});
				
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_subnet}  = "";
				$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = "";
				
				# We'll try the SN, then the BCN and finally the IFN to see which, if any, network we
				# can reach the peer on. This is needed because the DR host could be on a totally 
				# different network.
				my ($target_ip, $target_network) = $anvil->Network->find_target_ip({
					debug       => 2,
					host_uuid   => $this_host_uuid,
					networks    => 'sn,bcn,mn,ifn,any', 
					ping        => 1, 
					test_access => 1, 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					target_ip      => $target_ip,
					target_network => $target_network,
				}});
				if ($target_ip)
				{
					$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip}      = $target_ip;
					$anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network} = $target_network;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"hosts::host_uuid::${this_host_uuid}::network::use_ip"      => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip},
						"hosts::host_uuid::${this_host_uuid}::network::use_network" => $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_network},
					}});
				}
				
				if (not $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip})
				{
					# No access
					my $variables = {
						waiting   => $wait_for,
						host_name => $this_host_name,
					};
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0404", variables => $variables});
					$anvil->Job->update_progress({
						progress  => $anvil->Job->bump_progress({steps => 2}),
						message   => "job_0404", 
						variables => $variables, 
					});
					
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
			
			# If we're waiting for a peer, record as such.
			sleep $wait_for;
		}
	}

	# If we're protecting, make sure there's enough space on the DR host.
	if ($anvil->data->{switches}{protect})
	{
		process_protect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{remove})
	{
		process_remove($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{'connect'})
	{
		process_connect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{disconnect})
	{
		process_disconnect($anvil, $terminal);
	}
	elsif ($anvil->data->{switches}{update})
	{
		process_update($anvil, $terminal);
	}
	
	return(0);
}

sub process_update
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->data->{sys}{anvil_uuid} ? $anvil->data->{sys}{anvil_uuid} : $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{confirm})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0395", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 10}),
			message   => "job_0395", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "update=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0388", 
	});
	run_scanagent($anvil, "scan-drbd");
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		# Call 'scan-drbd' on the peer.
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
	}
	
	# Now watch until out volume(s) are UpToDate
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0390", 
	});
	
	my $waiting = 1;
	while ($waiting)
	{
		$anvil->DRBD->gather_data({debug => 2});
		   $waiting      = 0;
		my $time_to_sync = "";
		my $sync_speed   = "";
		my $sync_source  = "";
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "synctarget")
				{
					$sync_source = $peer;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_source => $sync_source }});
				}
				if ($replication_speed)
				{
					$sync_speed = $anvil->Convert->bytes_to_human_readable({'bytes' => $replication_speed});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sync_speed => $sync_speed }});
				}
				if ($estimated_time_to_sync)
				{
					$time_to_sync = $anvil->Convert->time({'time' => $estimated_time_to_sync, translate => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { time_to_sync => $time_to_sync }});
				}
				
				if ($local_disk_state ne "uptodate")
				{
					$waiting = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
				}
			}
		}
		if ($waiting)
		{
			if ($sync_source)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					sync_source  => $sync_source, 
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				}});
				my $variables = { 
					sync_source  => $sync_source,
					sync_speed   => $sync_speed, 
					time_to_sync => $time_to_sync, 
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0396", variables => $variables});
				$anvil->Job->update_progress({
					progress  => $anvil->Job->bump_progress({steps => 5}),
					message   => "job_0396", 
					variables => $variables, 
				});
			}
			else
			{
				$anvil->Job->update_progress({
					progress  => $anvil->Job->bump_progress({steps => 5}),
					message   => "job_0398", 
					variables => $variables, 
				});
			}
			sleep 5;
		}
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0397", variables => $variables});
	$anvil->Job->update_progress({
		debug     => 2,
		progress  => $anvil->Job->bump_progress({steps => 5}),
		message   => "job_0397", 
		variables => $variables, 
	});
	
	# Bring down the connection.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	$anvil->Job->bump_progress({set => 92});
	run_scanagent($anvil, "scan-drbd");
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		debug     => $anvil->Job->bump_progress({steps => 2}),
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_disconnect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->data->{sys}{anvil_uuid} ? $anvil->data->{sys}{anvil_uuid} : $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{confirm})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0392", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 10}),
			message   => "job_0392", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "disconnect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is up, bring it down.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	run_scanagent($anvil, "scan-drbd");
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0394", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0394", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_connect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->data->{sys}{anvil_uuid} ? $anvil->data->{sys}{anvil_uuid} : $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{confirm})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0386", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 10}),
			message   => "job_0386", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "connect=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# If the resource is down, bring it up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0387", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0387", 
		variables => $variables, 
	});
	
	# Bring up the connection locally, and then also bring up the connection on the nodes, in case the 
	# server is down.
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	
	# If this was an error, abort.
	if ($return_code)
	{
		# Failed.
		my $variables = {
			return_code => $return_code, 
			output      => $output, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0370", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 100,
			message   => "error_0370", 
			variables => $variables, 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	run_scanagent($anvil, "scan-drbd");
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0388", variables => $variables});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0388", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either/both nodes
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0389", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0389", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until we connect to at least one peer.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0390"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0390", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		$waiting = 0;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			my $volume_up = 0;
			foreach my $peer (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}})
			{
				my $replication_speed      = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{replication_speed};
				my $peer_role              = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_role};
				my $peer_disk_state        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{peer_disk_state};
				my $estimated_time_to_sync = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{estimated_time_to_sync};
				my $local_disk_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_disk_state};
				my $connection_state       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{connection_state};
				my $local_role             = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$peer}{local_role};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					peer                   => $peer,
					replication_speed      => $replication_speed, 
					peer_role              => $peer_role, 
					peer_disk_state        => $peer_disk_state, 
					estimated_time_to_sync => $estimated_time_to_sync, 
					local_disk_state       => $local_disk_state, 
					connection_state       => $connection_state, 
					local_role             => $local_role, 
				}});
				
				if ($connection_state eq "established")
				{
					$volume_up = 1;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume_up => $volume_up }});
				}
				last if $volume_up;
			}
			
			if (not $volume_up)
			{
				$waiting = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
		}
		
		if (not $waiting)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => $anvil->Job->bump_progress({steps => 5}),
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	run_scanagent($anvil, "scan-drbd");
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0391", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0391", 
		variables => $variables, 
	});
	
	return(0);
}

sub process_remove
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->data->{sys}{anvil_uuid} ? $anvil->data->{sys}{anvil_uuid} : $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{confirm})
	{
		my $variables = {
			server => $server_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0407", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 10}),
			message   => "job_0407", 
			variables => $variables, 
		});
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "remove=1\n";
		
		# Register the job with the DR host.
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $dr_host_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0406", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0406,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	# Bring down the resource, if it's up.
	my $variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0393", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 5}),
		message   => "job_0393", 
		variables => $variables, 
	});
	
	# Bring down the connection locally
	my $drbd_down_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." && ".$anvil->data->{path}{exe}{drbdadm}." down ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_down_call => $drbd_down_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $drbd_down_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	run_scanagent($anvil, "scan-drbd");
	
	# Read through the local DRBD config file, and remove any LVs that are found to still exist.
	$anvil->DRBD->gather_data({debug => 2});
	
	my $config_file = $anvil->data->{new}{resource}{$server_name}{config_file} ? $anvil->data->{new}{resource}{$server_name}{config_file} : "";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
	
	# If we don't have a config file, we could be re-running and the file was wiped. We need this though,
	# so try to pull it from a node.
	if (not $config_file)
	{
		# Try to read it from node 1.
		$config_file = $anvil->data->{path}{directories}{drbd_resources}."/".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_file => $config_file }});
		
		my $variables = { 
			config_file => $config_file,
			source      => $node1_short_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0417", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0417", 
			variables => $variables, 
		});
		
		# Rsync it over. 
		my $source = "root\@".$anvil->data->{lookup}{host_uuid}{$node1_host_uuid}{network}{use_ip}.":".$config_file;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { source => $source }});
		$anvil->Storage->rsync({
			source      => $source,
			destination => $anvil->data->{path}{directories}{drbd_resources}."/",
		});
		
		# Rescan DRBD
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0418"});
		run_scanagent($anvil, "scan-drbd");
		$anvil->Job->update_progress({
			progress => $anvil->Job->bump_progress({steps => 5}),
			message  => "job_0418", 
		});
		$anvil->DRBD->gather_data({debug => 2});
	}
	
	# Read the old config file.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_resource_config => $old_resource_config }});
	
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		my $local_lv = $anvil->data->{new}{resource}{$server_name}{host}{$dr_short_host_name}{volume}{$volume}{backing_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			volume   => $volume,
			local_lv => $local_lv, 
		}});
		
		my $variables = {
			server   => $server_name, 
			volume   => $volume,
			local_lv => $local_lv, 
		};
		if (-e $local_lv)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0408", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				message   => "job_0408", 
				variables => $variables, 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0591", variables => { device_path => $local_lv }});
			my $return_code = $anvil->DRBD->remove_backing_lv({
				debug        => 2, 
				backing_disk => $local_lv, 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
			if ($return_code)
			{
				# Should have been '0'
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0368", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "error_0368", 
					variables => $variables, 
				});
				$anvil->nice_exit({exit_code => 1});
			}
			else
			{
				# Gone.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0410"});
				$anvil->Job->update_progress({
					progress  => $anvil->Job->bump_progress({steps => 1}),
					message   => "job_0410", 
				});
				run_scanagent($anvil, "scan-lvm");
			}
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0409", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 1}),
				message   => "job_0409", 
				variables => $variables, 
			});
		}
	}
	
	# Tell the user we're generationg the new resource config now.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0411"});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0411", 
	});
	
	### If we're here, we can now update the resource config file to remove the DR host.
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses({debug => 2});
	
	# First loop to build the node sections, then we'll loop to build the connections
	my $node1_volumes    = "";
	my $node2_volumes    = "";
	my $node1_ip_address = "";
	my $node1_tcp_port   = "";
	my $node2_ip_address = "";
	my $node2_tcp_port   = "";
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
	{
		# Find the node 1 backing device, drbd device and minor numbers.
		my $node1_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_path};
		my $node1_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{backing_disk};
		my $node1_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{device_minor};
		my $node1_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node1_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node1_device_path'  => $node1_device_path,
			's2:node1_backing_disk' => $node1_backing_disk, 
			's3:node1_device_minor' => $node1_device_minor, 
			's4:node1_meta_disk'    => $node1_meta_disk, 
		}});
		
		$node1_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node1_device_path, 
			minor     => $node1_device_minor, 
			lv_path   => $node1_backing_disk, 
			meta_data => $node1_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_volumes => $node1_volumes }});
		
		my $node2_device_path  = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_path};
		my $node2_backing_disk = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{backing_disk};
		my $node2_device_minor = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{device_minor};
		my $node2_meta_disk    = $anvil->data->{new}{resource}{$server_name}{host}{$node2_short_host_name}{volume}{$volume}{meta_disk};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:node2_device_path'  => $node2_device_path,
			's2:node2_backing_disk' => $node2_backing_disk, 
			's3:node2_device_minor' => $node2_device_minor, 
			's4:node2_meta_disk'    => $node2_meta_disk, 
		}});
		
		$node2_volumes .= $anvil->Words->string({key => "file_0004", variables => {
			volume    => $volume,
			drbd_path => $node2_device_path, 
			minor     => $node2_device_minor, 
			lv_path   => $node2_backing_disk, 
			meta_data => $node2_meta_disk, 
		}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_volumes => $node2_volumes }});
		
		if (not $node1_ip_address)
		{
			$node1_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_ip_address => $node1_ip_address }});
		}
		if (not $node1_tcp_port)
		{
			$node1_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host1_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_tcp_port => $node1_tcp_port }});
		}
		if (not $node2_ip_address)
		{
			$node2_ip_address = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_ip_address};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_ip_address => $node2_ip_address }});
		}
		if (not $node2_tcp_port)
		{
			$node2_tcp_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$node1_short_host_name}{$node2_short_host_name}{host2_tcp_port};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_tcp_port => $node2_tcp_port }});
		}
	}

	# Put together the nodes section.
	my $hosts = $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node1_short_host_name, 
		node_id         => "0",
		volumes         => $node1_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	$hosts .= $anvil->Words->string({key => "file_0003", variables => {
		short_host_name => $node2_short_host_name, 
		node_id         => "1",
		volumes         => $node2_volumes, 
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }});

	# Build the connections now
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name,
		host1_ip         => $node1_ip_address,
		tcp_port         => $node1_tcp_port,
		host2_short_name => $node2_short_host_name,
		host2_ip         => $node2_ip_address,
		tcp_port         => $node2_tcp_port,
		'c-rate-maximum' => 500,
		protocol         => "C",
		fencing          => "resource-and-stonith"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});

	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
	
	# Test the config,
	test_config($anvil, $server_name, $config_file, $new_resource_config);
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0412"});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 2}),
		message   => "job_0412", 
	});
	
	# Copy the new config to the peers.
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid)
	{
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 2}),
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		else
		{
			# Call 'scan-drbd' on the peer.
			my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
			
			my ($output, $error, $return_code) = $anvil->Remote->call({
				target     => $peer_ip, 
				password   => $anvil_password,
				shell_call => $shell_call,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				error       => $error,
				output      => $output,
				return_code => $return_code, 
			}});
		}
	}
	
	# Call 'drbdadm adjust <server>' to update the running configs.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0413"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 2}),
		message  => "job_0413", 
	});
	
	my $drbdadm_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name.$anvil->Log->switches();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbdadm_call => $drbdadm_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbdadm_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	run_scanagent($anvil, "scan-drbd");
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0414", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 2}),
			message   => "job_0414", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbdadm_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		# Call 'scan-drbd' on the peer.
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now delete our resource config file.
	$variables = { config_file => $config_file };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0415", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0415", 
		variables => $variables, 
	});
	
	if (-e $config_file)
	{
		my $problem = $anvil->Storage->delete_file({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		if ($problem)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0369", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0369", 
				variables => $variables
			});
			$anvil->nice_exit({exit_code => 1});
		}
		# Call scan-drbd to update our view
		run_scanagent($anvil, "scan-drbd");
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0416", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0416", 
		variables => $variables, 
	});
	
	return(0);
}

sub test_config
{
	my ($anvil, $server_name, $config_file, $new_resource_config) = @_;
	
	# Write out a test file.
	my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
	my ($problem) = $anvil->Storage->write_file({
		debug     => 2,
		backup    => 0,
		overwrite => 1,
		file      => $test_file, 
		body      => $new_resource_config,
		user      => "root", 
		group     => "root", 
		mode      => "0644",
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
	
	# Validate.
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	if ($return_code)
	{
		# Something went wrong.
		my $variables = {
			return_code => $return_code, 
			output      => $output, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
		$anvil->Job->update_progress({
			progress  => 100,
			message   => "error_0345", 
			variables => $variables, 
		});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Remove the test file.
	unlink $test_file;
	
	return(0);
}

sub process_protect
{
	my ($anvil, $terminal) = @_;
	
	# Parse out the DRBD resource's backing the server and get their LV sizes. 
	$anvil->Database->get_server_definitions();
	my $anvil_uuid            = $anvil->data->{sys}{anvil_uuid} ? $anvil->data->{sys}{anvil_uuid} : $anvil->Cluster->get_anvil_uuid();
	my $anvil_password        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
	my $node1_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
	my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
	my $node2_host_uuid       = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $node2_host_name       = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{host_name};
	my $node2_short_host_name = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{short_host_name};
	my $dr_host_uuid          = $anvil->data->{sys}{dr_host_uuid};
	my $dr_host_name          = $anvil->data->{sys}{dr_host_name};
	my $dr_short_host_name    = $anvil->data->{hosts}{host_uuid}{$dr_host_uuid}{short_host_name};
	my $server_name           = $anvil->data->{server}{'server-name'};
	my $server_uuid           = $anvil->data->{server}{'server-uuid'};
	my $short_host_name       = $anvil->Get->short_host_name();
	my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid            => $anvil_uuid, 
		anvil_password        => $anvil->Log->is_secure($anvil_password), 
		node1_host_uuid       => $node1_host_uuid, 
		node1_host_name       => $node1_host_name, 
		node1_short_host_name => $node1_short_host_name, 
		node2_host_uuid       => $node2_host_uuid, 
		node2_host_name       => $node2_host_name, 
		node2_short_host_name => $node2_short_host_name, 
		dr_host_uuid          => $dr_host_uuid,
		dr_host_name          => $dr_host_name,
		dr_short_host_name    => $dr_short_host_name,
		server_name           => $server_name, 
		server_uuid           => $server_uuid, 
		server_definition_xml => $server_definition_xml, 
		short_host_name       => $short_host_name,
	}});
	
	# Sanity checks complete!
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0360", variables => { server => $anvil->data->{server}{'server-name'} }});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0360,!!server!".$anvil->data->{server}{'server-name'}."!!", 
	});
	
	$anvil->Server->parse_definition({
		debug      => 2,
		host       => $short_host_name,
		server     => $anvil->data->{server}{'server-name'}, 
		source     => "from_db",
		definition => $server_definition_xml, 
	});
	
	$anvil->DRBD->gather_data({debug => 2});
	
	my $server_ram = $anvil->data->{server}{$short_host_name}{$server_name}{'from_db'}{memory};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_ram => $anvil->Convert->add_commas({number => $server_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram}).")",
	}});
	
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
	{
		next if $resource eq "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			my $this_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $this_host_name});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				this_host_name => $this_host_name,
				this_host_uuid => $this_host_uuid, 
			}});
			
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}})
			{
				# Always get the LV sizes, as that factors metadata. DRBD size is 
				# minus metadata, and 0 when down.
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				my $tcp_port     = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:volume"       => $volume, 
					"s2:device_path"  => $device_path,
					"s3:backing_disk" => $backing_disk,
					"s4:device_minor" => $device_minor,
				}});
				
				my $this_size = $anvil->Storage->get_size_of_block_device({debug => 2, host_uuid => $this_host_uuid, path => $backing_disk});
				if ($this_size eq "")
				{
					# DRBD config file was updated, but LV not created yet.
					next;
				}
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					this_size => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")", 
				}});
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{size}) or (not $anvil->data->{server}{drbd}{$resource}{$volume}{size}))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->data->{server}{drbd}{$resource}{$volume}{size}, 
					}});
				}
				
				if (not exists $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid} = "";
				}
				
				### NOTE: This check make sense only under the assumption that the DRBD minor
				###       is common across both nodes. This should be the case, but doesn't 
				###       strictly have to be so.
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{minor_number} = $device_minor;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::minor_number" => $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number},
					}});
				}
				
				if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port})  or 
				    (not defined $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}) or 
				    ($anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} eq ""))
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port} = $tcp_port;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::tcp_port" => $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port},
					}});
				}
				
				# What storage group does this belong to?
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"server::drbd::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
				}});
				if (not $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid})
				{
					my $storage_key        = $resource."/".$volume;
					my $storage_group_uuid = $anvil->Storage->get_storage_group_from_path({
						debug      => 2,
						anvil_uuid => $anvil_uuid, 
						path       => $backing_disk,
					});
					my $storage_group_name = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{group_name};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						storage_key        => $storage_key, 
						storage_group_uuid => $storage_group_uuid, 
						storage_group_name => $storage_group_name, 
					}});
					
					# We'll need to sum up the volumes on each storage group, as 
					# it's possible the volumes are on different SGs.
					$anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}               = $storage_group_uuid;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key} = 1;
					$anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}    = $storage_group_uuid;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::storage_group_uuid"               => $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid}, 
						"server::storage_groups::${storage_group_name}::used_by::${storage_key}" => $anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}{$storage_key},
						"server::storage_groups::${storage_group_name}::storage_group_uuid"      => $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid}, 
					}});
				}
				
				if ($this_size > $anvil->data->{server}{drbd}{$resource}{$volume}{size})
				{
					$anvil->data->{server}{drbd}{$resource}{$volume}{size} = $this_size;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::drbd::${resource}::${volume}::size" => $anvil->Convert->add_commas({number => $anvil->data->{server}{drbd}{$resource}{$volume}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{server}{drbd}{$resource}{$volume}{size}}).")", 
					}});
				}
			}
		}
	}
	
	# Make sure there is enough space on DR for the volumes under this VM.
	my $problem     = 0;
	my $config_file = "";
	foreach my $storage_group_name (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}})
	{
		my $storage_group_uuid = $anvil->data->{server}{storage_groups}{$storage_group_name}{storage_group_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			storage_group_name => $storage_group_name, 
			storage_group_uuid => $storage_group_uuid, 
		}});
		
		# First, is this SG on DR?
		if (not exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid})
		{
			# The DR host doesn't appear to be storage group.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0343", variables => { 
				host_name     => $dr_host_name,
				storage_group => $storage_group_name,
			}});
			$anvil->Job->update_progress({
				progress => 100,
				message  => "error_0343,!!host_name!".$dr_host_name."!!,!!storage_group!".$storage_group_name."!!",
			});
			$problem = 1;
		}
		
		my $space_needed = 0;
		foreach my $resource_key (sort {$a cmp $b} keys %{$anvil->data->{server}{storage_groups}{$storage_group_name}{used_by}})
		{
			my ($resource, $volume) =  ($resource_key =~ /^(.*)\/(\d+)$/);
			my $volume_size         =  $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			   $space_needed        += $volume_size, 
			   $config_file         =  $anvil->data->{new}{resource}{$resource}{config_file};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				config_file  => $config_file, 
				resource_key => $resource_key, 
				resource     => $resource, 
				volume       => $volume, 
				volume_size  => $anvil->Convert->add_commas({number => $volume_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size}).")", 
				space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
			}});
			
		}
		
		# Is there enough space on DR?
		my $space_on_dr = $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid}{vg_free};
		if ((not defined $space_on_dr) or ($space_on_dr !~ /^\d+$/))
		{
			# The DR host doesn't have a VG in the SG.
			my $variables = {
				space_needed       => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}), 
				space_needed_bytes => $anvil->Convert->add_commas({number => $space_needed}),
				storage_group      => $storage_group_name, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0468", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0344", 
				variables => $variables
			});
			$problem = 1;
		}
		else
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				space_on_dr  => $anvil->Convert->add_commas({number => $space_on_dr})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}).")", 
				space_needed => $anvil->Convert->add_commas({number => $space_needed})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}).")", 
			}});
			if ($space_needed > $space_on_dr)
			{
				my $variables = {
					space_needed       => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_needed}), 
					space_needed_bytes => $anvil->Convert->add_commas({number => $space_needed}),
					storage_group      => $storage_group_name, 
					space_on_dr        => $anvil->Convert->bytes_to_human_readable({'bytes' => $space_on_dr}), 
					space_on_dr_bytes  => $anvil->Convert->add_commas({number => $space_on_dr}), 
				};
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0344", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "error_0344", 
					variables => $variables
				});
				$problem = 1;
			}
		}
	}
	if ($problem)
	{
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Have we already configured the DR? If so, what ports are already allocated.
	my $node1_to_dr_port         = "";
	my $node1_to_dr_port_inside  = "";
	my $node1_to_dr_port_outside = "";
	my $node2_to_dr_port         = "";
	my $node2_to_dr_port_inside  = "";
	my $node2_to_dr_port_outside = "";
	foreach my $host1_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host1_name => $host1_name }});
		foreach my $host2_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host2_name => $host2_name }});
			next if (($host1_name ne $dr_short_host_name) && ($host2_name ne $dr_short_host_name));
			
			if (($host1_name eq $node1_short_host_name) or ($host2_name eq $node1_short_host_name))
			{
				$node1_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
				
				if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port})
				{
					$node1_to_dr_port_inside  = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port};
					$node1_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						node1_to_dr_port_inside  => $node1_to_dr_port_inside,
						node1_to_dr_port_outside => $node1_to_dr_port_outside, 
					}});
				}
			}
			else
			{
				$node2_to_dr_port = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host1_tcp_port};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
				
				if (exists $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port})
				{
					$node2_to_dr_port_inside  = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_inside_tcp_port};
					$node2_to_dr_port_outside = $anvil->data->{new}{resource}{$server_name}{host1_to_host2}{$host1_name}{$host2_name}{host2_outside_tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						node2_to_dr_port_inside  => $node2_to_dr_port_inside,
						node2_to_dr_port_outside => $node2_to_dr_port_outside, 
					}});
				}
			}
		}
	}
	
	# Get net next pair of TCP ports, if needed.
	my $dr_tcp_ports     = 1;
	my $long_throw_ports = 0;
	if ($anvil->data->{switches}{protocol} eq "long-throw")
	{
		$dr_tcp_ports     = 0;
		$long_throw_ports = 1;
	}
	my (undef, $tcp_ports) = $anvil->DRBD->get_next_resource({
		debug            => 2,
		dr_tcp_ports     => $dr_tcp_ports, 
		long_throw_ports => $long_throw_ports, 
	});
	my @free_ports = split/,/, $tcp_ports;
	my $i          = 0;

	if ($node1_to_dr_port eq "")
	{
		$node1_to_dr_port = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port => $node1_to_dr_port }});
	}
	
	if (($long_throw_ports) && (not $node1_to_dr_port_inside))
	{
		$node1_to_dr_port_inside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_inside => $node1_to_dr_port_inside }});
	}
	if (($long_throw_ports) && (not $node1_to_dr_port_outside))
	{
		$node1_to_dr_port_outside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_to_dr_port_outside => $node1_to_dr_port_outside }});
	}
	
	if ($node2_to_dr_port eq "")
	{
		$node2_to_dr_port = $free_ports[$i];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port => $node2_to_dr_port }});
	}
	if (($long_throw_ports) && (not $node2_to_dr_port_inside))
	{
		$node2_to_dr_port_inside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_inside => $node2_to_dr_port_inside }});
	}
	if (($long_throw_ports) && (not $node2_to_dr_port_outside))
	{
		$node2_to_dr_port_outside = $free_ports[$i++];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_to_dr_port_outside => $node2_to_dr_port_outside }});
	}
	
	# Show what we're doing
	my $variables = {
		protocol                 => $anvil->data->{switches}{protocol}, 
		node1_to_dr_port         => $node1_to_dr_port, 
		node1_to_dr_port_inside  => $node1_to_dr_port_inside, 
		node1_to_dr_port_outside => $node1_to_dr_port_outside, 
		node2_to_dr_port         => $node2_to_dr_port, 
		node2_to_dr_port_inside  => $node2_to_dr_port_inside, 
		node2_to_dr_port_outside => $node2_to_dr_port_outside, 
		config_file              => $config_file, 
	};
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0361", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 10}),
		message   => "job_0361", 
		variables => $variables, 
	});
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
		{
			my $variables = {
				resource => $resource, 
				volume   => $volume, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0362", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 2}),
				message   => "job_0362", 
				variables => $variables, 
			});
			my $lv_size            = $anvil->data->{server}{drbd}{$resource}{$volume}{size};
			my $storage_group_uuid = $anvil->data->{server}{drbd}{$resource}{$volume}{storage_group_uuid};
			my $dr_lv_name         = $resource."_".$volume;
			my $dr_vg_name         = $anvil->Storage->get_vg_name({
				debug              => 3, 
				storage_group_uuid => $storage_group_uuid, 
				host_uuid          => $dr_host_uuid,
			});
			my $dr_lv_path         = "/dev/".$dr_vg_name."/".$dr_lv_name;
			my $extent_size        = $anvil->data->{storage_groups}{storage_group_uuid}{$storage_group_uuid}{host_uuid}{$dr_host_uuid}{vg_extent_size};
			my $extent_count       = int($lv_size / $extent_size);
			my $shell_call         = $anvil->data->{path}{exe}{lvcreate}." -l ".$extent_count." -n ".$dr_lv_name." ".$dr_vg_name." -y";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"s01:resource"           => $resource, 
				"s02:volume"             => $volume, 
				"s03:lv_size"            => $anvil->Convert->add_commas({number => $lv_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}).")", , 
				"s04:storage_group_uuid" => $storage_group_uuid, 
				"s05:dr_lv_name"         => $dr_lv_name, 
				"s06:dr_vg_name"         => $dr_vg_name, 
				"s07:dr_lv_path"         => $dr_lv_path, 
				"s08:extent_size"        => $anvil->Convert->add_commas({number => $extent_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $extent_size}).")", 
				"s09:extent_count"       => $extent_count, 
				"s10:shell_call"         => $shell_call, 
			}});
			
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}      = $shell_call;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}            = $dr_lv_path;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid} = $storage_group_uuid;
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}      = $anvil->data->{server}{drbd}{$resource}{$volume}{tcp_port}; 
			$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}         = $anvil->data->{server}{drbd}{$resource}{$volume}{minor_number};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"server::dr::volumes::${resource}::${volume}::lvcreate_call"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lvcreate_call}, 
				"server::dr::volumes::${resource}::${volume}::lv_path"            => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path}, 
				"server::dr::volumes::${resource}::${volume}::storage_group_uuid" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{storage_group_uuid}, 
				"server::dr::volumes::${resource}::${volume}::drbd_tcp_port"      => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_tcp_port}, 
				"server::dr::volumes::${resource}::${volume}::drbd_minor"         => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor}, 
			}});
			
			# Get the VG name that this volume will be created on.
			$variables = {
				lv_path       => $dr_lv_path, 
				lv_size       => $anvil->Convert->bytes_to_human_readable({'bytes' => $lv_size}), 
				lv_size_bytes => $anvil->Convert->add_commas({number => $lv_size}), 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0363", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 2}),
				message   => "job_0363", 
				variables => $variables, 
			});
		}
	}
	
	### NOTE: 'Yes' is set when a job is picked up, so this won't re-register the job.
	my $record_job = 0;
	if (not $anvil->data->{switches}{confirm})
	{
		# If there's no config file, this might be being run from DR which doesn't have info about 
		# the resource yet.
		if ((not $config_file) && (not $anvil->data->{switches}{'job-uuid'}))
		{
			# Bail out.
			print $anvil->Words->string({key => "message_0308"})."\n";
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Ask the user to confirm.
		print "\n".$anvil->Words->string({key => "message_0021"})."\n";
		my $answer = <STDIN>;
		chomp $answer;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});
		
		if ($answer =~ /^y/i)
		{
			print $anvil->Words->string({key => "message_0175"})."\n";
			$record_job = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
		}
		else
		{
			print $anvil->Words->string({key => "message_0022"})."\n";
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif (not $anvil->data->{switches}{'job-uuid'})
	{
		$record_job = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { record_job => $record_job }});
	}
	
	if ($record_job)
	{
		my $job_data =  "server=".$anvil->data->{switches}{server}."\n";
		   $job_data .= "protect=1\n";
		   $job_data .= "dr-host=".$anvil->data->{sys}{dr_host_name}."\n";
		   $job_data .= "protocol=".$anvil->data->{switches}{protocol}."\n";
		
		# We always record the job against node 1, as it has to use cluster info to run this, so we 
		# can't run it on the DR itself.
		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_host_uuid => $node1_host_uuid }});
		
		# Register the job with this host
		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
			debug           => 2, 
			job_command     => $anvil->data->{path}{exe}{'anvil-manage-dr'}.$anvil->Log->switches, 
			job_data        => $job_data, 
			job_name        => "server::dr", 
			job_title       => "job_0384", 
			job_description => "job_0385", 
			job_progress    => 0,
			job_host_uuid   => $node1_host_uuid, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		
		# Report the job UUID.
		print $anvil->Words->string({key => "job_0383", variables => { job_uuid => $job_uuid }})."\n";
		
		
		$anvil->nice_exit({exit_code => 0});
	}
	
	### If we're still here, time to get started.
	# Read in the old config and update it.
	my $old_resource_config = $anvil->Storage->read_file({file => $config_file});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_resource_config => $old_resource_config }});
	
	# Pull the data out of the old file
	my $hosts          = "";
	my $nodes_tcp_port = "";
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
		my $dr_seen = 0;
		foreach my $this_host_name (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$resource}{host}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_host_name => $this_host_name }});
			my $node_id = "";
			if (($this_host_name eq $node1_short_host_name) or ($this_host_name eq $node1_host_name))
			{
				$node_id = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $node2_short_host_name) or ($this_host_name eq $node2_host_name))
			{
				$node_id = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node_id => $node_id }});
				
				if ((not $nodes_tcp_port) && 
				    (exists $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}) && 
				    ($anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}))
				{
					$nodes_tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { nodes_tcp_port => $nodes_tcp_port }});
				}
			}
			elsif (($this_host_name eq $dr_short_host_name) or ($this_host_name eq $dr_host_name))
			{
				$node_id = 2;
				$dr_seen = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					node_id => $node_id,
					dr_seen => $dr_seen,
				}});
			}
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$resource}})
			{
				my $device_path  = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_path};
				my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
				my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
				
				# Record the DRBD device for adding DR.
				if (not exists $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path})
				{
					$anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path} = $device_path;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						"server::dr::volumes::${resource}::${volume}::device_path" => $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path},
					}});
				}
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $this_host_name, 
				node_id         => $node_id, 
				volumes         => $volumes,
			}});
		}
		if (not $dr_seen)
		{
			# Inject the DR.
			my $volumes = "";
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{dr}{volumes}{$resource}})
			{
				my $device_path  = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{device_path};
				my $backing_disk = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{lv_path};
				my $device_minor = $anvil->data->{server}{dr}{volumes}{$resource}{$volume}{drbd_minor};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:device_path"  => $device_path,
					"s2:backing_disk" => $backing_disk,
					"s3:device_minor" => $device_minor,
				}});
				
				$volumes .= $anvil->Words->string({key => "file_0004", variables => {
					volume    => $volume,
					drbd_path => $device_path, 
					minor     => $device_minor, 
					lv_path   => $backing_disk, 
				        meta_data => "internal", 
				}});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volumes => $volumes }});
			}
			$hosts .= $anvil->Words->string({key => "file_0003", variables => {
				short_host_name => $dr_short_host_name,
				node_id         => "2", 
				volumes         => $volumes,
			}});
		}
	}
	
	# Refresh the IP info (usually scrubbed by this point)
	$anvil->Database->get_ip_addresses({debug => 2});
	
	# The connections. Node 1 to 2 always uses the BCN, Either node to DR needs 
	my $storage_network = "sn1";
	my $dr_network      = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_network};
	my $dr_ip           = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
	my $node1_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$storage_network}{ip_address};
	my $node1_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{network}{$dr_network}{ip_address};
	my $node2_sn_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$storage_network}{ip_address};
	my $node2_dr_ip     = $anvil->data->{hosts}{host_uuid}{$node2_host_uuid}{network}{$dr_network}{ip_address};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's01:storage_network' => $storage_network,
		's02:dr_host_uuid'    => $dr_host_uuid,
		's03:dr_network'      => $dr_network, 
		's04:dr_ip'           => $dr_ip,
		's05:node1_host_uuid' => $node1_host_uuid, 
		's06:node1_sn_ip'     => $node1_sn_ip,
		's07:node1_dr_ip'     => $node1_dr_ip,
		's08:node2_host_uuid' => $node2_host_uuid, 
		's09:node2_sn_ip'     => $node2_sn_ip, 
		's10:node2_dr_ip'     => $node2_dr_ip, 
	}});
	
	# Choose the DR protocol
	my $use_drbd_proxy = 0;
	my $dr_protocol    = "A";
	if ($anvil->data->{switches}{protocol} eq "sync")
	{
		$dr_protocol = "C";
	}
	elsif ($anvil->data->{switches}{protocol} eq "long-throw")
	{
		$use_drbd_proxy = 1;
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		dr_protocol    => $dr_protocol,
		use_drbd_proxy => $use_drbd_proxy, 
	}});

	# Node 1 to Node 2 first
	my $proxy       = "";
	my $max_c_rate  = 500;
	my $message_key = "file_0005";
	if ($use_drbd_proxy)
	{
		$proxy       = $anvil->Words->string({key => "file_0008", variables => { memlimit => 256 }});
		$max_c_rate  = 100;
		$message_key = "file_0007";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		proxy      => $proxy, 
		max_c_rate => $max_c_rate,
		file_key   => $message_key, 
	}});
	
	# Node 1 to 2
	my $connections = $anvil->Words->string({key => "file_0005", variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_sn_ip, 
		host2_short_name => $node2_short_host_name, 
		host2_ip         => $node2_sn_ip, 
		tcp_port         => $nodes_tcp_port, 
		'c-rate-maximum' => $max_c_rate,
		protocol         => "C",
		fencing          => "resource-and-stonith"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});
	
	# Node 1 to DR
	$connections .= $anvil->Words->string({key => $message_key, variables => {
		host1_short_name => $node1_short_host_name, 
		host1_ip         => $node1_dr_ip, 
		host2_short_name => $dr_short_host_name,
		host2_ip         => $dr_ip,
		tcp_port         => $node1_to_dr_port, 
		inside_tcp_port  => $node1_to_dr_port_inside,
		outside_tcp_port => $node1_to_dr_port_outside,
		'c-rate-maximum' => $max_c_rate,
		protocol         => $dr_protocol,
		fencing          => "dont-care"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});

	# Node 2 to DR
	$connections .= $anvil->Words->string({key => $message_key, variables => {
		host1_short_name => $node2_short_host_name, 
		host1_ip         => $node2_dr_ip, 
		host2_short_name => $dr_short_host_name,
		host2_ip         => $dr_ip,
		tcp_port         => $node2_to_dr_port, 
		inside_tcp_port  => $node2_to_dr_port_inside,
		outside_tcp_port => $node2_to_dr_port_outside,
		'c-rate-maximum' => $max_c_rate,
		protocol         => $dr_protocol,
		fencing          => "dont-care"
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connections => $connections }});
	
	my $new_resource_config = $anvil->Words->string({key => "file_0006", variables => {
		server      => $server_name,
		proxy       => $proxy, 
		hosts       => $hosts,
		connections => $connections,
	}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
	
	# Is the new res file the same as the old one?
	my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
	
	# Is there a difference on the local version? There might be a difference on peers, but we'll deel 
	# with that below.
	if (not $difference)
	{
		# The resource file doesn't need to be updated.
		my $variables = {
			file => $config_file, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0364", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0364", 
			variables => $variables, 
		});
	}
	else
	{
		# Write out a test file.
		my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $test_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Validate.
		my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output,
			return_code => $return_code, 
		}});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0345", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Remove the test file.
		unlink $test_file;
		
		# Backup the res file so we can tell the user where the current config was backed up to in 
		# case they need to restore it.
		my ($backup_file) = $anvil->Storage->backup({file => $config_file});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		my $variables = { backup_file => $backup_file };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0365", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0365", 
			variables => $variables, 
		});
		
		# Write out the new file.
		($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 0,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config,
			user      => "root", 
			group     => "root", 
			mode      => "0644",
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0366"});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0366", 
		});
		
		# Call 'drbdadm dump-xml' to check that it's OK.
		($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
		if ($return_code)
		{
			# Something went wrong.
			my $variables = {
				return_code => $return_code, 
				output      => $output, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0345", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 5}),
				message   => "error_0345", 
				variables => $variables, 
			});
			
			# Restoring the old config now.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0346"});
			$anvil->Job->update_progress({
				progress => $anvil->Job->bump_progress({steps => 5}),
				message  => "error_0346", 
			});
			
			# Backup the bad file and worn the user.
			my ($backup_file) = $anvil->Storage->backup({file => $config_file});
			$variables = { file => $backup_file };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0347", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 5}),
				message   => "error_0347", 
				variables => $variables, 
			});
			
			# Write out the new file.
			my ($problem) = $anvil->Storage->write_file({
				debug     => 2,
				backup    => 1,
				overwrite => 1,
				file      => $config_file, 
				body      => $old_resource_config, 
				user      => "root", 
				group     => "root", 
				mode      => "0644",
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});

			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0348"});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0348", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		run_scanagent($anvil, "scan-drbd");
		
		# New config is good! Update the file on the peers.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0367"});
		$anvil->Job->update_progress({
			progress => $anvil->Job->bump_progress({steps => 5}),
			message  => "job_0367", 
		});
	}
	
	# New config is good! Update the file on the peers.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0368"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 2}),
		message  => "job_0368", 
	});
	
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host. It's not uncommon for the DR host to 
		# not have a connection over the SN or even the BCN. So we'll use the IFN1 to move files.
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = {
			file       => $config_file, 
			host_name  => $peer_host_name, 
			ip_address => $peer_ip, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0369", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0369", 
			variables => $variables, 
		});
		my ($problem) = $anvil->Storage->write_file({
			debug     => 2,
			backup    => 1,
			overwrite => 1,
			file      => $config_file, 
			body      => $new_resource_config, 
			user      => "root", 
			group     => "root", 
			mode      => "0644",
			target    => $peer_ip, 
			password  => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
		
		# Make sure the file exists now.
		my $check_resource_config = $anvil->Storage->read_file({
			file     => $config_file,
			target   => $peer_ip, 
			password => $anvil_password,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_resource_config => $check_resource_config }});
		
		my $difference = diff \$new_resource_config, \$check_resource_config, { STYLE => 'Unified' };
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
		
		# Failed to write the file.
		if ($difference)
		{
			$variables = {
				host_name             => $peer_host_name, 
				file                  => $config_file, 
				difference            => $difference, 
				new_resource_config   => $new_resource_config, 
				check_resource_config => $check_resource_config, 
			};
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0405", variables => $variables});
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "job_0405", 
				variables  => $variables, 
				job_status => "failed", 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		run_scanagent($anvil, "scan-drbd");
	}
	
	# Immediately call scan-drbd on all machines to ensure that if another run is about to happen for a 
	# different server, it knows the used ports list is updated.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0381"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 2}),
		message  => "job_0381", 
	});
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0382", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 2}),
			message   => "job_0382", 
			variables => $variables, 
		});
		
	        my $scan_drbd_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd --force ".$anvil->Log->switches();
	        $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_drbd_call => $scan_drbd_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $scan_drbd_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Create the LV and MD on DR.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0370"});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 2}),
		message   => "job_0370", 
	});
	my $create_md = 0;
	foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$server_name}})
	{
		my $dr_ip   = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
		my $lv_path = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lv_path};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:volume'  => $volume,
			's2:dr_ip'   => $dr_ip,
			's3:lv_path' => $lv_path, 
		}});
		
		my $variables = { 
			volume  => $volume, 
			lv_path => $lv_path, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0371", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 2}),
			message   => "job_0371", 
			variables => $variables, 
		});
		
		my $lv_check_call = "
if [ -e '".$lv_path."' ]; 
then 
    echo exists;
else 
    echo create;
fi";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lv_check_call => $lv_check_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "exists")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0372", variables => $variables});
			$anvil->Job->update_progress({
				progress  => $anvil->Job->bump_progress({steps => 2}),
				message   => "job_0372", 
				variables => $variables, 
			});
			next;
		}
		
		# Create the LV.
		my $lvcreate_call = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lvcreate_call};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lvcreate_call => $lvcreate_call }});
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lvcreate_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		sleep 1;
		# Does it exist now?
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $lv_check_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		if ($output eq "create")
		{
			my $variables = { lv_path => $lv_path };
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0349", variables => $variables});
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "error_0349", 
				variables => $variables, 
			});
			$anvil->nice_exit({exit_code => 1});
		}
		run_scanagent($anvil, "scan-lvm");
		
		# Create the DRBD metadata now.
		$create_md = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_md => $create_md }});
	}
	
	if ($create_md)
	{
		my $dr_ip        = $anvil->data->{lookup}{host_uuid}{$dr_host_uuid}{network}{use_ip};
		my $drbd_md_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$server_name;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:dr_ip'        => $dr_ip,
			's2:drbd_md_call' => $drbd_md_call, 
		}});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $dr_ip,
			password   => $anvil_password,
			shell_call => $drbd_md_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		run_scanagent($anvil, "scan-drbd");
	}
	
	# Reload the config.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0373"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 5}),
		message  => "job_0373", 
	});
	my $shell_call = $anvil->data->{path}{exe}{drbdadm}." adjust ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output,
		return_code => $return_code, 
	}});
	run_scanagent($anvil, "scan-drbd");
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		my $variables = { 
			server    => $server_name,
			host_name => $peer_host_name, 
		};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0374", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0374", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# If the resource is down, bring it up.
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0375", variables => $variables});
	$anvil->Job->update_progress({
		progress  => $anvil->Job->bump_progress({steps => 2}),
		message   => "job_0375", 
		variables => $variables, 
	});
	my $drbd_up_call = $anvil->data->{path}{exe}{drbdsetup}." status ".$server_name." || ".$anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_up_call => $drbd_up_call }});
	($output, $return_code) = $anvil->System->call({shell_call => $drbd_up_call});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code, 
	}});
	run_scanagent($anvil, "scan-drbd");
	foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr_host_uuid)
	{
		# "Peer" in this context is either a node or a DR host
		next if $this_host_uuid eq $anvil->Get->host_uuid();
		my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
		my $peer_ip        = $anvil->data->{lookup}{host_uuid}{$this_host_uuid}{network}{use_ip};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:this_host_uuid' => $this_host_uuid, 
			's2:peer_host_name' => $peer_host_name, 
			's3:peer_ip'        => $peer_ip, 
		}});
		
		$variables = { host_name => $peer_host_name };
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0376", variables => $variables});
		$anvil->Job->update_progress({
			progress  => $anvil->Job->bump_progress({steps => 5}),
			message   => "job_0376", 
			variables => $variables, 
		});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $drbd_up_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
		
		my $shell_call = $anvil->data->{path}{directories}{scan_agents}."/scan-drbd/scan-drbd".$anvil->Log->switches();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		($output, $error, $return_code) = $anvil->Remote->call({
			target     => $peer_ip, 
			password   => $anvil_password,
			shell_call => $shell_call,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			error       => $error,
			output      => $output,
			return_code => $return_code, 
		}});
	}
	
	# Now watch until the DR host shows up
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0377"});
	$anvil->Job->update_progress({
		progress => $anvil->Job->bump_progress({steps => 2}),
		message  => "job_0377", 
	});
	my $waiting = 1;
	while($waiting)
	{
		sleep 5;
		$anvil->DRBD->gather_data({debug => 2});
		
		my $dr_seen = 1;
		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
			if (exists $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name})
			{
				my $local_role       = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{local_role};
				my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{local_disk_state};
				my $peer_role        = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{peer_role};
				my $peer_disk_state  = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr_short_host_name}{peer_disk_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					local_role       => $local_role,
					local_disk_state => $local_disk_state, 
					peer_role        => $peer_role, 
					peer_disk_state  => $peer_disk_state, 
				}});
			}
			else
			{
				# Not up yet.
				my $next_check = $anvil->Get->date_and_time({offset => 5, time_only => 1});
				my $variables  = { next_check => $next_check };
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0378", variables => $variables});
				$anvil->Job->update_progress({
					progress  => 95,
					message   => "job_0378", 
					variables => $variables, 
				});
				$dr_seen = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_seen => $dr_seen }});
				last;
			}
		}
		
		if ($dr_seen)
		{
			# We're ready.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0379"});
			$anvil->Job->update_progress({
				progress => $anvil->Job->bump_progress({steps => 2}),
				message  => "job_0379", 
			});
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			run_scanagent($anvil, "scan-drbd");
		}
	}
	
	# Done!
	$variables = { server => $server_name };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0380", variables => $variables});
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0380", 
		variables => $variables, 
	});
	
	return(0);
}

sub load_job
{
	my ($anvil) = @_;
	
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => $anvil->Job->bump_progress({steps => 1}),
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0267", 
	});
	
	# We're a job, so '--confirm' is assumed.
	$anvil->data->{switches}{confirm} = 1;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::confirm" => $anvil->data->{switches}{confirm} }});
	
	#print "Loading the job: [".$anvil->data->{switches}{'job-uuid'}."]...\n";
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"jobs::job_command"  => $anvil->data->{jobs}{job_command},
		"jobs::job_data"     => $anvil->data->{jobs}{job_data}, 
		"jobs::job_progress" => $anvil->data->{jobs}{job_progress}, 
		"jobs::job_status"   => $anvil->data->{jobs}{job_status}, 
	}});
	
	# Break up the job data into switches.
	#print "- Set the switch: [--Yes] to true.\n";
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /(.*?)=(.*)$/)
		{
			my $key   = $1;
			my $value = $2;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				key   => $key,
				value => $value, 
			}});
			
			$anvil->data->{switches}{$key} = $value;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"switches::${key}" => $anvil->data->{switches}{$key}, 
			}});
			
			#print "* Set the switch: [--".$key."] to: [".$value."]\n";
		}
	}
	#print "Job loaded successfully.\n\n";
	
	return(0);
}

sub check_for_duplicate_dr_links
{
	my ($anvil) = @_;

	if (exists $anvil->data->{dr_links})
	{
		delete $anvil->data->{dr_links};
	}
	my $query = "
SELECT
    dr_link_uuid,
    dr_link_host_uuid,
    dr_link_anvil_uuid,
    dr_link_note
FROM
    dr_links
ORDER BY
    modified_date DESC;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
	my $count   = @{$results};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
		results => $results,
		count   => $count,
	}});
	foreach my $row (@{$results})
	{
		my $dr_link_uuid       = $row->[0];
		my $dr_link_host_uuid  = $row->[1];
		my $dr_link_anvil_uuid = $row->[2];
		my $dr_link_note       = $row->[3];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
			dr_link_uuid       => $dr_link_uuid,
			dr_link_host_uuid  => $dr_link_host_uuid,
			dr_link_anvil_uuid => $dr_link_anvil_uuid,
			dr_link_note       => $dr_link_note,
		}});

		if (exists $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note})
		{
			# Duplicate!
			my $previous_note = $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note};
			my $this_note     = $dr_link_note;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				previous_note => $previous_note,
				this_note     => $this_note,
			}});

			# If the previous note was "DELETED", delete it. Otherwise we'll delete this one.
			if ($previous_note eq "DELETED")
			{
				# Delete the previous one.
				my $old_dr_link_uuid = $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_dr_link_uuid => $old_dr_link_uuid }});

				my $queries = [];
				push @{$queries}, "DELETE FROM history.dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($old_dr_link_uuid).";";
				push @{$queries}, "DELETE FROM dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($old_dr_link_uuid).";";
				foreach my $query (@{$queries})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				}
				$anvil->Database->write({debug => 2, query => $queries, source => $THIS_FILE, line => __LINE__});

				# Update the local record to store this one.
				$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid} = $dr_link_uuid;
				$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note} = $dr_link_note;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
					"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid},
					"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_note" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note},
				}});
			}
			else
			{
				# Delete this one.
				my $queries = [];
				push @{$queries}, "DELETE FROM history.dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
				push @{$queries}, "DELETE FROM dr_links WHERE dr_link_uuid = ".$anvil->Database->quote($dr_link_uuid).";";
				foreach my $query (@{$queries})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
				}
				$anvil->Database->write({debug => 2, query => $queries, source => $THIS_FILE, line => __LINE__});
			}
		}
		else
		{
			$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid} = $dr_link_uuid;
			$anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note} = $dr_link_note;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
				"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_uuid" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_uuid},
				"dr_links::host_uuid::${dr_link_host_uuid}::${dr_link_anvil_uuid}::dr_link_note" => $anvil->data->{dr_links}{host_uuid}{$dr_link_host_uuid}{$dr_link_anvil_uuid}{dr_link_note},
			}});
		}
	}

	# Done, reload dr_links.
	delete $anvil->data->{dr_links};
	$anvil->Database->get_dr_links();
}

sub show_servers
{
	my ($anvil, $anvil_uuid) = @_;

	my $server_count = keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
	if ($server_count > 0)
	{
		print $anvil->Words->string({key => "error_0413"})."\n";
		foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
		{
			my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
			print "- ".$server_name." (".$server_uuid.")\n";
		}
		print "\n";
	}
}
