#!/usr/bin/perl
# 
# This server handles tasks like restoring servers on a rebuilt peer, and other (future) tasks.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 
# TODO: 
# 

use strict;
use warnings;
use Anvil::Tools;
require POSIX;
use Term::Cap;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->Get->switches({list => [
	"boot-after",
	"delay", 
	"resync", 
	"server", 
	"y"
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0305"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

### TODO: We don't yet run as a job. For now, the only thing this tool does is find/resync servers, which 
###       happens automatically.
# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "message_0251", 
	});
}

if ($anvil->data->{switches}{resync})
{
	handle_resync($anvil);
}

if ($anvil->data->{switches}{'boot-after'})
{
	handle_boot_order($anvil);
}

# If we were called with a job UUID, close it out.
$anvil->Job->update_progress({
	progress  => 100,
	message   => "job_0112", 
	log_level => 1, 
	'print'   => 1, 
});

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This handles setting / updating server boot order.
sub handle_boot_order
{
	my ($anvil) = @_;
	
	# If we're an Anvil! subnode, we'll limit the servers shown to this node.
	my $this_anvil_uuid = $anvil->Cluster->get_anvil_uuid({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_anvil_uuid => $this_anvil_uuid }});
	
	$anvil->Database->get_anvils({debug => 2});
	$anvil->Database->get_servers({debug => 2});
	if (not $anvil->data->{switches}{server})
	{
		show_boot_order($anvil, $this_anvil_uuid);
	}
	
	# Is the server valid?
	my ($server_name, $server_uuid) = $anvil->Get->server_from_switch({
		debug      => 2,
		server     => $anvil->data->{switches}{server},
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_name => $server_name,
		server_uuid => $server_uuid, 
	}});
	
	if (not $server_uuid)
	{
		# We now support boot dependencies across nodes.
		if (not $this_anvil_uuid)
		{
			# The server wasn't found anywhere on the Anvil!
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0007", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $anvil->data->{switches}{server}, 
				},
			});
		}
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Read to see if this server is set to stay off during auto boot.
	my $stay_off_key = "server::".$server_name."::stay-off";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { stay_off_key => $stay_off_key }});
	my ($server_stay_off, $variable_uuid, undef, undef) = $anvil->Database->read_variable({
		debug                 => 2,
		variable_name         => $stay_off_key, 
		variable_source_uuid  => $server_uuid, 
		variable_source_table => "servers", 
	});
	$server_stay_off = 0 if not $server_stay_off;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_stay_off => $server_stay_off, 
		variable_uuid   => $variable_uuid,
	}});
	
	my $boot_after_server_name         = "";
	my $server_start_delay             = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay};
	my $server_start_after_server_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid};
	   $server_start_after_server_uuid = "" if $server_start_after_server_uuid eq "NULL";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server_start_delay             => $server_start_delay, 
		server_start_after_server_uuid => $server_start_after_server_uuid,
	}});
	if ($server_stay_off)
	{
		# The server stays off, boot after not applicable
		$server_start_delay = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_start_delay => $server_start_delay }});
	}
	elsif ((not $server_start_after_server_uuid) or ($server_start_after_server_uuid eq "NULL"))
	{
		$server_start_after_server_uuid = "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_start_after_server_uuid => $server_start_after_server_uuid }});
	}
	else
	{
		$boot_after_server_name = $anvil->data->{servers}{server_uuid}{$server_start_after_server_uuid}{server_name};
		$server_start_delay     = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			boot_after_server_name => $boot_after_server_name,
			server_start_delay     => $server_start_delay, 
		}});
	}
	
	# Is the user asking to change the boot after machine?
	if ((not $anvil->data->{switches}{'boot-after'}) or ($anvil->data->{switches}{'boot-after'} eq "#!SET!#"))
	{
		# Just show the info about the server.
		if (not $server_start_after_server_uuid)
		{
			# The sever will boot without delay.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0079", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
		}
		elsif ($server_stay_off)
		{
			# The server will stay off.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0080", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
		}
		else
		{
			# The server will boot after another server
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0081", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server     => $server_name,
					delay      => $server_start_delay, 
					boot_after => $boot_after_server_name, 
				},
			});
		}
		$anvil->nice_exit({exit_code => 0});
	}
	
	my $boot_after_none                    = 0;
	my $dont_auto_boot                     = 0;
	my $new_server_start_delay             = 30;
	my $new_boot_after_server_name         = "";
	my $new_server_start_after_server_uuid = "";
	if ($anvil->data->{switches}{'boot-after'} eq "none")
	{
		$boot_after_none = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { boot_after_none => $boot_after_none }});
	}
	elsif ($anvil->data->{switches}{'boot-after'} eq "stay-off")
	{
		$dont_auto_boot = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dont_auto_boot => $dont_auto_boot }});
	}
	elsif ($anvil->data->{switches}{'boot-after'})
	{
		($new_boot_after_server_name, $new_server_start_after_server_uuid) = $anvil->Get->server_from_switch({
			server     => $anvil->data->{switches}{'boot-after'},
			anvil_uuid => $this_anvil_uuid,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_name => $server_name,
			server_uuid => $server_uuid, 
		}});
	}
	
	# If we don't have a boot after server, show the current config and exit.
	if ((not $boot_after_none) && (not $dont_auto_boot) && (not $new_boot_after_server_name))
	{
		if ($this_anvil_uuid)
		{
			# The boot-after server was not found on this Anvil! node.
			my $anvil_name        = $anvil->data->{anvils}{anvil_uuid}{$this_anvil_uuid}{anvil_name};
			my $anvil_description = $anvil->data->{anvils}{anvil_uuid}{$this_anvil_uuid}{anvil_description};
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0082", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					boot_after        => $anvil->data->{switches}{'boot-after'}, 
					anvil_name        => $anvil_name, 
					anvil_description => $anvil_description, 
				},
			});
		}
		else
		{
			# The boot-after server wasn't found anywhere on the Anvil!.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0083", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					boot_after => $anvil->data->{switches}{'boot-after'}, 
				},
			});
		}
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Still alive? ok, do we have a delay?
	if ($anvil->data->{switches}{delay})
	{
		# Is the value valid?
		if ($anvil->data->{switches}{delay} =~ /\D/)
		{
			# The delay isn't valid
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0084", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					delay => $anvil->data->{switches}{delay}, 
				},
			});
			$anvil->nice_exit({exit_code => 1});
		}
		$new_server_start_delay = $anvil->data->{switches}{delay};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_server_start_delay => $new_server_start_delay }});
	}
	
	# OK, now do the update, if needed.
	if ($dont_auto_boot)
	{
		if ($server_stay_off)
		{
			# Already configured to not auto-boot.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0085", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
		}
		else
		{
			# Updating to not automatically boot.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0086", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
			my $variable_uuid = $anvil->Database->insert_or_update_variables({
				debug                 => 2,
				variable_name         => $stay_off_key, 
				variable_value        => "1", 
				variable_default      => "0", 
				variable_description  => "striker_0001", 
				variable_section      => "servers", 
				variable_source_uuid  => $server_uuid, 
				variable_source_table => "servers", 
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
			
			if (($boot_after_none) or ($server_start_delay ne "30"))
			{
				# Clear the old boot-after
				my $query = "
UPDATE 
    servers
SET 
    server_start_after_server_uuid = NULL, 
    server_start_delay             = '30', 
    modified_date                  = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0123", variables => { query => $query }});
				$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			}
		}
		$anvil->nice_exit({exit_code => 0});
	}
	
	if ($boot_after_none)
	{
		if ((not $server_start_after_server_uuid) && (not $server_stay_off))
		{
			# I was asked to set the server to not wait on any other servers, which is already the case.
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0087", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
		}
		else
		{
			# Updating the not wait for any other servers
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0088", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server => $server_name, 
				},
			});
			if ($server_stay_off)
			{
				my $variable_uuid = $anvil->Database->insert_or_update_variables({
					debug                 => 2,
					variable_name         => $stay_off_key, 
					variable_value        => "0", 
					variable_default      => "0", 
					variable_description  => "striker_0001", 
					variable_section      => "servers", 
					variable_source_uuid  => $server_uuid, 
					variable_source_table => "servers", 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
			}
			if ($server_start_after_server_uuid)
			{
				my $query = "
UPDATE 
    servers
SET 
    server_start_after_server_uuid = NULL, 
    server_start_delay             = '30', 
    modified_date                  = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0123", variables => { query => $query }});
				$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			}
		}
		$anvil->nice_exit({exit_code => 0});
	}
	
	if ($new_boot_after_server_name)
	{
		my $update = 0;
		if ($new_boot_after_server_name eq $boot_after_server_name)
		{
			# Did the delay change?
			if ($server_start_delay ne $new_server_start_delay)
			{
				$update = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_server_start_delay => $new_server_start_delay }});
				
				# Updating the server to change the boot delay after
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "message_0089", 
					log_level => 1, 
					'print'   => 1, 
					variables => {
						server      => $server_name,
						start_delay => $server_start_delay, 
						boot_after  => $new_boot_after_server_name, 
					},
				});
			}
			else
			{
				# I was asked to set the server to boot after another server boots, which is already the case.
				$anvil->Job->update_progress({
					progress  => 100,
					message   => "message_0090", 
					log_level => 1, 
					'print'   => 1, 
					variables => {
						server      => $server_name,
						start_delay => $new_server_start_delay, 
						boot_after  => $new_boot_after_server_name, 
					},
				});
			}
		}
		else
		{
			# Updating the to boot after the new other boots
			$update = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_server_start_delay => $new_server_start_delay }});
			
			$anvil->Job->update_progress({
				progress  => 100,
				message   => "message_0091", 
				log_level => 1, 
				'print'   => 1, 
				variables => {
					server      => $server_name,
					start_delay => $new_server_start_delay, 
					boot_after  => $new_boot_after_server_name, 
				},
			});
		}
		
		if ($update)
		{
			my $query = "
UPDATE 
    servers
SET 
    server_start_after_server_uuid = ".$anvil->Database->quote($new_server_start_after_server_uuid).", 
    server_start_delay             = ".$anvil->Database->quote($new_server_start_delay).", 
    modified_date                  = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE 
    server_uuid = ".$anvil->Database->quote($server_uuid)."
;";
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0123", variables => { query => $query }});
			$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
			
			# Clear stay-off if needed
			if ($server_stay_off)
			{
				my $variable_uuid = $anvil->Database->insert_or_update_variables({
					debug                 => 2,
					variable_name         => $stay_off_key, 
					variable_value        => "0", 
					variable_default      => "0", 
					variable_description  => "striker_0001", 
					variable_section      => "servers", 
					variable_source_uuid  => $server_uuid, 
					variable_source_table => "servers", 
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
			}
		}
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

sub show_boot_order
{
	my ($anvil, $this_anvil_uuid) = @_;
	
	# There's no server, so show all servers and their boot orders
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:anvil_name' => $anvil_name,
			's2:anvil_uuid' => $anvil_uuid,
		}});
		next if (($this_anvil_uuid) && ($this_anvil_uuid ne $anvil_uuid));
		
		my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_description => $anvil_description }});
		
		print "Anvil! Node: [".$anvil_name."] - ".$anvil_description."\n";
		my $server_count = keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }});
		
		if ($server_count)
		{
			foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}})
			{
				my $server_uuid  = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid};
				my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:server_name'  => $server_name,
					's2:server_uuid'  => $server_uuid,
					's3:server_state' => $server_state,
				}});
				next if $server_state eq "DELETED";
				
				# Read to see if this server is set to stay off during auto boot.
				my $stay_off_key = "server::".$server_name."::stay-off";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { stay_off_key => $stay_off_key }});
				my ($server_stay_off, $variable_uuid, undef, undef) = $anvil->Database->read_variable({
					debug                 => 2,
					variable_name         => $stay_off_key, 
					variable_source_uuid  => $server_uuid, 
					variable_source_table => "servers", 
				});
				$server_stay_off = 0 if not $server_stay_off;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					server_stay_off => $server_stay_off, 
					variable_uuid   => $variable_uuid,
				}});
				
				my $server_start_after_server_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_after_server_uuid};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_start_after_server_uuid => $server_start_after_server_uuid }});
				if ($server_stay_off)
				{
					print " |- ".$server_name." will not boot automatically.\n";
					next;
				}
				elsif ((not $server_start_after_server_uuid) or ($server_start_after_server_uuid eq "NULL"))
				{
					print " |- ".$server_name." will boot without delay.\n";
					next;
				}

				my $boot_after_server_name  = $anvil->data->{servers}{server_uuid}{$server_start_after_server_uuid}{server_name};
				my $boot_after_server_state = $anvil->data->{servers}{server_uuid}{$server_start_after_server_uuid}{server_state};
				my $server_start_delay      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_start_delay};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:boot_after_server_name'  => $boot_after_server_name,
					's2:boot_after_server_state' => $boot_after_server_state,
					's3:server_start_delay'      => $server_start_delay, 
				}});
				
				if ($boot_after_server_state eq "DELETED")
				{
					print " |- ".$server_name." will boot without delay. It used to boot after the now-deleted: [".$boot_after_server_name."] server.\n";
				}
				else
				{
					$server_start_delay = 30 if not $server_start_delay;
					print " |- ".$server_name." will boot at least: [".$server_start_delay."s] after: [".$boot_after_server_name."].\n";
				}
			}
			print " \\- End List.\n\n";
		}
		else
		{
			print " \\- This node has no servers yet.\n\n";
		}
	}
	
	# We're done.
	$anvil->Job->update_progress({
		progress  => 100,
		message   => "job_0112", 
		log_level => 1, 
		'print'   => 1, 
	});
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}

# This looks for a server (or all servers) and if any aren't found locally, they're connected.
sub handle_resync
{
	my ($anvil) = @_;
	
	# If the cluster is up and both nodes are online, make sure all DRBD resources are connected.
	$anvil->Database->get_hosts();
	$anvil->DRBD->get_status();
	my $host_uuid       = $anvil->Get->host_uuid();
	my $host_type       = $anvil->Get->host_type();
	my $short_host_name = $anvil->Get->short_host_name();
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid       => $host_uuid,
		host_type       => $host_type,
		short_host_name => $short_host_name, 
	}});
	if ($host_type ne "node")
	{
		# Not a node, nothing to do.
		return(0);
	}
	
	# We can only proceed if the peer is online.
	my $problem = $anvil->Cluster->parse_cib({debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		problem                     => $problem,
		"cib::parsed::local::ready" => $anvil->data->{cib}{parsed}{'local'}{ready},
		"cib::parsed::peer::ready"  => $anvil->data->{cib}{parsed}{peer}{ready},
	}});
	
	if (($problem) or (not $anvil->data->{cib}{parsed}{peer}{ready}) or (not $anvil->data->{cib}{parsed}{'local'}{ready}))
	{
		# Not in the cluster, nothing to do.
		return(0);
	}
	
	# Walk through all resources and make sure the peer and local disks are attached and 
	# the connection established.
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
	}
	
	$anvil->Database->get_hosts;
	$anvil->Database->get_servers;
	
	
	return(0);
}
