#!/usr/bin/perl
# 
# This program will boot a target machine using either it's IPMI interface, if available, or one of the 
# (non-PDU) fence methods, if the target is in an Anvil! and we have a manifest for it.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 
# TODO: 
# 

use strict;
use warnings;
use Anvil::Tools;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

$anvil->Get->switches({list => [
	"confirm", 
	"force-off", 
	"force-reboot", 
	"host",
	"host-uuid",
	"job-uuid"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->clear();
	$anvil->Job->get_job_details();
	$anvil->Job->update_progress({
		progress         => 1,
		job_picked_up_by => $$, 
		job_picked_up_at => time, 
		message          => "job_0283", 
	});
	
	# Pull out the job data.
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		if ($line =~ /host-uuid=(.*?)$/)
		{
			$anvil->data->{switches}{'host-uuid'} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'},
			}});
		}
		if ($line =~ /host=(.*?)$/)
		{
			$anvil->data->{switches}{host} = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				'switches::host' => $anvil->data->{switches}{host},
			}});
		}
		if ($line =~ /task=(.*?)$/)
		{
			my $task = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { task => $task }});
			
			if ($task eq "force-off")
			{
				$anvil->data->{switches}{'force-off'} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::force-off' => $anvil->data->{switches}{'force-off'},
				}});
			}
			elsif ($task eq "force-reboot")
			{
				$anvil->data->{switches}{'force-reboot'} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					'switches::force-reboot' => $anvil->data->{switches}{'force-reboot'},
				}});
			}
		}
	}
}

# Get the host info (copy host-uuid to host)
if ((not $anvil->data->{switches}{host}) && ($anvil->data->{switches}{'host-uuid'}))
{
	$anvil->data->{switches}{host} = $anvil->data->{switches}{'host-uuid'};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'switches::host' => $anvil->data->{switches}{host},
	}});
}

# If the 'host' is 'all', then we can't be doing a force-off/reboot.
if (($anvil->data->{switches}{host} eq "all") && (($anvil->data->{switches}{'force-off'}) or ($anvil->data->{switches}{'force-reboot'})))
{
	# Not allowed
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0290"});
	$anvil->Job->update_progress({progress => 100, message => "error_0290"});
	$anvil->nice_exit({exit_code => 0});
}

# If the host is 'all', don't translate to the host_uuid. Otherwise, look for the host_uuid from the host string
if ((not $anvil->data->{switches}{'host-uuid'}) && ($anvil->data->{switches}{host} ne "all"))
{
	$anvil->data->{switches}{'host-uuid'} = $anvil->Database->get_host_uuid_from_string({string => $anvil->data->{switches}{host}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'switches::host-uuid' => $anvil->data->{switches}{'host-uuid'},
	}});
	
	# host name not found
	if (not $anvil->data->{switches}{'host-uuid'})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0291", variables => { host_name => $anvil->data->{switches}{'host-name'} }});
		$anvil->Job->update_progress({progress => 100, message => "error_0291,!!host_name!".$anvil->data->{switches}{'host-name'}."!!"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	$anvil->data->{switches}{'host-name'} = $anvil->Get->host_name_from_uuid({host_uuid => $anvil->data->{switches}{'host-uuid'}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'switches::host-name' => $anvil->data->{switches}{'host-name'},
	}});
}

find_power_control($anvil);

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This will try to boot or force off the node with host_ipmi data, if available. If not, it will try to find
# a (non-PDU) fence method to try
sub find_power_control
{
	my ($anvil) = @_;
	
	$anvil->Database->get_hosts_info({debug => 3});
	
	my $hosts = [];
	if ($anvil->data->{switches}{'host-uuid'})
	{
		push @{$hosts}, $anvil->data->{switches}{'host-name'};
	}
	elsif ($anvil->data->{switches}{host} eq "all")
	{
		$anvil->Database->get_hosts({debug => 2});
		foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
		{
			my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
			my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
			my $host_key  = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_key};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host_name' => $host_name,
				's2:host_uuid' => $host_uuid, 
				's3:host_ipmi' => $anvil->Log->is_secure($host_ipmi),
			}});
			next if $host_key eq "DELETED";
			next if not $host_ipmi;
			
			push @{$hosts}, $host_name;
		}
	}
	
	my $host_count = @{$hosts};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_count => $host_count }});
	if (not $host_count)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0357"});
		$anvil->Job->update_progress({progress => 100, message => "message_0357"});
		$anvil->nice_exit({exit_code => 0});
	}
	
	my $steps = int((80 / $host_count) / 3);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { steps => $steps }});
	
	$anvil->data->{sys}{progress} = 5;
	foreach my $host_name (sort {$a cmp $b} @{$hosts})
	{
		my $host_uuid = $anvil->Database->get_host_uuid_from_string({string => $host_name});
		my $host_ipmi = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
		my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host_uuid => $host_uuid, 
			host_name => $host_name,
			host_ipmi => $anvil->Log->is_secure($host_ipmi),
			host_type => $host_type, 
		}});
		
		$anvil->data->{sys}{progress} += $steps;
		
		if (not $host_ipmi)
		{
			# Sometimes the host_ipmi gets wiped. Can we find older entries? 
			my $tested = {};
			my $query  = "SELECT host_ipmi FROM history.hosts WHERE host_uuid = ".$anvil->Database->quote($host_uuid)." AND host_ipmi != '' ORDER BY modified_date DESC;";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
			my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				results => $results, 
				count   => $count, 
			}});
			foreach my $row (@{$results})
			{
				my $this_host_ipmi = $row->[0];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { this_host_ipmi => $this_host_ipmi }});
				
				if (exists $tested->{$this_host_ipmi})
				{
					next;
				}
				$tested->{$this_host_ipmi} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "tested->{".$this_host_ipmi."}" => $tested->{$this_host_ipmi} }});
				
				my $shell_call = $this_host_ipmi." -o status";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { this_host_ipmi => $this_host_ipmi }});
				
				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output, 
					return_code => $return_code,
				}});
				
				if (($return_code eq "0") or ($return_code eq "2"))
				{
					# Good IPMI, use it.
					$host_ipmi = $this_host_ipmi;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { host_ipmi => $host_ipmi }});
				
					if (($return_code eq "0") or ($return_code eq "2"))
					{
						# Good IPMI, use it.
						$host_ipmi = $this_host_ipmi;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { host_ipmi => $host_ipmi }});
						
						# Update the database. 
						my $query = "
UPDATE 
    hosts 
SET 
    host_ipmi     = ".$anvil->Database->quote($host_ipmi).", 
    modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." 
WHERE
    host_uuid     = ".$anvil->Database->quote($host_uuid)."
;";
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
						$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
					}
					last if $host_ipmi;
				}
			}
		}
		
		if ($host_ipmi)
		{
			# Got it.
			$anvil->data->{sys}{progress} += $steps;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0327", variables => { host_name => $host_name }});
			$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "job_0327,!!host_name!".$host_name."!!"});
			
			# First, is the node already on?
			$anvil->data->{sys}{progress} += $steps;
			my $problem = call_fence_agent($anvil, $host_ipmi, $anvil->data->{sys}{progress});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { problem => $problem }});
			
			# If there was no problem, we're done.
			if (not $problem)
			{
				$anvil->Job->update_progress({progress => 100, message => "message_0025"});
				$anvil->nice_exit({exit_code => 0});
			}
		}
		
		$anvil->data->{sys}{progress} += $steps;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"sys::progress" => $anvil->data->{sys}{progress},
		}});
		
		# If I am here, there is no IPMI. Can we control it using another fence method?
		# The machine would have to be in an Anvil! for this to work.
		my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({host_uuid => $host_uuid});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		if ($anvil_uuid)
		{
			my $anvil_name = $anvil->Cluster->get_anvil_name({anvil_uuid => $anvil_uuid});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0331", variables => { 
				host_name  => $host_name, 
				anvil_name => $anvil_name,
			}});
			$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "job_0331,!!host_name!".$host_name."!!,!!anvil_name!".$anvil_name."!!"});
			
			$anvil->Cluster->get_fence_methods({
				debug      => 2,
				anvil_uuid => $anvil_uuid, 
				host_uuid  => $host_uuid,
			});
			
			foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{fence_method}})
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_name => $target_host_name }});
				foreach my $order (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}})
				{
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { order => $order }});
					foreach my $method (sort {$a cmp $b} keys %{$anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}})
					{
						next if $method =~ /pdu/;
						my $shell_call = $anvil->data->{fence_method}{$target_host_name}{order}{$order}{method}{$method}{command};
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							method     => $method,
							shell_call => $anvil->Log->is_secure($shell_call),
						}});
						my $problem = call_fence_agent($anvil, $shell_call, $anvil->data->{sys}{progress});
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
						
						if ($problem)
						{
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0297"});
							$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "error_0297"});
						}
					}
				}
			}
		}
		else
		{
			# Nothing we can do to boot this machine.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0294"});
			$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}, message => "error_0294"});
		}
	}
	
	# Done.
	$anvil->Job->update_progress({progress => 100, message => "message_0025"});
	$anvil->nice_exit({exit_code => 0});
	
	return(0);
}

# This calls a fence agent and exits if it successfully boots or powers off the target
sub call_fence_agent
{
	my ($anvil, $shell_call, $progress) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		shell_call => $shell_call, 
		progress   => $progress,
	}});
	
	if (($shell_call !~ / -o /) && ($shell_call !~ / --action/))
	{
		$shell_call .= " -o status";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
	}
	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		output      => $output, 
		return_code => $return_code,
	}});
	
	if ($return_code eq "0")
	{
		# The machine is already on. Shall we turn it off?
		if (($anvil->data->{switches}{'force-off'}) or ($anvil->data->{switches}{'force-reboot'}))
		{
			# Yup, force it off. But first, ask to confirm?
			if ((not $anvil->data->{switches}{'job-uuid'}) && (not $anvil->data->{switches}{confirm}))
			{
				# Ask to confirm.
				print $anvil->Words->string({key => "message_0065", variables => { host => $anvil->data->{switches}{'host-name'} }})."\n";
				print $anvil->Words->string({key => "message_0021"})."\n";
				my $answer = <STDIN>;
				chomp $answer;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { answer => $answer }});

				if ($answer =~ /^y/i)
				{
					print $anvil->Words->string({key => "message_0175"})."\n";
				}
				else
				{
					print $anvil->Words->string({key => "message_0022"})."\n";
					$anvil->nice_exit({exit_code => 0});
				}
			}
			
			# Proceed!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0332"});
			$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress} += 4, message => "job_0332"});

			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call." -o off" }});
			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, secure => 1});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				output      => $output, 
				return_code => $return_code,
			}});
			
			if ($return_code eq "0")
			{
				# Success! Do we need to turn it back on?
				if ($anvil->data->{switches}{'force-reboot'})
				{
					# Boot it again.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0333"});
					$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0333"});
					
					# Wait a sec
					sleep 5;
					
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call." -o on" }});
					my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call." -o on", secure => 1});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						output      => $output, 
						return_code => $return_code,
					}});
					
					if ($return_code eq "0")
					{
						# Success!
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0334"});
						$anvil->Job->update_progress({progress => 100, message => "job_0334"});
						
						# Update the host's status to 'booting' and exit
						$anvil->Database->update_host_status({
							debug       => 2,
							host_uuid   => $anvil->data->{switches}{'host-uuid'}, 
							host_status => "booting", 
						});
						$anvil->nice_exit({exit_code => 0});
					}
					else
					{
						# Failed. 
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }});
						$anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"});
						$anvil->nice_exit({exit_code => 1});
					}
				}
				else
				{
					# We're done
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0335"});
					$anvil->Job->update_progress({progress => 100, message => "job_0335"});
					
					# Update the host's status to 'powered off' and exit
					$anvil->Database->update_host_status({
						debug       => 2,
						host_uuid   => $anvil->data->{switches}{'host-uuid'}, 
						host_status => "powered off", 
					});
					$anvil->nice_exit({exit_code => 0});
				}
			}
			else
			{
				# Failed. 
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }});
				$anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"});
				$anvil->nice_exit({exit_code => 1});
			}
			
		}
		else
		{
			# Nope, we're done.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0328"});
			$anvil->Job->update_progress({progress => 100, message => "job_0328"});
			$anvil->nice_exit({exit_code => 0});
		}
	}
	elsif ($return_code eq "1")
	{
		# Unable to connect to the fence device.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0293", variables => { output => $output }});
		$anvil->Job->update_progress({progress => 100, message => "error_0293,!!output!".$output."!!"});
		$anvil->nice_exit({exit_code => 1});
	}
	elsif ($return_code eq "2")
	{
		# The machine is off, Are we forcing it off?
		if ($anvil->data->{switches}{'force-off'})
		{
			# It's off, we're done.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0337"});
			$anvil->Job->update_progress({progress => 100, message => "job_job_0337"});
			$anvil->nice_exit({exit_code => 0});
		}
		elsif ($anvil->data->{switches}{'force-reboot'})
		{
			# It's off, but we were asked to reboot it, so proceed with turning it on.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0336"});
		}
		else
		{
			# We're booting it.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0329"});
		}
		
		$anvil->Job->update_progress({progress => $anvil->data->{sys}{progress}+=10, message => "job_0329"});

		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { shell_call => $shell_call." -o on" }});
		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call." -o on", secure => 1});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output      => $output, 
			return_code => $return_code,
		}});
		
		if ($return_code eq "0")
		{
			# Success!
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "job_0330"});
			$anvil->Job->update_progress({progress => 100, message => "job_0330"});
			
			# Update the host's status to 'booting' and exit
			$anvil->Database->update_host_status({
				debug       => 2,
				host_uuid   => $anvil->data->{switches}{'host-uuid'}, 
				host_status => "booting", 
			});
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			# Failed. 
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0292", variables => { output => $output }});
			$anvil->Job->update_progress({progress => 100, message => "error_0292,!!output!".$output."!!"});
			$anvil->nice_exit({exit_code => 1});
		}
	}
	
	return(0);
}
