#!/usr/bin/perl
# 
# This removes a bad key from a 
# 
# This program is setuid 'admin' and calls a (new) peer to read its host name and system UUID. It takes the 
# target's password in via a file.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = No database connection.
# 2 = Job not found.
# 3 = No offending keys found.
# 

use strict;
use warnings;
use Anvil::Tools;
use Text::Diff;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is 
# passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->Get->switches({list => [
	"confirm", 
	"job-uuid", 
	"test", 
	], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
	# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try 
	# again after we exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"});
	sleep 10;
	$anvil->nice_exit({exit_code => 1});
}

if ($anvil->data->{switches}{test})
{
	do_tests($anvil);
	$anvil->nice_exit({exit_code => 0});
}

# Pick up the job details
load_job_data($anvil);

# Process the bad keys
process_key($anvil);

# Done.
update_progress($anvil, 100, "job_0051");
$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub do_tests
{
	my ($anvil) = @_;
	
	# Tell the user we're starting.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0004"});
	
	# Does the known_hosts file exist?
	my $known_hosts    = "";
	my $home_directory = $anvil->Get->users_home({debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { home_directory => $home_directory }});
	if ($home_directory)
	{
		$known_hosts = $home_directory."/.ssh/known_hosts";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_hosts => $known_hosts }});
		
		if (-f $known_hosts)
		{
			# Found it
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0006", variables => { known_hosts => $known_hosts }});
		}
		else
		{
			# We're done.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0006", variables => { known_hosts => $known_hosts }});
			return(0);
		}
		
		my $bad_key_found = {};
		my $old_body      = $anvil->Remote->_check_known_hosts_for_bad_entries({
			debug       => 2,
			known_hosts => $known_hosts,
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_body => $old_body }});
		foreach my $line (split/\n/, $old_body)
		{
			$line = $anvil->Words->clean_spaces({string => $line});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
			
			$line =~ s/#.*$//;
			next if not $line;
			
			if ($line =~ /^(.*?)\s+(.*?)\s+(.*)$/)
			{
				my $host = $1;
				my $algo = $2;
				my $key  = $3;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's1:host' => $host,
					's2:algp' => $algo,
					's3:key'  => $key, 
				}});
				
				# Tell the user we're going to test this host.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0009", variables => { host => $host }});
				
				# Call the target
				my ($output, $error, $return_code) = $anvil->Remote->call({
					debug      => 2, 
					shell_call => $anvil->data->{path}{exe}{echo}." 1", 
					target     => $host,
					tries      => 1,
					'close'    => 1,
					no_cache   => 1,
					use_ip     => 0,
				});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					output      => $output,
					error       => $error,
					return_code => $return_code, 
				}});
				
				if ($output)
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0010", variables => { host => $host }});
				}
				else
				{
					# Failed to connect. See if there's a 'host_key_changed::<host>' 
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0029"});
					
					my $state_name = "host_key_changed::".$host;
					my $query      = "SELECT state_note FROM states WHERE state_name = ".$anvil->Database->quote($state_name)." AND state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid).";";
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
					
					my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
					my $count   = @{$results};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						results => $results, 
						count   => $count,
					}});
					if ($count)
					{
						my $state_note = $results->[0]->[0];
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							state_note    => $state_note,
							bad_key_found => $bad_key_found, 
						}});
						
						$bad_key_found->{$host} = $state_note;
						
						# Tell the user.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0008"});
					}
					else
					{
						# Nope, failed for another reason.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0030"});
					}
				}
			}
			else
			{
				next;
			}
		}
		
		# If we found a bad key, tell the user to re-run 
		my $bad_key_count = keys %{$bad_key_found};
		if ($bad_key_count)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0042", variables => { count => $bad_key_count }});
			foreach my $host (sort {$a cmp $b} keys %{$bad_key_found})
			{
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0062", variables => { host => $host }});
			}
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0032"});
		}
	}
	
	return(0);
}

sub process_key
{
	my ($anvil) = @_;
	
	# We need a list of both hosts and keys to delete. The job will have one key for one algo. Take the
	# key and figure out which host it belongs to. Once we know the host, we need to find all names and
	# IPs that the target host uses, and delete all keys that are used by any of those names or IPs.
	my $bad_key      = $anvil->data->{bad_key};
	my ($algo, $key) = ($bad_key =~ /^(.*?)\s+(.*)$/);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:bad_key' => $bad_key, 
		's2:algo'    => $algo,
		's3:key'     => $key, 
	}});
	
	$anvil->data->{bad_keys}{$key} = 1;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"bad_keys::${key}" => $anvil->data->{bad_keys}{$key},
	}});
	
	my $target_host_uuid = "";
	print "Key to remove: [".$bad_key."]\n";
	foreach my $host (sort {$a cmp $b} keys %{$anvil->{ssh_host}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host }});
		foreach my $this_algo (sort {$a cmp $b} keys %{$anvil->{ssh_host}{$host}{algo}})
		{
			my $this_key = $anvil->{ssh_host}{$host}{algo}{$this_algo}{key};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:this_algo' => $this_algo,
				's2:this_key'  => $this_key, 
			}});
			
			if ($this_key eq $key)
			{
				# Record this host, we'll want to find any other keys regardless of if we 
				# find the host_uuid
				$anvil->data->{bad_hosts}{$host} = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"bad_hosts::${host}" => $anvil->data->{bad_hosts}{$host},
				}});
				
				next if $target_host_uuid;
				my $is_ip = $anvil->Validate->ip({ip => $host});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { is_ip => $is_ip }});
				if ($is_ip)
				{
					# Get the host names
					$target_host_uuid = $anvil->Network->get_host_from_ip({ip => $host});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
				}
				else
				{
					# Find the IP from the hostname.
					$target_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $host});
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host_uuid => $target_host_uuid }});
				}
			}
		}
	}
	
	# If I have a host_uuid, look 
	my $target_host_name       = "";
	my $target_short_host_name = "";
	my $target_ips             = "";
	if ($target_host_uuid)
	{
		$anvil->data->{bad_host_uuid} = $target_host_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_host_uuid => $anvil->data->{bad_host_uuid} }});
		
		$target_host_name       = $anvil->Get->host_name_from_uuid({host_uuid => $target_host_uuid});
		$target_short_host_name = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{short_host_name};
		$target_ips             = $anvil->Network->get_ip_from_host({host => $target_host_name});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			's1:target_host_name'       => $target_host_name, 
			's2:target_short_host_name' => $target_short_host_name, 
			's3:target_ips'             => $target_ips,
		}});
		
		# Create the hash of IPs and names to look for.
		$anvil->data->{bad_hosts}{$target_short_host_name} = 1;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			"bad_hosts::${target_short_host_name}" => $anvil->data->{bad_hosts}{$target_short_host_name},
		}});
		foreach my $ip (split/,/, $target_ips)
		{
			$anvil->data->{bad_hosts}{$ip} = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"bad_hosts::${ip}" => $anvil->data->{bad_hosts}{$ip},
			}});
		}
	}
	
	# Walk through the keys we already processed and see if there's any other bad keys to remove.
	$anvil->data->{bad_keys}{$key} = $algo;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		"bad_keys::${key}" => $anvil->data->{bad_keys}{$key},
	}});
	foreach my $host (sort {$a cmp $b} keys %{$anvil->{ssh_host}})
	{
		next if ((not exists $anvil->data->{bad_hosts}{$host}) or (not $anvil->data->{bad_hosts}{$host}));
		foreach my $this_algo (sort {$a cmp $b} keys %{$anvil->{ssh_host}{$host}{algo}})
		{
			my $this_key = $anvil->{ssh_host}{$host}{algo}{$this_algo}{key};
			next if $anvil->data->{bad_keys}{$this_key};
			
			$anvil->data->{bad_keys}{$this_key} = $this_algo;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				"bad_keys::${key}" => $anvil->data->{bad_keys}{$key},
			}});
		}
	}
	
	# Delete the key now.
	my $user_home   = $anvil->Get->users_home();
	my $known_hosts = $user_home."/.ssh/known_hosts";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_hosts => $known_hosts }});
	process_file($anvil, $known_hosts);
	
	# If we're root, look through other users also
	if (($< == 0) or ($> == 0))
	{
		# Walk through any other users.
		my $directory = "/home";
		local(*DIRECTORY);
		opendir(DIRECTORY, $directory);
		while(my $file = readdir(DIRECTORY))
		{
			next if $file eq ".";
			next if $file eq "..";
			my $full_path = $directory."/".$file;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
				file      => $file,
				full_path => $full_path,
			}});
			
			# If we're looking at a directory, scan it. Otherwise, see if it's an executable and that it
			# starts with 'scan-*'.
			if (-d $full_path)
			{
				# Check for a known_hosts file.
				my $known_hosts = $full_path."/.ssh/known_hosts";
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { known_hosts => $known_hosts }});
				if (-e $known_hosts)
				{
					process_file($anvil, $known_hosts);
				}
			}
		}
		closedir(DIRECTORY);
	}
	
	# Delete old state_uuids now.
	my $state_note =  $anvil->Database->quote($anvil->data->{bad_key});
	   $state_note =~ s/^'(.*?)'$/'%$1%'/;
	my $query      =  "
SELECT 
    state_uuid, 
    state_name, 
    state_note 
FROM 
    states 
WHERE 
    state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)." 
AND 
    state_note LIKE ".$state_note."
;";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	# States aren't sync'ed, so we may need to check both/all DBs to find our data.
	foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
	{
		my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
		my $count   = @{$results};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			results => $results, 
			count   => $count, 
		}});
		if ($count)
		{
			foreach my $row (@{$results})
			{
				my $state_uuid = $row->[0];
				my $state_name = $row->[1];
				my $state_note = $row->[2];
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					's2:state_uuid' => $state_uuid, 
					's3:state_name' => $state_name, 
					's4:state_note' => $state_note, 
				}});
				
				# Delete this key.
				$anvil->data->{job}{progress} += 5;
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0004,!!state_uuid!".$state_uuid."!!,!!bad_key!".$anvil->data->{bad_key}."!!,!!db_uuid!".$uuid."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0004", variables => {
					state_uuid => $state_uuid, 
					db_uuid    => $uuid,
					bad_key    => $anvil->data->{bad_key},
				}});
				my $query = "DELETE FROM states	WHERE state_uuid = ".$anvil->Database->quote($state_uuid).";";
				$anvil->Database->write({debug => 3, query => $query, source => $THIS_FILE, line => __LINE__});
			}
		}
	}
	
	return(0);
}

# Look through the file for bad keys.
sub process_file
{
	my ($anvil, $file) = @_;
	my $bad_key        = $anvil->data->{bad_key};
	my ($algo, $key)   = ($bad_key =~ /^(.*?)\s+(.*)$/);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});

	$anvil->data->{job}{progress} += 5;
	update_progress($anvil, $anvil->data->{job}{progress}, "job_0049,!!bad_key!".$bad_key."!!");
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0049", variables => { bad_key => $bad_key }});
	
	# Read in the file, if it exists.
	if (not -e $file)
	{
		# File doesn't actually exist, wtf?
		$anvil->data->{job}{progress} += 10;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0050,!!file!".$file."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0050", variables => { file => $file }});
		
		return(1); 
	}
	
	# Read in the file
	my ($old_body) = $anvil->Storage->read_file({file => $file});
	if ($old_body eq "!!error!!")
	{
		# Failed to read the file
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0052,!!file!".$file."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0052", variables => { file => $file }});
		
		return(1);
	}
	
	# Find our key(s)
	my $line_number = 0;
	my $new_body    = "";
	foreach my $line (split/\n/, $old_body)
	{
		$line_number++;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line_number.":".$line }});
		
		if ($line =~ /^(\S.*)\s+(\S.*)\s+(\S.*)$/)
		{
			my $host = $1;
			my $algo = $2;
			my $key  = $3;
			my $bad  = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:host' => $host,
				's2:algo' => $algo,
				's3:key'  => $key,
			}});
			
			if ((exists $anvil->data->{bad_hosts}{$host}) && ($anvil->data->{bad_hosts}{$host}))
			{
				$bad = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad => $bad }});
			}
			elsif ((exists $anvil->data->{bad_keys}{$key}) && ($anvil->data->{bad_keys}{$key}))
			{
				$bad = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad => $bad }});
			}
			if ($bad)
			{
				update_progress($anvil, $anvil->data->{job}{progress}, "job_0054,!!line!".$line_number."!!");
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0054", variables => { line => $line_number }});
				next;
			}
		}
		
		$new_body .= $line."\n";
	}
	
	# Record the difference, if any.
	my $difference = diff \$old_body, \$new_body, { STYLE => 'Unified' };
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0003", variables => { 
		file       => $file, 
		difference => $difference,
	}});
	
	if ($difference)
	{
		# Write the file out.
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0055,!!file!".$file."!!");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0055", variables => { file => $file }});
		
		# Get the owning user and group.
		my ($owning_uid, $owning_gid) = (stat($file))[4,5];
		my $owning_user               = getpwuid($owning_uid);
		my $owning_group              = getpwuid($owning_gid);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			owning_uid   => $owning_uid, 
			owning_gid   => $owning_gid, 
			owning_user  => $owning_user, 
			owning_group => $owning_group,
		}});
		
		my $error = $anvil->Storage->write_file({
			body      => $new_body,
			debug     => 3,
			file      => $file,
			overwrite => 1,
			user      => $owning_user, 
			group     => $owning_group
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, list => { error => $error }});
		if ($error)
		{
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0059,!!file!".$file."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0059", variables => { file => $file }});
		}
		else
		{
			# Success!
			$anvil->data->{job}{progress} += 5;
			update_progress($anvil, $anvil->data->{job}{progress}, "job_0060,!!file!".$file."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0060", variables => { file => $file }});
		}
	}
	
	return(0);
}

# Load the job data or exit
sub load_job_data
{
	my ($anvil) = @_;
	
	# Create a job for all hosts.
	$anvil->Database->get_hosts({debug => 3});
	
	# Before I start. read in my known_hosts file so I can translate file/lines. 
	my $known_hosts    = "";
	my $home_directory = $anvil->Get->users_home({debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { home_directory => $home_directory }});
	if ($home_directory)
	{
		$known_hosts = $home_directory."/.ssh/known_hosts";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { known_hosts => $known_hosts }});
		
		if (-f $known_hosts)
		{
			my ($old_body) = $anvil->Storage->read_file({file => $known_hosts});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_body => $old_body }});
			
			my $line_number = 0;
			foreach my $line (split/\n/, $old_body)
			{
				$line_number++;
				$line = $anvil->Words->clean_spaces({string => $line});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line_number.":".$line }});
				
				$line =~ s/#.*$//;
				next if not $line;
				
				my ($host, $algo, $key) = ($line =~ /^(.*?)\s+(.*?)\s+(.*)$/);
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
					's1:host' => $host,
					's2:algp' => $algo,
					's3:key'  => $key, 
				}});
				
				$anvil->{ssh_keys}{$known_hosts}{host}{$host}{$algo}{key}  = $key;
				$anvil->{ssh_keys}{$known_hosts}{host}{$host}{$algo}{line} = $line_number;
				$anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{algo} = $algo;
				$anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{host} = $host;
				$anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{key}  = $key;
				$anvil->{ssh_key}{$key}{host}{$host}{algo}{$algo}{asked}   = 0;
				$anvil->{ssh_key}{$key}{algo}{$algo}{host}                 = $host;
				$anvil->{ssh_host}{$host}{algo}{$algo}{key}                = $key;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					"s1:ssh_keys::${known_hosts}::host::${host}::${algo}::key"  => $anvil->{ssh_keys}{$known_hosts}{host}{$host}{$algo}{key}, 
					"s2:ssh_keys::${known_hosts}::host::${host}::${algo}::line" => $anvil->{ssh_keys}{$known_hosts}{host}{$host}{$algo}{line}, 
					"s3:ssh_keys::${known_hosts}::line::${line_number}::algo"   => $anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{algo}, 
					"s4:ssh_keys::${known_hosts}::line::${line_number}::host"   => $anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{host}, 
					"s5:ssh_keys::${known_hosts}::line::${line_number}::key"    => $anvil->{ssh_keys}{$known_hosts}{line}{$line_number}{key}, 
					"s7:ssh_key::${key}::host::${host}::algo::${algo}::asked"   => $anvil->{ssh_key}{$key}{host}{$host}{algo}{$algo}{asked}, 
					"s8:ssh_key::${key}::algo::${algo}::host"                   => $anvil->{ssh_key}{$key}{algo}{$algo}{host}, 
					"s9:ssh_host::${host}::algo::${algo}::key"                  => $anvil->{ssh_host}{$host}{algo}{$algo}{key},
				}});
			}
		}
	}
	
	if ($anvil->data->{switches}{'job-uuid'})
	{
		# Get the job
		my $query = "SELECT job_data FROM jobs WHERE job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'}).";";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
		my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
		my $count   = @{$results};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			results => $results, 
			count   => $count, 
		}});
		
		if (not $count)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0079", variables => {
				job_uuid => $anvil->data->{switches}{'job-uuid'}, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Pick up the data.
		my $job_data = $results->[0]->[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_data => $job_data }});
		if ((not $job_data) or ($job_data !~ /^bad_key=/))
		{
			update_progress($anvil, 100, "error_0081,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0081", variables => {
				job_uuid => $anvil->data->{switches}{'job-uuid'}, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Pick up the job.
		$anvil->data->{job}{progress} = 0;
		update_progress($anvil, 0, "clear");
		
		$anvil->data->{job}{progress} += 5;
		update_progress($anvil, $anvil->data->{job}{progress}, "job_0048");
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0048"});
		
		# The job data only consists of 1 'bad_key=<key>' entry
		$anvil->data->{bad_key} = ($job_data =~ /bad_key=(.*)$/)[0];
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_key => $anvil->data->{bad_key} }});
		
		# Make sure we say this key.
		my ($algo, $key) = ($anvil->data->{bad_key} =~ /^(.*?)\s+(.*)$/);
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			algo => $algo,
			key  => $key,
		}});
		
		if ((not $algo) or (not $key))
		{
			update_progress($anvil, 100, "error_0322,!!job_data!".$job_data."!!");
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0322", variables => { job_data => $job_data }});
			$anvil->nice_exit({exit_code => 1});
		}
		
# 		if ((not exists $anvil->{ssh_key}{$key}) or (not $anvil->{ssh_key}{$key}{host}))
# 		{
# 			# Nothing to do.
# 			update_progress($anvil, 100, "error_0321,!!key!".$key."!!");
# 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0321", variables => { key => $key }});
# 			$anvil->nice_exit({exit_code => 1});
# 		}
	}
	else
	{
		# No job UUID. Are there known bad keys? If so, show them to the user, one at a time, and ask
		# if they want to remove them. We'll create a job for each one they say yes to.
		my $query = "
SELECT 
    state_uuid, 
    state_name, 
    state_note 
FROM 
    states 
WHERE 
    state_name 
LIKE 
    'host_key_changed::%' 
AND 
    state_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
;";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
		
		# Collect from all DBs.
		my $bad_keys_found = 0;
		foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
		{
			my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
			my $count   = @{$results};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				's1:uuid'    => $uuid ? $uuid." (".$anvil->Get->host_name_from_uuid({host_uuid => $uuid}).")" : "",
				's2:results' => $results, 
				's3:count'   => $count, 
			}});
			if ($count)
			{
				$bad_keys_found = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_keys_found => $bad_keys_found }});
				foreach my $row (@{$results})
				{
					my $state_uuid = $row->[0];
					my $state_name = $row->[1];
					my $state_note = $row->[2];
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						's1:state_uuid' => $state_uuid, 
						's2:state_name' => $state_name, 
						's2:state_note' => $state_note, 
					}});
					
					my $bad_key  = "";
					my $bad_file = "";
					my $bad_line = "";
					foreach my $pair (split/,/, $state_note)
					{
						my ($variable, $value) = ($pair =~ /^(.*?)=(.*)$/);
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
							pair     => $pair, 
							variable => $variable, 
							value    => $value,
						}});
						if ($variable eq "file")
						{
							$bad_file = $value;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_file => $bad_file }});
						}
						if ($variable eq "line")
						{
							$bad_line = $value;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_line => $bad_line }});
						}
						if ($variable eq "key")
						{
							$bad_key = $value;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_key => $bad_key }});
						}
					}
					
					if ((not $bad_key) && (($bad_file) && ($bad_line)))
					{
						# Find the key.
						if (exists $anvil->{ssh_keys}{$bad_file}{line}{$bad_line}{key})
						{
							$bad_key = $anvil->{ssh_keys}{$bad_file}{line}{$bad_line}{key};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_key => $bad_key }});
						}
					}
					
					if (not $bad_key)
					{
						# Failed to translate the bad file/line to a key.
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0001", variables => {
							file => $bad_file, 
							line => $bad_line,
						}});
					}
					else
					{
						# Have we asked about this key yet?
						if (not $anvil->{ssh_key}{$bad_key}{asked})
						{
							# Nope, ask now.
							my ($algo, $hash_key) = ($bad_key =~ /^(.*?)\s+(.*)$/);
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								's1:algo'     => $algo,
								's2:hash_key' => $hash_key, 
							}});
							
							$anvil->{ssh_key}{$hash_key}{asked} = 1;
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
								"ssh_key::${hash_key}::asked" => $anvil->{ssh_key}{$hash_key}{asked},
							}});
							
							my $host = $anvil->{ssh_key}{$hash_key}{algo}{$algo}{host};
							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host }});
							
							my $bad_key_host_uuid = "";
							### NOTE: We can only validate ipv4, ipv6 returns '1'
							###       for hostnames, and host_name and 
							###       domain_name checks are also unreliable. >_<
							if ($anvil->Validate->ipv4({ip => $host}))
							{
								($bad_key_host_uuid, my $host_name) = $anvil->Get->host_from_ip_address({ip_address => $host});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									bad_key_host_uuid => $bad_key_host_uuid,
									host_name         => $host_name, 
								}});
							}
							else
							{
								$bad_key_host_uuid = $anvil->Get->host_uuid_from_name({host_name => $host});
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bad_key_host_uuid => $bad_key_host_uuid }});
							}
							
							# Did the user confirm already?
							if ($anvil->data->{switches}{confirm})
							{
								# Yup.
								if ($anvil->data->{switches}{confirm} ne "2")
								{
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0359", variables => { host => $host }});
								}
							}
							else
							{
								# Ask the user to confirm.
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0362", variables => {
									host => $host, 
									algo => $algo, 
									key  => $hash_key,
								}});
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0363"});
								
								my $answer = <STDIN>;
								chomp $answer;
								$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0828", variables => { answer => $answer }});
								if ((lc($answer) eq "y") or (lc($answer) eq "yes"))
								{
									# Proceed.
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0175"});
									
									$anvil->data->{switches}{confirm} = 2;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										"switches::confirm" => $anvil->data->{switches}{confirm},
									}});
								}
								else
								{
									# Skip this key.
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0034"});
									next;
								}
							}
							
							# Create a job to remove the key from all hosts. Do 
							# it per-dashboard, because if the bad key is a 
							# striker, we'll fail to write to it.
							foreach my $db_uuid (keys %{$anvil->data->{cache}{database_handle}})
							{
								next if $db_uuid eq $bad_key_host_uuid;
								$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
									db_uuid           => $db_uuid           ? $db_uuid." (".$anvil->Get->host_name_from_uuid({host_uuid => $db_uuid}).")"                     : "",
									bad_key_host_uuid => $bad_key_host_uuid ? $bad_key_host_uuid." (".$anvil->Get->host_name_from_uuid({host_uuid => $bad_key_host_uuid}).")" : "",
								}});
								foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
								{
									my $this_host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
									next if $this_host_uuid eq $bad_key_host_uuid;
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
										host_name      => $host_name,
										this_host_uuid => $this_host_uuid,
									}});
									
									my $job_uuid = $anvil->Database->insert_or_update_jobs({
										debug           => 2,
										file            => $THIS_FILE, 
										line            => __LINE__, 
										uuid            => $db_uuid,
										job_title       => "job_0056",
										job_description => "job_0057",
										job_host_uuid   => $this_host_uuid, 
										job_data        => "bad_key=".$bad_key,
										job_command     => $anvil->data->{path}{exe}{'anvil-manage-keys'}.$anvil->Log->switches, 
										job_name        => "manage::broken_keys", 
										job_progress    => 0,
									});
									$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
									
									# Tell the user we've created a new job.
									$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0002", variables => {
										job_uuid  => $job_uuid, 
										host_name => $host_name, 
										target    => $host, 
									}});
								}
							}
						}
					}
				}
			}
		}
		
		if (not $bad_keys_found)
		{
			# No bad keys found, we're done.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0012"});
			$anvil->nice_exit({exit_code => 0});
		}
		
		# Done looking for keys
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

# This updates the progress if we were called with a job UUID.
sub update_progress
{
	my ($anvil, $progress, $message) = @_;
	
	$progress = 95 if $progress > 100;
	
	# Log the progress percentage.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		progress => $progress,
		message  => $message, 
	}});
	
	$anvil->Job->update_progress({
		debug    => 3, 
		progress => $progress, 
		message  => $message, 
		job_uuid => $anvil->data->{switches}{'job-uuid'},
	});
	
	return(0);
}
