/*
	 * Copyright (C) 2016-2025 Red Hat, Inc.  All rights reserved.
	 *
	 * Author: Fabio M. Di Nitto <fabbione@kronosnet.org>
	 *
	 * This software licensed under GPL-2.0+
	 */
	
	#include "config.h"
	
	#include <errno.h>
	#include <stdio.h>
	#include <string.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <fcntl.h>
	#include <pthread.h>
	#include <dirent.h>
	#include <sys/select.h>
	#include <poll.h>
	
	#include "libknet.h"
	#include "test-common.h"
	
	static pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
	static int log_init = 0;
	static pthread_mutex_t log_thread_mutex = PTHREAD_MUTEX_INITIALIZER;
	static pthread_t log_thread;
	static int log_thread_init = 0;
	static int log_fds[2];
	struct log_thread_data {
		int logfd;
		FILE *std;
	};
	static struct log_thread_data data;
	static char plugin_path[PATH_MAX];
	
	static int _read_pipe(int fd, char **file, size_t *length)
	{
		char buf[4096];
		int n;
		int done = 0;
	
		*file = NULL;
		*length = 0;
	
		memset(buf, 0, sizeof(buf));
	
		while (!done) {
	
			n = read(fd, buf, sizeof(buf));
	
			if (n < 0) {
				if (errno == EINTR)
					continue;
	
				if (*file)
					free(*file);
	
				return n;
			}
	
			if (n == 0 && (!*length))
				return 0;
	
			if (n == 0)
				done = 1;
	
			if (*file)
				*file = realloc(*file, (*length) + n + done);
			else
				*file = malloc(n + done);
	
			if (!*file)
				return -1;
	
			memmove((*file) + (*length), buf, n);
			*length += (done + n);
		}
	
		/* Null terminator */
		(*file)[(*length) - 1] = 0;
	
		return 0;
	}
	
	int execute_shell(const char *command, char **error_string)
	{
		pid_t pid;
		int status, err = 0;
		int fd[2];
		size_t size = 0;
	
		if ((command == NULL) || (!error_string)) {
			errno = EINVAL;
			return FAIL;
		}
	
		*error_string = NULL;
	
		err = pipe(fd);
		if (err)
			goto out_clean;
	
		pid = fork();
		if (pid < 0) {
			err = pid;
			goto out_clean;
		}
	
		if (pid) { /* parent */
	
			close(fd[1]);
			err = _read_pipe(fd[0], error_string, &size);
			if (err)
				goto out_clean0;
	
			waitpid(pid, &status, 0);
			if (!WIFEXITED(status)) {
				err = -1;
				goto out_clean0;
			}
			if (WIFEXITED(status) && WEXITSTATUS(status) != 0) {
				err = WEXITSTATUS(status);
				goto out_clean0;
			}
			goto out_clean0;
		} else { /* child */
			close(0);
			close(1);
			close(2);
	
			close(fd[0]);
			dup2(fd[1], 1);
			dup2(fd[1], 2);
			close(fd[1]);
	
			execlp("/bin/sh", "/bin/sh", "-c", command, NULL);
			exit(FAIL);
		}
	
	out_clean:
		close(fd[1]);
	out_clean0:
		close(fd[0]);
	
		return err;
	}
	
	int is_memcheck(void)
	{
		char *val;
	
		val = getenv("KNETMEMCHECK");
	
		if (val) {
			if (!strncmp(val, "yes", 3)) {
				return 1;
			}
		}
	
		return 0;
	}
	
	int is_helgrind(void)
	{
		char *val;
	
		val = getenv("KNETHELGRIND");
	
		if (val) {
			if (!strncmp(val, "yes", 3)) {
				return 1;
			}
		}
	
		return 0;
	}
	
	void set_scheduler(int policy)
	{
		struct sched_param sched_param;
		int err;
	
		err = sched_get_priority_max(policy);
		if (err < 0) {
			printf("Could not get maximum scheduler priority\n");
			exit(FAIL);
		}
		sched_param.sched_priority = err;
		err = sched_setscheduler(0, policy, &sched_param);
		if (err < 0) {
			printf("Could not set priority\n");
			exit(FAIL);
		}
		return;
	}
	
	int setup_logpipes(int *logfds)
	{
		if (pipe2(logfds, O_CLOEXEC | O_NONBLOCK) < 0) {
			printf("Unable to setup logging pipe\n");
			exit(FAIL);
		}
	
		return PASS;
	}
	
	void close_logpipes(int *logfds)
	{
		close(logfds[0]);
		logfds[0] = 0;
		close(logfds[1]);
		logfds[1] = 0;
	}
	
	void flush_logs(int logfd, FILE *std)
	{
		struct knet_log_msg msg;
		int len;
	
		while (1) {
			len = read(logfd, &msg, sizeof(msg));
			if (len != sizeof(msg)) {
				/*
				 * clear errno to avoid incorrect propagation
				 */
				errno = 0;
				return;
			}
	
			msg.msg[sizeof(msg.msg) - 1] = 0;
	
			fprintf(std, "[knet]: [%s] %s: %.*s\n",
				knet_log_get_loglevel_name(msg.msglevel),
				knet_log_get_subsystem_name(msg.subsystem),
				KNET_MAX_LOG_MSG_SIZE, msg.msg);
		}
	}
	
	static void *_logthread(void *args)
	{
		while (1) {
			int num;
			struct timeval tv = { 60, 0 };
			fd_set rfds;
	
			FD_ZERO(&rfds);
			FD_SET(data.logfd, &rfds);
	
			num = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
			if (num < 0) {
				fprintf(data.std, "Unable select over logfd!\nHALTING LOGTHREAD!\n");
				return NULL;
			}
			if (num == 0) {
				fprintf(data.std, "[knet]: No logs in the last 60 seconds\n");
				continue;
			}
			if (FD_ISSET(data.logfd, &rfds)) {
				flush_logs(data.logfd, data.std);
			}
		}
	}
	
	int start_logthread(int logfd, FILE *std)
	{
		int savederrno = 0;
	
		savederrno = pthread_mutex_lock(&log_thread_mutex);
		if (savederrno) {
			printf("Unable to get log_thread mutex lock\n");
			return -1;
		}
	
		if (!log_thread_init) {
			data.logfd = logfd;
			data.std = std;
	
			savederrno = pthread_create(&log_thread, 0, _logthread, NULL);
			if (savederrno) {
				printf("Unable to start logging thread: %s\n", strerror(savederrno));
				pthread_mutex_unlock(&log_thread_mutex);
				return -1;
			}
			log_thread_init = 1;
		}
	
		pthread_mutex_unlock(&log_thread_mutex);
		return 0;
	}
	
	int stop_logthread(void)
	{
		int savederrno = 0;
		void *retval;
	
		savederrno = pthread_mutex_lock(&log_thread_mutex);
		if (savederrno) {
			printf("Unable to get log_thread mutex lock\n");
			return -1;
		}
	
		if (log_thread_init) {
			pthread_cancel(log_thread);
			pthread_join(log_thread, &retval);
			log_thread_init = 0;
		}
	
		pthread_mutex_unlock(&log_thread_mutex);
		return 0;
	}
	
	static void stop_logging(void)
	{
		stop_logthread();
		flush_logs(log_fds[0], stdout);
		close_logpipes(log_fds);
	}
	
	int start_logging(FILE *std)
	{
		int savederrno = 0;
	
		savederrno = pthread_mutex_lock(&log_mutex);
		if (savederrno) {
			printf("Unable to get log_mutex lock\n");
			return -1;
		}
	
		if (!log_init) {
			setup_logpipes(log_fds);
	
			if (atexit(&stop_logging) != 0) {
				printf("Unable to register atexit handler to stop logging: %s\n",
				       strerror(errno));
				exit(FAIL);
			}
	
			if (start_logthread(log_fds[0], std) < 0) {
				exit(FAIL);
			}
	
			log_init = 1;
		}
	
		pthread_mutex_unlock(&log_mutex);
	
		return log_fds[1];
	}
	
	static int dir_filter(const struct dirent *dname)
	{
		if ( (strcmp(dname->d_name + strlen(dname->d_name)-3, ".so") == 0) &&
		    ((strncmp(dname->d_name,"crypto", 6) == 0) ||
		     (strncmp(dname->d_name,"compress", 8) == 0))) {
			return 1;
		}
		return 0;
	}
	
	/* Make sure the proposed plugin path has at least 1 of each plugin available
	   - just as a sanity check really */
	static int contains_plugins(char *path)
	{
		struct dirent **namelist;
		int n,i;
		size_t j;
		struct knet_compress_info compress_list[256];
		struct knet_crypto_info crypto_list[256];
		size_t num_compress, num_crypto;
		size_t compress_found = 0;
		size_t crypto_found = 0;
	
		if (knet_get_compress_list(compress_list, &num_compress) == -1) {
			return 0;
		}
		if (knet_get_crypto_list(crypto_list, &num_crypto) == -1) {
			return 0;
		}
	
		n = scandir(path, &namelist, dir_filter, alphasort);
		if (n == -1) {
			return 0;
		}
	
		/* Look for plugins in the list */
		for (i=0; i<n; i++) {
			for (j=0; j<num_crypto; j++) {
				if (strlen(namelist[i]->d_name) >= 7 &&
				    strncmp(crypto_list[j].name, namelist[i]->d_name+7,
					    strlen(crypto_list[j].name)) == 0) {
					crypto_found++;
				}
			}
			for (j=0; j<num_compress; j++) {
				if (strlen(namelist[i]->d_name) >= 9 &&
				    strncmp(compress_list[j].name, namelist[i]->d_name+9,
					    strlen(compress_list[j].name)) == 0) {
					compress_found++;
				}
			}
			free(namelist[i]);
		}
		free(namelist);
		/* If at least one plugin was found (or none were built) */
		if ((crypto_found || num_crypto == 0) &&
		    (compress_found || num_compress == 0)) {
			return 1;
		} else {
			return 0;
		}
	}
	
	
	/* libtool sets LD_LIBRARY_PATH to the build tree when running test in-tree */
	char *find_plugins_path(void)
	{
		char *ld_libs_env = getenv("LD_LIBRARY_PATH");
		if (ld_libs_env) {
			char *ld_libs = strdup(ld_libs_env);
			char *str = strtok(ld_libs, ":");
			while (str) {
				if (contains_plugins(str)) {
					strncpy(plugin_path, str, sizeof(plugin_path)-1);
					free(ld_libs);
					printf("Using plugins from %s\n", plugin_path);
					return plugin_path;
				}
				str = strtok(NULL, ":");
			}
			free(ld_libs);
		}
		return NULL;
	}
	
	
	knet_handle_t knet_handle_start(int logfds[2], uint8_t log_level, knet_handle_t knet_h_array[])
	{
		knet_handle_t knet_h = knet_handle_new_ex(1, logfds[1], log_level, 0);
		char *plugins_path;
	
		if (knet_h) {
			printf("knet_handle_new at %p\n", knet_h);
			plugins_path = find_plugins_path();
			/* Use plugins from the build tree */
			if (plugins_path) {
				knet_h->plugin_path = plugins_path;
			}
			knet_h_array[1] = knet_h;
			flush_logs(logfds[0], stdout);
			return knet_h;
		} else {
			printf("knet_handle_new failed: %s\n", strerror(errno));
			flush_logs(logfds[0], stdout);
			close_logpipes(logfds);
			exit(FAIL);
		}
	}
	
	int knet_handle_reconnect_links(knet_handle_t knet_h)
	{
		size_t i, j;
		knet_node_id_t host_ids[KNET_MAX_HOST];
		uint8_t link_ids[KNET_MAX_LINK];
		size_t host_ids_entries = 0, link_ids_entries = 0;
		unsigned int enabled;
	
		if (!knet_h) {
			errno = EINVAL;
			return -1;
		}
	
		if (knet_host_get_host_list(knet_h, host_ids, &host_ids_entries) < 0) {
			printf("knet_host_get_host_list failed: %s\n", strerror(errno));
			return -1;
		}
	
		for (i = 0; i < host_ids_entries; i++) {
			if (knet_link_get_link_list(knet_h, host_ids[i], link_ids, &link_ids_entries)) {
				printf("knet_link_get_link_list failed: %s\n", strerror(errno));
				return -1;
			}
			for (j = 0; j < link_ids_entries; j++) {
				if (knet_link_get_enable(knet_h, host_ids[i], link_ids[j], &enabled)) {
					printf("knet_link_get_enable failed: %s\n", strerror(errno));
					return -1;
				}
				if (!enabled) {
					if (knet_link_set_enable(knet_h, host_ids[i], j, 1)) {
						printf("knet_link_set_enable failed: %s\n", strerror(errno));
						return -1;
					}
				}
			}
		}
	
		return 0;
	}
	
	static int _make_local_sockaddr(struct sockaddr_storage *lo, int offset, int family)
	{
		in_port_t port;
		char portstr[32];
	
		if (offset < 0) {
			/*
			 * api_knet_link_set_config needs to access the API directly, but
			 * it does not send any traffic, so it´s safe to ask the kernel
			 * for a random port.
			 */
			port = 0;
		} else {
			/* Use the pid if we can. but makes sure its in a sensible range */
			port = (getpid() + offset) % (65536-1024) + 1024;
		}
		sprintf(portstr, "%u", port);
		memset(lo, 0, sizeof(struct sockaddr_storage));
		printf("Using port %u\n", port);
	
		if (family == AF_INET6) {
			return knet_strtoaddr("::1", portstr, lo, sizeof(struct sockaddr_storage));
		}
		return knet_strtoaddr("127.0.0.1", portstr, lo, sizeof(struct sockaddr_storage));
	}
	
	int make_local_sockaddr(struct sockaddr_storage *lo, int offset)
	{
		return _make_local_sockaddr(lo, offset, AF_INET);
	}
	
	int make_local_sockaddr6(struct sockaddr_storage *lo, int offset)
	{
		return _make_local_sockaddr(lo, offset, AF_INET6);
	}
	
	int _knet_link_set_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id,
				  uint8_t transport, uint64_t flags, int family, int dynamic,
				  struct sockaddr_storage *lo)
	{
		int err = 0, savederrno = 0;
		uint32_t port;
		char portstr[32];
	
		for (port = 1025; port < 65536; port++) {
			sprintf(portstr, "%u", port);
			memset(lo, 0, sizeof(struct sockaddr_storage));
			if (family == AF_INET6) {
				err = knet_strtoaddr("::1", portstr, lo, sizeof(struct sockaddr_storage));
			} else {
				err = knet_strtoaddr("127.0.0.1", portstr, lo, sizeof(struct sockaddr_storage));
			}
			if (err < 0) {
				printf("Unable to convert loopback to sockaddr: %s\n", strerror(errno));
				goto out;
			}
			errno = 0;
			if (dynamic) {
				err = knet_link_set_config(knet_h, host_id, link_id, transport, lo, NULL, flags);
			} else {
				err = knet_link_set_config(knet_h, host_id, link_id, transport, lo, lo, flags);
			}
			savederrno = errno;
			if ((err < 0)  && (savederrno != EADDRINUSE)) {
				printf("Unable to configure link: %s\n", strerror(savederrno));
				goto out;
			}
			if (!err) {
				printf("Using port %u\n", port);
				goto out;
			}
		}
	
		if (err) {
			printf("No more ports available\n");
		}
	out:
		errno = savederrno;
		return err;
	}
	
	void test_sleep(knet_handle_t knet_h, int seconds)
	{
		if (is_memcheck() || is_helgrind()) {
			printf("Test suite is running under valgrind, adjusting sleep timers\n");
			seconds = seconds * 16;
		}
		sleep(seconds);
	}
	
	
	int wait_for_packet(knet_handle_t knet_h, int seconds, int datafd, int logfd, FILE *std)
	{
		fd_set rfds;
		struct timeval tv;
		int err = 0, i = 0;
	
		if (is_memcheck() || is_helgrind()) {
			printf("Test suite is running under valgrind, adjusting wait_for_packet timeout\n");
			seconds = seconds * 16;
		}
	
	try_again:
		FD_ZERO(&rfds);
		FD_SET(datafd, &rfds);
	
		tv.tv_sec = 1;
		tv.tv_usec = 0;
	
		err = select(datafd+1, &rfds, NULL, NULL, &tv);
		/*
		 * on slow arches the first call to select can return 0.
		 * pick an arbitrary 10 times loop (multiplied by waiting seconds)
		 * before failing.
		 */
		if ((!err) && (i < seconds)) {
			flush_logs(logfd, std);
			i++;
			goto try_again;
		}
		if ((err > 0) && (FD_ISSET(datafd, &rfds))) {
			return 0;
		}
	
		errno = ETIMEDOUT;
		return -1;
	}
	
	/*
	 * functional tests helpers
	 */
	
	void knet_handle_start_nodes(knet_handle_t knet_h[], uint8_t numnodes, int logfds[2], uint8_t log_level)
	{
		uint8_t i;
		char *plugins_path = find_plugins_path();
	
		for (i = 1; i <= numnodes; i++) {
			knet_h[i] = knet_handle_new_ex(i, logfds[1], log_level, 0);
			if (!knet_h[i]) {
				printf("failed to create handle: %s\n", strerror(errno));
				break;
			} else {
				printf("knet_h[%u] at %p\n", i, knet_h[i]);
			}
			/* Use plugins from the build tree */
			if (plugins_path) {
				knet_h[i]->plugin_path = plugins_path;
			}
		}
	
		if (i < numnodes) {
			knet_handle_stop_everything(knet_h, i);
			exit(FAIL);
		}
	
		return;
	}
	
	void knet_handle_join_nodes(knet_handle_t knet_h[], uint8_t numnodes, uint8_t numlinks, int family, uint8_t transport)
	{
		uint8_t i, x, j;
		struct sockaddr_storage src, dst;
		int offset = 0;
		int res;
	
		for (i = 1; i <= numnodes; i++) {
			for (j = 1; j <= numnodes; j++) {
				/*
				 * don´t connect to itself
				 */
				if (j == i) {
					continue;
				}
	
				printf("host %u adding host: %u\n", i, j);
	
				if (knet_host_add(knet_h[i], j) < 0) {
					printf("Unable to add host: %s\n", strerror(errno));
					knet_handle_stop_everything(knet_h, numnodes);
					exit(FAIL);
				}
	
				for (x = 0; x < numlinks; x++) {
					res = -1;
					offset = 0;
					while (i + x + offset++ < 65535 && res != 0) {
						if (_make_local_sockaddr(&src, i + x + offset, family) < 0) {
							printf("Unable to convert src to sockaddr: %s\n", strerror(errno));
							knet_handle_stop_everything(knet_h, numnodes);
							exit(FAIL);
						}
	
						if (_make_local_sockaddr(&dst, j + x + offset, family) < 0) {
							printf("Unable to convert dst to sockaddr: %s\n", strerror(errno));
							knet_handle_stop_everything(knet_h, numnodes);
							exit(FAIL);
						}
	
						res = knet_link_set_config(knet_h[i], j, x, transport, &src, &dst, 0);
					}
					printf("joining node %u with node %u via link %u src offset: %u dst offset: %u\n", i, j, x, i+x, j+x);
					if (knet_link_set_enable(knet_h[i], j, x, 1) < 0) {
						printf("unable to enable link: %s\n", strerror(errno));
						knet_handle_stop_everything(knet_h, numnodes);
						exit(FAIL);
					}
				}
			}
		}
	
		for (i = 1; i <= numnodes; i++) {
			wait_for_nodes_state(knet_h[i], numnodes, 1, 600, knet_h[1]->logfd, stdout);
		}
		return;
	}
	
	
	static int target=0;
	
	static int state_wait_pipe[2] = {0,0};
	static int host_wait_pipe[2] = {0,0};
	
	static int count_nodes(knet_handle_t knet_h)
	{
		int nodes = 0;
		int i;
	
		for (i=0; i< KNET_MAX_HOST; i++) {
			if (knet_h->host_index[i] && knet_h->host_index[i]->status.reachable == 1) {
				nodes++;
			}
		}
		return nodes;
	}
	
	static void nodes_notify_callback(void *private_data,
					  knet_node_id_t host_id,
					  uint8_t reachable, uint8_t remote, uint8_t external)
	{
		knet_handle_t knet_h = (knet_handle_t) private_data;
		int nodes;
		int res;
	
		nodes = count_nodes(knet_h);
	
		if (nodes == target) {
			res = write(state_wait_pipe[1], ".", 1);
			if (res != 1) {
				printf("***FAILed to signal wait_for_nodes_state: %s\n", strerror(errno));
			}
		}
	}
	
	/* Called atexit() */
	static void finish_state_pipes()
	{
		if (state_wait_pipe[0] != 0) {
			close(state_wait_pipe[0]);
			close(state_wait_pipe[1]);
			state_wait_pipe[0] = 0;
		}
		if (host_wait_pipe[0] != 0) {
			close(host_wait_pipe[0]);
			close(host_wait_pipe[1]);
			host_wait_pipe[0] = 0;
		}
	}
	
	static void host_notify_callback(void *private_data,
					 knet_node_id_t host_id,
					 uint8_t reachable, uint8_t remote, uint8_t external)
	{
		knet_handle_t knet_h = (knet_handle_t) private_data;
		int res;
	
		if (knet_h->host_index[host_id]->status.reachable == 1) {
			res = write(host_wait_pipe[1], ".", 1);
			if (res != 1) {
				printf("***FAILed to signal wait_for_host: %s\n", strerror(errno));
			}
		}
	}
	
	static int wait_for_reply(int seconds, int pipefd)
	{
		int res;
		struct pollfd pfds;
		char tmpbuf[32];
	
		pfds.fd = pipefd;
		pfds.events = POLLIN | POLLERR | POLLHUP;
		pfds.revents = 0;
	
		res = poll(&pfds, 1, seconds*1000);
		if (res == 1) {
			if (pfds.revents & POLLIN) {
				res = read(pipefd, tmpbuf, sizeof(tmpbuf));
				if (res > 0) {
					return 0;
				}
			} else {
				printf("Error on pipe poll revent = 0x%x\n", pfds.revents);
				errno = EIO;
			}
		}
		if (res == 0) {
			errno = ETIMEDOUT;
			return -1;
		}
	
		return -1;
	}
	
	/* Wait for a cluster of 'numnodes' to come up/go down */
	int wait_for_nodes_state(knet_handle_t knet_h, size_t numnodes,
				 uint8_t state, uint32_t timeout,
				 int logfd, FILE *std)
	{
		int res, savederrno = 0;
	
		if (state_wait_pipe[0] == 0) {
			res = pipe(state_wait_pipe);
			if (res == -1) {
				savederrno = errno;
				printf("Error creating host reply pipe: %s\n", strerror(errno));
				errno = savederrno;
				return -1;
			}
			if (atexit(finish_state_pipes)) {
				printf("Unable to register atexit handler to close pipes: %s\n",
				       strerror(errno));
				exit(FAIL);
			}
	
		}
	
		if (state) {
			target = numnodes-1; /* exclude us */
		} else {
			target = 0; /* Wait for all to go down */
		}
	
		/* Set this before checking existing status or there's a race condition */
		knet_host_enable_status_change_notify(knet_h,
						      (void *)(long)knet_h,
						      nodes_notify_callback);
	
		/* Check we haven't already got all the nodes in the correct state */
		if (count_nodes(knet_h) == target) {
			fprintf(stderr, "target already reached\n");
			knet_host_enable_status_change_notify(knet_h, (void *)(long)0, NULL);
			flush_logs(logfd, std);
			return 0;
		}
	
		res = wait_for_reply(timeout, state_wait_pipe[0]);
		if (res == -1) {
			savederrno = errno;
			printf("Error waiting for nodes status reply: %s\n", strerror(errno));
		}
	
		knet_host_enable_status_change_notify(knet_h, (void *)(long)0, NULL);
		flush_logs(logfd, std);
		errno = savederrno;
		return res;
	}
	
	/* Wait for a single node to come up */
	int wait_for_host(knet_handle_t knet_h, uint16_t host_id, int seconds, int logfd, FILE *std)
	{
		int res = 0;
		int savederrno = 0;
	
		if (is_memcheck() || is_helgrind()) {
			printf("Test suite is running under valgrind, adjusting wait_for_host timeout\n");
			seconds = seconds * 16;
		}
	
		if (host_wait_pipe[0] == 0) {
			res = pipe(host_wait_pipe);
			if (res == -1) {
				savederrno = errno;
				printf("Error creating host reply pipe: %s\n", strerror(errno));
				errno = savederrno;
				return -1;
			}
			if (atexit(finish_state_pipes)) {
				printf("Unable to register atexit handler to close pipes: %s\n",
				       strerror(errno));
				exit(FAIL);
			}
	
		}
	
		/* Set this before checking existing status or there's a race condition */
		knet_host_enable_status_change_notify(knet_h,
						      (void *)(long)knet_h,
						      host_notify_callback);
	
		/* Check it's not already reachable */
		if (knet_h->host_index[host_id]->status.reachable == 1) {
			knet_host_enable_status_change_notify(knet_h, (void *)(long)0, NULL);
			flush_logs(logfd, std);
			return 0;
		}
	
		res = wait_for_reply(seconds, host_wait_pipe[0]);
		if (res == -1) {
			savederrno = errno;
			printf("Error waiting for host status reply: %s\n", strerror(errno));
		}
	
		knet_host_enable_status_change_notify(knet_h, (void *)(long)0, NULL);
	
		/* Still wait for it to settle */
		flush_logs(logfd, std);
		test_sleep(knet_h, 1);
		errno = savederrno;
		return res;
	}
	
	void clean_exit(knet_handle_t *knet_h, int testnodes, int *logfds, int exit_status)
	{
		knet_handle_stop_everything(knet_h, testnodes);
		stop_logthread();
		flush_logs(logfds[0], stdout);
		close_logpipes(logfds);
		if (exit_status != CONTINUE) {
			exit(exit_status);
		}
	}
	
	/* Shutdown all nodes and links attached to an array of knet handles.
	 * Mostly stolen from corosync code (that I wrote, before anyone complains about licences)
	 */
	void knet_handle_stop_everything(knet_handle_t knet_h[], uint8_t numnodes)
	{
		int res = 0;
		int h;
		size_t i,j;
		static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
		uint8_t links[KNET_MAX_LINK];
		size_t num_nodes;
		size_t num_links;
	

		for (h=1; h<numnodes+1; h++) {

			res = knet_handle_setfwd(knet_h[h], 0);

			if (res) {
				perror("knet_handle_setfwd failed");

			}
	
			res = knet_host_get_host_list(knet_h[h], nodes, &num_nodes);

			if (res) {
				perror("Cannot get knet node list for shutdown");
				continue;

			}
	
			/* Tidily shut down all nodes & links. */

			for (i=0; i<num_nodes; i++) {
	
				res = knet_link_get_link_list(knet_h[h], nodes[i], links, &num_links);

				if (res) {
					fprintf(stderr, "Cannot get knet link list for node %u  %s\n", nodes[i], strerror(errno));

					goto finalise_error;

				}

				for (j=0; j<num_links; j++) {
					res = knet_link_set_enable(knet_h[h], nodes[i], links[j], 0);

					if (res) {
						fprintf(stderr, "knet_link_set_enable(node %u, link %d) failed: %s\n", nodes[i], links[j], strerror(errno));
					}
					res = knet_link_clear_config(knet_h[h], nodes[i], links[j]);

					if (res) {
						fprintf(stderr, "knet_link_clear_config(node %u, link %d) failed: %s\n", nodes[i], links[j], strerror(errno));

					}

				}
				res = knet_host_remove(knet_h[h], nodes[i]);

				if (res) {
					fprintf(stderr, "knet_host_remove(node %u) failed: %s\n", nodes[i], strerror(errno));
				}

			}
	

		finalise_error:
			res = knet_handle_free(knet_h[h]);

			if (res) {
				fprintf(stderr, "knet_handle_free failed: %s\n", strerror(errno));
			}

		}
	}