#!/usr/bin/sh
#
# License:      GNU General Public License (GPL)
# Support:      zfs@lists.illumos.org
# Written by:   Saso Kiselkov
#
#	This script manages ZFS pools
#	It can import a ZFS pool or export it
#
#	usage: $0 {start|stop|status|monitor|validate-all|meta-data}
#
#	The "start" arg imports a ZFS pool.
#	The "stop" arg exports it.
#
#       OCF parameters are as follows
#       OCF_RESKEY_pool - the pool to import/export
#
#######################################################################
# Initialization:

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

# Defaults
OCF_RESKEY_pool_default=""
OCF_RESKEY_importargs_default=""
OCF_RESKEY_importforce_default=true
OCF_RESKEY_settle_timeout_default="10"

: ${OCF_RESKEY_pool=${OCF_RESKEY_pool_default}}
: ${OCF_RESKEY_importargs=${OCF_RESKEY_importargs_default}}
: ${OCF_RESKEY_importforce=${OCF_RESKEY_importforce_default}}
: ${OCF_RESKEY_settle_timeout=${OCF_RESKEY_settle_timeout_default}}

USAGE="usage: $0 {start|stop|status|monitor|validate-all|meta-data}";

#######################################################################

meta_data() {
	cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="ZFS" version="1.0">
<version>1.0</version>
<longdesc lang="en">
This script manages ZFS pools
It can import a ZFS pool or export it
</longdesc>
<shortdesc lang="en">Manages ZFS pools</shortdesc>

<parameters>
<parameter name="pool" unique="1" required="1">
<longdesc lang="en">
The name of the ZFS pool to manage, e.g. "tank".
</longdesc>
<shortdesc lang="en">ZFS pool name</shortdesc>
<content type="string" default="${OCF_RESKEY_pool_default}" />
</parameter>
<parameter name="importargs" unique="0" required="0">
<longdesc lang="en">
Arguments to zpool import, e.g. "-d /dev/disk/by-id".
</longdesc>
<shortdesc lang="en">Import arguments</shortdesc>
<content type="string" default="${OCF_RESKEY_importargs_default}" />
</parameter>
<parameter name="importforce" unique="0" required="0">
<longdesc lang="en">
zpool import is given the -f option.
</longdesc>
<shortdesc lang="en">Import is forced</shortdesc>
<content type="boolean" default="${OCF_RESKEY_importforce_default}" />
</parameter>
<parameter name="settle_timeout" unique="0" required="0">
<longdesc lang="en">
Seconds to wait after a successful import for the pool state to settle before
reporting the pool as started.

/proc/spl/kstat/zfs/&lt;pool&gt;/state is a lock-free snapshot of the root vdev
state, so shortly after an import it can still read OFFLINE or TRANSITIONING
while vdevs are being reopened. Pacemaker starts the recurring monitor as soon
as start returns, and the monitor treats anything other than ONLINE, DEGRADED
or FAULTED as a hard error, so a reading taken inside that window fails a
healthy pool and triggers a needless recovery.

The pool state is read once before any waiting, so a pool that is already
settled - the common case - costs nothing. Set to 0 to disable the wait.
</longdesc>
<shortdesc lang="en">Seconds to wait for the pool state to settle after import</shortdesc>
<content type="integer" default="${OCF_RESKEY_settle_timeout_default}" />
</parameter>
</parameters>

<actions>
<action name="start"   timeout="60s" />
<action name="stop"    timeout="60s" />
<action name="monitor" depth="0"  timeout="30s" interval="5s" />
<action name="validate-all"  timeout="30s" />
<action name="meta-data"  timeout="5s" />
</actions>
</resource-agent>
END
	exit $OCF_SUCCESS
}

zpool_is_imported () {
	# Check if ZFS kstats exists
	if [ -d  /proc/spl/kstat/zfs/ ] ; then
		# Check the existence of kstats for the pool. If the stats exists, the pool was imported.
		[ -d  /proc/spl/kstat/zfs/"${OCF_RESKEY_pool}" ]
		rc=$?
	else
		# If ZFS kstats do not exists, fallback to the standard check
		zpool list -H "$OCF_RESKEY_pool" > /dev/null
		rc=$?
	fi
	return $rc
}

# Reads the pool state kstat into $zpool_state.
# Returns 0 when the pool is usable (ONLINE or DEGRADED), 1 otherwise.
zpool_state_is_settled () {
	zpool_state=$(cat "/proc/spl/kstat/zfs/$OCF_RESKEY_pool/state" 2>/dev/null)
	case "$zpool_state" in
		ONLINE|DEGRADED) return 0;;
	esac
	return 1
}

# Waits for the pool state to settle after an import.
#
# /proc/spl/kstat/zfs/<pool>/state is a lock-free snapshot of the root vdev
# state, so right after an import it can still read OFFLINE (the root vdev is
# briefly VDEV_STATE_CLOSED while vdevs are reopened) or TRANSITIONING.
# Pacemaker starts the recurring monitor as soon as start returns, which is
# precisely when that window is open, and zpool_monitor maps anything that is
# not ONLINE, DEGRADED or FAULTED to OCF_ERR_GENERIC.
#
# This never fails the start. Judging the pool is the monitor's job, and the
# wait must not become a new way for start to fail.
zpool_settle () {
	# Older ZFS has no state kstat, and a pool only has one once imported,
	# so this is a runtime condition rather than a configuration check.
	[ -f "/proc/spl/kstat/zfs/$OCF_RESKEY_pool/state" ] || return 0

	[ "$OCF_RESKEY_settle_timeout" -gt 0 ] || return 0

	# Never spend the whole start timeout waiting: settle_timeout is only an
	# upper bound the admin asked for, the operation timeout is the real one.
	settle_max="$OCF_RESKEY_settle_timeout"
	# CRM_meta_timeout is supplied per operation by pacemaker, not configured,
	# and is absent during validate-all, so it is checked here.
	if ocf_is_decimal "$OCF_RESKEY_CRM_meta_timeout"; then
		op_budget=$((OCF_RESKEY_CRM_meta_timeout / 1000 - 5))
		if [ "$op_budget" -lt 1 ]; then
			op_budget=1
		fi
		if [ "$settle_max" -gt "$op_budget" ]; then
			settle_max="$op_budget"
		fi
	fi

	waited=0
	while [ "$waited" -lt "$settle_max" ]; do
		zpool_state_is_settled && return 0
		sleep 1
		waited=$((waited + 1))
	done
	zpool_state_is_settled && return 0

	ocf_log warn "${OCF_RESKEY_pool}: pool state is still '$zpool_state' after ${settle_max}s"
	return 0
}

# Forcibly imports a ZFS pool, mounting all of its auto-mounted filesystems
# (as configured in the 'mountpoint' and 'canmount' properties)
# If the pool is already imported, no operation is taken.
zpool_import () {
	if ! zpool_is_imported; then
		ocf_log debug "${OCF_RESKEY_pool}:starting import"

		# The meanings of the options to import are as follows:
		#   -f : import even if the pool is marked as imported to another
		#        system - the system may have failed and not exported it
		#        cleanly.
		#   -o cachefile=none : the import should be temporary, so do not
		#        cache it persistently (across machine reboots). We want
		#        the CRM to explicitly control imports of this pool.
	if ocf_is_true "${OCF_RESKEY_importforce}"; then
		FORCE=-f
	else
		FORCE=""
	fi
		if zpool import $FORCE $OCF_RESKEY_importargs -o cachefile=none "$OCF_RESKEY_pool" ; then
			ocf_log debug "${OCF_RESKEY_pool}:import successful"
			zpool_settle
			return $OCF_SUCCESS
		else
			ocf_log debug "${OCF_RESKEY_pool}:import failed"
			return $OCF_ERR_GENERIC
		fi
	fi
}

# Forcibly exports a ZFS pool, unmounting all of its filesystems in the process
# If the pool is not imported, no operation is taken.
zpool_export () {
	if zpool_is_imported; then
		ocf_log debug "${OCF_RESKEY_pool}:starting export"

		# -f : force the export, even if we have mounted filesystems
		# Please note that this may fail with a "busy" error if there are
		# other kernel subsystems accessing the pool (e.g. SCSI targets).
		# Always make sure the pool export is last in your failover logic.
		if zpool export -f "$OCF_RESKEY_pool" ; then
			ocf_log debug "${OCF_RESKEY_pool}:export successful"
			return $OCF_SUCCESS
        else
			ocf_log debug "${OCF_RESKEY_pool}:export failed"
			return $OCF_ERR_GENERIC
        fi
	fi
}

# Monitors the health of a ZFS pool resource. Please note that this only
# checks whether the pool is imported and functional, not whether it has
# any degraded devices (use monitoring systems such as Zabbix for that).
zpool_monitor () {
	# If the pool is not imported, then we can't monitor its health
	if ! zpool_is_imported; then
		return $OCF_NOT_RUNNING
	fi

	# Check the pool status
	# Since version 0.7.10 status can be obtained without locks
	# https://github.com/zfsonlinux/zfs/pull/7563
	if [ -f /proc/spl/kstat/zfs/$OCF_RESKEY_pool/state ] ; then
		HEALTH=$(cat /proc/spl/kstat/zfs/$OCF_RESKEY_pool/state)
	else
		HEALTH=$(zpool list -H -o health "$OCF_RESKEY_pool")
	fi

	case "$HEALTH" in
		ONLINE|DEGRADED) return $OCF_SUCCESS;;
		FAULTED)         return $OCF_NOT_RUNNING;;
		*)               return $OCF_ERR_GENERIC;;
	esac
}

# Validates whether we can import a given ZFS pool
zpool_validate () {
	# Check that the 'zpool' command is known
	if ! which zpool > /dev/null; then
		return $OCF_ERR_INSTALLED
	fi

	if ! ocf_is_decimal "$OCF_RESKEY_settle_timeout"; then
		ocf_exit_reason "Invalid settle_timeout='${OCF_RESKEY_settle_timeout}': expected a non-negative integer"
		return $OCF_ERR_CONFIGURED
	fi

	if [ -z "$OCF_RESKEY_pool" ]; then
		ocf_exit_reason "pool is required"
		return $OCF_ERR_CONFIGURED
	fi

	# If the pool is imported, then it is obviously valid
	if zpool_is_imported; then
		return $OCF_SUCCESS
	fi

	# Check that the pool can be imported
	if zpool import $OCF_RESKEY_importargs | grep 'pool:' | grep "\\<$OCF_RESKEY_pool\\>" > /dev/null;
	then
		return $OCF_SUCCESS
	else
		return $OCF_ERR_CONFIGURED
	fi
}

usage () {
	echo "$USAGE" >&2
	return $1
}

if [ $# -ne 1 ]; then
	usage $OCF_ERR_ARGS
fi

case $1 in
	meta-data)      meta_data;;
	start)          zpool_validate || exit $?
	                zpool_import;;
	stop)           zpool_export;;
	status|monitor) zpool_monitor;;
	validate-all)   zpool_validate;;
	usage)          usage $OCF_SUCCESS;;
	*)              usage $OCF_ERR_UNIMPLEMENTED;;
esac

exit $?
