1    	/*
2    	 * Copyright 2004-2023 the Pacemaker project contributors
3    	 *
4    	 * The version control history for this file may have further details.
5    	 *
6    	 * This source code is licensed under the GNU General Public License version 2
7    	 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8    	 */
9    	
10   	#include <crm_internal.h>
11   	
12   	#include <crm_resource.h>
13   	#include <crm/lrmd_internal.h>
14   	#include <crm/common/cmdline_internal.h>
15   	#include <crm/common/ipc_attrd_internal.h>
16   	#include <crm/common/lists_internal.h>
17   	#include <crm/common/output.h>
18   	#include <pacemaker-internal.h>
19   	
20   	#include <sys/param.h>
21   	#include <stdio.h>
22   	#include <sys/types.h>
23   	#include <unistd.h>
24   	#include <stdlib.h>
25   	#include <errno.h>
26   	#include <fcntl.h>
27   	#include <libgen.h>
28   	#include <time.h>
29   	
30   	#include <crm/crm.h>
31   	#include <crm/stonith-ng.h>
32   	#include <crm/common/ipc_controld.h>
33   	#include <crm/cib/internal.h>
34   	
35   	#define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"
36   	
37   	enum rsc_command {
38   	    cmd_none = 0,           // No command option given (yet)
39   	    cmd_ban,
40   	    cmd_cleanup,
41   	    cmd_clear,
42   	    cmd_colocations,
43   	    cmd_cts,
44   	    cmd_delete,
45   	    cmd_delete_param,
46   	    cmd_digests,
47   	    cmd_execute_agent,
48   	    cmd_fail,
49   	    cmd_get_param,
50   	    cmd_get_property,
51   	    cmd_list_active_ops,
52   	    cmd_list_agents,
53   	    cmd_list_all_ops,
54   	    cmd_list_alternatives,
55   	    cmd_list_instances,
56   	    cmd_list_providers,
57   	    cmd_list_resources,
58   	    cmd_list_standards,
59   	    cmd_locate,
60   	    cmd_metadata,
61   	    cmd_move,
62   	    cmd_query_raw_xml,
63   	    cmd_query_xml,
64   	    cmd_refresh,
65   	    cmd_restart,
66   	    cmd_set_param,
67   	    cmd_set_property,
68   	    cmd_wait,
69   	    cmd_why,
70   	};
71   	
72   	struct {
73   	    enum rsc_command rsc_cmd;     // crm_resource command to perform
74   	
75   	    // Infrastructure that given command needs to work
76   	    gboolean require_cib;         // Whether command requires CIB IPC
77   	    int cib_options;              // Options to use with CIB IPC calls
78   	    gboolean require_crmd;        // Whether command requires controller IPC
79   	    gboolean require_scheduler;   // Whether command requires scheduler data
80   	    gboolean require_resource;    // Whether command requires resource specified
81   	    gboolean require_node;        // Whether command requires node specified
82   	    int find_flags;               // Flags to use when searching for resource
83   	
84   	    // Command-line option values
85   	    gchar *rsc_id;                // Value of --resource
86   	    gchar *rsc_type;              // Value of --resource-type
87   	    gboolean force;               // --force was given
88   	    gboolean clear_expired;       // --expired was given
89   	    gboolean recursive;           // --recursive was given
90   	    gboolean promoted_role_only;  // --promoted was given
91   	    gchar *host_uname;            // Value of --node
92   	    gchar *interval_spec;         // Value of --interval
93   	    gchar *move_lifetime;         // Value of --lifetime
94   	    gchar *operation;             // Value of --operation
95   	    const char *attr_set_type;    // Instance, meta, utilization, or element attribute
96   	    gchar *prop_id;               // --nvpair (attribute XML ID)
97   	    char *prop_name;              // Attribute name
98   	    gchar *prop_set;              // --set-name (attribute block XML ID)
99   	    gchar *prop_value;            // --parameter-value (attribute value)
100  	    int timeout_ms;               // Parsed from --timeout value
101  	    char *agent_spec;             // Standard and/or provider and/or agent
102  	    gchar *xml_file;              // Value of (deprecated) --xml-file
103  	    int check_level;              // Optional value of --validate or --force-check
104  	
105  	    // Resource configuration specified via command-line arguments
106  	    gboolean cmdline_config;      // Resource configuration was via arguments
107  	    char *v_agent;                // Value of --agent
108  	    char *v_class;                // Value of --class
109  	    char *v_provider;             // Value of --provider
110  	    GHashTable *cmdline_params;   // Resource parameters specified
111  	
112  	    // Positional command-line arguments
113  	    gchar **remainder;            // Positional arguments as given
114  	    GHashTable *override_params;  // Resource parameter values that override config
115  	} options = {
116  	    .attr_set_type = XML_TAG_ATTR_SETS,
117  	    .check_level = -1,
118  	    .cib_options = cib_sync_call,
119  	    .require_cib = TRUE,
120  	    .require_scheduler = TRUE,
121  	    .require_resource = TRUE,
122  	};
123  	
124  	#if 0
125  	// @COMPAT @TODO enable this at next backward compatibility break
126  	#define SET_COMMAND(cmd) do {                                               \
127  	        if (options.rsc_cmd != cmd_none) {                                  \
128  	            g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,             \
129  	                        "Only one command option may be specified");        \
130  	            return FALSE;                                                   \
131  	        }                                                                   \
132  	        options.rsc_cmd = (cmd);                                            \
133  	    } while (0)
134  	#else
135  	#define SET_COMMAND(cmd) do {                                               \
136  	        if (options.rsc_cmd != cmd_none) {                                  \
137  	            reset_options();                                                \
138  	        }                                                                   \
139  	        options.rsc_cmd = (cmd);                                            \
140  	    } while (0)
141  	#endif
142  	
143  	gboolean agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
144  	gboolean attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
145  	gboolean class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
146  	gboolean cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
147  	gboolean delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
148  	gboolean expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
149  	gboolean list_agents_cb(const gchar *option_name, const gchar *optarg,
150  	                        gpointer data, GError **error);
151  	gboolean list_providers_cb(const gchar *option_name, const gchar *optarg,
152  	                           gpointer data, GError **error);
153  	gboolean list_standards_cb(const gchar *option_name, const gchar *optarg,
154  	                           gpointer data, GError **error);
155  	gboolean list_alternatives_cb(const gchar *option_name, const gchar *optarg,
156  	                              gpointer data, GError **error);
157  	gboolean metadata_cb(const gchar *option_name, const gchar *optarg,
158  	                     gpointer data, GError **error);
159  	gboolean option_cb(const gchar *option_name, const gchar *optarg,
160  	                   gpointer data, GError **error);
161  	gboolean fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
162  	gboolean flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
163  	gboolean get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
164  	gboolean list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
165  	gboolean set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
166  	gboolean set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
167  	gboolean timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
168  	gboolean validate_or_force_cb(const gchar *option_name, const gchar *optarg,
169  	                              gpointer data, GError **error);
170  	gboolean restart_cb(const gchar *option_name, const gchar *optarg,
171  	                    gpointer data, GError **error);
172  	gboolean digests_cb(const gchar *option_name, const gchar *optarg,
173  	                    gpointer data, GError **error);
174  	gboolean wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
175  	gboolean why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
176  	
177  	static crm_exit_t exit_code = CRM_EX_OK;
178  	static pcmk__output_t *out = NULL;
179  	static pcmk__common_args_t *args = NULL;
180  	
181  	// Things that should be cleaned up on exit
182  	static GError *error = NULL;
183  	static GMainLoop *mainloop = NULL;
184  	static cib_t *cib_conn = NULL;
185  	static pcmk_ipc_api_t *controld_api = NULL;
186  	static pcmk_scheduler_t *scheduler = NULL;
187  	
188  	#define MESSAGE_TIMEOUT_S 60
189  	
190  	#define INDENT "                                    "
191  	
192  	static pcmk__supported_format_t formats[] = {
193  	    PCMK__SUPPORTED_FORMAT_NONE,
194  	    PCMK__SUPPORTED_FORMAT_TEXT,
195  	    PCMK__SUPPORTED_FORMAT_XML,
196  	    { NULL, NULL, NULL }
197  	};
198  	
199  	// Clean up and exit
200  	static crm_exit_t
201  	bye(crm_exit_t ec)
202  	{
203  	    pcmk__output_and_clear_error(&error, out);
204  	
205  	    if (out != NULL) {
206  	        out->finish(out, ec, true, NULL);
207  	        pcmk__output_free(out);
208  	    }
209  	    pcmk__unregister_formats();
210  	
211  	    if (cib_conn != NULL) {
212  	        cib_t *save_cib_conn = cib_conn;
213  	
214  	        cib_conn = NULL; // Ensure we can't free this twice
215  	        cib__clean_up_connection(&save_cib_conn);
216  	    }
217  	
218  	    if (controld_api != NULL) {
219  	        pcmk_ipc_api_t *save_controld_api = controld_api;
220  	
221  	        controld_api = NULL; // Ensure we can't free this twice
222  	        pcmk_free_ipc_api(save_controld_api);
223  	    }
224  	
225  	    if (mainloop != NULL) {
226  	        g_main_loop_unref(mainloop);
227  	        mainloop = NULL;
228  	    }
229  	
230  	    pe_free_working_set(scheduler);
231  	    scheduler = NULL;
232  	    crm_exit(ec);
233  	    return ec;
234  	}
235  	
236  	static void
237  	quit_main_loop(crm_exit_t ec)
238  	{
239  	    exit_code = ec;
240  	    if (mainloop != NULL) {
241  	        GMainLoop *mloop = mainloop;
242  	
243  	        mainloop = NULL; // Don't re-enter this block
244  	        pcmk_quit_main_loop(mloop, 10);
245  	        g_main_loop_unref(mloop);
246  	    }
247  	}
248  	
249  	static gboolean
250  	resource_ipc_timeout(gpointer data)
251  	{
252  	    // Start with newline because "Waiting for ..." message doesn't have one
253  	    if (error != NULL) {
254  	        g_clear_error(&error);
255  	    }
256  	
257  	    g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
258  	                _("Aborting because no messages received in %d seconds"), MESSAGE_TIMEOUT_S);
259  	
260  	    quit_main_loop(CRM_EX_TIMEOUT);
261  	    return FALSE;
262  	}
263  	
264  	static void
265  	controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
266  	                          crm_exit_t status, void *event_data, void *user_data)
267  	{
268  	    switch (event_type) {
269  	        case pcmk_ipc_event_disconnect:
270  	            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
271  	                crm_info("Connection to controller was terminated");
272  	            }
273  	            quit_main_loop(exit_code);
274  	            break;
275  	
276  	        case pcmk_ipc_event_reply:
277  	            if (status != CRM_EX_OK) {
278  	                out->err(out, "Error: bad reply from controller: %s",
279  	                         crm_exit_str(status));
280  	                pcmk_disconnect_ipc(api);
281  	                quit_main_loop(status);
282  	            } else {
283  	                if ((pcmk_controld_api_replies_expected(api) == 0)
284  	                    && mainloop && g_main_loop_is_running(mainloop)) {
285  	                    out->info(out, "... got reply (done)");
286  	                    crm_debug("Got all the replies we expected");
287  	                    pcmk_disconnect_ipc(api);
288  	                    quit_main_loop(CRM_EX_OK);
289  	                } else {
290  	                    out->info(out, "... got reply");
291  	                }
292  	            }
293  	            break;
294  	
295  	        default:
296  	            break;
297  	    }
298  	}
299  	
300  	static void
301  	start_mainloop(pcmk_ipc_api_t *capi)
302  	{
303  	    unsigned int count = pcmk_controld_api_replies_expected(capi);
304  	
305  	    if (count > 0) {
306  	        out->info(out, "Waiting for %u %s from the controller",
307  	                  count, pcmk__plural_alt(count, "reply", "replies"));
308  	        exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
309  	        mainloop = g_main_loop_new(NULL, FALSE);
310  	        g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
311  	        g_main_loop_run(mainloop);
312  	    }
313  	}
314  	
315  	static int
316  	compare_id(gconstpointer a, gconstpointer b)
317  	{
318  	    return strcmp((const char *)a, (const char *)b);
319  	}
320  	
321  	static GList *
322  	build_constraint_list(xmlNode *root)
323  	{
324  	    GList *retval = NULL;
325  	    xmlNode *cib_constraints = NULL;
326  	    xmlXPathObjectPtr xpathObj = NULL;
327  	    int ndx = 0;
328  	
329  	    cib_constraints = pcmk_find_cib_element(root, XML_CIB_TAG_CONSTRAINTS);
330  	    xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
331  	
332  	    for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
333  	        xmlNode *match = getXpathResult(xpathObj, ndx);
334  	        retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
335  	    }
336  	
337  	    freeXpathObject(xpathObj);
338  	    return retval;
339  	}
340  	
341  	/* short option letters still available: eEJkKXyYZ */
342  	
343  	static GOptionEntry query_entries[] = {
344  	    { "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
345  	      "List all cluster resources with status",
346  	      NULL },
347  	    { "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
348  	      "List IDs of all instantiated resources (individual members\n"
349  	      INDENT "rather than groups etc.)",
350  	      NULL },
351  	    { "list-cts", 'c', G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
352  	      NULL,
353  	      NULL },
354  	    { "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
355  	      "List active resource operations, optionally filtered by\n"
356  	      INDENT "--resource and/or --node",
357  	      NULL },
358  	    { "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
359  	      "List all resource operations, optionally filtered by\n"
360  	      INDENT "--resource and/or --node",
361  	      NULL },
362  	    { "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
363  	      list_standards_cb,
364  	      "List supported standards",
365  	      NULL },
366  	    { "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
367  	      list_providers_cb,
368  	      "List all available OCF providers",
369  	      NULL },
370  	    { "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
371  	      list_agents_cb,
372  	      "List all agents available for the named standard and/or provider",
373  	      "STD:PROV" },
374  	    { "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
375  	      list_alternatives_cb,
376  	      "List all available providers for the named OCF agent",
377  	      "AGENT" },
378  	    { "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
379  	      metadata_cb,
380  	      "Show the metadata for the named class:provider:agent",
381  	      "SPEC" },
382  	    { "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
383  	      "Show XML configuration of resource (after any template expansion)",
384  	      NULL },
385  	    { "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
386  	      "Show XML configuration of resource (before any template expansion)",
387  	      NULL },
388  	    { "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
389  	      "Display named parameter for resource (use instance attribute\n"
390  	      INDENT "unless --element, --meta, or --utilization is specified)",
391  	      "PARAM" },
392  	    { "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
393  	      "Display named property of resource ('class', 'type', or 'provider') "
394  	      "(requires --resource)",
395  	      "PROPERTY" },
396  	    { "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
397  	      "Show node(s) currently running resource",
398  	      NULL },
399  	    { "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
400  	      "Display the location and colocation constraints that apply to a\n"
401  	      INDENT "resource, and if --recursive is specified, to the resources\n"
402  	      INDENT "directly or indirectly involved in those colocations.\n"
403  	      INDENT "If the named resource is part of a group, or a clone or\n"
404  	      INDENT "bundle instance, constraints for the collective resource\n"
405  	      INDENT "will be shown unless --force is given.",
406  	      NULL },
407  	    { "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
408  	      "Equivalent to --constraints --recursive",
409  	      NULL },
410  	    { "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
411  	      "Show why resources are not running, optionally filtered by\n"
412  	      INDENT "--resource and/or --node",
413  	      NULL },
414  	
415  	    { NULL }
416  	};
417  	
418  	static GOptionEntry command_entries[] = {
419  	    { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
420  	      validate_or_force_cb,
421  	      "Validate resource configuration by calling agent's validate-all\n"
422  	      INDENT "action. The configuration may be specified either by giving an\n"
423  	      INDENT "existing resource name with -r, or by specifying --class,\n"
424  	      INDENT "--agent, and --provider arguments, along with any number of\n"
425  	      INDENT "--option arguments. An optional LEVEL argument can be given\n"
426  	      INDENT "to control the level of checking performed.",
427  	      "LEVEL" },
428  	    { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
429  	      "If resource has any past failures, clear its history and fail\n"
430  	      INDENT "count. Optionally filtered by --resource, --node, --operation\n"
431  	      INDENT "and --interval (otherwise all). --operation and --interval\n"
432  	      INDENT "apply to fail counts, but entire history is always clear, to\n"
433  	      INDENT "allow current state to be rechecked. If the named resource is\n"
434  	      INDENT "part of a group, or one numbered instance of a clone or bundled\n"
435  	      INDENT "resource, the clean-up applies to the whole collective resource\n"
436  	      INDENT "unless --force is given.",
437  	      NULL },
438  	    { "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
439  	      "Delete resource's history (including failures) so its current state\n"
440  	      INDENT "is rechecked. Optionally filtered by --resource and --node\n"
441  	      INDENT "(otherwise all). If the named resource is part of a group, or one\n"
442  	      INDENT "numbered instance of a clone or bundled resource, the refresh\n"
443  	      INDENT "applies to the whole collective resource unless --force is given.",
444  	      NULL },
445  	    { "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
446  	      "Set named parameter for resource (requires -v). Use instance\n"
447  	      INDENT "attribute unless --element, --meta, or --utilization is "
448  	      "specified.",
449  	      "PARAM" },
450  	    { "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
451  	      "Delete named parameter for resource. Use instance attribute\n"
452  	      INDENT "unless --element, --meta or, --utilization is specified.",
453  	      "PARAM" },
454  	    { "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
455  	      "Set named property of resource ('class', 'type', or 'provider') "
456  	      "(requires -r, -t, -v)",
457  	      "PROPERTY" },
458  	
459  	    { NULL }
460  	};
461  	
462  	static GOptionEntry location_entries[] = {
463  	    { "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
464  	      "Create a constraint to move resource. If --node is specified,\n"
465  	      INDENT "the constraint will be to move to that node, otherwise it\n"
466  	      INDENT "will be to ban the current node. Unless --force is specified\n"
467  	      INDENT "this will return an error if the resource is already running\n"
468  	      INDENT "on the specified node. If --force is specified, this will\n"
469  	      INDENT "always ban the current node.\n"
470  	      INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
471  	      INDENT "resource from running on its previous location until the\n"
472  	      INDENT "implicit constraint expires or is removed with --clear.",
473  	      NULL },
474  	    { "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
475  	      "Create a constraint to keep resource off a node.\n"
476  	      INDENT "Optional: --node, --lifetime, --promoted.\n"
477  	      INDENT "NOTE: This will prevent the resource from running on the\n"
478  	      INDENT "affected node until the implicit constraint expires or is\n"
479  	      INDENT "removed with --clear. If --node is not specified, it defaults\n"
480  	      INDENT "to the node currently running the resource for primitives\n"
481  	      INDENT "and groups, or the promoted instance of promotable clones with\n"
482  	      INDENT "promoted-max=1 (all other situations result in an error as\n"
483  	      INDENT "there is no sane default).",
484  	      NULL },
485  	    { "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
486  	      "Remove all constraints created by the --ban and/or --move\n"
487  	      INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
488  	      INDENT "--expired. If --node is not specified, all constraints created\n"
489  	      INDENT "by --ban and --move will be removed for the named resource. If\n"
490  	      INDENT "--node and --force are specified, any constraint created by\n"
491  	      INDENT "--move will be cleared, even if it is not for the specified\n"
492  	      INDENT "node. If --expired is specified, only those constraints whose\n"
493  	      INDENT "lifetimes have expired will be removed.",
494  	      NULL },
495  	    { "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
496  	      "Modifies the --clear argument to remove constraints with\n"
497  	      INDENT "expired lifetimes.",
498  	      NULL },
499  	    { "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
500  	      "Lifespan (as ISO 8601 duration) of created constraints (with\n"
501  	      INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
502  	      "TIMESPEC" },
503  	    { "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
504  	      &options.promoted_role_only,
505  	      "Limit scope of command to promoted role (with -B, -M, -U). For\n"
506  	      INDENT "-B and -M, previously promoted instances may remain\n"
507  	      INDENT "active in the unpromoted role.",
508  	      NULL },
509  	
510  	    // Deprecated since 2.1.0
511  	    { "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
512  	      &options.promoted_role_only,
513  	      "Deprecated: Use --promoted instead", NULL },
514  	
515  	    { NULL }
516  	};
517  	
518  	static GOptionEntry advanced_entries[] = {
519  	    { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
520  	      "(Advanced) Delete a resource from the CIB. Required: -t",
521  	      NULL },
522  	    { "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
523  	      "(Advanced) Tell the cluster this resource has failed",
524  	      NULL },
525  	    { "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
526  	      "(Advanced) Tell the cluster to restart this resource and\n"
527  	      INDENT "anything that depends on it",
528  	      NULL },
529  	    { "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
530  	      "(Advanced) Wait until the cluster settles into a stable state",
531  	      NULL },
532  	    { "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, digests_cb,
533  	      "(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
534  	      INDENT "configuration changes (only accurate if there is resource\n"
535  	      INDENT "history on the specified node). Required: --resource, --node.\n"
536  	      INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
537  	      INDENT "the configuration (to see what the hash would be with those\n"
538  	      INDENT "changes).",
539  	      NULL },
540  	    { "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
541  	      validate_or_force_cb,
542  	      "(Advanced) Bypass the cluster and demote a resource on the local\n"
543  	      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
544  	      INDENT "the cluster believes the resource is a clone instance already\n"
545  	      INDENT "running on the local node.",
546  	      NULL },
547  	    { "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
548  	      validate_or_force_cb,
549  	      "(Advanced) Bypass the cluster and stop a resource on the local node",
550  	      NULL },
551  	    { "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
552  	      validate_or_force_cb,
553  	      "(Advanced) Bypass the cluster and start a resource on the local\n"
554  	      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
555  	      INDENT "the cluster believes the resource is a clone instance already\n"
556  	      INDENT "running on the local node.",
557  	      NULL },
558  	    { "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
559  	      validate_or_force_cb,
560  	      "(Advanced) Bypass the cluster and promote a resource on the local\n"
561  	      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
562  	      INDENT "the cluster believes the resource is a clone instance already\n"
563  	      INDENT "running on the local node.",
564  	      NULL },
565  	    { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
566  	      validate_or_force_cb,
567  	      "(Advanced) Bypass the cluster and check the state of a resource on\n"
568  	      INDENT "the local node. An optional LEVEL argument can be given\n"
569  	      INDENT "to control the level of checking performed.",
570  	      "LEVEL" },
571  	
572  	    { NULL }
573  	};
574  	
575  	static GOptionEntry addl_entries[] = {
576  	    { "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
577  	      "Node name",
578  	      "NAME" },
579  	    { "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
580  	      "Follow colocation chains when using --set-parameter or --constraints",
581  	      NULL },
582  	    { "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
583  	      "Resource XML element (primitive, group, etc.) (with -D)",
584  	      "ELEMENT" },
585  	    { "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
586  	      "Value to use with -p",
587  	      "PARAM" },
588  	    { "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
589  	      "Use resource meta-attribute instead of instance attribute\n"
590  	      INDENT "(with -p, -g, -d)",
591  	      NULL },
592  	    { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
593  	      "Use resource utilization attribute instead of instance attribute\n"
594  	      INDENT "(with -p, -g, -d)",
595  	      NULL },
596  	    { "element", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
597  	      "Use resource element attribute instead of instance attribute\n"
598  	      INDENT "(with -p, -g, -d)",
599  	      NULL },
600  	    { "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
601  	      "Operation to clear instead of all (with -C -r)",
602  	      "OPERATION" },
603  	    { "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
604  	      "Interval of operation to clear (default 0) (with -C -r -n)",
605  	      "N" },
606  	    { "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
607  	      "The standard the resource agent conforms to (for example, ocf).\n"
608  	      INDENT "Use with --agent, --provider, --option, and --validate.",
609  	      "CLASS" },
610  	    { "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
611  	      "The agent to use (for example, IPaddr). Use with --class,\n"
612  	      INDENT "--provider, --option, and --validate.",
613  	      "AGENT" },
614  	    { "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
615  	      "The vendor that supplies the resource agent (for example,\n"
616  	      INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
617  	      "PROVIDER" },
618  	    { "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
619  	      "Specify a device configuration parameter as NAME=VALUE (may be\n"
620  	      INDENT "specified multiple times). Use with --validate and without the\n"
621  	      INDENT "-r option.",
622  	      "PARAM" },
623  	    { "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
624  	      "(Advanced) XML ID of attributes element to use (with -p, -d)",
625  	      "ID" },
626  	    { "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
627  	      "(Advanced) XML ID of nvpair element to use (with -p, -d)",
628  	      "ID" },
629  	    { "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
630  	      "(Advanced) Abort if command does not finish in this time (with\n"
631  	      INDENT "--restart, --wait, --force-*)",
632  	      "N" },
633  	    { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
634  	      "Force the action to be performed. See help for individual commands for\n"
635  	      INDENT "additional behavior.",
636  	      NULL },
637  	    { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
638  	      NULL,
639  	      "FILE" },
640  	    { "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
641  	      NULL,
642  	      "HOST" },
643  	
644  	    { NULL }
645  	};
646  	
647  	static void
648  	reset_options(void) {
649  	    options.require_crmd = FALSE;
650  	    options.require_node = FALSE;
651  	
652  	    options.require_cib = TRUE;
653  	    options.require_scheduler = TRUE;
654  	    options.require_resource = TRUE;
655  	
656  	    options.find_flags = 0;
657  	}
658  	
659  	gboolean
660  	agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
661  	    options.cmdline_config = TRUE;
662  	    options.require_resource = FALSE;
663  	
664  	    if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
665  	        pcmk__str_update(&options.v_provider, optarg);
666  	    } else {
667  	        pcmk__str_update(&options.v_agent, optarg);
668  	    }
669  	
670  	    return TRUE;
671  	}
672  	
673  	gboolean
674  	attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
675  	    if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
676  	        options.attr_set_type = XML_TAG_META_SETS;
677  	    } else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
678  	        options.attr_set_type = XML_TAG_UTILIZATION;
679  	    } else if (pcmk__str_eq(option_name, "--element", pcmk__str_casei)) {
680  	        options.attr_set_type = ATTR_SET_ELEMENT;
681  	    }
682  	    return TRUE;
683  	}
684  	
685  	gboolean
686  	class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
687  	    pcmk__str_update(&options.v_class, optarg);
688  	    options.cmdline_config = TRUE;
689  	    options.require_resource = FALSE;
690  	    return TRUE;
691  	}
692  	
693  	gboolean
694  	cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
695  	    if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
696  	        SET_COMMAND(cmd_cleanup);
697  	    } else {
698  	        SET_COMMAND(cmd_refresh);
699  	    }
700  	
701  	    options.require_resource = FALSE;
702  	    if (getenv("CIB_file") == NULL) {
703  	        options.require_crmd = TRUE;
704  	    }
705  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
706  	    return TRUE;
707  	}
708  	
709  	gboolean
710  	delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
711  	    SET_COMMAND(cmd_delete);
712  	    options.require_scheduler = FALSE;
713  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
714  	    return TRUE;
715  	}
716  	
717  	gboolean
718  	expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
719  	    options.clear_expired = TRUE;
720  	    options.require_resource = FALSE;
721  	    return TRUE;
722  	}
723  	
724  	static void
725  	get_agent_spec(const gchar *optarg)
726  	{
727  	    options.require_cib = FALSE;
728  	    options.require_scheduler = FALSE;
729  	    options.require_resource = FALSE;
730  	    pcmk__str_update(&options.agent_spec, optarg);
731  	}
732  	
733  	gboolean
734  	list_agents_cb(const gchar *option_name, const gchar *optarg, gpointer data,
735  	               GError **error)
736  	{
737  	    SET_COMMAND(cmd_list_agents);
738  	    get_agent_spec(optarg);
739  	    return TRUE;
740  	}
741  	
742  	gboolean
743  	list_providers_cb(const gchar *option_name, const gchar *optarg, gpointer data,
744  	                  GError **error)
745  	{
746  	    SET_COMMAND(cmd_list_providers);
747  	    get_agent_spec(optarg);
748  	    return TRUE;
749  	}
750  	
751  	gboolean
752  	list_standards_cb(const gchar *option_name, const gchar *optarg, gpointer data,
753  	                  GError **error)
754  	{
755  	    SET_COMMAND(cmd_list_standards);
756  	    options.require_cib = FALSE;
757  	    options.require_scheduler = FALSE;
758  	    options.require_resource = FALSE;
759  	    return TRUE;
760  	}
761  	
762  	gboolean
763  	list_alternatives_cb(const gchar *option_name, const gchar *optarg,
764  	                     gpointer data, GError **error)
765  	{
766  	    SET_COMMAND(cmd_list_alternatives);
767  	    get_agent_spec(optarg);
768  	    return TRUE;
769  	}
770  	
771  	gboolean
772  	metadata_cb(const gchar *option_name, const gchar *optarg, gpointer data,
773  	            GError **error)
774  	{
775  	    SET_COMMAND(cmd_metadata);
776  	    get_agent_spec(optarg);
777  	    return TRUE;
778  	}
779  	
780  	gboolean
781  	option_cb(const gchar *option_name, const gchar *optarg, gpointer data,
782  	          GError **error)
783  	{
784  	    char *name = NULL;
785  	    char *value = NULL;
786  	
(1) Event alloc_arg: "pcmk__scan_nvpair" allocates memory that is stored into "name". [details]
(2) Event path: Condition "pcmk__scan_nvpair(optarg, &name, &value) != 2", taking true branch.
Also see events: [leaked_storage]
787  	    if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
CID (unavailable; MK=95bd789a27e4e738e34dcf3b3e37953a) (#1 of 1): Resource leak (RESOURCE_LEAK):
(3) Event leaked_storage: Variable "name" going out of scope leaks the storage it points to.
Also see events: [alloc_arg]
788  	        return FALSE;
789  	    }
790  	    if (options.cmdline_params == NULL) {
791  	        options.cmdline_params = pcmk__strkey_table(free, free);
792  	    }
793  	    g_hash_table_replace(options.cmdline_params, name, value);
794  	    return TRUE;
795  	}
796  	
797  	gboolean
798  	fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
799  	    SET_COMMAND(cmd_fail);
800  	    options.require_crmd = TRUE;
801  	    options.require_node = TRUE;
802  	    return TRUE;
803  	}
804  	
805  	gboolean
806  	flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
807  	    if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
808  	        SET_COMMAND(cmd_clear);
809  	        options.find_flags = pcmk_rsc_match_history
810  	                             |pcmk_rsc_match_anon_basename;
811  	    } else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
812  	        SET_COMMAND(cmd_ban);
813  	        options.find_flags = pcmk_rsc_match_history
814  	                             |pcmk_rsc_match_anon_basename;
815  	    } else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
816  	        SET_COMMAND(cmd_move);
817  	        options.find_flags = pcmk_rsc_match_history
818  	                             |pcmk_rsc_match_anon_basename;
819  	    } else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
820  	        SET_COMMAND(cmd_query_xml);
821  	        options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
822  	    } else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
823  	        SET_COMMAND(cmd_query_raw_xml);
824  	        options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
825  	    } else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
826  	        SET_COMMAND(cmd_locate);
827  	        options.find_flags = pcmk_rsc_match_history
828  	                             |pcmk_rsc_match_anon_basename;
829  	
830  	    } else if (pcmk__str_any_of(option_name, "-a", "--constraints", NULL)) {
831  	        SET_COMMAND(cmd_colocations);
832  	        options.find_flags = pcmk_rsc_match_history
833  	                             |pcmk_rsc_match_anon_basename;
834  	
835  	    } else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
836  	        SET_COMMAND(cmd_colocations);
837  	        options.find_flags = pcmk_rsc_match_history
838  	                             |pcmk_rsc_match_anon_basename;
839  	        options.recursive = TRUE;
840  	    }
841  	
842  	    return TRUE;
843  	}
844  	
845  	gboolean
846  	get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
847  	    if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
848  	        SET_COMMAND(cmd_get_param);
849  	    } else {
850  	        SET_COMMAND(cmd_get_property);
851  	    }
852  	
853  	    pcmk__str_update(&options.prop_name, optarg);
854  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
855  	    return TRUE;
856  	}
857  	
858  	gboolean
859  	list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
860  	    if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
861  	        SET_COMMAND(cmd_cts);
862  	    } else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
863  	        SET_COMMAND(cmd_list_resources);
864  	    } else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
865  	        SET_COMMAND(cmd_list_instances);
866  	    } else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
867  	        SET_COMMAND(cmd_list_active_ops);
868  	    } else {
869  	        SET_COMMAND(cmd_list_all_ops);
870  	    }
871  	
872  	    options.require_resource = FALSE;
873  	    return TRUE;
874  	}
875  	
876  	gboolean
877  	set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
878  	    if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
879  	        SET_COMMAND(cmd_set_param);
880  	    } else {
881  	        SET_COMMAND(cmd_delete_param);
882  	    }
883  	
884  	    pcmk__str_update(&options.prop_name, optarg);
885  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
886  	    return TRUE;
887  	}
888  	
889  	gboolean
890  	set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
891  	    SET_COMMAND(cmd_set_property);
892  	    options.require_scheduler = FALSE;
893  	    pcmk__str_update(&options.prop_name, optarg);
894  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
895  	    return TRUE;
896  	}
897  	
898  	gboolean
899  	timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
900  	    options.timeout_ms = crm_get_msec(optarg);
901  	    return TRUE;
902  	}
903  	
904  	gboolean
905  	validate_or_force_cb(const gchar *option_name, const gchar *optarg,
906  	                     gpointer data, GError **error)
907  	{
908  	    SET_COMMAND(cmd_execute_agent);
909  	    if (options.operation) {
910  	        g_free(options.operation);
911  	    }
912  	    options.operation = g_strdup(option_name + 2); // skip "--"
913  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
914  	    if (options.override_params == NULL) {
915  	        options.override_params = pcmk__strkey_table(free, free);
916  	    }
917  	
918  	    if (optarg != NULL) {
919  	        if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
920  	            g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
921  	                        _("Invalid check level setting: %s"), optarg);
922  	            return FALSE;
923  	        }
924  	    }
925  	
926  	    return TRUE;
927  	}
928  	
929  	gboolean
930  	restart_cb(const gchar *option_name, const gchar *optarg, gpointer data,
931  	           GError **error)
932  	{
933  	    SET_COMMAND(cmd_restart);
934  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
935  	    return TRUE;
936  	}
937  	
938  	gboolean
939  	digests_cb(const gchar *option_name, const gchar *optarg, gpointer data,
940  	           GError **error)
941  	{
942  	    SET_COMMAND(cmd_digests);
943  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
944  	    if (options.override_params == NULL) {
945  	        options.override_params = pcmk__strkey_table(free, free);
946  	    }
947  	    options.require_node = TRUE;
948  	    options.require_scheduler = TRUE;
949  	    return TRUE;
950  	}
951  	
952  	gboolean
953  	wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
954  	    SET_COMMAND(cmd_wait);
955  	    options.require_resource = FALSE;
956  	    options.require_scheduler = FALSE;
957  	    return TRUE;
958  	}
959  	
960  	gboolean
961  	why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
962  	    SET_COMMAND(cmd_why);
963  	    options.require_resource = FALSE;
964  	    options.find_flags = pcmk_rsc_match_history|pcmk_rsc_match_anon_basename;
965  	    return TRUE;
966  	}
967  	
968  	static int
969  	ban_or_move(pcmk__output_t *out, pcmk_resource_t *rsc,
970  	            const char *move_lifetime)
971  	{
972  	    int rc = pcmk_rc_ok;
973  	    pcmk_node_t *current = NULL;
974  	    unsigned int nactive = 0;
975  	
976  	    CRM_CHECK(rsc != NULL, return EINVAL);
977  	
978  	    current = pe__find_active_requires(rsc, &nactive);
979  	
980  	    if (nactive == 1) {
981  	        rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime,
982  	                              cib_conn, options.cib_options, options.promoted_role_only,
983  	                              PCMK__ROLE_PROMOTED);
984  	
985  	    } else if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
986  	        int count = 0;
987  	        GList *iter = NULL;
988  	
989  	        current = NULL;
990  	        for(iter = rsc->children; iter; iter = iter->next) {
991  	            pcmk_resource_t *child = (pcmk_resource_t *)iter->data;
992  	            enum rsc_role_e child_role = child->fns->state(child, TRUE);
993  	
994  	            if (child_role == pcmk_role_promoted) {
995  	                count++;
996  	                current = pe__current_node(child);
997  	            }
998  	        }
999  	
1000 	        if(count == 1 && current) {
1001 	            rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime,
1002 	                                  cib_conn, options.cib_options, options.promoted_role_only,
1003 	                                  PCMK__ROLE_PROMOTED);
1004 	
1005 	        } else {
1006 	            rc = EINVAL;
1007 	            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1008 	                        _("Resource '%s' not moved: active in %d locations (promoted in %d).\n"
1009 	                        "To prevent '%s' from running on a specific location, "
1010 	                        "specify a node."
1011 	                        "To prevent '%s' from being promoted at a specific "
1012 	                        "location, specify a node and the --promoted option."),
1013 	                        options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
1014 	        }
1015 	
1016 	    } else {
1017 	        rc = EINVAL;
1018 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1019 	                    _("Resource '%s' not moved: active in %d locations.\n"
1020 	                    "To prevent '%s' from running on a specific location, "
1021 	                    "specify a node."),
1022 	                    options.rsc_id, nactive, options.rsc_id);
1023 	    }
1024 	
1025 	    return rc;
1026 	}
1027 	
1028 	static void
1029 	cleanup(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
1030 	{
1031 	    int rc = pcmk_rc_ok;
1032 	
1033 	    if (options.force == FALSE) {
1034 	        rsc = uber_parent(rsc);
1035 	    }
1036 	
1037 	    crm_debug("Erasing failures of %s (%s requested) on %s",
1038 	              rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
1039 	    rc = cli_resource_delete(controld_api, options.host_uname, rsc,
1040 	                             options.operation, options.interval_spec, TRUE,
1041 	                             scheduler, options.force);
1042 	
1043 	    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
1044 	        // Show any reasons why resource might stay stopped
1045 	        cli_resource_check(out, rsc, node);
1046 	    }
1047 	
1048 	    if (rc == pcmk_rc_ok) {
1049 	        start_mainloop(controld_api);
1050 	    }
1051 	}
1052 	
1053 	static int
1054 	clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
1055 	{
1056 	    GList *before = NULL;
1057 	    GList *after = NULL;
1058 	    GList *remaining = NULL;
1059 	    GList *ele = NULL;
1060 	    pcmk_node_t *dest = NULL;
1061 	    int rc = pcmk_rc_ok;
1062 	
1063 	    if (!out->is_quiet(out)) {
1064 	        before = build_constraint_list(scheduler->input);
1065 	    }
1066 	
1067 	    if (options.clear_expired) {
1068 	        rc = cli_resource_clear_all_expired(scheduler->input, cib_conn,
1069 	                                            options.cib_options, options.rsc_id,
1070 	                                            options.host_uname,
1071 	                                            options.promoted_role_only);
1072 	
1073 	    } else if (options.host_uname) {
1074 	        dest = pe_find_node(scheduler->nodes, options.host_uname);
1075 	        if (dest == NULL) {
1076 	            rc = pcmk_rc_node_unknown;
1077 	            if (!out->is_quiet(out)) {
1078 	                g_list_free(before);
1079 	            }
1080 	            return rc;
1081 	        }
1082 	        rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
1083 	                                cib_conn, options.cib_options, TRUE, options.force);
1084 	
1085 	    } else {
1086 	        rc = cli_resource_clear(options.rsc_id, NULL, scheduler->nodes,
1087 	                                cib_conn, options.cib_options, TRUE, options.force);
1088 	    }
1089 	
1090 	    if (!out->is_quiet(out)) {
1091 	        rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
1092 	        rc = pcmk_legacy2rc(rc);
1093 	
1094 	        if (rc != pcmk_rc_ok) {
1095 	            g_set_error(&error, PCMK__RC_ERROR, rc,
1096 	                        _("Could not get modified CIB: %s\n"), pcmk_rc_str(rc));
1097 	            g_list_free(before);
1098 	            free_xml(*cib_xml_copy);
1099 	            *cib_xml_copy = NULL;
1100 	            return rc;
1101 	        }
1102 	
1103 	        scheduler->input = *cib_xml_copy;
1104 	        cluster_status(scheduler);
1105 	
1106 	        after = build_constraint_list(scheduler->input);
1107 	        remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);
1108 	
1109 	        for (ele = remaining; ele != NULL; ele = ele->next) {
1110 	            out->info(out, "Removing constraint: %s", (char *) ele->data);
1111 	        }
1112 	
1113 	        g_list_free(before);
1114 	        g_list_free(after);
1115 	        g_list_free(remaining);
1116 	    }
1117 	
1118 	    return rc;
1119 	}
1120 	
1121 	static int
1122 	delete(void)
1123 	{
1124 	    int rc = pcmk_rc_ok;
1125 	    xmlNode *msg_data = NULL;
1126 	
1127 	    if (options.rsc_type == NULL) {
1128 	        rc = ENXIO;
1129 	        g_set_error(&error, PCMK__RC_ERROR, rc,
1130 	                    _("You need to specify a resource type with -t"));
1131 	        return rc;
1132 	    }
1133 	
1134 	    msg_data = create_xml_node(NULL, options.rsc_type);
1135 	    crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
1136 	
1137 	    rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
1138 	                                options.cib_options);
1139 	    rc = pcmk_legacy2rc(rc);
1140 	    free_xml(msg_data);
1141 	    return rc;
1142 	}
1143 	
1144 	static int
1145 	initialize_scheduler_data(xmlNodePtr *cib_xml_copy)
1146 	{
1147 	    int rc = pcmk_rc_ok;
1148 	
1149 	    if (options.xml_file != NULL) {
1150 	        *cib_xml_copy = filename2xml(options.xml_file);
1151 	        if (*cib_xml_copy == NULL) {
1152 	            rc = pcmk_rc_cib_corrupt;
1153 	        }
1154 	    } else {
1155 	        rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
1156 	        rc = pcmk_legacy2rc(rc);
1157 	    }
1158 	
1159 	    if (rc == pcmk_rc_ok) {
1160 	        scheduler = pe_new_working_set();
1161 	        if (scheduler == NULL) {
1162 	            rc = ENOMEM;
1163 	        } else {
1164 	            pe__set_working_set_flags(scheduler,
1165 	                                      pcmk_sched_no_counts
1166 	                                      |pcmk_sched_no_compat);
1167 	            scheduler->priv = out;
1168 	            rc = update_scheduler_input(scheduler, cib_xml_copy);
1169 	        }
1170 	    }
1171 	
1172 	    if (rc != pcmk_rc_ok) {
1173 	        free_xml(*cib_xml_copy);
1174 	        *cib_xml_copy = NULL;
1175 	        return rc;
1176 	    }
1177 	
1178 	    cluster_status(scheduler);
1179 	    return pcmk_rc_ok;
1180 	}
1181 	
1182 	static int
1183 	refresh(pcmk__output_t *out)
1184 	{
1185 	    int rc = pcmk_rc_ok;
1186 	    const char *router_node = options.host_uname;
1187 	    int attr_options = pcmk__node_attr_none;
1188 	
1189 	    if (options.host_uname) {
1190 	        pcmk_node_t *node = pe_find_node(scheduler->nodes, options.host_uname);
1191 	
1192 	        if (pe__is_guest_or_remote_node(node)) {
1193 	            node = pe__current_node(node->details->remote_rsc);
1194 	            if (node == NULL) {
1195 	                rc = ENXIO;
1196 	                g_set_error(&error, PCMK__RC_ERROR, rc,
1197 	                            _("No cluster connection to Pacemaker Remote node %s detected"),
1198 	                            options.host_uname);
1199 	                return rc;
1200 	            }
1201 	            router_node = node->details->uname;
1202 	            attr_options |= pcmk__node_attr_remote;
1203 	        }
1204 	    }
1205 	
1206 	    if (controld_api == NULL) {
1207 	        out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
1208 	                  options.host_uname? options.host_uname : "all nodes");
1209 	        rc = pcmk_rc_ok;
1210 	        return rc;
1211 	    }
1212 	
1213 	    crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");
1214 	
1215 	    rc = pcmk__attrd_api_clear_failures(NULL, options.host_uname, NULL,
1216 	                                        NULL, NULL, NULL, attr_options);
1217 	
1218 	    if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
1219 	                                  router_node) == pcmk_rc_ok) {
1220 	        start_mainloop(controld_api);
1221 	    }
1222 	
1223 	    return rc;
1224 	}
1225 	
1226 	static void
1227 	refresh_resource(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
1228 	{
1229 	    int rc = pcmk_rc_ok;
1230 	
1231 	    if (options.force == FALSE) {
1232 	        rsc = uber_parent(rsc);
1233 	    }
1234 	
1235 	    crm_debug("Re-checking the state of %s (%s requested) on %s",
1236 	              rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
1237 	    rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL, 0,
1238 	                             FALSE, scheduler, options.force);
1239 	
1240 	    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
1241 	        // Show any reasons why resource might stay stopped
1242 	        cli_resource_check(out, rsc, node);
1243 	    }
1244 	
1245 	    if (rc == pcmk_rc_ok) {
1246 	        start_mainloop(controld_api);
1247 	    }
1248 	}
1249 	
1250 	static int
1251 	set_property(void)
1252 	{
1253 	    int rc = pcmk_rc_ok;
1254 	    xmlNode *msg_data = NULL;
1255 	
1256 	    if (pcmk__str_empty(options.rsc_type)) {
1257 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1258 	                    _("Must specify -t with resource type"));
1259 	        rc = ENXIO;
1260 	        return rc;
1261 	
1262 	    } else if (pcmk__str_empty(options.prop_value)) {
1263 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1264 	                    _("Must supply -v with new value"));
1265 	        rc = ENXIO;
1266 	        return rc;
1267 	    }
1268 	
1269 	    CRM_LOG_ASSERT(options.prop_name != NULL);
1270 	
1271 	    msg_data = create_xml_node(NULL, options.rsc_type);
1272 	    crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
1273 	    crm_xml_add(msg_data, options.prop_name, options.prop_value);
1274 	
1275 	    rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
1276 	                                options.cib_options);
1277 	    rc = pcmk_legacy2rc(rc);
1278 	    free_xml(msg_data);
1279 	
1280 	    return rc;
1281 	}
1282 	
1283 	static int
1284 	show_metadata(pcmk__output_t *out, const char *agent_spec)
1285 	{
1286 	    int rc = pcmk_rc_ok;
1287 	    char *standard = NULL;
1288 	    char *provider = NULL;
1289 	    char *type = NULL;
1290 	    char *metadata = NULL;
1291 	    lrmd_t *lrmd_conn = NULL;
1292 	
1293 	    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
1294 	    if (rc != pcmk_rc_ok) {
1295 	        g_set_error(&error, PCMK__RC_ERROR, rc,
1296 	                    _("Could not create executor connection"));
1297 	        lrmd_api_delete(lrmd_conn);
1298 	        return rc;
1299 	    }
1300 	
1301 	    rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
1302 	    rc = pcmk_legacy2rc(rc);
1303 	
1304 	    if (rc == pcmk_rc_ok) {
1305 	        rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
1306 	                                           provider, type,
1307 	                                           &metadata, 0);
1308 	        rc = pcmk_legacy2rc(rc);
1309 	
1310 	        if (metadata) {
1311 	            out->output_xml(out, "metadata", metadata);
1312 	            free(metadata);
1313 	        } else {
1314 	            /* We were given a validly formatted spec, but it doesn't necessarily
1315 	             * match up with anything that exists.  Use ENXIO as the return code
1316 	             * here because that maps to an exit code of CRM_EX_NOSUCH, which
1317 	             * probably is the most common reason to get here.
1318 	             */
1319 	            rc = ENXIO;
1320 	            g_set_error(&error, PCMK__RC_ERROR, rc,
1321 	                        _("Metadata query for %s failed: %s"),
1322 	                        agent_spec, pcmk_rc_str(rc));
1323 	        }
1324 	    } else {
1325 	        rc = ENXIO;
1326 	        g_set_error(&error, PCMK__RC_ERROR, rc,
1327 	                    _("'%s' is not a valid agent specification"), agent_spec);
1328 	    }
1329 	
1330 	    lrmd_api_delete(lrmd_conn);
1331 	    return rc;
1332 	}
1333 	
1334 	static void
1335 	validate_cmdline_config(void)
1336 	{
1337 	    // Cannot use both --resource and command-line resource configuration
1338 	    if (options.rsc_id != NULL) {
1339 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1340 	                    _("--resource cannot be used with --class, --agent, and --provider"));
1341 	
1342 	    // Not all commands support command-line resource configuration
1343 	    } else if (options.rsc_cmd != cmd_execute_agent) {
1344 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1345 	                    _("--class, --agent, and --provider can only be used with "
1346 	                    "--validate and --force-*"));
1347 	
1348 	    // Not all of --class, --agent, and --provider need to be given.  Not all
1349 	    // classes support the concept of a provider.  Check that what we were given
1350 	    // is valid.
1351 	    } else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
1352 	        if (options.v_provider != NULL) {
1353 	            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1354 	                        _("stonith does not support providers"));
1355 	
1356 	        } else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
1357 	            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1358 	                        _("%s is not a known stonith agent"), options.v_agent ? options.v_agent : "");
1359 	        }
1360 	
1361 	    } else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
1362 	        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1363 	                    _("%s:%s:%s is not a known resource"),
1364 	                    options.v_class ? options.v_class : "",
1365 	                    options.v_provider ? options.v_provider : "",
1366 	                    options.v_agent ? options.v_agent : "");
1367 	    }
1368 	
1369 	    if (error != NULL) {
1370 	        return;
1371 	    }
1372 	
1373 	    if (options.cmdline_params == NULL) {
1374 	        options.cmdline_params = pcmk__strkey_table(free, free);
1375 	    }
1376 	    options.require_resource = FALSE;
1377 	    options.require_scheduler = FALSE;
1378 	    options.require_cib = FALSE;
1379 	}
1380 	
1381 	static GOptionContext *
1382 	build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
1383 	    GOptionContext *context = NULL;
1384 	
1385 	    GOptionEntry extra_prog_entries[] = {
1386 	        { "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
1387 	          "Be less descriptive in output.",
1388 	          NULL },
1389 	        { "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
1390 	          "Resource ID",
1391 	          "ID" },
1392 	        { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
1393 	          NULL,
1394 	          NULL },
1395 	
1396 	        { NULL }
1397 	    };
1398 	
1399 	    const char *description = "Examples:\n\n"
1400 	                              "List the available OCF agents:\n\n"
1401 	                              "\t# crm_resource --list-agents ocf\n\n"
1402 	                              "List the available OCF agents from the linux-ha project:\n\n"
1403 	                              "\t# crm_resource --list-agents ocf:heartbeat\n\n"
1404 	                              "Move 'myResource' to a specific node:\n\n"
1405 	                              "\t# crm_resource --resource myResource --move --node altNode\n\n"
1406 	                              "Allow (but not force) 'myResource' to move back to its original "
1407 	                              "location:\n\n"
1408 	                              "\t# crm_resource --resource myResource --clear\n\n"
1409 	                              "Stop 'myResource' (and anything that depends on it):\n\n"
1410 	                              "\t# crm_resource --resource myResource --set-parameter target-role "
1411 	                              "--meta --parameter-value Stopped\n\n"
1412 	                              "Tell the cluster not to manage 'myResource' (the cluster will not "
1413 	                              "attempt to start or stop the\n"
1414 	                              "resource under any circumstances; useful when performing maintenance "
1415 	                              "tasks on a resource):\n\n"
1416 	                              "\t# crm_resource --resource myResource --set-parameter is-managed "
1417 	                              "--meta --parameter-value false\n\n"
1418 	                              "Erase the operation history of 'myResource' on 'aNode' (the cluster "
1419 	                              "will 'forget' the existing\n"
1420 	                              "resource state, including any errors, and attempt to recover the"
1421 	                              "resource; useful when a resource\n"
1422 	                              "had failed permanently and has been repaired by an administrator):\n\n"
1423 	                              "\t# crm_resource --resource myResource --cleanup --node aNode\n\n";
1424 	
1425 	    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
1426 	    g_option_context_set_description(context, description);
1427 	
1428 	    /* Add the -Q option, which cannot be part of the globally supported options
1429 	     * because some tools use that flag for something else.
1430 	     */
1431 	    pcmk__add_main_args(context, extra_prog_entries);
1432 	
1433 	    pcmk__add_arg_group(context, "queries", "Queries:",
1434 	                        "Show query help", query_entries);
1435 	    pcmk__add_arg_group(context, "commands", "Commands:",
1436 	                        "Show command help", command_entries);
1437 	    pcmk__add_arg_group(context, "locations", "Locations:",
1438 	                        "Show location help", location_entries);
1439 	    pcmk__add_arg_group(context, "advanced", "Advanced:",
1440 	                        "Show advanced option help", advanced_entries);
1441 	    pcmk__add_arg_group(context, "additional", "Additional Options:",
1442 	                        "Show additional options", addl_entries);
1443 	    return context;
1444 	}
1445 	
1446 	int
1447 	main(int argc, char **argv)
1448 	{
1449 	    xmlNode *cib_xml_copy = NULL;
1450 	    pcmk_resource_t *rsc = NULL;
1451 	    pcmk_node_t *node = NULL;
1452 	    int rc = pcmk_rc_ok;
1453 	
1454 	    GOptionGroup *output_group = NULL;
1455 	    gchar **processed_args = NULL;
1456 	    GOptionContext *context = NULL;
1457 	
1458 	    /*
1459 	     * Parse command line arguments
1460 	     */
1461 	
1462 	    args = pcmk__new_common_args(SUMMARY);
1463 	    processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
1464 	    context = build_arg_context(args, &output_group);
1465 	
1466 	    pcmk__register_formats(output_group, formats);
1467 	    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
1468 	        exit_code = CRM_EX_USAGE;
1469 	        goto done;
1470 	    }
1471 	
1472 	    pcmk__cli_init_logging("crm_resource", args->verbosity);
1473 	
1474 	    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
1475 	    if (rc != pcmk_rc_ok) {
1476 	        exit_code = CRM_EX_ERROR;
1477 	        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("Error creating output format %s: %s"),
1478 	                    args->output_ty, pcmk_rc_str(rc));
1479 	        goto done;
1480 	    }
1481 	
1482 	    pe__register_messages(out);
1483 	    crm_resource_register_messages(out);
1484 	    lrmd__register_messages(out);
1485 	    pcmk__register_lib_messages(out);
1486 	
1487 	    out->quiet = args->quiet;
1488 	
1489 	    crm_log_args(argc, argv);
1490 	
1491 	    /*
1492 	     * Validate option combinations
1493 	     */
1494 	
1495 	    // If the user didn't explicitly specify a command, list resources
1496 	    if (options.rsc_cmd == cmd_none) {
1497 	        options.rsc_cmd = cmd_list_resources;
1498 	        options.require_resource = FALSE;
1499 	    }
1500 	
1501 	    // --expired without --clear/-U doesn't make sense
1502 	    if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
1503 	        exit_code = CRM_EX_USAGE;
1504 	        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("--expired requires --clear or -U"));
1505 	        goto done;
1506 	    }
1507 	
1508 	    if ((options.remainder != NULL) && (options.override_params != NULL)) {
1509 	        // Commands that use positional arguments will create override_params
1510 	        for (gchar **s = options.remainder; *s; s++) {
1511 	            char *name = calloc(1, strlen(*s));
1512 	            char *value = calloc(1, strlen(*s));
1513 	            int rc = sscanf(*s, "%[^=]=%s", name, value);
1514 	
1515 	            if (rc == 2) {
1516 	                g_hash_table_replace(options.override_params, name, value);
1517 	
1518 	            } else {
1519 	                exit_code = CRM_EX_USAGE;
1520 	                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1521 	                            _("Error parsing '%s' as a name=value pair"),
1522 	                            argv[optind]);
1523 	                free(value);
1524 	                free(name);
1525 	                goto done;
1526 	            }
1527 	        }
1528 	
1529 	    } else if (options.remainder != NULL) {
1530 	        gchar **strv = NULL;
1531 	        gchar *msg = NULL;
1532 	        int i = 1;
1533 	        int len = 0;
1534 	
1535 	        for (gchar **s = options.remainder; *s; s++) {
1536 	            len++;
1537 	        }
1538 	
1539 	        CRM_ASSERT(len > 0);
1540 	
1541 	        /* Add 1 for the strv[0] string below, and add another 1 for the NULL
1542 	         * at the end of the array so g_strjoinv knows when to stop.
1543 	         */
1544 	        strv = calloc(len+2, sizeof(char *));
1545 	        strv[0] = strdup("non-option ARGV-elements:\n");
1546 	
1547 	        for (gchar **s = options.remainder; *s; s++) {
1548 	            strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
1549 	            i++;
1550 	        }
1551 	
1552 	        strv[i] = NULL;
1553 	
1554 	        exit_code = CRM_EX_USAGE;
1555 	        msg = g_strjoinv("", strv);
1556 	        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
1557 	        g_free(msg);
1558 	
1559 	        /* Don't try to free the last element, which is just NULL. */
1560 	        for(i = 0; i < len+1; i++) {
1561 	            free(strv[i]);
1562 	        }
1563 	        free(strv);
1564 	
1565 	        goto done;
1566 	    }
1567 	
1568 	    if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
1569 	        /* Kind of a hack to display XML lists using a real tag instead of <list>.  This just
1570 	         * saves from having to write custom messages to build the lists around all these things
1571 	         */
1572 	        switch (options.rsc_cmd) {
1573 	            case cmd_execute_agent:
1574 	            case cmd_list_resources:
1575 	            case cmd_query_xml:
1576 	            case cmd_query_raw_xml:
1577 	            case cmd_list_active_ops:
1578 	            case cmd_list_all_ops:
1579 	            case cmd_colocations:
1580 	                pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
1581 	                break;
1582 	
1583 	            default:
1584 	                pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
1585 	                break;
1586 	        }
1587 	    } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
1588 	        if ((options.rsc_cmd == cmd_colocations) ||
1589 	            options.rsc_cmd == cmd_list_resources) {
1590 	            pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
1591 	        }
1592 	    }
1593 	
1594 	    if (args->version) {
1595 	        out->version(out, false);
1596 	        goto done;
1597 	    }
1598 	
1599 	    if (options.cmdline_config) {
1600 	        /* A resource configuration was given on the command line. Sanity-check
1601 	         * the values and set error if they don't make sense.
1602 	         */
1603 	        validate_cmdline_config();
1604 	        if (error != NULL) {
1605 	            exit_code = CRM_EX_USAGE;
1606 	            goto done;
1607 	        }
1608 	
1609 	    } else if (options.cmdline_params != NULL) {
1610 	        // @COMPAT @TODO error out here when we can break backward compatibility
1611 	        g_hash_table_destroy(options.cmdline_params);
1612 	        options.cmdline_params = NULL;
1613 	    }
1614 	
1615 	    if (options.require_resource && (options.rsc_id == NULL)) {
1616 	        exit_code = CRM_EX_USAGE;
1617 	        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1618 	                    _("Must supply a resource id with -r"));
1619 	        goto done;
1620 	    }
1621 	    if (options.require_node && (options.host_uname == NULL)) {
1622 	        exit_code = CRM_EX_USAGE;
1623 	        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1624 	                    _("Must supply a node name with -N"));
1625 	        goto done;
1626 	    }
1627 	
1628 	    /*
1629 	     * Set up necessary connections
1630 	     */
1631 	
1632 	    if (options.find_flags && options.rsc_id) {
1633 	        options.require_scheduler = TRUE;
1634 	    }
1635 	
1636 	    // Establish a connection to the CIB if needed
1637 	    if (options.require_cib) {
1638 	        cib_conn = cib_new();
1639 	        if ((cib_conn == NULL) || (cib_conn->cmds == NULL)) {
1640 	            exit_code = CRM_EX_DISCONNECT;
1641 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1642 	                        _("Could not create CIB connection"));
1643 	            goto done;
1644 	        }
1645 	        rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
1646 	        rc = pcmk_legacy2rc(rc);
1647 	        if (rc != pcmk_rc_ok) {
1648 	            exit_code = pcmk_rc2exitc(rc);
1649 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1650 	                        _("Could not connect to the CIB: %s"), pcmk_rc_str(rc));
1651 	            goto done;
1652 	        }
1653 	    }
1654 	
1655 	    // Populate scheduler data from XML file if specified or CIB query otherwise
1656 	    if (options.require_scheduler) {
1657 	        rc = initialize_scheduler_data(&cib_xml_copy);
1658 	        if (rc != pcmk_rc_ok) {
1659 	            exit_code = pcmk_rc2exitc(rc);
1660 	            goto done;
1661 	        }
1662 	    }
1663 	
1664 	    // If command requires that resource exist if specified, find it
1665 	    if (options.find_flags && options.rsc_id) {
1666 	        rsc = pe_find_resource_with_flags(scheduler->resources, options.rsc_id,
1667 	                                          options.find_flags);
1668 	        if (rsc == NULL) {
1669 	            exit_code = CRM_EX_NOSUCH;
1670 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1671 	                        _("Resource '%s' not found"), options.rsc_id);
1672 	            goto done;
1673 	        }
1674 	
1675 	        /* The --ban, --clear, --move, and --restart commands do not work with
1676 	         * instances of clone resourcs.
1677 	         */
1678 	        if (strchr(options.rsc_id, ':') != NULL && pe_rsc_is_clone(rsc->parent) &&
1679 	            (options.rsc_cmd == cmd_ban || options.rsc_cmd == cmd_clear ||
1680 	             options.rsc_cmd == cmd_move || options.rsc_cmd == cmd_restart)) {
1681 	            exit_code = CRM_EX_INVALID_PARAM;
1682 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1683 	                        _("Cannot operate on clone resource instance '%s'"), options.rsc_id);
1684 	            goto done;
1685 	        }
1686 	    }
1687 	
1688 	    // If user supplied a node name, check whether it exists
1689 	    if ((options.host_uname != NULL) && (scheduler != NULL)) {
1690 	        node = pe_find_node(scheduler->nodes, options.host_uname);
1691 	
1692 	        if (node == NULL) {
1693 	            exit_code = CRM_EX_NOSUCH;
1694 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1695 	                        _("Node '%s' not found"), options.host_uname);
1696 	            goto done;
1697 	        }
1698 	    }
1699 	
1700 	    // Establish a connection to the controller if needed
1701 	    if (options.require_crmd) {
1702 	        rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
1703 	        if (rc != pcmk_rc_ok) {
1704 	            exit_code = pcmk_rc2exitc(rc);
1705 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1706 	                        _("Error connecting to the controller: %s"), pcmk_rc_str(rc));
1707 	            goto done;
1708 	        }
1709 	        pcmk_register_ipc_callback(controld_api, controller_event_callback,
1710 	                                   NULL);
1711 	        rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
1712 	        if (rc != pcmk_rc_ok) {
1713 	            exit_code = pcmk_rc2exitc(rc);
1714 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1715 	                        _("Error connecting to %s: %s"),
1716 	                        pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
1717 	            goto done;
1718 	        }
1719 	    }
1720 	
1721 	    /*
1722 	     * Handle requested command
1723 	     */
1724 	
1725 	    switch (options.rsc_cmd) {
1726 	        case cmd_list_resources: {
1727 	            GList *all = NULL;
1728 	            all = g_list_prepend(all, (gpointer) "*");
1729 	            rc = out->message(out, "resource-list", scheduler,
1730 	                              pcmk_show_inactive_rscs | pcmk_show_rsc_only | pcmk_show_pending,
1731 	                              true, all, all, false);
1732 	            g_list_free(all);
1733 	
1734 	            if (rc == pcmk_rc_no_output) {
1735 	                rc = ENXIO;
1736 	            }
1737 	            break;
1738 	        }
1739 	
1740 	        case cmd_list_instances:
1741 	            rc = out->message(out, "resource-names-list", scheduler->resources);
1742 	
1743 	            if (rc != pcmk_rc_ok) {
1744 	                rc = ENXIO;
1745 	            }
1746 	
1747 	            break;
1748 	
1749 	        case cmd_list_alternatives:
1750 	            rc = pcmk__list_alternatives(out, options.agent_spec);
1751 	            break;
1752 	
1753 	        case cmd_list_agents:
1754 	            rc = pcmk__list_agents(out, options.agent_spec);
1755 	            break;
1756 	
1757 	        case cmd_list_standards:
1758 	            rc = pcmk__list_standards(out);
1759 	            break;
1760 	
1761 	        case cmd_list_providers:
1762 	            rc = pcmk__list_providers(out, options.agent_spec);
1763 	            break;
1764 	
1765 	        case cmd_metadata:
1766 	            rc = show_metadata(out, options.agent_spec);
1767 	            break;
1768 	
1769 	        case cmd_restart:
1770 	            /* We don't pass scheduler because rsc needs to stay valid for the
1771 	             * entire lifetime of cli_resource_restart(), but it will reset and
1772 	             * update the scheduler data multiple times, so it needs to use its
1773 	             * own copy.
1774 	             */
1775 	            rc = cli_resource_restart(out, rsc, node, options.move_lifetime,
1776 	                                      options.timeout_ms, cib_conn,
1777 	                                      options.cib_options, options.promoted_role_only,
1778 	                                      options.force);
1779 	            break;
1780 	
1781 	        case cmd_wait:
1782 	            rc = wait_till_stable(out, options.timeout_ms, cib_conn);
1783 	            break;
1784 	
1785 	        case cmd_execute_agent:
1786 	            if (options.cmdline_config) {
1787 	                exit_code = cli_resource_execute_from_params(out, NULL,
1788 	                    options.v_class, options.v_provider, options.v_agent,
1789 	                    options.operation, options.cmdline_params,
1790 	                    options.override_params, options.timeout_ms,
1791 	                    args->verbosity, options.force, options.check_level);
1792 	            } else {
1793 	                exit_code = cli_resource_execute(rsc, options.rsc_id,
1794 	                    options.operation, options.override_params,
1795 	                    options.timeout_ms, cib_conn, scheduler,
1796 	                    args->verbosity, options.force, options.check_level);
1797 	            }
1798 	            goto done;
1799 	
1800 	        case cmd_digests:
1801 	            node = pe_find_node(scheduler->nodes, options.host_uname);
1802 	            if (node == NULL) {
1803 	                rc = pcmk_rc_node_unknown;
1804 	            } else {
1805 	                rc = pcmk__resource_digests(out, rsc, node,
1806 	                                            options.override_params);
1807 	            }
1808 	            break;
1809 	
1810 	        case cmd_colocations:
1811 	            rc = out->message(out, "locations-and-colocations", rsc,
1812 	                              options.recursive, (bool) options.force);
1813 	            break;
1814 	
1815 	        case cmd_cts:
1816 	            rc = pcmk_rc_ok;
1817 	            g_list_foreach(scheduler->resources, (GFunc) cli_resource_print_cts,
1818 	                           out);
1819 	            cli_resource_print_cts_constraints(scheduler);
1820 	            break;
1821 	
1822 	        case cmd_fail:
1823 	            rc = cli_resource_fail(controld_api, options.host_uname,
1824 	                                   options.rsc_id, scheduler);
1825 	            if (rc == pcmk_rc_ok) {
1826 	                start_mainloop(controld_api);
1827 	            }
1828 	            break;
1829 	
1830 	        case cmd_list_active_ops:
1831 	            rc = cli_resource_print_operations(options.rsc_id,
1832 	                                               options.host_uname, TRUE,
1833 	                                               scheduler);
1834 	            break;
1835 	
1836 	        case cmd_list_all_ops:
1837 	            rc = cli_resource_print_operations(options.rsc_id,
1838 	                                               options.host_uname, FALSE,
1839 	                                               scheduler);
1840 	            break;
1841 	
1842 	        case cmd_locate: {
1843 	            GList *nodes = cli_resource_search(rsc, options.rsc_id, scheduler);
1844 	            rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
1845 	            g_list_free_full(nodes, free);
1846 	            break;
1847 	        }
1848 	
1849 	        case cmd_query_xml:
1850 	            rc = cli_resource_print(rsc, scheduler, true);
1851 	            break;
1852 	
1853 	        case cmd_query_raw_xml:
1854 	            rc = cli_resource_print(rsc, scheduler, false);
1855 	            break;
1856 	
1857 	        case cmd_why:
1858 	            if ((options.host_uname != NULL) && (node == NULL)) {
1859 	                rc = pcmk_rc_node_unknown;
1860 	            } else {
1861 	                rc = out->message(out, "resource-reasons-list",
1862 	                                  scheduler->resources, rsc, node);
1863 	            }
1864 	            break;
1865 	
1866 	        case cmd_clear:
1867 	            rc = clear_constraints(out, &cib_xml_copy);
1868 	            break;
1869 	
1870 	        case cmd_move:
1871 	            if (options.host_uname == NULL) {
1872 	                rc = ban_or_move(out, rsc, options.move_lifetime);
1873 	            } else {
1874 	                rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
1875 	                                       options.move_lifetime, cib_conn,
1876 	                                       options.cib_options, scheduler,
1877 	                                       options.promoted_role_only,
1878 	                                       options.force);
1879 	            }
1880 	
1881 	            if (rc == EINVAL) {
1882 	                exit_code = CRM_EX_USAGE;
1883 	                goto done;
1884 	            }
1885 	
1886 	            break;
1887 	
1888 	        case cmd_ban:
1889 	            if (options.host_uname == NULL) {
1890 	                rc = ban_or_move(out, rsc, options.move_lifetime);
1891 	            } else if (node == NULL) {
1892 	                rc = pcmk_rc_node_unknown;
1893 	            } else {
1894 	                rc = cli_resource_ban(out, options.rsc_id, node->details->uname,
1895 	                                      options.move_lifetime, cib_conn,
1896 	                                      options.cib_options,
1897 	                                      options.promoted_role_only,
1898 	                                      PCMK__ROLE_PROMOTED);
1899 	            }
1900 	
1901 	            if (rc == EINVAL) {
1902 	                exit_code = CRM_EX_USAGE;
1903 	                goto done;
1904 	            }
1905 	
1906 	            break;
1907 	
1908 	        case cmd_get_property:
1909 	            rc = out->message(out, "property-list", rsc, options.prop_name);
1910 	            if (rc == pcmk_rc_no_output) {
1911 	                rc = ENXIO;
1912 	            }
1913 	
1914 	            break;
1915 	
1916 	        case cmd_set_property:
1917 	            rc = set_property();
1918 	            break;
1919 	
1920 	        case cmd_get_param: {
1921 	            unsigned int count = 0;
1922 	            GHashTable *params = NULL;
1923 	            pcmk_node_t *current = rsc->fns->active_node(rsc, &count, NULL);
1924 	            bool free_params = true;
1925 	            const char* value = NULL;
1926 	
1927 	            if (count > 1) {
1928 	                out->err(out, "%s is active on more than one node,"
1929 	                         " returning the default value for %s", rsc->id,
1930 	                         pcmk__s(options.prop_name, "unspecified property"));
1931 	                current = NULL;
1932 	            }
1933 	
1934 	            crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
1935 	
1936 	            if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_none)) {
1937 	                params = pe_rsc_params(rsc, current, scheduler);
1938 	                free_params = false;
1939 	
1940 	                value = g_hash_table_lookup(params, options.prop_name);
1941 	
1942 	            } else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_none)) {
1943 	                params = pcmk__strkey_table(free, free);
1944 	                get_meta_attributes(params, rsc, current, scheduler);
1945 	
1946 	                value = g_hash_table_lookup(params, options.prop_name);
1947 	
1948 	            } else if (pcmk__str_eq(options.attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
1949 	
1950 	                value = crm_element_value(rsc->xml, options.prop_name);
1951 	                free_params = false;
1952 	
1953 	            } else {
1954 	                params = pcmk__strkey_table(free, free);
1955 	                pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
1956 	                                           NULL, FALSE, scheduler);
1957 	
1958 	                value = g_hash_table_lookup(params, options.prop_name);
1959 	            }
1960 	
1961 	            rc = out->message(out, "attribute-list", rsc, options.prop_name, value);
1962 	            if (free_params) {
1963 	                g_hash_table_destroy(params);
1964 	            }
1965 	
1966 	            break;
1967 	        }
1968 	
1969 	        case cmd_set_param:
1970 	            if (pcmk__str_empty(options.prop_value)) {
1971 	                exit_code = CRM_EX_USAGE;
1972 	                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
1973 	                            _("You need to supply a value with the -v option"));
1974 	                goto done;
1975 	            }
1976 	
1977 	            /* coverity[var_deref_model] False positive */
1978 	            rc = cli_resource_update_attribute(rsc, options.rsc_id,
1979 	                                               options.prop_set,
1980 	                                               options.attr_set_type,
1981 	                                               options.prop_id,
1982 	                                               options.prop_name,
1983 	                                               options.prop_value,
1984 	                                               options.recursive, cib_conn,
1985 	                                               options.cib_options,
1986 	                                               options.force);
1987 	            break;
1988 	
1989 	        case cmd_delete_param:
1990 	            /* coverity[var_deref_model] False positive */
1991 	            rc = cli_resource_delete_attribute(rsc, options.rsc_id,
1992 	                                               options.prop_set,
1993 	                                               options.attr_set_type,
1994 	                                               options.prop_id,
1995 	                                               options.prop_name, cib_conn,
1996 	                                               options.cib_options,
1997 	                                               options.force);
1998 	            break;
1999 	
2000 	        case cmd_cleanup:
2001 	            if (rsc == NULL) {
2002 	                rc = cli_cleanup_all(controld_api, options.host_uname,
2003 	                                     options.operation, options.interval_spec,
2004 	                                     scheduler);
2005 	                if (rc == pcmk_rc_ok) {
2006 	                    start_mainloop(controld_api);
2007 	                }
2008 	            } else {
2009 	                cleanup(out, rsc, node);
2010 	            }
2011 	            break;
2012 	
2013 	        case cmd_refresh:
2014 	            if (rsc == NULL) {
2015 	                rc = refresh(out);
2016 	            } else {
2017 	                refresh_resource(out, rsc, node);
2018 	            }
2019 	            break;
2020 	
2021 	        case cmd_delete:
2022 	            rc = delete();
2023 	            break;
2024 	
2025 	        default:
2026 	            exit_code = CRM_EX_USAGE;
2027 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
2028 	                        _("Unimplemented command: %d"), (int) options.rsc_cmd);
2029 	            goto done;
2030 	    }
2031 	
2032 	    /* Convert rc into an exit code. */
2033 	    if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
2034 	        exit_code = pcmk_rc2exitc(rc);
2035 	    }
2036 	
2037 	    /*
2038 	     * Clean up and exit
2039 	     */
2040 	
2041 	done:
2042 	    /* When we get here, exit_code has been set one of two ways - either at one of
2043 	     * the spots where there's a "goto done" (which itself could have happened either
2044 	     * directly or by calling pcmk_rc2exitc), or just up above after any of the break
2045 	     * statements.
2046 	     *
2047 	     * Thus, we can use just exit_code here to decide what to do.
2048 	     */
2049 	    if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
2050 	        if (error != NULL) {
2051 	            char *msg = crm_strdup_printf("%s\nError performing operation: %s",
2052 	                                          error->message, crm_exit_str(exit_code));
2053 	            g_clear_error(&error);
2054 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
2055 	            free(msg);
2056 	        } else {
2057 	            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
2058 	                        _("Error performing operation: %s"), crm_exit_str(exit_code));
2059 	        }
2060 	    }
2061 	
2062 	    g_free(options.host_uname);
2063 	    g_free(options.interval_spec);
2064 	    g_free(options.move_lifetime);
2065 	    g_free(options.operation);
2066 	    g_free(options.prop_id);
2067 	    free(options.prop_name);
2068 	    g_free(options.prop_set);
2069 	    g_free(options.prop_value);
2070 	    g_free(options.rsc_id);
2071 	    g_free(options.rsc_type);
2072 	    free(options.agent_spec);
2073 	    free(options.v_agent);
2074 	    free(options.v_class);
2075 	    free(options.v_provider);
2076 	    g_free(options.xml_file);
2077 	    g_strfreev(options.remainder);
2078 	
2079 	    if (options.override_params != NULL) {
2080 	        g_hash_table_destroy(options.override_params);
2081 	    }
2082 	
2083 	    /* options.cmdline_params does not need to be destroyed here.  See the
2084 	     * comments in cli_resource_execute_from_params.
2085 	     */
2086 	
2087 	    g_strfreev(processed_args);
2088 	    g_option_context_free(context);
2089 	
2090 	    return bye(exit_code);
2091 	}
2092