1    	/*
2    	 * Copyright 2004-2023 the Pacemaker project contributors
3    	 *
4    	 * The version control history for this file may have further details.
5    	 *
6    	 * This source code is licensed under the GNU General Public License version 2
7    	 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8    	 */
9    	
10   	#include <crm_internal.h>
11   	
12   	#include <inttypes.h>               // PRIx32
13   	#include <stdbool.h>
14   	#include <glib.h>
15   	
16   	#include <crm/crm.h>
17   	#include <pacemaker-internal.h>
18   	#include "libpacemaker_private.h"
19   	
20   	enum pe_order_kind {
21   	    pe_order_kind_optional,
22   	    pe_order_kind_mandatory,
23   	    pe_order_kind_serialize,
24   	};
25   	
26   	enum ordering_symmetry {
27   	    ordering_asymmetric,        // the only relation in an asymmetric ordering
28   	    ordering_symmetric,         // the normal relation in a symmetric ordering
29   	    ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
30   	};
31   	
32   	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {                  \
33   	        __rsc = pcmk__find_constraint_resource(scheduler->resources,        \
34   	                                               __name);                     \
35   	        if (__rsc == NULL) {                                                \
36   	            pcmk__config_err("%s: No resource found for %s", __set, __name);\
37   	            return pcmk_rc_unpack_error;                                    \
38   	        }                                                                   \
39   	    } while (0)
40   	
41   	static const char *
42   	invert_action(const char *action)
43   	{
44   	    if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
45   	        return PCMK_ACTION_STOP;
46   	
47   	    } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
48   	        return PCMK_ACTION_START;
49   	
50   	    } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
51   	        return PCMK_ACTION_DEMOTE;
52   	
53   	    } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
54   	        return PCMK_ACTION_PROMOTE;
55   	
56   	    } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
57   	        return PCMK_ACTION_DEMOTED;
58   	
59   	    } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
60   	        return PCMK_ACTION_PROMOTED;
61   	
62   	    } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
63   	        return PCMK_ACTION_STOPPED;
64   	
65   	    } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
66   	        return PCMK_ACTION_RUNNING;
67   	    }
68   	    crm_warn("Unknown action '%s' specified in order constraint", action);
69   	    return NULL;
70   	}
71   	
72   	static enum pe_order_kind
73   	get_ordering_type(const xmlNode *xml_obj)
74   	{
75   	    enum pe_order_kind kind_e = pe_order_kind_mandatory;
76   	    const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
77   	
78   	    if (kind == NULL) {
79   	        const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
80   	
81   	        kind_e = pe_order_kind_mandatory;
82   	
83   	        if (score) {
84   	            // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
85   	            int score_i = char2score(score);
86   	
87   	            if (score_i == 0) {
88   	                kind_e = pe_order_kind_optional;
89   	            }
90   	            pe_warn_once(pcmk__wo_order_score,
91   	                         "Support for 'score' in rsc_order is deprecated "
92   	                         "and will be removed in a future release "
93   	                         "(use 'kind' instead)");
94   	        }
95   	
96   	    } else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
97   	        kind_e = pe_order_kind_mandatory;
98   	
99   	    } else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
100  	        kind_e = pe_order_kind_optional;
101  	
102  	    } else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
103  	        kind_e = pe_order_kind_serialize;
104  	
105  	    } else {
106  	        pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
107  	                         "%s to 'Mandatory' because '%s' is not valid",
108  	                         pcmk__s(ID(xml_obj), "missing ID"), kind);
109  	    }
110  	    return kind_e;
111  	}
112  	
113  	/*!
114  	 * \internal
115  	 * \brief Get ordering symmetry from XML
116  	 *
117  	 * \param[in] xml_obj               Ordering XML
118  	 * \param[in] parent_kind           Default ordering kind
119  	 * \param[in] parent_symmetrical_s  Parent element's symmetrical setting, if any
120  	 *
121  	 * \retval ordering_symmetric   Ordering is symmetric
122  	 * \retval ordering_asymmetric  Ordering is asymmetric
123  	 */
124  	static enum ordering_symmetry
125  	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
126  	                      const char *parent_symmetrical_s)
127  	{
128  	    int rc = pcmk_rc_ok;
129  	    bool symmetric = false;
130  	    enum pe_order_kind kind = parent_kind; // Default to parent's kind
131  	
132  	    // Check ordering XML for explicit kind
133  	    if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
134  	        || (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
135  	        kind = get_ordering_type(xml_obj);
136  	    }
137  	
138  	    // Check ordering XML (and parent) for explicit symmetrical setting
139  	    rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);
140  	
141  	    if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
142  	        symmetric = crm_is_true(parent_symmetrical_s);
143  	        rc = pcmk_rc_ok;
144  	    }
145  	
146  	    if (rc == pcmk_rc_ok) {
147  	        if (symmetric) {
148  	            if (kind == pe_order_kind_serialize) {
149  	                pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
150  	                                  " for '%s' because not valid with "
151  	                                  XML_ORDER_ATTR_KIND " of 'Serialize'",
152  	                                  ID(xml_obj));
153  	            } else {
154  	                return ordering_symmetric;
155  	            }
156  	        }
157  	        return ordering_asymmetric;
158  	    }
159  	
160  	    // Use default symmetry
161  	    if (kind == pe_order_kind_serialize) {
162  	        return ordering_asymmetric;
163  	    }
164  	    return ordering_symmetric;
165  	}
166  	
167  	/*!
168  	 * \internal
169  	 * \brief Get ordering flags appropriate to ordering kind
170  	 *
171  	 * \param[in] kind      Ordering kind
172  	 * \param[in] first     Action name for 'first' action
173  	 * \param[in] symmetry  This ordering's symmetry role
174  	 *
175  	 * \return Minimal ordering flags appropriate to \p kind
176  	 */
177  	static uint32_t
178  	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
179  	                        enum ordering_symmetry symmetry)
180  	{
181  	    uint32_t flags = pcmk__ar_none; // so we trace-log all flags set
182  	
183  	    switch (kind) {
184  	        case pe_order_kind_optional:
185  	            pe__set_order_flags(flags, pcmk__ar_ordered);
186  	            break;
187  	
188  	        case pe_order_kind_serialize:
189  	            /* This flag is not used anywhere directly but means the relation
190  	             * will not match an equality comparison against pcmk__ar_none or
191  	             * pcmk__ar_ordered.
192  	             */
193  	            pe__set_order_flags(flags, pcmk__ar_serialize);
194  	            break;
195  	
196  	        case pe_order_kind_mandatory:
197  	            pe__set_order_flags(flags, pcmk__ar_ordered);
198  	            switch (symmetry) {
199  	                case ordering_asymmetric:
200  	                    pe__set_order_flags(flags, pcmk__ar_asymmetric);
201  	                    break;
202  	
203  	                case ordering_symmetric:
204  	                    pe__set_order_flags(flags, pcmk__ar_first_implies_then);
205  	                    if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
206  	                                             PCMK_ACTION_PROMOTE, NULL)) {
207  	                        pe__set_order_flags(flags,
208  	                                            pcmk__ar_unrunnable_first_blocks);
209  	                    }
210  	                    break;
211  	
212  	                case ordering_symmetric_inverse:
213  	                    pe__set_order_flags(flags, pcmk__ar_then_implies_first);
214  	                    break;
215  	            }
216  	            break;
217  	    }
218  	    return flags;
219  	}
220  	
221  	/*!
222  	 * \internal
223  	 * \brief Find resource corresponding to ID specified in ordering
224  	 *
225  	 * \param[in] xml            Ordering XML
226  	 * \param[in] resource_attr  XML attribute name for resource ID
227  	 * \param[in] instance_attr  XML attribute name for instance number.
228  	 *                           This option is deprecated and will be removed in a
229  	 *                           future release.
230  	 * \param[in] scheduler      Scheduler data
231  	 *
232  	 * \return Resource corresponding to \p id, or NULL if none
233  	 */
234  	static pcmk_resource_t *
235  	get_ordering_resource(const xmlNode *xml, const char *resource_attr,
236  	                      const char *instance_attr,
237  	                      const pcmk_scheduler_t *scheduler)
238  	{
239  	    // @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
240  	    pcmk_resource_t *rsc = NULL;
241  	    const char *rsc_id = crm_element_value(xml, resource_attr);
242  	    const char *instance_id = crm_element_value(xml, instance_attr);
243  	
244  	    if (rsc_id == NULL) {
245  	        pcmk__config_err("Ignoring constraint '%s' without %s",
246  	                         ID(xml), resource_attr);
247  	        return NULL;
248  	    }
249  	
250  	    rsc = pcmk__find_constraint_resource(scheduler->resources, rsc_id);
251  	    if (rsc == NULL) {
252  	        pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
253  	                         "does not exist", ID(xml), rsc_id);
254  	        return NULL;
255  	    }
256  	
257  	    if (instance_id != NULL) {
258  	        pe_warn_once(pcmk__wo_order_inst,
259  	                     "Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
260  	                     XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
261  	                     "removed in a future release.");
262  	
263  	        if (!pe_rsc_is_clone(rsc)) {
264  	            pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
265  	                             "is not a clone but instance '%s' was requested",
266  	                             ID(xml), rsc_id, instance_id);
267  	            return NULL;
268  	        }
269  	        rsc = find_clone_instance(rsc, instance_id);
270  	        if (rsc == NULL) {
271  	            pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
272  	                             "does not have an instance '%s'",
273  	                             "'%s'", ID(xml), rsc_id, instance_id);
274  	            return NULL;
275  	        }
276  	    }
277  	    return rsc;
278  	}
279  	
280  	/*!
281  	 * \internal
282  	 * \brief Determine minimum number of 'first' instances required in ordering
283  	 *
284  	 * \param[in] rsc  'First' resource in ordering
285  	 * \param[in] xml  Ordering XML
286  	 *
287  	 * \return Minimum 'first' instances required (or 0 if not applicable)
288  	 */
289  	static int
290  	get_minimum_first_instances(const pcmk_resource_t *rsc, const xmlNode *xml)
291  	{
292  	    const char *clone_min = NULL;
293  	    bool require_all = false;
294  	
295  	    if (!pe_rsc_is_clone(rsc)) {
296  	        return 0;
297  	    }
298  	
299  	    clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
300  	    if (clone_min != NULL) {
301  	        int clone_min_int = 0;
302  	
303  	        pcmk__scan_min_int(clone_min, &clone_min_int, 0);
304  	        return clone_min_int;
305  	    }
306  	
307  	    /* @COMPAT 1.1.13:
308  	     * require-all=false is deprecated equivalent of clone-min=1
309  	     */
310  	    if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
311  	        pe_warn_once(pcmk__wo_require_all,
312  	                     "Support for require-all in ordering constraints "
313  	                     "is deprecated and will be removed in a future release"
314  	                     " (use clone-min clone meta-attribute instead)");
315  	        if (!require_all) {
316  	            return 1;
317  	        }
318  	    }
319  	
320  	    return 0;
321  	}
322  	
323  	/*!
324  	 * \internal
325  	 * \brief Create orderings for a constraint with clone-min > 0
326  	 *
327  	 * \param[in]     id            Ordering ID
328  	 * \param[in,out] rsc_first     'First' resource in ordering (a clone)
329  	 * \param[in]     action_first  'First' action in ordering
330  	 * \param[in]     rsc_then      'Then' resource in ordering
331  	 * \param[in]     action_then   'Then' action in ordering
332  	 * \param[in]     flags         Ordering flags
333  	 * \param[in]     clone_min     Minimum required instances of 'first'
334  	 */
335  	static void
336  	clone_min_ordering(const char *id,
337  	                   pcmk_resource_t *rsc_first, const char *action_first,
338  	                   pcmk_resource_t *rsc_then, const char *action_then,
339  	                   uint32_t flags, int clone_min)
340  	{
341  	    // Create a pseudo-action for when the minimum instances are active
342  	    char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
343  	    pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);
344  	
345  	    free(task);
346  	
347  	    /* Require the pseudo-action to have the required number of actions to be
348  	     * considered runnable before allowing the pseudo-action to be runnable.
349  	     */
350  	    clone_min_met->required_runnable_before = clone_min;
351  	    pe__set_action_flags(clone_min_met, pcmk_action_min_runnable);
352  	
353  	    // Order the actions for each clone instance before the pseudo-action
354  	    for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
355  	        pcmk_resource_t *child = iter->data;
356  	
357  	        pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
358  	                           NULL, NULL, NULL, clone_min_met,
359  	                           pcmk__ar_min_runnable
360  	                           |pcmk__ar_first_implies_then_graphed,
361  	                           rsc_first->cluster);
362  	    }
363  	
364  	    // Order "then" action after the pseudo-action (if runnable)
365  	    pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
366  	                       pcmk__op_key(rsc_then->id, action_then, 0),
367  	                       NULL, flags|pcmk__ar_unrunnable_first_blocks,
368  	                       rsc_first->cluster);
369  	}
370  	
371  	/*!
372  	 * \internal
373  	 * \brief Update ordering flags for restart-type=restart
374  	 *
375  	 * \param[in]     rsc    'Then' resource in ordering
376  	 * \param[in]     kind   Ordering kind
377  	 * \param[in]     flag   Ordering flag to set (when applicable)
378  	 * \param[in,out] flags  Ordering flag set to update
379  	 *
380  	 * \compat The restart-type resource meta-attribute is deprecated. Eventually,
381  	 *         it will be removed, and pe_restart_ignore will be the only behavior,
382  	 *         at which time this can just be removed entirely.
383  	 */
384  	#define handle_restart_type(rsc, kind, flag, flags) do {        \
385  	        if (((kind) == pe_order_kind_optional)                  \
386  	            && ((rsc)->restart_type == pe_restart_restart)) {   \
387  	            pe__set_order_flags((flags), (flag));               \
388  	        }                                                       \
389  	    } while (0)
390  	
391  	/*!
392  	 * \internal
393  	 * \brief Create new ordering for inverse of symmetric constraint
394  	 *
395  	 * \param[in]     id            Ordering ID (for logging only)
396  	 * \param[in]     kind          Ordering kind
397  	 * \param[in]     rsc_first     'First' resource in ordering (a clone)
398  	 * \param[in]     action_first  'First' action in ordering
399  	 * \param[in,out] rsc_then      'Then' resource in ordering
400  	 * \param[in]     action_then   'Then' action in ordering
401  	 */
402  	static void
403  	inverse_ordering(const char *id, enum pe_order_kind kind,
404  	                 pcmk_resource_t *rsc_first, const char *action_first,
405  	                 pcmk_resource_t *rsc_then, const char *action_then)
406  	{
407  	    action_then = invert_action(action_then);
408  	    action_first = invert_action(action_first);
409  	    if ((action_then == NULL) || (action_first == NULL)) {
410  	        pcmk__config_warn("Cannot invert constraint '%s' "
411  	                          "(please specify inverse manually)", id);
412  	    } else {
413  	        uint32_t flags = ordering_flags_for_kind(kind, action_first,
414  	                                                 ordering_symmetric_inverse);
415  	
416  	        handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
417  	        pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
418  	                                     action_first, flags);
419  	    }
420  	}
421  	
422  	static void
423  	unpack_simple_rsc_order(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
424  	{
425  	    pcmk_resource_t *rsc_then = NULL;
426  	    pcmk_resource_t *rsc_first = NULL;
427  	    int min_required_before = 0;
428  	    enum pe_order_kind kind = pe_order_kind_mandatory;
429  	    uint32_t flags = pcmk__ar_none;
430  	    enum ordering_symmetry symmetry;
431  	
432  	    const char *action_then = NULL;
433  	    const char *action_first = NULL;
434  	    const char *id = NULL;
435  	
436  	    CRM_CHECK(xml_obj != NULL, return);
437  	
438  	    id = crm_element_value(xml_obj, XML_ATTR_ID);
439  	    if (id == NULL) {
440  	        pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
441  	                         xml_obj->name);
442  	        return;
443  	    }
444  	
445  	    rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
446  	                                      XML_ORDER_ATTR_FIRST_INSTANCE,
447  	                                      scheduler);
448  	    if (rsc_first == NULL) {
449  	        return;
450  	    }
451  	
452  	    rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
453  	                                     XML_ORDER_ATTR_THEN_INSTANCE,
454  	                                     scheduler);
455  	    if (rsc_then == NULL) {
456  	        return;
457  	    }
458  	
459  	    action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
460  	    if (action_first == NULL) {
461  	        action_first = PCMK_ACTION_START;
462  	    }
463  	
464  	    action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
465  	    if (action_then == NULL) {
466  	        action_then = action_first;
467  	    }
468  	
469  	    kind = get_ordering_type(xml_obj);
470  	
471  	    symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
472  	    flags = ordering_flags_for_kind(kind, action_first, symmetry);
473  	
474  	    handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);
475  	
476  	    /* If there is a minimum number of instances that must be runnable before
477  	     * the 'then' action is runnable, we use a pseudo-action for convenience:
478  	     * minimum number of clone instances have runnable actions ->
479  	     * pseudo-action is runnable -> dependency is runnable.
480  	     */
481  	    min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
482  	    if (min_required_before > 0) {
483  	        clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
484  	                           flags, min_required_before);
485  	    } else {
486  	        pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
487  	                                     action_then, flags);
488  	    }
489  	
490  	    if (symmetry == ordering_symmetric) {
491  	        inverse_ordering(id, kind, rsc_first, action_first,
492  	                         rsc_then, action_then);
493  	    }
494  	}
495  	
496  	/*!
497  	 * \internal
498  	 * \brief Create a new ordering between two actions
499  	 *
500  	 * \param[in,out] first_rsc          Resource for 'first' action (if NULL and
501  	 *                                   \p first_action is a resource action, that
502  	 *                                   resource will be used)
503  	 * \param[in,out] first_action_task  Action key for 'first' action (if NULL and
504  	 *                                   \p first_action is not NULL, its UUID will
505  	 *                                   be used)
506  	 * \param[in,out] first_action       'first' action (if NULL, \p first_rsc and
507  	 *                                   \p first_action_task must be set)
508  	 *
509  	 * \param[in]     then_rsc           Resource for 'then' action (if NULL and
510  	 *                                   \p then_action is a resource action, that
511  	 *                                   resource will be used)
512  	 * \param[in,out] then_action_task   Action key for 'then' action (if NULL and
513  	 *                                   \p then_action is not NULL, its UUID will
514  	 *                                   be used)
515  	 * \param[in]     then_action        'then' action (if NULL, \p then_rsc and
516  	 *                                   \p then_action_task must be set)
517  	 *
518  	 * \param[in]     flags              Group of enum pcmk__action_relation_flags
519  	 * \param[in,out] sched              Scheduler data to add ordering to
520  	 *
521  	 * \note This function takes ownership of first_action_task and
522  	 *       then_action_task, which do not need to be freed by the caller.
523  	 */
524  	void
525  	pcmk__new_ordering(pcmk_resource_t *first_rsc, char *first_action_task,
526  	                   pcmk_action_t *first_action, pcmk_resource_t *then_rsc,
527  	                   char *then_action_task, pcmk_action_t *then_action,
528  	                   uint32_t flags, pcmk_scheduler_t *sched)
529  	{
530  	    pe__ordering_t *order = NULL;
531  	
532  	    // One of action or resource must be specified for each side
533  	    CRM_CHECK(((first_action != NULL) || (first_rsc != NULL))
534  	              && ((then_action != NULL) || (then_rsc != NULL)),
535  	              free(first_action_task); free(then_action_task); return);
536  	
537  	    if ((first_rsc == NULL) && (first_action != NULL)) {
538  	        first_rsc = first_action->rsc;
539  	    }
540  	    if ((then_rsc == NULL) && (then_action != NULL)) {
541  	        then_rsc = then_action->rsc;
542  	    }
543  	
544  	    order = calloc(1, sizeof(pe__ordering_t));
545  	    CRM_ASSERT(order != NULL);
546  	
547  	    order->id = sched->order_id++;
548  	    order->flags = flags;
549  	    order->lh_rsc = first_rsc;
550  	    order->rh_rsc = then_rsc;
551  	    order->lh_action = first_action;
552  	    order->rh_action = then_action;
553  	    order->lh_action_task = first_action_task;
554  	    order->rh_action_task = then_action_task;
555  	
556  	    if ((order->lh_action_task == NULL) && (first_action != NULL)) {
557  	        order->lh_action_task = strdup(first_action->uuid);
558  	    }
559  	
560  	    if ((order->rh_action_task == NULL) && (then_action != NULL)) {
561  	        order->rh_action_task = strdup(then_action->uuid);
562  	    }
563  	
(16) Event example_checked: Example 2: "first_action->rsc" has its value checked in "order->lh_rsc == NULL".
Also see events: [null_field][dereference][example_checked][example_checked][example_checked]
564  	    if ((order->lh_rsc == NULL) && (first_action != NULL)) {
565  	        order->lh_rsc = first_action->rsc;
566  	    }
567  	
568  	    if ((order->rh_rsc == NULL) && (then_action != NULL)) {
569  	        order->rh_rsc = then_action->rsc;
570  	    }
571  	
572  	    pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
573  	                 (sched->order_id - 1),
574  	                 pcmk__s(order->lh_action_task, "an underspecified action"),
575  	                 pcmk__s(order->rh_action_task, "an underspecified action"));
576  	
577  	    sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
578  	                                                 order);
579  	    pcmk__order_migration_equivalents(order);
580  	}
581  	
582  	/*!
583  	 * \brief Unpack a set in an ordering constraint
584  	 *
585  	 * \param[in]     set                   Set XML to unpack
586  	 * \param[in]     parent_kind           rsc_order XML "kind" attribute
587  	 * \param[in]     parent_symmetrical_s  rsc_order XML "symmetrical" attribute
588  	 * \param[in,out] scheduler             Scheduler data
589  	 *
590  	 * \return Standard Pacemaker return code
591  	 */
592  	static int
593  	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
594  	                 const char *parent_symmetrical_s, pcmk_scheduler_t *scheduler)
595  	{
596  	    GList *set_iter = NULL;
597  	    GList *resources = NULL;
598  	
599  	    pcmk_resource_t *last = NULL;
600  	    pcmk_resource_t *resource = NULL;
601  	
602  	    int local_kind = parent_kind;
603  	    bool sequential = false;
604  	    uint32_t flags = pcmk__ar_ordered;
605  	    enum ordering_symmetry symmetry;
606  	
607  	    char *key = NULL;
608  	    const char *id = ID(set);
609  	    const char *action = crm_element_value(set, "action");
610  	    const char *sequential_s = crm_element_value(set, "sequential");
611  	    const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);
612  	
613  	    if (action == NULL) {
614  	        action = PCMK_ACTION_START;
615  	    }
616  	
617  	    if (kind_s) {
618  	        local_kind = get_ordering_type(set);
619  	    }
620  	    if (sequential_s == NULL) {
621  	        sequential_s = "1";
622  	    }
623  	
624  	    sequential = crm_is_true(sequential_s);
625  	
626  	    symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
627  	    flags = ordering_flags_for_kind(local_kind, action, symmetry);
628  	
629  	    for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
630  	         xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
631  	
632  	        EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
633  	        resources = g_list_append(resources, resource);
634  	    }
635  	
636  	    if (pcmk__list_of_1(resources)) {
637  	        crm_trace("Single set: %s", id);
638  	        goto done;
639  	    }
640  	
641  	    set_iter = resources;
642  	    while (set_iter != NULL) {
643  	        resource = (pcmk_resource_t *) set_iter->data;
644  	        set_iter = set_iter->next;
645  	
646  	        key = pcmk__op_key(resource->id, action, 0);
647  	
648  	        if (local_kind == pe_order_kind_serialize) {
649  	            /* Serialize before everything that comes after */
650  	
651  	            for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
652  	                pcmk_resource_t *then_rsc = iter->data;
653  	                char *then_key = pcmk__op_key(then_rsc->id, action, 0);
654  	
655  	                pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
656  	                                   then_key, NULL, flags, scheduler);
657  	            }
658  	
659  	        } else if (sequential) {
660  	            if (last != NULL) {
661  	                pcmk__order_resource_actions(last, action, resource, action,
662  	                                             flags);
663  	            }
664  	            last = resource;
665  	        }
666  	        free(key);
667  	    }
668  	
669  	    if (symmetry == ordering_asymmetric) {
670  	        goto done;
671  	    }
672  	
673  	    last = NULL;
674  	    action = invert_action(action);
675  	
676  	    flags = ordering_flags_for_kind(local_kind, action,
677  	                                    ordering_symmetric_inverse);
678  	
679  	    set_iter = resources;
680  	    while (set_iter != NULL) {
681  	        resource = (pcmk_resource_t *) set_iter->data;
682  	        set_iter = set_iter->next;
683  	
684  	        if (sequential) {
685  	            if (last != NULL) {
686  	                pcmk__order_resource_actions(resource, action, last, action,
687  	                                             flags);
688  	            }
689  	            last = resource;
690  	        }
691  	    }
692  	
693  	  done:
694  	    g_list_free(resources);
695  	    return pcmk_rc_ok;
696  	}
697  	
698  	/*!
699  	 * \brief Order two resource sets relative to each other
700  	 *
701  	 * \param[in]     id         Ordering ID (for logging)
702  	 * \param[in]     set1       First listed set
703  	 * \param[in]     set2       Second listed set
704  	 * \param[in]     kind       Ordering kind
705  	 * \param[in,out] scheduler  Scheduler data
706  	 * \param[in]     symmetry   Which ordering symmetry applies to this relation
707  	 *
708  	 * \return Standard Pacemaker return code
709  	 */
710  	static int
711  	order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2,
712  	               enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
713  	               enum ordering_symmetry symmetry)
714  	{
715  	
716  	    const xmlNode *xml_rsc = NULL;
717  	    const xmlNode *xml_rsc_2 = NULL;
718  	
719  	    pcmk_resource_t *rsc_1 = NULL;
720  	    pcmk_resource_t *rsc_2 = NULL;
721  	
722  	    const char *action_1 = crm_element_value(set1, "action");
723  	    const char *action_2 = crm_element_value(set2, "action");
724  	
725  	    uint32_t flags = pcmk__ar_none;
726  	
727  	    bool require_all = true;
728  	
729  	    (void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);
730  	
731  	    if (action_1 == NULL) {
732  	        action_1 = PCMK_ACTION_START;
733  	    }
734  	
735  	    if (action_2 == NULL) {
736  	        action_2 = PCMK_ACTION_START;
737  	    }
738  	
739  	    if (symmetry == ordering_symmetric_inverse) {
740  	        action_1 = invert_action(action_1);
741  	        action_2 = invert_action(action_2);
742  	    }
743  	
744  	    if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
745  	        || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
746  	        /* Assuming: A -> ( B || C) -> D
747  	         * The one-or-more logic only applies during the start/promote phase.
748  	         * During shutdown neither B nor can shutdown until D is down, so simply
749  	         * turn require_all back on.
750  	         */
751  	        require_all = true;
752  	    }
753  	
754  	    flags = ordering_flags_for_kind(kind, action_1, symmetry);
755  	
756  	    /* If we have an unordered set1, whether it is sequential or not is
757  	     * irrelevant in regards to set2.
758  	     */
759  	    if (!require_all) {
760  	        char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
761  	        pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);
762  	
763  	        free(task);
764  	        pe__set_action_flags(unordered_action, pcmk_action_min_runnable);
765  	
766  	        for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
767  	             xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
768  	
769  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
770  	
771  	            /* Add an ordering constraint between every element in set1 and the
772  	             * pseudo action. If any action in set1 is runnable the pseudo
773  	             * action will be runnable.
774  	             */
775  	            pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
776  	                               NULL, NULL, NULL, unordered_action,
777  	                               pcmk__ar_min_runnable
778  	                               |pcmk__ar_first_implies_then_graphed,
779  	                               scheduler);
780  	        }
781  	        for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
782  	             xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
783  	
784  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
785  	
786  	            /* Add an ordering constraint between the pseudo-action and every
787  	             * element in set2. If the pseudo-action is runnable, every action
788  	             * in set2 will be runnable.
789  	             */
790  	            pcmk__new_ordering(NULL, NULL, unordered_action,
791  	                               rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
792  	                               NULL, flags|pcmk__ar_unrunnable_first_blocks,
793  	                               scheduler);
794  	        }
795  	
796  	        return pcmk_rc_ok;
797  	    }
798  	
799  	    if (pcmk__xe_attr_is_true(set1, "sequential")) {
800  	        if (symmetry == ordering_symmetric_inverse) {
801  	            // Get the first one
802  	            xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
803  	            if (xml_rsc != NULL) {
804  	                EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
805  	            }
806  	
807  	        } else {
808  	            // Get the last one
809  	            const char *rid = NULL;
810  	
811  	            for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
812  	                 xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
813  	
814  	                rid = ID(xml_rsc);
815  	            }
816  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
817  	        }
818  	    }
819  	
820  	    if (pcmk__xe_attr_is_true(set2, "sequential")) {
821  	        if (symmetry == ordering_symmetric_inverse) {
822  	            // Get the last one
823  	            const char *rid = NULL;
824  	
825  	            for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
826  	                 xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
827  	
828  	                rid = ID(xml_rsc);
829  	            }
830  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
831  	
832  	        } else {
833  	            // Get the first one
834  	            xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
835  	            if (xml_rsc != NULL) {
836  	                EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
837  	            }
838  	        }
839  	    }
840  	
841  	    if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
842  	        pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);
843  	
844  	    } else if (rsc_1 != NULL) {
845  	        for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
846  	             xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
847  	
848  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
849  	            pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
850  	                                         flags);
851  	        }
852  	
853  	    } else if (rsc_2 != NULL) {
854  	        for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
855  	             xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
856  	
857  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
858  	            pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
859  	                                         flags);
860  	        }
861  	
862  	    } else {
863  	        for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
864  	             xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
865  	
866  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
867  	
868  	            for (xmlNode *xml_rsc_2 = first_named_child(set2,
869  	                                                        XML_TAG_RESOURCE_REF);
870  	                 xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
871  	
872  	                EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
873  	                pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
874  	                                             action_2, flags);
875  	            }
876  	        }
877  	    }
878  	
879  	    return pcmk_rc_ok;
880  	}
881  	
882  	/*!
883  	 * \internal
884  	 * \brief If an ordering constraint uses resource tags, expand them
885  	 *
886  	 * \param[in,out] xml_obj       Ordering constraint XML
887  	 * \param[out]    expanded_xml  Equivalent XML with tags expanded
888  	 * \param[in]     scheduler     Scheduler data
889  	 *
890  	 * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
891  	 *         and pcmk_rc_unpack_error on invalid configuration)
892  	 */
893  	static int
894  	unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml,
895  	                  const pcmk_scheduler_t *scheduler)
896  	{
897  	    const char *id_first = NULL;
898  	    const char *id_then = NULL;
899  	    const char *action_first = NULL;
900  	    const char *action_then = NULL;
901  	
902  	    pcmk_resource_t *rsc_first = NULL;
903  	    pcmk_resource_t *rsc_then = NULL;
904  	    pcmk_tag_t *tag_first = NULL;
905  	    pcmk_tag_t *tag_then = NULL;
906  	
907  	    xmlNode *rsc_set_first = NULL;
908  	    xmlNode *rsc_set_then = NULL;
909  	    bool any_sets = false;
910  	
911  	    // Check whether there are any resource sets with template or tag references
912  	    *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
913  	    if (*expanded_xml != NULL) {
914  	        crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
915  	        return pcmk_rc_ok;
916  	    }
917  	
918  	    id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
919  	    id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
920  	    if ((id_first == NULL) || (id_then == NULL)) {
921  	        return pcmk_rc_ok;
922  	    }
923  	
924  	    if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
925  	                                     &tag_first)) {
926  	        pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
927  	                         "valid resource or tag", ID(xml_obj), id_first);
928  	        return pcmk_rc_unpack_error;
929  	    }
930  	
931  	    if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
932  	                                     &tag_then)) {
933  	        pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
934  	                         "valid resource or tag", ID(xml_obj), id_then);
935  	        return pcmk_rc_unpack_error;
936  	    }
937  	
938  	    if ((rsc_first != NULL) && (rsc_then != NULL)) {
939  	        // Neither side references a template or tag
940  	        return pcmk_rc_ok;
941  	    }
942  	
943  	    action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
944  	    action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
945  	
946  	    *expanded_xml = copy_xml(xml_obj);
947  	
948  	    // Convert template/tag reference in "first" into constraint resource_set
949  	    if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
950  	                          true, scheduler)) {
951  	        free_xml(*expanded_xml);
952  	        *expanded_xml = NULL;
953  	        return pcmk_rc_unpack_error;
954  	    }
955  	
956  	    if (rsc_set_first != NULL) {
957  	        if (action_first != NULL) {
958  	            // Move "first-action" into converted resource_set as "action"
959  	            crm_xml_add(rsc_set_first, "action", action_first);
960  	            xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
961  	        }
962  	        any_sets = true;
963  	    }
964  	
965  	    // Convert template/tag reference in "then" into constraint resource_set
966  	    if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
967  	                          true, scheduler)) {
968  	        free_xml(*expanded_xml);
969  	        *expanded_xml = NULL;
970  	        return pcmk_rc_unpack_error;
971  	    }
972  	
973  	    if (rsc_set_then != NULL) {
974  	        if (action_then != NULL) {
975  	            // Move "then-action" into converted resource_set as "action"
976  	            crm_xml_add(rsc_set_then, "action", action_then);
977  	            xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
978  	        }
979  	        any_sets = true;
980  	    }
981  	
982  	    if (any_sets) {
983  	        crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
984  	    } else {
985  	        free_xml(*expanded_xml);
986  	        *expanded_xml = NULL;
987  	    }
988  	
989  	    return pcmk_rc_ok;
990  	}
991  	
992  	/*!
993  	 * \internal
994  	 * \brief Unpack ordering constraint XML
995  	 *
996  	 * \param[in,out] xml_obj    Ordering constraint XML to unpack
997  	 * \param[in,out] scheduler  Scheduler data
998  	 */
999  	void
1000 	pcmk__unpack_ordering(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
1001 	{
1002 	    xmlNode *set = NULL;
1003 	    xmlNode *last = NULL;
1004 	
1005 	    xmlNode *orig_xml = NULL;
1006 	    xmlNode *expanded_xml = NULL;
1007 	
1008 	    const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
1009 	    const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
1010 	    enum pe_order_kind kind = get_ordering_type(xml_obj);
1011 	
1012 	    enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
1013 	                                                            NULL);
1014 	
1015 	    // Expand any resource tags in the constraint XML
1016 	    if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
1017 	        return;
1018 	    }
1019 	    if (expanded_xml != NULL) {
1020 	        orig_xml = xml_obj;
1021 	        xml_obj = expanded_xml;
1022 	    }
1023 	
1024 	    // If the constraint has resource sets, unpack them
1025 	    for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
1026 	         set != NULL; set = crm_next_same_xml(set)) {
1027 	
1028 	        set = expand_idref(set, scheduler->input);
1029 	        if ((set == NULL) // Configuration error, message already logged
1030 	            || (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {
1031 	
1032 	            if (expanded_xml != NULL) {
1033 	                free_xml(expanded_xml);
1034 	            }
1035 	            return;
1036 	        }
1037 	
1038 	        if (last != NULL) {
1039 	
1040 	            if (order_rsc_sets(id, last, set, kind, scheduler,
1041 	                               symmetry) != pcmk_rc_ok) {
1042 	                if (expanded_xml != NULL) {
1043 	                    free_xml(expanded_xml);
1044 	                }
1045 	                return;
1046 	            }
1047 	
1048 	            if ((symmetry == ordering_symmetric)
1049 	                && (order_rsc_sets(id, set, last, kind, scheduler,
1050 	                                   ordering_symmetric_inverse) != pcmk_rc_ok)) {
1051 	                if (expanded_xml != NULL) {
1052 	                    free_xml(expanded_xml);
1053 	                }
1054 	                return;
1055 	            }
1056 	
1057 	        }
1058 	        last = set;
1059 	    }
1060 	
1061 	    if (expanded_xml) {
1062 	        free_xml(expanded_xml);
1063 	        xml_obj = orig_xml;
1064 	    }
1065 	
1066 	    // If the constraint has no resource sets, unpack it as a simple ordering
1067 	    if (last == NULL) {
1068 	        return unpack_simple_rsc_order(xml_obj, scheduler);
1069 	    }
1070 	}
1071 	
1072 	static bool
1073 	ordering_is_invalid(pcmk_action_t *action, pcmk__related_action_t *input)
1074 	{
1075 	    /* Prevent user-defined ordering constraints between resources
1076 	     * running in a guest node and the resource that defines that node.
1077 	     */
1078 	    if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed)
1079 	        && (input->action->rsc != NULL)
1080 	        && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {
1081 	
1082 	        crm_warn("Invalid ordering constraint between %s and %s",
1083 	                 input->action->rsc->id, action->rsc->id);
1084 	        return true;
1085 	    }
1086 	
1087 	    /* If there's an order like
1088 	     * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
1089 	     *
1090 	     * then rscA is being migrated from node1 to node2, while rscB is being
1091 	     * migrated from node2 to node1. If there would be a graph loop,
1092 	     * break the order "load_stopped_node2" -> "rscA_migrate_to node1".
1093 	     */
1094 	    if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target)
1095 	        && (action->rsc != NULL)
1096 	        && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
1097 	        && pcmk__graph_has_loop(action, action, input)) {
1098 	        return true;
1099 	    }
1100 	
1101 	    return false;
1102 	}
1103 	
1104 	void
1105 	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
1106 	{
1107 	    for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
1108 	        pcmk_action_t *action = (pcmk_action_t *) iter->data;
1109 	        pcmk__related_action_t *input = NULL;
1110 	
1111 	        for (GList *input_iter = action->actions_before;
1112 	             input_iter != NULL; input_iter = input_iter->next) {
1113 	
1114 	            input = input_iter->data;
1115 	            if (ordering_is_invalid(action, input)) {
1116 	                input->type = (enum pe_ordering) pcmk__ar_none;
1117 	            }
1118 	        }
1119 	    }
1120 	}
1121 	
1122 	/*!
1123 	 * \internal
1124 	 * \brief Order stops on a node before the node's shutdown
1125 	 *
1126 	 * \param[in,out] node         Node being shut down
1127 	 * \param[in]     shutdown_op  Shutdown action for node
1128 	 */
1129 	void
1130 	pcmk__order_stops_before_shutdown(pcmk_node_t *node, pcmk_action_t *shutdown_op)
1131 	{
1132 	    for (GList *iter = node->details->data_set->actions;
1133 	         iter != NULL; iter = iter->next) {
1134 	
1135 	        pcmk_action_t *action = (pcmk_action_t *) iter->data;
1136 	
1137 	        // Only stops on the node shutting down are relevant
1138 	        if (!pe__same_node(action->node, node)
1139 	            || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
1140 	            continue;
1141 	        }
1142 	
1143 	        // Resources and nodes in maintenance mode won't be touched
1144 	
1145 	        if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
1146 	            pe_rsc_trace(action->rsc,
1147 	                         "Not ordering %s before shutdown of %s because "
1148 	                         "resource in maintenance mode",
1149 	                         action->uuid, pe__node_name(node));
1150 	            continue;
1151 	
1152 	        } else if (node->details->maintenance) {
1153 	            pe_rsc_trace(action->rsc,
1154 	                         "Not ordering %s before shutdown of %s because "
1155 	                         "node in maintenance mode",
1156 	                         action->uuid, pe__node_name(node));
1157 	            continue;
1158 	        }
1159 	
1160 	        /* Don't touch a resource that is unmanaged or blocked, to avoid
1161 	         * blocking the shutdown (though if another action depends on this one,
1162 	         * we may still end up blocking)
1163 	         */
1164 	        if (!pcmk_any_flags_set(action->rsc->flags,
1165 	                                pcmk_rsc_managed|pcmk_rsc_blocked)) {
1166 	            pe_rsc_trace(action->rsc,
1167 	                         "Not ordering %s before shutdown of %s because "
1168 	                         "resource is unmanaged or blocked",
1169 	                         action->uuid, pe__node_name(node));
1170 	            continue;
1171 	        }
1172 	
1173 	        pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
1174 	                     action->uuid, pe__node_name(node));
1175 	        pe__clear_action_flags(action, pcmk_action_optional);
1176 	        pcmk__new_ordering(action->rsc, NULL, action, NULL,
1177 	                           strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
1178 	                           pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks,
1179 	                           node->details->data_set);
1180 	    }
1181 	}
1182 	
1183 	/*!
1184 	 * \brief Find resource actions matching directly or as child
1185 	 *
1186 	 * \param[in] rsc           Resource to check
1187 	 * \param[in] original_key  Action key to search for (possibly referencing
1188 	 *                          parent of \rsc)
1189 	 *
1190 	 * \return Newly allocated list of matching actions
1191 	 * \note It is the caller's responsibility to free the result with g_list_free()
1192 	 */
1193 	static GList *
1194 	find_actions_by_task(const pcmk_resource_t *rsc, const char *original_key)
1195 	{
1196 	    // Search under given task key directly
1197 	    GList *list = find_actions(rsc->actions, original_key, NULL);
1198 	
1199 	    if (list == NULL) {
1200 	        // Search again using this resource's ID
1201 	        char *key = NULL;
1202 	        char *task = NULL;
1203 	        guint interval_ms = 0;
1204 	
1205 	        if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
1206 	            key = pcmk__op_key(rsc->id, task, interval_ms);
1207 	            list = find_actions(rsc->actions, key, NULL);
1208 	            free(key);
1209 	            free(task);
1210 	        } else {
1211 	            crm_err("Invalid operation key (bug?): %s", original_key);
1212 	        }
1213 	    }
1214 	    return list;
1215 	}
1216 	
1217 	/*!
1218 	 * \internal
1219 	 * \brief Order relevant resource actions after a given action
1220 	 *
1221 	 * \param[in,out] first_action  Action to order after (or NULL if none runnable)
1222 	 * \param[in]     rsc           Resource whose actions should be ordered
1223 	 * \param[in,out] order         Ordering constraint being applied
1224 	 */
1225 	static void
1226 	order_resource_actions_after(pcmk_action_t *first_action,
1227 	                             const pcmk_resource_t *rsc, pe__ordering_t *order)
1228 	{
1229 	    GList *then_actions = NULL;
1230 	    uint32_t flags = pcmk__ar_none;
1231 	
(1) Event path: Condition "rsc != NULL", taking true branch.
(2) Event path: Condition "order != NULL", taking true branch.
1232 	    CRM_CHECK((rsc != NULL) && (order != NULL), return);
1233 	
1234 	    flags = order->flags;
(3) Event path: Switch case default.
(4) Event path: Condition "trace_tag_cs == NULL", taking true branch.
(5) Event path: Condition "crm_is_callsite_active(trace_tag_cs, _level, converted_tag)", taking false branch.
1235 	    pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
1236 	                 order->id, rsc->id);
1237 	
(6) Event path: Condition "order->rh_action != NULL", taking false branch.
1238 	    if (order->rh_action != NULL) {
1239 	        then_actions = g_list_prepend(NULL, order->rh_action);
1240 	
1241 	    } else {
1242 	        then_actions = find_actions_by_task(rsc, order->rh_action_task);
1243 	    }
1244 	
(7) Event path: Condition "then_actions == NULL", taking false branch.
1245 	    if (then_actions == NULL) {
1246 	        pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
1247 	                     order->id, order->rh_action_task, rsc->id);
1248 	        return;
1249 	    }
1250 	
(8) Event path: Condition "first_action != NULL", taking false branch.
1251 	    if ((first_action != NULL) && (first_action->rsc == rsc)
1252 	        && pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {
1253 	
1254 	        pe_rsc_trace(rsc,
1255 	                     "Detected dangling migration ordering (%s then %s %s)",
1256 	                     first_action->uuid, order->rh_action_task, rsc->id);
1257 	        pe__clear_order_flags(flags, pcmk__ar_first_implies_then);
1258 	    }
1259 	
(9) Event path: Condition "first_action == NULL", taking true branch.
(10) Event path: Condition "!pcmk_all_flags_set(flags, pcmk__ar_first_implies_then)", taking false branch.
1260 	    if ((first_action == NULL)
1261 	        && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {
1262 	
1263 	        pe_rsc_debug(rsc,
1264 	                     "Ignoring ordering %d for %s: No first action found",
1265 	                     order->id, rsc->id);
1266 	        g_list_free(then_actions);
1267 	        return;
1268 	    }
1269 	
(11) Event path: Condition "iter != NULL", taking true branch.
1270 	    for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
1271 	        pcmk_action_t *then_action_iter = (pcmk_action_t *) iter->data;
1272 	
(12) Event path: Condition "first_action != NULL", taking false branch.
1273 	        if (first_action != NULL) {
1274 	            order_actions(first_action, then_action_iter, flags);
1275 	        } else {
1276 	            pe__clear_action_flags(then_action_iter, pcmk_action_runnable);
CID (unavailable; MK=829cb3cad741a494438096aea37a6413) (#1 of 1): Dereference of potentially null field (NULL_FIELD):
(13) Event null_field: Reading field "lh_rsc", which is expected to possibly be "NULL" in "order->lh_rsc" (checked 4 out of 5 times).
(14) Event dereference: Dereferencing "order->lh_rsc", which is known to be "NULL".
Also see events: [example_checked][example_checked][example_checked][example_checked]
1277 	            crm_warn("%s of %s is unrunnable because there is no %s of %s "
1278 	                     "to order it after", then_action_iter->task, rsc->id,
1279 	                     order->lh_action_task, order->lh_rsc->id);
1280 	        }
1281 	    }
1282 	
1283 	    g_list_free(then_actions);
1284 	}
1285 	
1286 	static void
1287 	rsc_order_first(pcmk_resource_t *first_rsc, pe__ordering_t *order)
1288 	{
1289 	    GList *first_actions = NULL;
1290 	    pcmk_action_t *first_action = order->lh_action;
1291 	    pcmk_resource_t *then_rsc = order->rh_rsc;
1292 	
1293 	    CRM_ASSERT(first_rsc != NULL);
1294 	    pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
1295 	                 order->id, first_rsc->id);
1296 	
1297 	    if (first_action != NULL) {
1298 	        first_actions = g_list_prepend(NULL, first_action);
1299 	
1300 	    } else {
1301 	        first_actions = find_actions_by_task(first_rsc, order->lh_action_task);
1302 	    }
1303 	
1304 	    if ((first_actions == NULL) && (first_rsc == then_rsc)) {
1305 	        pe_rsc_trace(first_rsc,
1306 	                     "Ignoring constraint %d: first (%s for %s) not found",
1307 	                     order->id, order->lh_action_task, first_rsc->id);
1308 	
1309 	    } else if (first_actions == NULL) {
1310 	        char *key = NULL;
1311 	        char *op_type = NULL;
1312 	        guint interval_ms = 0;
1313 	
1314 	        parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
1315 	        key = pcmk__op_key(first_rsc->id, op_type, interval_ms);
1316 	
1317 	        if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
1318 	            && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
1319 	            free(key);
1320 	            pe_rsc_trace(first_rsc,
1321 	                         "Ignoring constraint %d: first (%s for %s) not found",
1322 	                         order->id, order->lh_action_task, first_rsc->id);
1323 	
1324 	        } else if ((first_rsc->fns->state(first_rsc,
1325 	                                          TRUE) == pcmk_role_unpromoted)
1326 	                   && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
1327 	                                   pcmk__str_none)) {
1328 	            free(key);
1329 	            pe_rsc_trace(first_rsc,
1330 	                         "Ignoring constraint %d: first (%s for %s) not found",
1331 	                         order->id, order->lh_action_task, first_rsc->id);
1332 	
1333 	        } else {
1334 	            pe_rsc_trace(first_rsc,
1335 	                         "Creating first (%s for %s) for constraint %d ",
1336 	                         order->lh_action_task, first_rsc->id, order->id);
1337 	            first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
1338 	                                         first_rsc->cluster);
1339 	            first_actions = g_list_prepend(NULL, first_action);
1340 	        }
1341 	
1342 	        free(op_type);
1343 	    }
1344 	
1345 	    if (then_rsc == NULL) {
1346 	        if (order->rh_action == NULL) {
1347 	            pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
1348 	                         order->id);
1349 	            return;
1350 	        }
1351 	        then_rsc = order->rh_action->rsc;
1352 	    }
1353 	    for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
1354 	        first_action = iter->data;
1355 	
1356 	        if (then_rsc == NULL) {
1357 	            order_actions(first_action, order->rh_action, order->flags);
1358 	
1359 	        } else {
1360 	            order_resource_actions_after(first_action, then_rsc, order);
1361 	        }
1362 	    }
1363 	
1364 	    g_list_free(first_actions);
1365 	}
1366 	
1367 	// GFunc to call pcmk__block_colocation_dependents()
1368 	static void
1369 	block_colocation_dependents(gpointer data, gpointer user_data)
1370 	{
1371 	    pcmk__block_colocation_dependents(data);
1372 	}
1373 	
1374 	// GFunc to call pcmk__update_action_for_orderings()
1375 	static void
1376 	update_action_for_orderings(gpointer data, gpointer user_data)
1377 	{
1378 	    pcmk__update_action_for_orderings((pcmk_action_t *) data,
1379 	                                      (pcmk_scheduler_t *) user_data);
1380 	}
1381 	
1382 	/*!
1383 	 * \internal
1384 	 * \brief Apply all ordering constraints
1385 	 *
1386 	 * \param[in,out] sched  Scheduler data
1387 	 */
1388 	void
1389 	pcmk__apply_orderings(pcmk_scheduler_t *sched)
1390 	{
1391 	    crm_trace("Applying ordering constraints");
1392 	
1393 	    /* Ordering constraints need to be processed in the order they were created.
1394 	     * rsc_order_first() and order_resource_actions_after() require the relevant
1395 	     * actions to already exist in some cases, but rsc_order_first() will create
1396 	     * the 'first' action in certain cases. Thus calling rsc_order_first() can
1397 	     * change the behavior of later-created orderings.
1398 	     *
1399 	     * Also, g_list_append() should be avoided for performance reasons, so we
1400 	     * prepend orderings when creating them and reverse the list here.
1401 	     *
1402 	     * @TODO This is brittle and should be carefully redesigned so that the
1403 	     * order of creation doesn't matter, and the reverse becomes unneeded.
1404 	     */
1405 	    sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);
1406 	
1407 	    for (GList *iter = sched->ordering_constraints;
1408 	         iter != NULL; iter = iter->next) {
1409 	
1410 	        pe__ordering_t *order = iter->data;
1411 	        pcmk_resource_t *rsc = order->lh_rsc;
1412 	
(17) Event example_checked: Example 3: "order->lh_rsc" has its value checked in "rsc != NULL".
Also see events: [null_field][dereference][example_checked][example_checked][example_checked]
1413 	        if (rsc != NULL) {
1414 	            rsc_order_first(rsc, order);
1415 	            continue;
1416 	        }
1417 	
1418 	        rsc = order->rh_rsc;
1419 	        if (rsc != NULL) {
1420 	            order_resource_actions_after(order->lh_action, rsc, order);
1421 	
1422 	        } else {
1423 	            crm_trace("Applying ordering constraint %d (non-resource actions)",
1424 	                      order->id);
1425 	            order_actions(order->lh_action, order->rh_action, order->flags);
1426 	        }
1427 	    }
1428 	
1429 	    g_list_foreach(sched->actions, block_colocation_dependents, NULL);
1430 	
1431 	    crm_trace("Ordering probes");
1432 	    pcmk__order_probes(sched);
1433 	
1434 	    crm_trace("Updating %d actions", g_list_length(sched->actions));
1435 	    g_list_foreach(sched->actions, update_action_for_orderings, sched);
1436 	
1437 	    pcmk__disable_invalid_orderings(sched);
1438 	}
1439 	
1440 	/*!
1441 	 * \internal
1442 	 * \brief Order a given action after each action in a given list
1443 	 *
1444 	 * \param[in,out] after  "After" action
1445 	 * \param[in,out] list   List of "before" actions
1446 	 */
1447 	void
1448 	pcmk__order_after_each(pcmk_action_t *after, GList *list)
1449 	{
1450 	    const char *after_desc = (after->task == NULL)? after->uuid : after->task;
1451 	
1452 	    for (GList *iter = list; iter != NULL; iter = iter->next) {
1453 	        pcmk_action_t *before = (pcmk_action_t *) iter->data;
1454 	        const char *before_desc = before->task? before->task : before->uuid;
1455 	
1456 	        crm_debug("Ordering %s on %s before %s on %s",
1457 	                  before_desc, pe__node_name(before->node),
1458 	                  after_desc, pe__node_name(after->node));
1459 	        order_actions(before, after, pcmk__ar_ordered);
1460 	    }
1461 	}
1462 	
1463 	/*!
1464 	 * \internal
1465 	 * \brief Order promotions and demotions for restarts of a clone or bundle
1466 	 *
1467 	 * \param[in,out] rsc  Clone or bundle to order
1468 	 */
1469 	void
1470 	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
1471 	{
1472 	    // Order start and promote after all instances are stopped
1473 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1474 	                                 rsc, PCMK_ACTION_START,
1475 	                                 pcmk__ar_ordered);
1476 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1477 	                                 rsc, PCMK_ACTION_PROMOTE,
1478 	                                 pcmk__ar_ordered);
1479 	
1480 	    // Order stop, start, and promote after all instances are demoted
1481 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1482 	                                 rsc, PCMK_ACTION_STOP,
1483 	                                 pcmk__ar_ordered);
1484 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1485 	                                 rsc, PCMK_ACTION_START,
1486 	                                 pcmk__ar_ordered);
1487 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1488 	                                 rsc, PCMK_ACTION_PROMOTE,
1489 	                                 pcmk__ar_ordered);
1490 	
1491 	    // Order promote after all instances are started
1492 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
1493 	                                 rsc, PCMK_ACTION_PROMOTE,
1494 	                                 pcmk__ar_ordered);
1495 	
1496 	    // Order demote after all instances are demoted
1497 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
1498 	                                 rsc, PCMK_ACTION_DEMOTED,
1499 	                                 pcmk__ar_ordered);
1500 	}
1501