1    	/*
2    	 * Copyright 2004-2025 the Pacemaker project contributors
3    	 *
4    	 * The version control history for this file may have further details.
5    	 *
6    	 * This source code is licensed under the GNU General Public License version 2
7    	 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8    	 */
9    	
10   	#include <crm_internal.h>
11   	
12   	#include <inttypes.h>               // PRIx32
13   	#include <stdbool.h>                // bool, true, false
14   	#include <glib.h>
15   	
16   	#include <crm/crm.h>
17   	#include <pacemaker-internal.h>
18   	#include "libpacemaker_private.h"
19   	
20   	enum pe_order_kind {
21   	    pe_order_kind_optional,
22   	    pe_order_kind_mandatory,
23   	    pe_order_kind_serialize,
24   	};
25   	
26   	enum ordering_symmetry {
27   	    ordering_asymmetric,        // the only relation in an asymmetric ordering
28   	    ordering_symmetric,         // the normal relation in a symmetric ordering
29   	    ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
30   	};
31   	
32   	// @TODO de-functionize this for readability and possibly better log messages
33   	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {                  \
34   	        __rsc = pcmk__find_constraint_resource(scheduler->priv->resources,  \
35   	                                               __name);                     \
36   	        if (__rsc == NULL) {                                                \
37   	            pcmk__config_err("%s: No resource found for %s", __set, __name);\
38   	            return pcmk_rc_unpack_error;                                    \
39   	        }                                                                   \
40   	    } while (0)
41   	
42   	static const char *
43   	invert_action(const char *action)
44   	{
45   	    if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
46   	        return PCMK_ACTION_STOP;
47   	
48   	    } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
49   	        return PCMK_ACTION_START;
50   	
51   	    } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
52   	        return PCMK_ACTION_DEMOTE;
53   	
54   	    } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
55   	        return PCMK_ACTION_PROMOTE;
56   	
57   	    } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
58   	        return PCMK_ACTION_DEMOTED;
59   	
60   	    } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
61   	        return PCMK_ACTION_PROMOTED;
62   	
63   	    } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
64   	        return PCMK_ACTION_STOPPED;
65   	
66   	    } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
67   	        return PCMK_ACTION_RUNNING;
68   	    }
69   	    pcmk__config_warn("Unknown action '%s' specified in order constraint",
70   	                      action);
71   	    return NULL;
72   	}
73   	
74   	static enum pe_order_kind
75   	get_ordering_type(const xmlNode *xml_obj)
76   	{
77   	    enum pe_order_kind kind_e = pe_order_kind_mandatory;
78   	    const char *kind = crm_element_value(xml_obj, PCMK_XA_KIND);
79   	
80   	    if (kind == NULL) {
81   	        const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);
82   	
83   	        kind_e = pe_order_kind_mandatory;
84   	
85   	        if (score) {
86   	            // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
87   	            int score_i = 0;
88   	
89   	            (void) pcmk_parse_score(score, &score_i, 0);
90   	            if (score_i == 0) {
91   	                kind_e = pe_order_kind_optional;
92   	            }
93   	            pcmk__warn_once(pcmk__wo_order_score,
94   	                            "Support for '" PCMK_XA_SCORE "' in "
95   	                            PCMK_XE_RSC_ORDER " is deprecated and will be "
96   	                            "removed in a future release "
97   	                            "(use '" PCMK_XA_KIND "' instead)");
98   	        }
99   	
100  	    } else if (pcmk__str_eq(kind, PCMK_VALUE_MANDATORY, pcmk__str_none)) {
101  	        kind_e = pe_order_kind_mandatory;
102  	
103  	    } else if (pcmk__str_eq(kind, PCMK_VALUE_OPTIONAL, pcmk__str_none)) {
104  	        kind_e = pe_order_kind_optional;
105  	
106  	    } else if (pcmk__str_eq(kind, PCMK_VALUE_SERIALIZE, pcmk__str_none)) {
107  	        kind_e = pe_order_kind_serialize;
108  	
109  	    } else {
110  	        pcmk__config_err("Resetting '" PCMK_XA_KIND "' for constraint %s to "
111  	                         "'" PCMK_VALUE_MANDATORY "' because '%s' is not valid",
112  	                         pcmk__s(pcmk__xe_id(xml_obj), "missing ID"), kind);
113  	    }
114  	    return kind_e;
115  	}
116  	
117  	/*!
118  	 * \internal
119  	 * \brief Get ordering symmetry from XML
120  	 *
121  	 * \param[in] xml_obj               Ordering XML
122  	 * \param[in] parent_kind           Default ordering kind
123  	 * \param[in] parent_symmetrical_s  Parent element's \c PCMK_XA_SYMMETRICAL
124  	 *                                  setting, if any
125  	 *
126  	 * \retval ordering_symmetric   Ordering is symmetric
127  	 * \retval ordering_asymmetric  Ordering is asymmetric
128  	 */
129  	static enum ordering_symmetry
130  	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
131  	                      const char *parent_symmetrical_s)
132  	{
133  	    int rc = pcmk_rc_ok;
134  	    bool symmetric = false;
135  	    enum pe_order_kind kind = parent_kind; // Default to parent's kind
136  	
137  	    // Check ordering XML for explicit kind
138  	    if ((crm_element_value(xml_obj, PCMK_XA_KIND) != NULL)
139  	        || (crm_element_value(xml_obj, PCMK_XA_SCORE) != NULL)) {
140  	        kind = get_ordering_type(xml_obj);
141  	    }
142  	
143  	    // Check ordering XML (and parent) for explicit PCMK_XA_SYMMETRICAL setting
144  	    rc = pcmk__xe_get_bool_attr(xml_obj, PCMK_XA_SYMMETRICAL, &symmetric);
145  	
146  	    if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
147  	        symmetric = crm_is_true(parent_symmetrical_s);
148  	        rc = pcmk_rc_ok;
149  	    }
150  	
151  	    if (rc == pcmk_rc_ok) {
152  	        if (symmetric) {
153  	            if (kind == pe_order_kind_serialize) {
154  	                pcmk__config_warn("Ignoring " PCMK_XA_SYMMETRICAL
155  	                                  " for '%s' because not valid with "
156  	                                  PCMK_XA_KIND " of '" PCMK_VALUE_SERIALIZE "'",
157  	                                  pcmk__xe_id(xml_obj));
158  	            } else {
159  	                return ordering_symmetric;
160  	            }
161  	        }
162  	        return ordering_asymmetric;
163  	    }
164  	
165  	    // Use default symmetry
166  	    if (kind == pe_order_kind_serialize) {
167  	        return ordering_asymmetric;
168  	    }
169  	    return ordering_symmetric;
170  	}
171  	
172  	/*!
173  	 * \internal
174  	 * \brief Get ordering flags appropriate to ordering kind
175  	 *
176  	 * \param[in] kind      Ordering kind
177  	 * \param[in] first     Action name for 'first' action
178  	 * \param[in] symmetry  This ordering's symmetry role
179  	 *
180  	 * \return Minimal ordering flags appropriate to \p kind
181  	 */
182  	static uint32_t
183  	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
184  	                        enum ordering_symmetry symmetry)
185  	{
186  	    uint32_t flags = pcmk__ar_none; // so we trace-log all flags set
187  	
188  	    switch (kind) {
189  	        case pe_order_kind_optional:
190  	            pcmk__set_relation_flags(flags, pcmk__ar_ordered);
191  	            break;
192  	
193  	        case pe_order_kind_serialize:
194  	            /* This flag is not used anywhere directly but means the relation
195  	             * will not match an equality comparison against pcmk__ar_none or
196  	             * pcmk__ar_ordered.
197  	             */
198  	            pcmk__set_relation_flags(flags, pcmk__ar_serialize);
199  	            break;
200  	
201  	        case pe_order_kind_mandatory:
202  	            pcmk__set_relation_flags(flags, pcmk__ar_ordered);
203  	            switch (symmetry) {
204  	                case ordering_asymmetric:
205  	                    pcmk__set_relation_flags(flags, pcmk__ar_asymmetric);
206  	                    break;
207  	
208  	                case ordering_symmetric:
209  	                    pcmk__set_relation_flags(flags,
210  	                                             pcmk__ar_first_implies_then);
211  	                    if (pcmk__is_up_action(first)) {
212  	                        pcmk__set_relation_flags(flags,
213  	                                                 pcmk__ar_unrunnable_first_blocks);
214  	                    }
215  	                    break;
216  	
217  	                case ordering_symmetric_inverse:
218  	                    pcmk__set_relation_flags(flags,
219  	                                             pcmk__ar_then_implies_first);
220  	                    break;
221  	            }
222  	            break;
223  	    }
224  	    return flags;
225  	}
226  	
227  	/*!
228  	 * \internal
229  	 * \brief Find resource corresponding to ID specified in ordering
230  	 *
231  	 * \param[in] xml            Ordering XML
232  	 * \param[in] resource_attr  XML attribute name for resource ID
233  	 * \param[in] scheduler      Scheduler data
234  	 *
235  	 * \return Resource corresponding to \p id, or NULL if none
236  	 */
237  	static pcmk_resource_t *
238  	get_ordering_resource(const xmlNode *xml, const char *resource_attr,
239  	                      const pcmk_scheduler_t *scheduler)
240  	{
241  	    pcmk_resource_t *rsc = NULL;
242  	    const char *rsc_id = crm_element_value(xml, resource_attr);
243  	
244  	    if (rsc_id == NULL) {
245  	        pcmk__config_err("Ignoring constraint '%s' without %s",
246  	                         pcmk__xe_id(xml), resource_attr);
247  	        return NULL;
248  	    }
249  	
250  	    rsc = pcmk__find_constraint_resource(scheduler->priv->resources, rsc_id);
251  	    if (rsc == NULL) {
252  	        pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
253  	                         "does not exist", pcmk__xe_id(xml), rsc_id);
254  	        return NULL;
255  	    }
256  	
257  	    return rsc;
258  	}
259  	
260  	/*!
261  	 * \internal
262  	 * \brief Determine minimum number of 'first' instances required in ordering
263  	 *
264  	 * \param[in] rsc  'First' resource in ordering
265  	 * \param[in] xml  Ordering XML
266  	 *
267  	 * \return Minimum 'first' instances required (or 0 if not applicable)
268  	 */
269  	static int
270  	get_minimum_first_instances(const pcmk_resource_t *rsc, const xmlNode *xml)
271  	{
272  	    const char *clone_min = NULL;
273  	    bool require_all = false;
274  	
275  	    if (!pcmk__is_clone(rsc)) {
276  	        return 0;
277  	    }
278  	
279  	    clone_min = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CLONE_MIN);
280  	    if (clone_min != NULL) {
281  	        int clone_min_int = 0;
282  	
283  	        pcmk__scan_min_int(clone_min, &clone_min_int, 0);
284  	        return clone_min_int;
285  	    }
286  	
287  	    /* @COMPAT 1.1.13:
288  	     * PCMK_XA_REQUIRE_ALL=PCMK_VALUE_FALSE is deprecated equivalent of
289  	     * PCMK_META_CLONE_MIN=1
290  	     */
291  	    if (pcmk__xe_get_bool_attr(xml, PCMK_XA_REQUIRE_ALL,
292  	                               &require_all) != ENODATA) {
293  	        pcmk__warn_once(pcmk__wo_require_all,
294  	                        "Support for " PCMK_XA_REQUIRE_ALL " in ordering "
295  	                        "constraints is deprecated and will be removed in a "
296  	                        "future release (use " PCMK_META_CLONE_MIN " clone "
297  	                        "meta-attribute instead)");
298  	        if (!require_all) {
299  	            return 1;
300  	        }
301  	    }
302  	
303  	    return 0;
304  	}
305  	
306  	/*!
307  	 * \internal
308  	 * \brief Create orderings for a constraint with \c PCMK_META_CLONE_MIN > 0
309  	 *
310  	 * \param[in]     id            Ordering ID
311  	 * \param[in,out] rsc_first     'First' resource in ordering (a clone)
312  	 * \param[in]     action_first  'First' action in ordering
313  	 * \param[in]     rsc_then      'Then' resource in ordering
314  	 * \param[in]     action_then   'Then' action in ordering
315  	 * \param[in]     flags         Ordering flags
316  	 * \param[in]     clone_min     Minimum required instances of 'first'
317  	 */
318  	static void
319  	clone_min_ordering(const char *id,
320  	                   pcmk_resource_t *rsc_first, const char *action_first,
321  	                   pcmk_resource_t *rsc_then, const char *action_then,
322  	                   uint32_t flags, int clone_min)
323  	{
324  	    // Create a pseudo-action for when the minimum instances are active
325  	    char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
326  	    pcmk_action_t *clone_min_met = get_pseudo_op(task,
327  	                                                 rsc_first->priv->scheduler);
328  	
329  	    free(task);
330  	
331  	    /* Require the pseudo-action to have the required number of actions to be
332  	     * considered runnable before allowing the pseudo-action to be runnable.
333  	     */
334  	    clone_min_met->required_runnable_before = clone_min;
335  	
336  	    // Order the actions for each clone instance before the pseudo-action
337  	    for (GList *iter = rsc_first->priv->children;
338  	         iter != NULL; iter = iter->next) {
339  	
340  	        pcmk_resource_t *child = iter->data;
341  	
342  	        pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
343  	                           NULL, NULL, NULL, clone_min_met,
344  	                           pcmk__ar_min_runnable
345  	                           |pcmk__ar_first_implies_then_graphed,
346  	                           rsc_first->priv->scheduler);
347  	    }
348  	
349  	    // Order "then" action after the pseudo-action (if runnable)
350  	    pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
351  	                       pcmk__op_key(rsc_then->id, action_then, 0),
352  	                       NULL, flags|pcmk__ar_unrunnable_first_blocks,
353  	                       rsc_first->priv->scheduler);
354  	}
355  	
356  	/*!
357  	 * \internal
358  	 * \brief Create new ordering for inverse of symmetric constraint
359  	 *
360  	 * \param[in]     id            Ordering ID (for logging only)
361  	 * \param[in]     kind          Ordering kind
362  	 * \param[in]     rsc_first     'First' resource in ordering (a clone)
363  	 * \param[in]     action_first  'First' action in ordering
364  	 * \param[in,out] rsc_then      'Then' resource in ordering
365  	 * \param[in]     action_then   'Then' action in ordering
366  	 */
367  	static void
368  	inverse_ordering(const char *id, enum pe_order_kind kind,
369  	                 pcmk_resource_t *rsc_first, const char *action_first,
370  	                 pcmk_resource_t *rsc_then, const char *action_then)
371  	{
372  	    uint32_t flags;
373  	    const char *inverted_first = invert_action(action_first);
374  	    const char *inverted_then = invert_action(action_then);
375  	
376  	    if ((inverted_then == NULL) || (inverted_first == NULL)) {
377  	        pcmk__config_warn("Cannot invert constraint '%s' "
378  	                          "(please specify inverse manually)", id);
379  	        return;
380  	    }
381  	
382  	    // Order inverted actions
383  	    flags = ordering_flags_for_kind(kind, inverted_first,
384  	                                    ordering_symmetric_inverse);
385  	    pcmk__order_resource_actions(rsc_then, inverted_then,
386  	                                 rsc_first, inverted_first, flags);
387  	}
388  	
389  	static void
390  	unpack_simple_rsc_order(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
391  	{
392  	    pcmk_resource_t *rsc_then = NULL;
393  	    pcmk_resource_t *rsc_first = NULL;
394  	    int min_required_before = 0;
395  	    enum pe_order_kind kind = pe_order_kind_mandatory;
396  	    uint32_t flags = pcmk__ar_none;
397  	    enum ordering_symmetry symmetry;
398  	
399  	    const char *action_then = NULL;
400  	    const char *action_first = NULL;
401  	    const char *id = NULL;
402  	
403  	    CRM_CHECK(xml_obj != NULL, return);
404  	
405  	    id = crm_element_value(xml_obj, PCMK_XA_ID);
406  	    if (id == NULL) {
407  	        pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
408  	                         xml_obj->name);
409  	        return;
410  	    }
411  	
412  	    rsc_first = get_ordering_resource(xml_obj, PCMK_XA_FIRST, scheduler);
413  	    if (rsc_first == NULL) {
414  	        return;
415  	    }
416  	
417  	    rsc_then = get_ordering_resource(xml_obj, PCMK_XA_THEN, scheduler);
418  	    if (rsc_then == NULL) {
419  	        return;
420  	    }
421  	
422  	    action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
423  	    if (action_first == NULL) {
424  	        action_first = PCMK_ACTION_START;
425  	    }
426  	
427  	    action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);
428  	    if (action_then == NULL) {
429  	        action_then = action_first;
430  	    }
431  	
432  	    kind = get_ordering_type(xml_obj);
433  	
434  	    symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
435  	    flags = ordering_flags_for_kind(kind, action_first, symmetry);
436  	
437  	    /* If there is a minimum number of instances that must be runnable before
438  	     * the 'then' action is runnable, we use a pseudo-action for convenience:
439  	     * minimum number of clone instances have runnable actions ->
440  	     * pseudo-action is runnable -> dependency is runnable.
441  	     */
442  	    min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
443  	    if (min_required_before > 0) {
444  	        clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
445  	                           flags, min_required_before);
446  	    } else {
447  	        pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
448  	                                     action_then, flags);
449  	    }
450  	
451  	    if (symmetry == ordering_symmetric) {
452  	        inverse_ordering(id, kind, rsc_first, action_first,
453  	                         rsc_then, action_then);
454  	    }
455  	}
456  	
457  	/*!
458  	 * \internal
459  	 * \brief Create a new ordering between two actions
460  	 *
461  	 * \param[in,out] first_rsc          Resource for 'first' action (if NULL and
462  	 *                                   \p first_action is a resource action, that
463  	 *                                   resource will be used)
464  	 * \param[in,out] first_action_task  Action key for 'first' action (if NULL and
465  	 *                                   \p first_action is not NULL, its UUID will
466  	 *                                   be used)
467  	 * \param[in,out] first_action       'first' action (if NULL, \p first_rsc and
468  	 *                                   \p first_action_task must be set)
469  	 *
470  	 * \param[in]     then_rsc           Resource for 'then' action (if NULL and
471  	 *                                   \p then_action is a resource action, that
472  	 *                                   resource will be used)
473  	 * \param[in,out] then_action_task   Action key for 'then' action (if NULL and
474  	 *                                   \p then_action is not NULL, its UUID will
475  	 *                                   be used)
476  	 * \param[in]     then_action        'then' action (if NULL, \p then_rsc and
477  	 *                                   \p then_action_task must be set)
478  	 *
479  	 * \param[in]     flags              Group of enum pcmk__action_relation_flags
480  	 * \param[in,out] sched              Scheduler data to add ordering to
481  	 *
482  	 * \note This function takes ownership of first_action_task and
483  	 *       then_action_task, which do not need to be freed by the caller.
484  	 */
485  	void
486  	pcmk__new_ordering(pcmk_resource_t *first_rsc, char *first_action_task,
487  	                   pcmk_action_t *first_action, pcmk_resource_t *then_rsc,
488  	                   char *then_action_task, pcmk_action_t *then_action,
489  	                   uint32_t flags, pcmk_scheduler_t *sched)
490  	{
491  	    pcmk__action_relation_t *order = NULL;
492  	
493  	    // One of action or resource must be specified for each side
494  	    CRM_CHECK(((first_action != NULL) || (first_rsc != NULL))
495  	              && ((then_action != NULL) || (then_rsc != NULL)),
496  	              free(first_action_task); free(then_action_task); return);
497  	
498  	    if ((first_rsc == NULL) && (first_action != NULL)) {
499  	        first_rsc = first_action->rsc;
500  	    }
501  	    if ((then_rsc == NULL) && (then_action != NULL)) {
502  	        then_rsc = then_action->rsc;
503  	    }
504  	
505  	    order = pcmk__assert_alloc(1, sizeof(pcmk__action_relation_t));
506  	
507  	    order->id = sched->priv->next_ordering_id++;
508  	    order->flags = flags;
509  	    order->rsc1 = first_rsc;
510  	    order->rsc2 = then_rsc;
511  	    order->action1 = first_action;
512  	    order->action2 = then_action;
513  	    order->task1 = first_action_task;
514  	    order->task2 = then_action_task;
515  	
516  	    if ((order->task1 == NULL) && (first_action != NULL)) {
517  	        order->task1 = strdup(first_action->uuid);
518  	    }
519  	
520  	    if ((order->task2 == NULL) && (then_action != NULL)) {
521  	        order->task2 = strdup(then_action->uuid);
522  	    }
523  	
(16) Event example_checked: Example 2: "first_action->rsc" has its value checked in "order->rsc1 == NULL".
Also see events: [null_field][dereference][example_checked][example_checked][example_checked]
524  	    if ((order->rsc1 == NULL) && (first_action != NULL)) {
525  	        order->rsc1 = first_action->rsc;
526  	    }
527  	
528  	    if ((order->rsc2 == NULL) && (then_action != NULL)) {
529  	        order->rsc2 = then_action->rsc;
530  	    }
531  	
532  	    pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s",
533  	                    (sched->priv->next_ordering_id - 1),
534  	                    pcmk__s(order->task1, "an underspecified action"),
535  	                    pcmk__s(order->task2, "an underspecified action"));
536  	
537  	    sched->priv->ordering_constraints =
538  	        g_list_prepend(sched->priv->ordering_constraints, order);
539  	    pcmk__order_migration_equivalents(order);
540  	}
541  	
542  	/*!
543  	 * \brief Unpack a set in an ordering constraint
544  	 *
545  	 * \param[in]     set                   Set XML to unpack
546  	 * \param[in]     parent_kind           \c PCMK_XE_RSC_ORDER XML \c PCMK_XA_KIND
547  	 *                                      attribute
548  	 * \param[in]     parent_symmetrical_s  \c PCMK_XE_RSC_ORDER XML
549  	 *                                      \c PCMK_XA_SYMMETRICAL attribute
550  	 * \param[in,out] scheduler             Scheduler data
551  	 *
552  	 * \return Standard Pacemaker return code
553  	 */
554  	static int
555  	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
556  	                 const char *parent_symmetrical_s, pcmk_scheduler_t *scheduler)
557  	{
558  	    GList *set_iter = NULL;
559  	    GList *resources = NULL;
560  	
561  	    pcmk_resource_t *last = NULL;
562  	    pcmk_resource_t *resource = NULL;
563  	
564  	    int local_kind = parent_kind;
565  	    bool sequential = false;
566  	    uint32_t flags = pcmk__ar_ordered;
567  	    enum ordering_symmetry symmetry;
568  	
569  	    char *key = NULL;
570  	    const char *id = pcmk__xe_id(set);
571  	    const char *action = crm_element_value(set, PCMK_XA_ACTION);
572  	    const char *sequential_s = crm_element_value(set, PCMK_XA_SEQUENTIAL);
573  	    const char *kind_s = crm_element_value(set, PCMK_XA_KIND);
574  	
575  	    if (action == NULL) {
576  	        action = PCMK_ACTION_START;
577  	    }
578  	
579  	    if (kind_s) {
580  	        local_kind = get_ordering_type(set);
581  	    }
582  	    if (sequential_s == NULL) {
583  	        sequential_s = "1";
584  	    }
585  	
586  	    sequential = crm_is_true(sequential_s);
587  	
588  	    symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
589  	    flags = ordering_flags_for_kind(local_kind, action, symmetry);
590  	
591  	    for (const xmlNode *xml_rsc = pcmk__xe_first_child(set,
592  	                                                       PCMK_XE_RESOURCE_REF,
593  	                                                       NULL, NULL);
594  	         xml_rsc != NULL;
595  	         xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
596  	
597  	        EXPAND_CONSTRAINT_IDREF(id, resource, pcmk__xe_id(xml_rsc));
598  	        resources = g_list_append(resources, resource);
599  	    }
600  	
601  	    if (pcmk__list_of_1(resources)) {
602  	        crm_trace("Single set: %s", id);
603  	        goto done;
604  	    }
605  	
606  	    set_iter = resources;
607  	    while (set_iter != NULL) {
608  	        resource = (pcmk_resource_t *) set_iter->data;
609  	        set_iter = set_iter->next;
610  	
611  	        key = pcmk__op_key(resource->id, action, 0);
612  	
613  	        if (local_kind == pe_order_kind_serialize) {
614  	            /* Serialize before everything that comes after */
615  	
616  	            for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
617  	                pcmk_resource_t *then_rsc = iter->data;
618  	                char *then_key = pcmk__op_key(then_rsc->id, action, 0);
619  	
620  	                pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
621  	                                   then_key, NULL, flags, scheduler);
622  	            }
623  	
624  	        } else if (sequential) {
625  	            if (last != NULL) {
626  	                pcmk__order_resource_actions(last, action, resource, action,
627  	                                             flags);
628  	            }
629  	            last = resource;
630  	        }
631  	        free(key);
632  	    }
633  	
634  	    if (symmetry == ordering_asymmetric) {
635  	        goto done;
636  	    }
637  	
638  	    last = NULL;
639  	    action = invert_action(action);
640  	
641  	    flags = ordering_flags_for_kind(local_kind, action,
642  	                                    ordering_symmetric_inverse);
643  	
644  	    set_iter = resources;
645  	    while (set_iter != NULL) {
646  	        resource = (pcmk_resource_t *) set_iter->data;
647  	        set_iter = set_iter->next;
648  	
649  	        if (sequential) {
650  	            if (last != NULL) {
651  	                pcmk__order_resource_actions(resource, action, last, action,
652  	                                             flags);
653  	            }
654  	            last = resource;
655  	        }
656  	    }
657  	
658  	  done:
659  	    g_list_free(resources);
660  	    return pcmk_rc_ok;
661  	}
662  	
663  	/*!
664  	 * \brief Order two resource sets relative to each other
665  	 *
666  	 * \param[in]     id         Ordering ID (for logging)
667  	 * \param[in]     set1       First listed set
668  	 * \param[in]     set2       Second listed set
669  	 * \param[in]     kind       Ordering kind
670  	 * \param[in,out] scheduler  Scheduler data
671  	 * \param[in]     symmetry   Which ordering symmetry applies to this relation
672  	 *
673  	 * \return Standard Pacemaker return code
674  	 */
675  	static int
676  	order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2,
677  	               enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
678  	               enum ordering_symmetry symmetry)
679  	{
680  	
681  	    const xmlNode *xml_rsc = NULL;
682  	    const xmlNode *xml_rsc_2 = NULL;
683  	
684  	    pcmk_resource_t *rsc_1 = NULL;
685  	    pcmk_resource_t *rsc_2 = NULL;
686  	
687  	    const char *action_1 = crm_element_value(set1, PCMK_XA_ACTION);
688  	    const char *action_2 = crm_element_value(set2, PCMK_XA_ACTION);
689  	
690  	    uint32_t flags = pcmk__ar_none;
691  	
692  	    bool require_all = true;
693  	
694  	    (void) pcmk__xe_get_bool_attr(set1, PCMK_XA_REQUIRE_ALL, &require_all);
695  	
696  	    if (action_1 == NULL) {
697  	        action_1 = PCMK_ACTION_START;
698  	    }
699  	
700  	    if (action_2 == NULL) {
701  	        action_2 = PCMK_ACTION_START;
702  	    }
703  	
704  	    if (symmetry == ordering_symmetric_inverse) {
705  	        action_1 = invert_action(action_1);
706  	        action_2 = invert_action(action_2);
707  	    }
708  	
709  	    if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
710  	        || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
711  	        /* Assuming: A -> ( B || C) -> D
712  	         * The one-or-more logic only applies during the start/promote phase.
713  	         * During shutdown neither B nor can shutdown until D is down, so simply
714  	         * turn require_all back on.
715  	         */
716  	        require_all = true;
717  	    }
718  	
719  	    flags = ordering_flags_for_kind(kind, action_1, symmetry);
720  	
721  	    /* If we have an unordered set1, whether it is sequential or not is
722  	     * irrelevant in regards to set2.
723  	     */
724  	    if (!require_all) {
725  	        char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s",
726  	                                       pcmk__xe_id(set1));
727  	        pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);
728  	
729  	        free(task);
730  	        unordered_action->required_runnable_before = 1;
731  	
732  	        for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
733  	                                            NULL);
734  	             xml_rsc != NULL;
735  	             xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
736  	
737  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
738  	
739  	            /* Add an ordering constraint between every element in set1 and the
740  	             * pseudo action. If any action in set1 is runnable the pseudo
741  	             * action will be runnable.
742  	             */
743  	            pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
744  	                               NULL, NULL, NULL, unordered_action,
745  	                               pcmk__ar_min_runnable
746  	                               |pcmk__ar_first_implies_then_graphed,
747  	                               scheduler);
748  	        }
749  	        for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
750  	                                              NULL);
751  	             xml_rsc_2 != NULL;
752  	             xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) {
753  	
754  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));
755  	
756  	            /* Add an ordering constraint between the pseudo-action and every
757  	             * element in set2. If the pseudo-action is runnable, every action
758  	             * in set2 will be runnable.
759  	             */
760  	            pcmk__new_ordering(NULL, NULL, unordered_action,
761  	                               rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
762  	                               NULL, flags|pcmk__ar_unrunnable_first_blocks,
763  	                               scheduler);
764  	        }
765  	
766  	        return pcmk_rc_ok;
767  	    }
768  	
769  	    if (pcmk__xe_attr_is_true(set1, PCMK_XA_SEQUENTIAL)) {
770  	        if (symmetry == ordering_symmetric_inverse) {
771  	            // Get the first one
772  	            xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
773  	                                           NULL);
774  	            if (xml_rsc != NULL) {
775  	                EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
776  	            }
777  	
778  	        } else {
779  	            // Get the last one
780  	            const char *rid = NULL;
781  	
782  	            for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF,
783  	                                                NULL, NULL);
784  	                 xml_rsc != NULL;
785  	                 xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
786  	
787  	                rid = pcmk__xe_id(xml_rsc);
788  	            }
789  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
790  	        }
791  	    }
792  	
793  	    if (pcmk__xe_attr_is_true(set2, PCMK_XA_SEQUENTIAL)) {
794  	        if (symmetry == ordering_symmetric_inverse) {
795  	            // Get the last one
796  	            const char *rid = NULL;
797  	
798  	            for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF,
799  	                                                NULL, NULL);
800  	                 xml_rsc != NULL;
801  	                 xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
802  	
803  	                rid = pcmk__xe_id(xml_rsc);
804  	            }
805  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
806  	
807  	        } else {
808  	            // Get the first one
809  	            xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
810  	                                           NULL);
811  	            if (xml_rsc != NULL) {
812  	                EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
813  	            }
814  	        }
815  	    }
816  	
817  	    if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
818  	        pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);
819  	
820  	    } else if (rsc_1 != NULL) {
821  	        for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
822  	                                            NULL);
823  	             xml_rsc != NULL;
824  	             xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
825  	
826  	            EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
827  	            pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
828  	                                         flags);
829  	        }
830  	
831  	    } else if (rsc_2 != NULL) {
832  	        for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
833  	                                            NULL);
834  	             xml_rsc != NULL;
835  	             xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
836  	
837  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
838  	            pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
839  	                                         flags);
840  	        }
841  	
842  	    } else {
843  	        for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
844  	                                            NULL);
845  	             xml_rsc != NULL;
846  	             xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
847  	
848  	            EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
849  	
850  	            for (xmlNode *xml_rsc_2 = pcmk__xe_first_child(set2,
851  	                                                           PCMK_XE_RESOURCE_REF,
852  	                                                           NULL, NULL);
853  	                 xml_rsc_2 != NULL;
854  	                 xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) {
855  	
856  	                EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));
857  	                pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
858  	                                             action_2, flags);
859  	            }
860  	        }
861  	    }
862  	
863  	    return pcmk_rc_ok;
864  	}
865  	
866  	/*!
867  	 * \internal
868  	 * \brief If an ordering constraint uses resource tags, expand them
869  	 *
870  	 * \param[in,out] xml_obj       Ordering constraint XML
871  	 * \param[out]    expanded_xml  Equivalent XML with tags expanded
872  	 * \param[in]     scheduler     Scheduler data
873  	 *
874  	 * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
875  	 *         and pcmk_rc_unpack_error on invalid configuration)
876  	 */
877  	static int
878  	unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml,
879  	                  const pcmk_scheduler_t *scheduler)
880  	{
881  	    const char *id_first = NULL;
882  	    const char *id_then = NULL;
883  	    const char *action_first = NULL;
884  	    const char *action_then = NULL;
885  	
886  	    pcmk_resource_t *rsc_first = NULL;
887  	    pcmk_resource_t *rsc_then = NULL;
888  	    pcmk__idref_t *tag_first = NULL;
889  	    pcmk__idref_t *tag_then = NULL;
890  	
891  	    xmlNode *rsc_set_first = NULL;
892  	    xmlNode *rsc_set_then = NULL;
893  	    bool any_sets = false;
894  	
895  	    // Check whether there are any resource sets with template or tag references
896  	    *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
897  	    if (*expanded_xml != NULL) {
898  	        crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
899  	        return pcmk_rc_ok;
900  	    }
901  	
902  	    id_first = crm_element_value(xml_obj, PCMK_XA_FIRST);
903  	    id_then = crm_element_value(xml_obj, PCMK_XA_THEN);
904  	    if ((id_first == NULL) || (id_then == NULL)) {
905  	        return pcmk_rc_ok;
906  	    }
907  	
908  	    if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
909  	                                     &tag_first)) {
910  	        pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
911  	                         "valid resource or tag",
912  	                         pcmk__xe_id(xml_obj), id_first);
913  	        return pcmk_rc_unpack_error;
914  	    }
915  	
916  	    if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
917  	                                     &tag_then)) {
918  	        pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
919  	                         "valid resource or tag",
920  	                         pcmk__xe_id(xml_obj), id_then);
921  	        return pcmk_rc_unpack_error;
922  	    }
923  	
924  	    if ((rsc_first != NULL) && (rsc_then != NULL)) {
925  	        // Neither side references a template or tag
926  	        return pcmk_rc_ok;
927  	    }
928  	
929  	    action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
930  	    action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);
931  	
932  	    *expanded_xml = pcmk__xml_copy(NULL, xml_obj);
933  	
934  	    /* Convert template/tag reference in PCMK_XA_FIRST into constraint
935  	     * PCMK_XE_RESOURCE_SET
936  	     */
937  	    if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, PCMK_XA_FIRST, true,
938  	                          scheduler)) {
939  	        pcmk__xml_free(*expanded_xml);
940  	        *expanded_xml = NULL;
941  	        return pcmk_rc_unpack_error;
942  	    }
943  	
944  	    if (rsc_set_first != NULL) {
945  	        if (action_first != NULL) {
946  	            /* Move PCMK_XA_FIRST_ACTION into converted PCMK_XE_RESOURCE_SET as
947  	             * PCMK_XA_ACTION
948  	             */
949  	            crm_xml_add(rsc_set_first, PCMK_XA_ACTION, action_first);
950  	            pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_FIRST_ACTION);
951  	        }
952  	        any_sets = true;
953  	    }
954  	
955  	    /* Convert template/tag reference in PCMK_XA_THEN into constraint
956  	     * PCMK_XE_RESOURCE_SET
957  	     */
958  	    if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, PCMK_XA_THEN, true,
959  	                          scheduler)) {
960  	        pcmk__xml_free(*expanded_xml);
961  	        *expanded_xml = NULL;
962  	        return pcmk_rc_unpack_error;
963  	    }
964  	
965  	    if (rsc_set_then != NULL) {
966  	        if (action_then != NULL) {
967  	            /* Move PCMK_XA_THEN_ACTION into converted PCMK_XE_RESOURCE_SET as
968  	             * PCMK_XA_ACTION
969  	             */
970  	            crm_xml_add(rsc_set_then, PCMK_XA_ACTION, action_then);
971  	            pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_THEN_ACTION);
972  	        }
973  	        any_sets = true;
974  	    }
975  	
976  	    if (any_sets) {
977  	        crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
978  	    } else {
979  	        pcmk__xml_free(*expanded_xml);
980  	        *expanded_xml = NULL;
981  	    }
982  	
983  	    return pcmk_rc_ok;
984  	}
985  	
986  	/*!
987  	 * \internal
988  	 * \brief Unpack ordering constraint XML
989  	 *
990  	 * \param[in,out] xml_obj    Ordering constraint XML to unpack
991  	 * \param[in,out] scheduler  Scheduler data
992  	 */
993  	void
994  	pcmk__unpack_ordering(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
995  	{
996  	    xmlNode *set = NULL;
997  	    xmlNode *last = NULL;
998  	
999  	    xmlNode *orig_xml = NULL;
1000 	    xmlNode *expanded_xml = NULL;
1001 	
1002 	    const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
1003 	    const char *invert = crm_element_value(xml_obj, PCMK_XA_SYMMETRICAL);
1004 	    enum pe_order_kind kind = get_ordering_type(xml_obj);
1005 	
1006 	    enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
1007 	                                                            NULL);
1008 	
1009 	    // Expand any resource tags in the constraint XML
1010 	    if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
1011 	        return;
1012 	    }
1013 	    if (expanded_xml != NULL) {
1014 	        orig_xml = xml_obj;
1015 	        xml_obj = expanded_xml;
1016 	    }
1017 	
1018 	    // If the constraint has resource sets, unpack them
1019 	    for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
1020 	         set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) {
1021 	
1022 	        set = pcmk__xe_resolve_idref(set, scheduler->input);
1023 	        if ((set == NULL) // Configuration error, message already logged
1024 	            || (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {
1025 	
1026 	            if (expanded_xml != NULL) {
1027 	                pcmk__xml_free(expanded_xml);
1028 	            }
1029 	            return;
1030 	        }
1031 	
1032 	        if (last != NULL) {
1033 	
1034 	            if (order_rsc_sets(id, last, set, kind, scheduler,
1035 	                               symmetry) != pcmk_rc_ok) {
1036 	                if (expanded_xml != NULL) {
1037 	                    pcmk__xml_free(expanded_xml);
1038 	                }
1039 	                return;
1040 	            }
1041 	
1042 	            if ((symmetry == ordering_symmetric)
1043 	                && (order_rsc_sets(id, set, last, kind, scheduler,
1044 	                                   ordering_symmetric_inverse) != pcmk_rc_ok)) {
1045 	                if (expanded_xml != NULL) {
1046 	                    pcmk__xml_free(expanded_xml);
1047 	                }
1048 	                return;
1049 	            }
1050 	
1051 	        }
1052 	        last = set;
1053 	    }
1054 	
1055 	    if (expanded_xml) {
1056 	        pcmk__xml_free(expanded_xml);
1057 	        xml_obj = orig_xml;
1058 	    }
1059 	
1060 	    // If the constraint has no resource sets, unpack it as a simple ordering
1061 	    if (last == NULL) {
1062 	        return unpack_simple_rsc_order(xml_obj, scheduler);
1063 	    }
1064 	}
1065 	
1066 	static bool
1067 	ordering_is_invalid(pcmk_action_t *action, pcmk__related_action_t *input)
1068 	{
1069 	    /* Prevent user-defined ordering constraints between resources
1070 	     * running in a guest node and the resource that defines that node.
1071 	     */
1072 	    if (!pcmk_is_set(input->flags, pcmk__ar_guest_allowed)
1073 	        && (input->action->rsc != NULL)
1074 	        && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {
1075 	
1076 	        pcmk__config_warn("Invalid ordering constraint between %s and %s",
1077 	                          input->action->rsc->id, action->rsc->id);
1078 	        return true;
1079 	    }
1080 	
1081 	    /* If there's an order like
1082 	     * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
1083 	     *
1084 	     * then rscA is being migrated from node1 to node2, while rscB is being
1085 	     * migrated from node2 to node1. If there would be a graph loop,
1086 	     * break the order "load_stopped_node2" -> "rscA_migrate_to node1".
1087 	     */
1088 	    if ((input->flags == pcmk__ar_if_on_same_node_or_target)
1089 	        && (action->rsc != NULL)
1090 	        && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
1091 	        && pcmk__graph_has_loop(action, action, input)) {
1092 	        return true;
1093 	    }
1094 	
1095 	    return false;
1096 	}
1097 	
1098 	void
1099 	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
1100 	{
1101 	    for (GList *iter = scheduler->priv->actions;
1102 	         iter != NULL; iter = iter->next) {
1103 	
1104 	        pcmk_action_t *action = (pcmk_action_t *) iter->data;
1105 	        pcmk__related_action_t *input = NULL;
1106 	
1107 	        for (GList *input_iter = action->actions_before;
1108 	             input_iter != NULL; input_iter = input_iter->next) {
1109 	
1110 	            input = input_iter->data;
1111 	            if (ordering_is_invalid(action, input)) {
1112 	                input->flags = pcmk__ar_none;
1113 	            }
1114 	        }
1115 	    }
1116 	}
1117 	
1118 	/*!
1119 	 * \internal
1120 	 * \brief Order stops on a node before the node's shutdown
1121 	 *
1122 	 * \param[in,out] node         Node being shut down
1123 	 * \param[in]     shutdown_op  Shutdown action for node
1124 	 */
1125 	void
1126 	pcmk__order_stops_before_shutdown(pcmk_node_t *node, pcmk_action_t *shutdown_op)
1127 	{
1128 	    for (GList *iter = node->priv->scheduler->priv->actions;
1129 	         iter != NULL; iter = iter->next) {
1130 	
1131 	        pcmk_action_t *action = (pcmk_action_t *) iter->data;
1132 	
1133 	        // Only stops on the node shutting down are relevant
1134 	        if (!pcmk__same_node(action->node, node)
1135 	            || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
1136 	            continue;
1137 	        }
1138 	
1139 	        // Resources and nodes in maintenance mode won't be touched
1140 	
1141 	        if (pcmk_is_set(action->rsc->flags, pcmk__rsc_maintenance)) {
1142 	            pcmk__rsc_trace(action->rsc,
1143 	                            "Not ordering %s before shutdown of %s because "
1144 	                            "resource in maintenance mode",
1145 	                            action->uuid, pcmk__node_name(node));
1146 	            continue;
1147 	
1148 	        } else if (node->details->maintenance) {
1149 	            pcmk__rsc_trace(action->rsc,
1150 	                            "Not ordering %s before shutdown of %s because "
1151 	                            "node in maintenance mode",
1152 	                            action->uuid, pcmk__node_name(node));
1153 	            continue;
1154 	        }
1155 	
1156 	        /* Don't touch a resource that is unmanaged or blocked, to avoid
1157 	         * blocking the shutdown (though if another action depends on this one,
1158 	         * we may still end up blocking)
1159 	         *
1160 	         * @TODO This "if" looks wrong, create a regression test for these cases
1161 	         */
1162 	        if (!pcmk_any_flags_set(action->rsc->flags,
1163 	                                pcmk__rsc_managed|pcmk__rsc_blocked)) {
1164 	            pcmk__rsc_trace(action->rsc,
1165 	                            "Not ordering %s before shutdown of %s because "
1166 	                            "resource is unmanaged or blocked",
1167 	                            action->uuid, pcmk__node_name(node));
1168 	            continue;
1169 	        }
1170 	
1171 	        pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
1172 	                        action->uuid, pcmk__node_name(node));
1173 	        pcmk__clear_action_flags(action, pcmk__action_optional);
1174 	        pcmk__new_ordering(action->rsc, NULL, action, NULL,
1175 	                           strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
1176 	                           pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks,
1177 	                           node->priv->scheduler);
1178 	    }
1179 	}
1180 	
1181 	/*!
1182 	 * \brief Find resource actions matching directly or as child
1183 	 *
1184 	 * \param[in] rsc           Resource to check
1185 	 * \param[in] original_key  Action key to search for (possibly referencing
1186 	 *                          parent of \rsc)
1187 	 *
1188 	 * \return Newly allocated list of matching actions
1189 	 * \note It is the caller's responsibility to free the result with g_list_free()
1190 	 */
1191 	static GList *
1192 	find_actions_by_task(const pcmk_resource_t *rsc, const char *original_key)
1193 	{
1194 	    // Search under given task key directly
1195 	    GList *list = find_actions(rsc->priv->actions, original_key, NULL);
1196 	
1197 	    if (list == NULL) {
1198 	        // Search again using this resource's ID
1199 	        char *key = NULL;
1200 	        char *task = NULL;
1201 	        guint interval_ms = 0;
1202 	
1203 	        CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms),
1204 	                  return NULL);
1205 	        key = pcmk__op_key(rsc->id, task, interval_ms);
1206 	        list = find_actions(rsc->priv->actions, key, NULL);
1207 	        free(key);
1208 	        free(task);
1209 	    }
1210 	    return list;
1211 	}
1212 	
1213 	/*!
1214 	 * \internal
1215 	 * \brief Order relevant resource actions after a given action
1216 	 *
1217 	 * \param[in,out] first_action  Action to order after (or NULL if none runnable)
1218 	 * \param[in]     rsc           Resource whose actions should be ordered
1219 	 * \param[in,out] order         Ordering constraint being applied
1220 	 */
1221 	static void
1222 	order_resource_actions_after(pcmk_action_t *first_action,
1223 	                             const pcmk_resource_t *rsc,
1224 	                             pcmk__action_relation_t *order)
1225 	{
1226 	    GList *then_actions = NULL;
1227 	    uint32_t flags = pcmk__ar_none;
1228 	
(1) Event path: Condition "rsc != NULL", taking true branch.
(2) Event path: Condition "order != NULL", taking true branch.
1229 	    CRM_CHECK((rsc != NULL) && (order != NULL), return);
1230 	
1231 	    flags = order->flags;
(3) Event path: Switch case default.
(4) Event path: Condition "trace_tag_cs == NULL", taking true branch.
(5) Event path: Condition "crm_is_callsite_active(trace_tag_cs, _level, converted_tag)", taking false branch.
1232 	    pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
1233 	                    order->id, rsc->id);
1234 	
(6) Event path: Condition "order->action2 != NULL", taking false branch.
1235 	    if (order->action2 != NULL) {
1236 	        then_actions = g_list_prepend(NULL, order->action2);
1237 	
1238 	    } else {
1239 	        then_actions = find_actions_by_task(rsc, order->task2);
1240 	    }
1241 	
(7) Event path: Condition "then_actions == NULL", taking false branch.
1242 	    if (then_actions == NULL) {
1243 	        pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
1244 	                        order->id, order->task2, rsc->id);
1245 	        return;
1246 	    }
1247 	
(8) Event path: Condition "first_action != NULL", taking false branch.
1248 	    if ((first_action != NULL) && (first_action->rsc == rsc)
1249 	        && pcmk_is_set(first_action->flags, pcmk__action_migration_abort)) {
1250 	
1251 	        pcmk__rsc_trace(rsc,
1252 	                        "Detected dangling migration ordering (%s then %s %s)",
1253 	                        first_action->uuid, order->task2, rsc->id);
1254 	        pcmk__clear_relation_flags(flags, pcmk__ar_first_implies_then);
1255 	    }
1256 	
(9) Event path: Condition "first_action == NULL", taking true branch.
(10) Event path: Condition "!pcmk_all_flags_set(flags, pcmk__ar_first_implies_then)", taking false branch.
1257 	    if ((first_action == NULL)
1258 	        && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {
1259 	
1260 	        pcmk__rsc_debug(rsc,
1261 	                        "Ignoring ordering %d for %s: No first action found",
1262 	                        order->id, rsc->id);
1263 	        g_list_free(then_actions);
1264 	        return;
1265 	    }
1266 	
(11) Event path: Condition "iter != NULL", taking true branch.
1267 	    for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
1268 	        pcmk_action_t *then_action_iter = (pcmk_action_t *) iter->data;
1269 	
(12) Event path: Condition "first_action != NULL", taking false branch.
1270 	        if (first_action != NULL) {
1271 	            order_actions(first_action, then_action_iter, flags);
1272 	        } else {
1273 	            pcmk__clear_action_flags(then_action_iter, pcmk__action_runnable);
CID (unavailable; MK=1fb5ec2bf267e10e8dbdd7cb7733e393) (#1 of 1): Dereference of potentially null field (NULL_FIELD):
(13) Event null_field: Reading field "rsc1", which is expected to possibly be "NULL" in "order->rsc1" (checked 4 out of 5 times).
(14) Event dereference: Dereferencing "order->rsc1", which is known to be "NULL".
Also see events: [example_checked][example_checked][example_checked][example_checked]
1274 	            crm_warn("%s of %s is unrunnable because there is no %s of %s "
1275 	                     "to order it after", then_action_iter->task, rsc->id,
1276 	                     order->task1, order->rsc1->id);
1277 	        }
1278 	    }
1279 	
1280 	    g_list_free(then_actions);
1281 	}
1282 	
1283 	static void
1284 	rsc_order_first(pcmk_resource_t *first_rsc, pcmk__action_relation_t *order)
1285 	{
1286 	    GList *first_actions = NULL;
1287 	    pcmk_action_t *first_action = order->action1;
1288 	    pcmk_resource_t *then_rsc = order->rsc2;
1289 	
1290 	    pcmk__assert(first_rsc != NULL);
1291 	    pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
1292 	                    order->id, first_rsc->id);
1293 	
1294 	    if (first_action != NULL) {
1295 	        first_actions = g_list_prepend(NULL, first_action);
1296 	
1297 	    } else {
1298 	        first_actions = find_actions_by_task(first_rsc, order->task1);
1299 	    }
1300 	
1301 	    if ((first_actions == NULL) && (first_rsc == then_rsc)) {
1302 	        pcmk__rsc_trace(first_rsc,
1303 	                        "Ignoring constraint %d: first (%s for %s) not found",
1304 	                        order->id, order->task1, first_rsc->id);
1305 	
1306 	    } else if (first_actions == NULL) {
1307 	        char *key = NULL;
1308 	        char *op_type = NULL;
1309 	        guint interval_ms = 0;
1310 	        enum rsc_role_e first_role;
1311 	
1312 	        parse_op_key(order->task1, NULL, &op_type, &interval_ms);
1313 	        key = pcmk__op_key(first_rsc->id, op_type, interval_ms);
1314 	
1315 	        first_role = first_rsc->priv->fns->state(first_rsc, true);
1316 	        if ((first_role == pcmk_role_stopped)
1317 	            && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
1318 	            free(key);
1319 	            pcmk__rsc_trace(first_rsc,
1320 	                            "Ignoring constraint %d: first (%s for %s) "
1321 	                            "not found",
1322 	                            order->id, order->task1, first_rsc->id);
1323 	
1324 	        } else if ((first_role == pcmk_role_unpromoted)
1325 	                   && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
1326 	                                   pcmk__str_none)) {
1327 	            free(key);
1328 	            pcmk__rsc_trace(first_rsc,
1329 	                            "Ignoring constraint %d: first (%s for %s) "
1330 	                            "not found",
1331 	                            order->id, order->task1, first_rsc->id);
1332 	
1333 	        } else {
1334 	            pcmk__rsc_trace(first_rsc,
1335 	                            "Creating first (%s for %s) for constraint %d ",
1336 	                            order->task1, first_rsc->id, order->id);
1337 	            first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
1338 	                                         first_rsc->priv->scheduler);
1339 	            first_actions = g_list_prepend(NULL, first_action);
1340 	        }
1341 	
1342 	        free(op_type);
1343 	    }
1344 	
1345 	    if (then_rsc == NULL) {
1346 	        if (order->action2 == NULL) {
1347 	            pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
1348 	                            order->id);
1349 	            return;
1350 	        }
1351 	        then_rsc = order->action2->rsc;
1352 	    }
1353 	    for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
1354 	        first_action = iter->data;
1355 	
1356 	        if (then_rsc == NULL) {
1357 	            order_actions(first_action, order->action2, order->flags);
1358 	
1359 	        } else {
1360 	            order_resource_actions_after(first_action, then_rsc, order);
1361 	        }
1362 	    }
1363 	
1364 	    g_list_free(first_actions);
1365 	}
1366 	
1367 	// GFunc to call pcmk__block_colocation_dependents()
1368 	static void
1369 	block_colocation_dependents(gpointer data, gpointer user_data)
1370 	{
1371 	    pcmk__block_colocation_dependents(data);
1372 	}
1373 	
1374 	// GFunc to call pcmk__update_action_for_orderings()
1375 	static void
1376 	update_action_for_orderings(gpointer data, gpointer user_data)
1377 	{
1378 	    pcmk__update_action_for_orderings((pcmk_action_t *) data,
1379 	                                      (pcmk_scheduler_t *) user_data);
1380 	}
1381 	
1382 	/*!
1383 	 * \internal
1384 	 * \brief Apply all ordering constraints
1385 	 *
1386 	 * \param[in,out] sched  Scheduler data
1387 	 */
1388 	void
1389 	pcmk__apply_orderings(pcmk_scheduler_t *sched)
1390 	{
1391 	    crm_trace("Applying ordering constraints");
1392 	
1393 	    /* Ordering constraints need to be processed in the order they were created.
1394 	     * rsc_order_first() and order_resource_actions_after() require the relevant
1395 	     * actions to already exist in some cases, but rsc_order_first() will create
1396 	     * the 'first' action in certain cases. Thus calling rsc_order_first() can
1397 	     * change the behavior of later-created orderings.
1398 	     *
1399 	     * Also, g_list_append() should be avoided for performance reasons, so we
1400 	     * prepend orderings when creating them and reverse the list here.
1401 	     *
1402 	     * @TODO This is brittle and should be carefully redesigned so that the
1403 	     * order of creation doesn't matter, and the reverse becomes unneeded.
1404 	     */
1405 	    sched->priv->ordering_constraints =
1406 	        g_list_reverse(sched->priv->ordering_constraints);
1407 	
1408 	    for (GList *iter = sched->priv->ordering_constraints;
1409 	         iter != NULL; iter = iter->next) {
1410 	
1411 	        pcmk__action_relation_t *order = iter->data;
1412 	        pcmk_resource_t *rsc = order->rsc1;
1413 	
(17) Event example_checked: Example 3: "order->rsc1" has its value checked in "rsc != NULL".
Also see events: [null_field][dereference][example_checked][example_checked][example_checked]
1414 	        if (rsc != NULL) {
1415 	            rsc_order_first(rsc, order);
1416 	            continue;
1417 	        }
1418 	
1419 	        rsc = order->rsc2;
1420 	        if (rsc != NULL) {
1421 	            order_resource_actions_after(order->action1, rsc, order);
1422 	
1423 	        } else {
1424 	            crm_trace("Applying ordering constraint %d (non-resource actions)",
1425 	                      order->id);
1426 	            order_actions(order->action1, order->action2, order->flags);
1427 	        }
1428 	    }
1429 	
1430 	    g_list_foreach(sched->priv->actions, block_colocation_dependents, NULL);
1431 	
1432 	    crm_trace("Ordering probes");
1433 	    pcmk__order_probes(sched);
1434 	
1435 	    crm_trace("Updating %d actions", g_list_length(sched->priv->actions));
1436 	    g_list_foreach(sched->priv->actions, update_action_for_orderings, sched);
1437 	
1438 	    pcmk__disable_invalid_orderings(sched);
1439 	}
1440 	
1441 	/*!
1442 	 * \internal
1443 	 * \brief Order a given action after each action in a given list
1444 	 *
1445 	 * \param[in,out] after  "After" action
1446 	 * \param[in,out] list   List of "before" actions
1447 	 */
1448 	void
1449 	pcmk__order_after_each(pcmk_action_t *after, GList *list)
1450 	{
1451 	    const char *after_desc = (after->task == NULL)? after->uuid : after->task;
1452 	
1453 	    for (GList *iter = list; iter != NULL; iter = iter->next) {
1454 	        pcmk_action_t *before = (pcmk_action_t *) iter->data;
1455 	        const char *before_desc = before->task? before->task : before->uuid;
1456 	
1457 	        crm_debug("Ordering %s on %s before %s on %s",
1458 	                  before_desc, pcmk__node_name(before->node),
1459 	                  after_desc, pcmk__node_name(after->node));
1460 	        order_actions(before, after, pcmk__ar_ordered);
1461 	    }
1462 	}
1463 	
1464 	/*!
1465 	 * \internal
1466 	 * \brief Order promotions and demotions for restarts of a clone or bundle
1467 	 *
1468 	 * \param[in,out] rsc  Clone or bundle to order
1469 	 */
1470 	void
1471 	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
1472 	{
1473 	    // Order start and promote after all instances are stopped
1474 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1475 	                                 rsc, PCMK_ACTION_START,
1476 	                                 pcmk__ar_ordered);
1477 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1478 	                                 rsc, PCMK_ACTION_PROMOTE,
1479 	                                 pcmk__ar_ordered);
1480 	
1481 	    // Order stop, start, and promote after all instances are demoted
1482 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1483 	                                 rsc, PCMK_ACTION_STOP,
1484 	                                 pcmk__ar_ordered);
1485 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1486 	                                 rsc, PCMK_ACTION_START,
1487 	                                 pcmk__ar_ordered);
1488 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1489 	                                 rsc, PCMK_ACTION_PROMOTE,
1490 	                                 pcmk__ar_ordered);
1491 	
1492 	    // Order promote after all instances are started
1493 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
1494 	                                 rsc, PCMK_ACTION_PROMOTE,
1495 	                                 pcmk__ar_ordered);
1496 	
1497 	    // Order demote after all instances are demoted
1498 	    pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
1499 	                                 rsc, PCMK_ACTION_DEMOTED,
1500 	                                 pcmk__ar_ordered);
1501 	}
1502