1 /*
2 * Copyright 2004-2023 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU General Public License version 2
7 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <sys/stat.h>
13
14 #include <crm/crm.h>
15 #include <crm/common/xml.h>
16 #include <crm/common/xml_internal.h>
17 #include <crm/msg_xml.h>
18 #include <crm/cluster.h> /* For ONLINESTATUS etc */
19
20 #include <pacemaker-controld.h>
21
22 void te_update_confirm(const char *event, xmlNode * msg);
23
24 #define RSC_OP_PREFIX "//" XML_TAG_DIFF_ADDED "//" XML_TAG_CIB \
25 "//" XML_LRM_TAG_RSC_OP "[@" XML_ATTR_ID "='"
26
27 // An explicit shutdown-lock of 0 means the lock has been cleared
28 static bool
29 shutdown_lock_cleared(xmlNode *lrm_resource)
30 {
31 time_t shutdown_lock = 0;
32
33 return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
34 &shutdown_lock) == pcmk_ok)
35 && (shutdown_lock == 0);
36 }
37
38 static void
39 te_update_diff_v1(const char *event, xmlNode *diff)
40 {
41 int lpc, max;
42 xmlXPathObject *xpathObj = NULL;
43 GString *rsc_op_xpath = NULL;
44
45 CRM_CHECK(diff != NULL, return);
46
47 pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
48 controld_globals.logger_out->message(controld_globals.logger_out,
49 "xml-patchset", diff);
50
51 if (cib__config_changed_v1(NULL, NULL, &diff)) {
52 abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
53 diff);
54 goto bail; /* configuration changed */
55 }
56
57 /* Tickets Attributes - Added/Updated */
58 xpathObj =
59 xpath_search(diff,
60 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
61 if (numXpathResults(xpathObj) > 0) {
62 xmlNode *aborted = getXpathResult(xpathObj, 0);
63
64 abort_transition(INFINITY, pcmk__graph_restart,
65 "Ticket attribute: update", aborted);
66 goto bail;
67
68 }
69 freeXpathObject(xpathObj);
70
71 /* Tickets Attributes - Removed */
72 xpathObj =
73 xpath_search(diff,
74 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
75 if (numXpathResults(xpathObj) > 0) {
76 xmlNode *aborted = getXpathResult(xpathObj, 0);
77
78 abort_transition(INFINITY, pcmk__graph_restart,
79 "Ticket attribute: removal", aborted);
80 goto bail;
81 }
82 freeXpathObject(xpathObj);
83
84 /* Transient Attributes - Removed */
85 xpathObj =
86 xpath_search(diff,
87 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
88 XML_TAG_TRANSIENT_NODEATTRS);
89 if (numXpathResults(xpathObj) > 0) {
90 xmlNode *aborted = getXpathResult(xpathObj, 0);
91
92 abort_transition(INFINITY, pcmk__graph_restart,
93 "Transient attribute: removal", aborted);
94 goto bail;
95
96 }
97 freeXpathObject(xpathObj);
98
99 // Check for lrm_resource entries
100 xpathObj = xpath_search(diff,
101 "//" F_CIB_UPDATE_RESULT
102 "//" XML_TAG_DIFF_ADDED
103 "//" XML_LRM_TAG_RESOURCE);
104 max = numXpathResults(xpathObj);
105
106 /*
107 * Updates by, or in response to, graph actions will never affect more than
108 * one resource at a time, so such updates indicate an LRM refresh. In that
109 * case, start a new transition rather than check each result individually,
110 * which can result in _huge_ speedups in large clusters.
111 *
112 * Unfortunately, we can only do so when there are no pending actions.
113 * Otherwise, we could mistakenly throw away those results here, and
114 * the cluster will stall waiting for them and time out the operation.
115 */
116 if ((controld_globals.transition_graph->pending == 0) && (max > 1)) {
117 crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
118 max);
119 crm_log_xml_trace(diff, "lrm-refresh");
120 abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
121 NULL);
122 goto bail;
123 }
124
125 if (max == 1) {
126 xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
127
128 if (shutdown_lock_cleared(lrm_resource)) {
129 // @TODO would be more efficient to abort once after transition done
130 abort_transition(INFINITY, pcmk__graph_restart,
131 "Shutdown lock cleared", lrm_resource);
132 // Still process results, so we stop timers and update failcounts
133 }
134 }
135 freeXpathObject(xpathObj);
136
137 /* Process operation updates */
138 xpathObj =
139 xpath_search(diff,
140 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
141 max = numXpathResults(xpathObj);
142 if (max > 0) {
143 int lpc = 0;
144
145 for (lpc = 0; lpc < max; lpc++) {
146 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
147 const char *node = get_node_id(rsc_op);
148
149 process_graph_event(rsc_op, node);
150 }
151 }
152 freeXpathObject(xpathObj);
153
154 /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
155 xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
156 max = numXpathResults(xpathObj);
157 for (lpc = 0; lpc < max; lpc++) {
158 const char *op_id = NULL;
159 xmlXPathObject *op_match = NULL;
160 xmlNode *match = getXpathResult(xpathObj, lpc);
161
162 CRM_LOG_ASSERT(match != NULL);
163 if(match == NULL) { continue; };
164
165 op_id = ID(match);
166
167 if (rsc_op_xpath == NULL) {
168 rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
169 } else {
170 g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
171 }
172 pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);
173
174 op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
175 if (numXpathResults(op_match) == 0) {
176 /* Prevent false positives by matching cancelations too */
177 const char *node = get_node_id(match);
178 pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);
179
180 if (cancelled == NULL) {
181 crm_debug("No match for deleted action %s (%s on %s)",
182 (const char *) rsc_op_xpath->str, op_id, node);
183 abort_transition(INFINITY, pcmk__graph_restart,
184 "Resource op removal", match);
185 freeXpathObject(op_match);
186 goto bail;
187
188 } else {
189 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
190 op_id, node, cancelled->id);
191 }
192 }
193
194 freeXpathObject(op_match);
195 }
196
197 bail:
198 freeXpathObject(xpathObj);
199 if (rsc_op_xpath != NULL) {
200 g_string_free(rsc_op_xpath, TRUE);
201 }
202 }
203
204 static void
205 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
206 {
207 for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
208 rsc_op = pcmk__xml_next(rsc_op)) {
209 process_graph_event(rsc_op, node);
210 }
211 if (shutdown_lock_cleared(lrm_resource)) {
212 // @TODO would be more efficient to abort once after transition done
213 abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
214 lrm_resource);
215 }
216 }
217
218 static void
219 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
220 const char *op, const char *xpath)
221 {
222 xmlNode *rsc = NULL;
223
224 if (xml == NULL) {
225 return;
226 }
227
228 if (pcmk__xe_is(xml, XML_CIB_TAG_LRM)) {
229 xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
230 CRM_CHECK(xml != NULL, return);
231 }
232
233 CRM_CHECK(pcmk__xe_is(xml, XML_LRM_TAG_RESOURCES), return);
234
235 /*
236 * Updates by, or in response to, TE actions will never contain updates
237 * for more than one resource at a time, so such updates indicate an
238 * LRM refresh.
239 *
240 * In that case, start a new transition rather than check each result
241 * individually, which can result in _huge_ speedups in large clusters.
242 *
243 * Unfortunately, we can only do so when there are no pending actions.
244 * Otherwise, we could mistakenly throw away those results here, and
245 * the cluster will stall waiting for them and time out the operation.
246 */
247 if ((controld_globals.transition_graph->pending == 0)
248 && (xml->children != NULL) && (xml->children->next != NULL)) {
249
250 crm_log_xml_trace(change, "lrm-refresh");
251 abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
252 NULL);
253 return;
254 }
255
256 for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
257 rsc = pcmk__xml_next(rsc)) {
258 crm_trace("Processing %s", ID(rsc));
259 process_lrm_resource_diff(rsc, node);
260 }
261 }
262
263 static char *extract_node_uuid(const char *xpath)
264 {
265 char *mutable_path = strdup(xpath);
266 char *node_uuid = NULL;
267 char *search = NULL;
268 char *match = NULL;
269
270 match = strstr(mutable_path, "node_state[@" XML_ATTR_ID "=\'");
271 if (match == NULL) {
272 free(mutable_path);
273 return NULL;
274 }
275 match += strlen("node_state[@" XML_ATTR_ID "=\'");
276
277 search = strchr(match, '\'');
278 if (search == NULL) {
279 free(mutable_path);
280 return NULL;
281 }
282 search[0] = 0;
283
284 node_uuid = strdup(match);
285 free(mutable_path);
286 return node_uuid;
287 }
288
289 static void
290 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
291 const char *reason)
292 {
293 char *node_uuid = NULL;
294 pcmk__graph_action_t *down = NULL;
295
296 if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
297 abort_transition(INFINITY, pcmk__graph_restart, reason, change);
298 return;
299 }
300
301 node_uuid = extract_node_uuid(xpath);
302 if(node_uuid == NULL) {
303 crm_err("Could not extract node ID from %s", xpath);
304 abort_transition(INFINITY, pcmk__graph_restart, reason, change);
305 return;
306 }
307
308 down = match_down_event(node_uuid);
309 if (down == NULL) {
310 crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
311 abort_transition(INFINITY, pcmk__graph_restart, reason, change);
312 } else {
313 crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
314 }
315 free(node_uuid);
316 }
317
318 static void
319 process_op_deletion(const char *xpath, xmlNode *change)
320 {
321 char *mutable_key = strdup(xpath);
322 char *key;
323 char *node_uuid;
324
325 // Extract the part of xpath between last pair of single quotes
326 key = strrchr(mutable_key, '\'');
327 if (key != NULL) {
328 *key = '\0';
329 key = strrchr(mutable_key, '\'');
330 }
331 if (key == NULL) {
332 crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
333 xpath);
334 free(mutable_key);
335 return;
336 }
337 ++key;
338
339 node_uuid = extract_node_uuid(xpath);
340 if (confirm_cancel_action(key, node_uuid) == FALSE) {
341 abort_transition(INFINITY, pcmk__graph_restart,
342 "Resource operation removal", change);
343 }
344 free(mutable_key);
345 free(node_uuid);
346 }
347
348 static void
349 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
350 {
351 if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
352 process_op_deletion(xpath, change);
353
354 } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
355 abort_unless_down(xpath, op, change, "Resource state removal");
356
357 } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
358 abort_unless_down(xpath, op, change, "Node state removal");
359
360 } else {
361 crm_trace("Ignoring delete of %s", xpath);
362 }
363 }
364
365 static void
366 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
367 const char *xpath)
368 {
369 xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
370
371 process_resource_updates(ID(state), lrm, change, op, xpath);
372 }
373
374 static void
375 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
376 const char *xpath)
377 {
378 for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
379 state = pcmk__xml_next(state)) {
380 process_node_state_diff(state, change, op, xpath);
381 }
382 }
383
384 static void
385 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
386 const char *xpath)
387 {
388 xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
389 xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
390
391 if (status) {
392 process_status_diff(status, change, op, xpath);
393 }
394 if (config) {
395 abort_transition(INFINITY, pcmk__graph_restart,
396 "Non-status-only change", change);
397 }
398 }
399
400 static void
401 te_update_diff_v2(xmlNode *diff)
402 {
403 crm_log_xml_trace(diff, "Patch:Raw");
404
405 for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
406 change = pcmk__xml_next(change)) {
407
408 xmlNode *match = NULL;
409 const char *name = NULL;
410 const char *xpath = crm_element_value(change, XML_DIFF_PATH);
411
412 // Possible ops: create, modify, delete, move
413 const char *op = crm_element_value(change, XML_DIFF_OP);
414
415 // Ignore uninteresting updates
416 if (op == NULL) {
417 continue;
418
419 } else if (xpath == NULL) {
420 crm_trace("Ignoring %s change for version field", op);
421 continue;
422
423 } else if ((strcmp(op, "move") == 0)
424 && (strstr(xpath,
425 "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
426 "/" XML_CIB_TAG_RESOURCES) == NULL)) {
427 /* We still need to consider moves within the resources section,
428 * since they affect placement order.
429 */
430 crm_trace("Ignoring move change at %s", xpath);
431 continue;
432 }
433
434 // Find the result of create/modify ops
435 if (strcmp(op, "create") == 0) {
436 match = change->children;
437
438 } else if (strcmp(op, "modify") == 0) {
439 match = first_named_child(change, XML_DIFF_RESULT);
440 if(match) {
441 match = match->children;
442 }
443
444 } else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
445 crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
446 op, xpath);
447 continue;
448 }
449
450 if (match) {
451 if (match->type == XML_COMMENT_NODE) {
452 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
453 continue;
454 }
455 name = (const char *)match->name;
456 }
457
458 crm_trace("Handling %s operation for %s%s%s",
459 op, (xpath? xpath : "CIB"),
460 (name? " matched by " : ""), (name? name : ""));
461
462 if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
463 abort_transition(INFINITY, pcmk__graph_restart,
464 "Configuration change", change);
465 break; // Won't be packaged with operation results we may be waiting for
466
467 } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
468 || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
469 abort_transition(INFINITY, pcmk__graph_restart,
470 "Ticket attribute change", change);
471 break; // Won't be packaged with operation results we may be waiting for
472
473 } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
474 || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
475 pcmk__str_none)) {
476 abort_unless_down(xpath, op, change, "Transient attribute change");
477 break; // Won't be packaged with operation results we may be waiting for
478
479 } else if (strcmp(op, "delete") == 0) {
480 process_delete_diff(xpath, op, change);
481
482 } else if (name == NULL) {
483 crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
484 op, xpath);
485
486 } else if (strcmp(name, XML_TAG_CIB) == 0) {
487 process_cib_diff(match, change, op, xpath);
488
489 } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
490 process_status_diff(match, change, op, xpath);
491
492 } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
493 process_node_state_diff(match, change, op, xpath);
494
495 } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
496 process_resource_updates(ID(match), match, change, op, xpath);
497
498 } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
499 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
500
501 process_resource_updates(local_node, match, change, op, xpath);
502 free(local_node);
503
504 } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
505 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
506
507 process_lrm_resource_diff(match, local_node);
508 free(local_node);
509
510 } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
511 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
512
513 process_graph_event(match, local_node);
514 free(local_node);
515
516 } else {
517 crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
518 op, xpath, name);
519 }
520 }
521 }
522
523 void
524 te_update_diff(const char *event, xmlNode * msg)
525 {
526 xmlNode *diff = NULL;
527 const char *op = NULL;
528 int rc = -EINVAL;
529 int format = 1;
530 int p_add[] = { 0, 0, 0 };
531 int p_del[] = { 0, 0, 0 };
532
533 CRM_CHECK(msg != NULL, return);
534 crm_element_value_int(msg, F_CIB_RC, &rc);
535
536 if (controld_globals.transition_graph == NULL) {
537 crm_trace("No graph");
538 return;
539
540 } else if (rc < pcmk_ok) {
541 crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
542 return;
543
544 } else if (controld_globals.transition_graph->complete
545 && (controld_globals.fsa_state != S_IDLE)
546 && (controld_globals.fsa_state != S_TRANSITION_ENGINE)
547 && (controld_globals.fsa_state != S_POLICY_ENGINE)) {
548 crm_trace("Filter state=%s (complete)",
549 fsa_state2string(controld_globals.fsa_state));
550 return;
551 }
552
553 op = crm_element_value(msg, F_CIB_OPERATION);
554 diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
555
556 xml_patch_versions(diff, p_add, p_del);
557 crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
558 p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
559 fsa_state2string(controld_globals.fsa_state));
560
561 crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
562 switch (format) {
563 case 1:
564 te_update_diff_v1(event, diff);
565 break;
566 case 2:
567 te_update_diff_v2(diff);
568 break;
569 default:
570 crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
571 format);
572 }
573 controld_remove_all_outside_events();
574 }
575
576 void
577 process_te_message(xmlNode * msg, xmlNode * xml_data)
578 {
579 const char *value = NULL;
580 xmlXPathObject *xpathObj = NULL;
581 int nmatches = 0;
582
583 CRM_CHECK(msg != NULL, return);
584
585 // Transition requests must specify transition engine as subsystem
586 value = crm_element_value(msg, F_CRM_SYS_TO);
587 if (pcmk__str_empty(value)
588 || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
589 crm_info("Received invalid transition request: subsystem '%s' not '"
590 CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
591 return;
592 }
593
594 // Only the lrm_invoke command is supported as a transition request
595 value = crm_element_value(msg, F_CRM_TASK);
596 if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
597 crm_info("Received invalid transition request: command '%s' not '"
598 CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
599 return;
600 }
601
602 // Transition requests must be marked as coming from the executor
603 value = crm_element_value(msg, F_CRM_SYS_FROM);
604 if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
605 crm_info("Received invalid transition request: from '%s' not '"
606 CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
607 return;
608 }
609
610 crm_debug("Processing transition request with ref='%s' origin='%s'",
611 pcmk__s(crm_element_value(msg, F_CRM_REFERENCE), ""),
612 pcmk__s(crm_element_value(msg, F_ORIG), ""));
613
614 xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
615 nmatches = numXpathResults(xpathObj);
616 if (nmatches == 0) {
617 crm_err("Received transition request with no results (bug?)");
618 } else {
619 for (int lpc = 0; lpc < nmatches; lpc++) {
620 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
621 const char *node = get_node_id(rsc_op);
622
623 process_graph_event(rsc_op, node);
624 }
625 }
626 freeXpathObject(xpathObj);
627 }
628
629 void
630 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
631 {
632 if (rc < pcmk_ok) {
633 crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
634 }
635 }
636
637 /*!
638 * \brief Handle a timeout in node-to-node communication
639 *
640 * \param[in,out] data Pointer to graph action
641 *
642 * \return FALSE (indicating that source should be not be re-added)
643 */
644 gboolean
645 action_timer_callback(gpointer data)
646 {
647 pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
648 const char *task = NULL;
649 const char *on_node = NULL;
650 const char *via_node = NULL;
651
652 CRM_CHECK(data != NULL, return FALSE);
653
654 stop_te_timer(action);
655
656 task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
657 on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
658 via_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
659
660 if (controld_globals.transition_graph->complete) {
661 crm_notice("Node %s did not send %s result (via %s) within %dms "
662 "(ignoring because transition not in progress)",
663 (on_node? on_node : ""), (task? task : "unknown action"),
664 (via_node? via_node : "controller"), action->timeout);
665 } else {
666 /* fail the action */
667
668 crm_err("Node %s did not send %s result (via %s) within %dms "
669 "(action timeout plus cluster-delay)",
670 (on_node? on_node : ""), (task? task : "unknown action"),
671 (via_node? via_node : "controller"),
672 (action->timeout
673 + controld_globals.transition_graph->network_delay));
674 pcmk__log_graph_action(LOG_ERR, action);
675
676 pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
677
678 te_action_confirmed(action, controld_globals.transition_graph);
679 abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);
680
681 // Record timeout in the CIB if appropriate
682 if ((action->type == pcmk__rsc_graph_action)
683 && controld_action_is_recordable(task)) {
684 controld_record_action_timeout(action);
685 }
686 }
687
688 return FALSE;
689 }
690