1 /*
2 * Copyright 2009-2026 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU General Public License version 2
7 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <stdbool.h>
13 #include <stdio.h>
14 #include <libxml/tree.h> // xmlNode
15 #include <libxml/xpath.h> // xmlXPathObject, etc.
16
17 #include <crm/crm.h>
18 #include <crm/common/xml.h>
19
20 #include <crm/cluster/internal.h>
21
22 #include <crm/cib.h>
23 #include <crm/cib/internal.h>
24
25 #include <pacemaker-fenced.h>
26
27 static xmlNode *local_cib = NULL;
28 static cib_t *cib_api = NULL;
29 static bool have_cib_devices = FALSE;
30
31 /*!
32 * \internal
33 * \brief Check whether a node has a specific attribute name/value
34 *
35 * \param[in] node Name of node to check
36 * \param[in] name Name of an attribute to look for
37 * \param[in] value The value the named attribute needs to be set to in order to be considered a match
38 *
39 * \return TRUE if the locally cached CIB has the specified node attribute
40 */
41 gboolean
42 node_has_attr(const char *node, const char *name, const char *value)
43 {
44 GString *xpath = NULL;
45 xmlNode *match;
46
47 CRM_CHECK((local_cib != NULL) && (node != NULL) && (name != NULL)
48 && (value != NULL), return FALSE);
49
50 /* Search for the node's attributes in the CIB. While the schema allows
51 * multiple sets of instance attributes, and allows instance attributes to
52 * use id-ref to reference values elsewhere, that is intended for resources,
53 * so we ignore that here.
54 */
55 xpath = g_string_sized_new(256);
56 pcmk__g_strcat(xpath,
57 "//" PCMK_XE_NODES "/" PCMK_XE_NODE
58 "[@" PCMK_XA_UNAME "='", node, "']"
59 "/" PCMK_XE_INSTANCE_ATTRIBUTES
60 "/" PCMK_XE_NVPAIR
61 "[@" PCMK_XA_NAME "='", name, "' "
62 "and @" PCMK_XA_VALUE "='", value, "']", NULL);
63
64 match = pcmk__xpath_find_one(local_cib->doc, xpath->str, PCMK__LOG_NEVER);
65
66 g_string_free(xpath, TRUE);
67 return (match != NULL);
68 }
69
70 static void
71 remove_topology_level(xmlNode *match)
72 {
73 int index = 0;
74 char *key = NULL;
75 xmlNode *data = NULL;
76
77 CRM_CHECK(match != NULL, return);
78
79 key = stonith_level_key(match, fenced_target_by_unknown);
80 pcmk__xe_get_int(match, PCMK_XA_INDEX, &index);
81
82 data = pcmk__xe_create(NULL, PCMK_XE_FENCING_LEVEL);
83 pcmk__xe_set(data, PCMK__XA_ST_ORIGIN, __func__);
84 pcmk__xe_set(data, PCMK_XA_TARGET, key);
85 pcmk__xe_set_int(data, PCMK_XA_INDEX, index);
86
87 fenced_unregister_level(data, NULL);
88
89 free(key);
90 pcmk__xml_free(data);
91 }
92
93 static void
94 register_fencing_topology(xmlXPathObjectPtr xpathObj)
95 {
96 int max = pcmk__xpath_num_results(xpathObj);
97
98 for (int lpc = 0; lpc < max; lpc++) {
99 xmlNode *match = pcmk__xpath_result(xpathObj, lpc);
100
101 if (match == NULL) {
102 continue;
103 }
104 remove_topology_level(match);
105 fenced_register_level(match, NULL);
106 }
107 }
108
109 /* Fencing
110 <diff crm_feature_set="3.0.6">
111 <diff-removed>
112 <fencing-topology>
113 <fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
114 <fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
115 <fencing-level devices="disk,network" id="f-p2.1"/>
116 </fencing-topology>
117 </diff-removed>
118 <diff-added>
119 <fencing-topology>
120 <fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
121 <fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
122 <fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
123 </fencing-topology>
124 </diff-added>
125 </diff>
126 */
127
128 void
129 fencing_topology_init(void)
130 {
131 xmlXPathObject *xpathObj = NULL;
132 const char *xpath = "//" PCMK_XE_FENCING_LEVEL;
133
134 pcmk__trace("Full topology refresh");
135 free_topology_list();
136 init_topology_list();
137
138 /* Grab everything */
139 xpathObj = pcmk__xpath_search(local_cib->doc, xpath);
140 register_fencing_topology(xpathObj);
141
142 xmlXPathFreeObject(xpathObj);
143 }
144
145 #define XPATH_WATCHDOG_TIMEOUT "//" PCMK_XE_NVPAIR \
146 "[@" PCMK_XA_NAME "='" \
147 PCMK_OPT_FENCING_WATCHDOG_TIMEOUT "']"
148
149 /* @COMPAT The "stonith-watchdog-timeout" option has been a deprecated alias
150 * for "fencing-watchdog-timeout" since 3.0.2. Make sure it still works as a
151 * fallback until it's dropped in a future release.
152 */
153 #define XPATH_STONITH_WATCHDOG_TIMEOUT "//" PCMK_XE_NVPAIR \
154 "[@" PCMK_XA_NAME "='" \
155 PCMK_OPT_STONITH_WATCHDOG_TIMEOUT "']"
156
157 static long long
158 get_fencing_watchdog_timeout(xmlNode *cib)
159 {
160 xmlNode *stonith_watchdog_xml = NULL;
161 const char *value = NULL;
162 int rc = pcmk_rc_ok;
163 long long timeout_ms = 0;
164
165 // @TODO An XPath search can't handle multiple instances or rules
166 stonith_watchdog_xml = pcmk__xpath_find_one(cib->doc,
167 XPATH_WATCHDOG_TIMEOUT,
168 PCMK__LOG_NEVER);
169
170 /* @COMPAT The "stonith-watchdog-timeout" option has been a deprecated alias
171 * for "fencing-watchdog-timeout" since 3.0.2. Make sure it still works as a
172 * fallback until it's dropped in a future release.
173 */
174 if (stonith_watchdog_xml == NULL) {
175 stonith_watchdog_xml = pcmk__xpath_find_one(cib->doc,
176 XPATH_STONITH_WATCHDOG_TIMEOUT,
177 PCMK__LOG_NEVER);
178 }
179
180 if (stonith_watchdog_xml == NULL) {
181 return 0;
182 }
183
184 value = pcmk__xe_get(stonith_watchdog_xml, PCMK_XA_VALUE);
185 if (value == NULL) {
186 return 0;
187 }
188
189 rc = pcmk__parse_ms(value, &timeout_ms);
190 if ((rc == pcmk_rc_ok) && (timeout_ms >= 0)) {
191 return timeout_ms;
192 }
193
194 return pcmk__auto_fencing_watchdog_timeout();
195 }
196
197 /*!
198 * \internal
199 * \brief Mark a fence device dirty if its \c fenced_df_cib_registered flag is
200 * set
201 *
202 * \param[in] key Ignored
203 * \param[in,out] value Fence device (<tt>fenced_device_t *</tt>)
204 * \param[in] user_data Ignored
205 *
206 * \note This function is suitable for use with \c g_hash_table_foreach().
207 */
208 static void
209 mark_dirty_if_cib_registered(gpointer key, gpointer value, gpointer user_data)
210 {
211 fenced_device_t *device = value;
212
213 if (pcmk__is_set(device->flags, fenced_df_cib_registered)) {
214 fenced_device_set_flags(device, fenced_df_dirty);
215 }
216 }
217
218 /*!
219 * \internal
220 * \brief Return the value of a fence device's \c dirty flag
221 *
222 * \param[in] key Ignored
223 * \param[in] value Fence device (<tt>fenced_device_t *</tt>)
224 * \param[in] user_data Ignored
225 *
226 * \return \c dirty flag of \p value
227 *
228 * \note This function is suitable for use with
229 * \c g_hash_table_foreach_remove().
230 */
231 static gboolean
232 device_is_dirty(gpointer key, gpointer value, gpointer user_data)
233 {
234 fenced_device_t *device = value;
235
236 return pcmk__is_set(device->flags, fenced_df_dirty);
237 }
238
239 /*!
240 * \internal
241 * \brief Update all STONITH device definitions based on current CIB
242 */
243 static void
244 cib_devices_update(void)
245 {
246 pcmk__info("Updating devices to version %s.%s.%s",
247 pcmk__xe_get(local_cib, PCMK_XA_ADMIN_EPOCH),
248 pcmk__xe_get(local_cib, PCMK_XA_EPOCH),
249 pcmk__xe_get(local_cib, PCMK_XA_NUM_UPDATES));
250
251 fenced_foreach_device(mark_dirty_if_cib_registered, NULL);
252
253 /* have list repopulated if cib has a watchdog-fencing-resource
254 TODO: keep a cached list for queries happening while we are refreshing
255 */
256 g_list_free_full(stonith_watchdog_targets, free);
257 stonith_watchdog_targets = NULL;
258
259 fenced_scheduler_run(local_cib);
260
261 fenced_foreach_device_remove(device_is_dirty);
262 }
263
264 #define PRIMITIVE_ID_XP_FRAGMENT "/" PCMK_XE_PRIMITIVE "[@" PCMK_XA_ID "='"
265
266 static void
267 update_cib_stonith_devices(const xmlNode *patchset)
268 {
269 char *reason = NULL;
270
271 for (const xmlNode *change = pcmk__xe_first_child(patchset, NULL, NULL,
272 NULL);
273 change != NULL; change = pcmk__xe_next(change, NULL)) {
274
275 const char *op = pcmk__xe_get(change, PCMK_XA_OPERATION);
276 const char *xpath = pcmk__xe_get(change, PCMK_XA_PATH);
277 const char *primitive_xpath = NULL;
278
279 if (pcmk__str_eq(op, PCMK_VALUE_MOVE, pcmk__str_null_matches)
280 || (strstr(xpath, "/" PCMK_XE_STATUS) != NULL)) {
281 continue;
282 }
283
284 primitive_xpath = strstr(xpath, PRIMITIVE_ID_XP_FRAGMENT);
285 if ((primitive_xpath != NULL)
286 && pcmk__str_eq(op, PCMK_VALUE_DELETE, pcmk__str_none)) {
287
288 const char *rsc_id = NULL;
289 const char *end_quote = NULL;
290
291 if ((strstr(primitive_xpath, PCMK_XE_INSTANCE_ATTRIBUTES) != NULL)
292 || (strstr(primitive_xpath, PCMK_XE_META_ATTRIBUTES) != NULL)) {
293
294 reason = pcmk__str_copy("(meta) attribute deleted from "
295 "resource");
296 break;
297 }
298
299 rsc_id = primitive_xpath + sizeof(PRIMITIVE_ID_XP_FRAGMENT) - 1;
300 end_quote = strchr(rsc_id, '\'');
301
302 CRM_LOG_ASSERT(end_quote != NULL);
303 if (end_quote == NULL) {
304 pcmk__err("Bug: Malformed item in Pacemaker-generated patchset");
305 continue;
306 }
307
308 if (strchr(end_quote, '/') == NULL) {
309 /* The primitive element itself was deleted. If this was a
310 * fencing resource, it's faster to remove it directly than to
311 * run the scheduler and update all device registrations.
312 */
313 char *copy = strndup(rsc_id, end_quote - rsc_id);
314
315 pcmk__assert(copy != NULL);
316 stonith_device_remove(copy, true);
317
318 /* watchdog_device_update called afterwards
319 to fall back to implicit definition if needed */
320
321 free(copy);
322 continue;
323 }
324 }
325
326 if (strstr(xpath, "/" PCMK_XE_RESOURCES)
327 || strstr(xpath, "/" PCMK_XE_CONSTRAINTS)
328 || strstr(xpath, "/" PCMK_XE_RSC_DEFAULTS)) {
329
330 const char *shortpath = strrchr(xpath, '/');
331
332 reason = pcmk__assert_asprintf("%s %s", op, shortpath + 1);
333 break;
334 }
335 }
336
337 if (reason != NULL) {
338 pcmk__info("Updating device list from CIB: %s", reason);
339 cib_devices_update();
340 free(reason);
341 } else {
342 pcmk__trace("No updates for device list found in CIB");
343 }
344 }
345
346 static void
347 watchdog_device_update(void)
348 {
349 if (fencing_watchdog_timeout_ms > 0) {
350 if (!fenced_has_watchdog_device()
351 && (stonith_watchdog_targets == NULL)) {
352 /* getting here watchdog-fencing enabled, no device there yet
353 and reason isn't stonith_watchdog_targets preventing that
354 */
355 int rc;
356 xmlNode *xml;
357
358 xml = create_device_registration_xml(
359 STONITH_WATCHDOG_ID,
360 st_namespace_internal,
361 STONITH_WATCHDOG_AGENT,
362 NULL, /* fenced_device_register() will add our
363 own name as PCMK_FENCING_HOST_LIST param
364 so we can skip that here
365 */
366 NULL);
367 rc = fenced_device_register(xml, true);
368 pcmk__xml_free(xml);
369 if (rc != pcmk_rc_ok) {
370 exit_code = CRM_EX_FATAL;
371 pcmk__crit("Cannot register watchdog pseudo fence agent: %s",
372 pcmk_rc_str(rc));
373 stonith_shutdown(0);
374 }
375 }
376
377 } else if (fenced_has_watchdog_device()) {
378 /* be silent if no device - todo parameter to stonith_device_remove */
379 stonith_device_remove(STONITH_WATCHDOG_ID, true);
380 }
381 }
382
383 /*!
384 * \internal
385 * \brief Query the full CIB
386 *
387 * \return Standard Pacemaker return code
388 */
389 static int
390 fenced_query_cib(void)
391 {
392 int rc = pcmk_ok;
393
394 pcmk__trace("Re-requesting full CIB");
395 rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_sync_call);
396 rc = pcmk_legacy2rc(rc);
397 if (rc == pcmk_rc_ok) {
398 pcmk__assert(local_cib != NULL);
399 } else {
400 pcmk__err("Couldn't retrieve the CIB: %s " QB_XS " rc=%d",
401 pcmk_rc_str(rc), rc);
402 }
403 return rc;
404 }
405
406 static void
407 update_fencing_topology(const char *event, xmlNode *msg)
408 {
409 xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT,
410 NULL, NULL);
411 xmlNode *patchset = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
412
413 int format = 1;
414
415 int add[] = { 0, 0, 0 };
416 int del[] = { 0, 0, 0 };
417
418 CRM_CHECK(patchset != NULL, return);
419
420 pcmk__xe_get_int(patchset, PCMK_XA_FORMAT, &format);
421 if (format != 2) {
422 pcmk__warn("Unknown patch format: %d", format);
423 return;
424 }
425
426 pcmk__xml_patchset_versions(patchset, del, add);
427
428 for (xmlNode *change = pcmk__xe_first_child(patchset, NULL, NULL, NULL);
429 change != NULL; change = pcmk__xe_next(change, NULL)) {
430
431 const char *op = pcmk__xe_get(change, PCMK_XA_OPERATION);
432 const char *xpath = pcmk__xe_get(change, PCMK_XA_PATH);
433
434 if (op == NULL) {
435 continue;
436 }
437
438 if (strstr(xpath, "/" PCMK_XE_FENCING_LEVEL) != NULL) {
439 // Change to a specific entry
440 pcmk__trace("Handling %s operation %d.%d.%d for %s", op,
441 add[0], add[1], add[2], xpath);
442
443 if (strcmp(op, PCMK_VALUE_DELETE) == 0) {
444 /* We have only path and ID, which is not enough info to remove
445 * a specific entry. Re-initialize the whole topology.
446 */
447 pcmk__info("Re-initializing fencing topology after %s "
448 "operation %d.%d.%d for %s",
449 op, add[0], add[1], add[2], xpath);
450 fencing_topology_init();
451 return;
452 }
453
454 if (strcmp(op, PCMK_VALUE_CREATE) == 0) {
455 fenced_register_level(change->children, NULL);
456
457 } else if (strcmp(op, PCMK_VALUE_MODIFY) == 0) {
458 xmlNode *match = pcmk__xe_first_child(change,
459 PCMK_XE_CHANGE_RESULT,
460 NULL, NULL);
461
462 if (match != NULL) {
463 remove_topology_level(match->children);
464 fenced_register_level(match->children, NULL);
465 }
466 }
467 continue;
468 }
469
470 if (strstr(xpath, "/" PCMK_XE_FENCING_TOPOLOGY) != NULL) {
471 // Change to the topology in general
472 pcmk__info("Re-initializing fencing topology after top-level %s "
473 "operation %d.%d.%d for %s",
474 op, add[0], add[1], add[2], xpath);
475 fencing_topology_init();
476 return;
477 }
478
479 if ((strstr(xpath, "/" PCMK_XE_CONFIGURATION) != NULL)
480 && (pcmk__xe_first_child(change, PCMK_XE_FENCING_TOPOLOGY, NULL,
481 NULL) != NULL)
482 && pcmk__str_any_of(op, PCMK_VALUE_CREATE, PCMK_VALUE_DELETE,
483 NULL)) {
484
485 // Topology was created or entire configuration section was deleted
486 pcmk__info("Re-initializing fencing topology after top-level %s "
487 "operation %d.%d.%d for %s",
488 op, add[0], add[1], add[2], xpath);
489 fencing_topology_init();
490 return;
491 }
492
493 pcmk__trace("Nothing for us in %s operation %d.%d.%d for %s", op,
494 add[0], add[1], add[2], xpath);
495 }
496 }
497
498 static void
499 update_cib_cache_cb(const char *event, xmlNode * msg)
500 {
501 xmlNode *patchset = NULL;
502 long long timeout_ms_saved = fencing_watchdog_timeout_ms;
503 bool need_full_refresh = false;
504
|
(1) Event path: |
Condition "!have_cib_devices", taking false branch. |
505 if(!have_cib_devices) {
506 pcmk__trace("Skipping updates until we get a full dump");
507 return;
508
|
(2) Event path: |
Condition "msg == NULL", taking false branch. |
509 } else if(msg == NULL) {
510 pcmk__trace("Missing %s update", event);
511 return;
512 }
513
514 /* Maintain a local copy of the CIB so that we have full access
515 * to device definitions, location constraints, and node attributes
516 */
|
(3) Event path: |
Condition "local_cib != NULL", taking true branch. |
517 if (local_cib != NULL) {
518 int rc = pcmk_ok;
519 xmlNode *wrapper = NULL;
520
521 pcmk__xe_get_int(msg, PCMK__XA_CIB_RC, &rc);
|
(4) Event path: |
Condition "rc != 0", taking false branch. |
522 if (rc != pcmk_ok) {
523 return;
524 }
525
526 wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT, NULL,
527 NULL);
528 patchset = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
529
530 rc = xml_apply_patchset(local_cib, patchset, TRUE);
531
|
(5) Event path: |
Condition "rc != 0", taking true branch. |
532 if (rc != pcmk_ok) {
|
(6) Event path: |
Condition "rc == -205", taking false branch. |
|
(7) Event path: |
Condition "rc == -206", taking true branch. |
533 if ((rc == -pcmk_err_old_data) || (rc == -pcmk_err_diff_failed)) {
534 pcmk__notice("[%s] Patch aborted: %s (%d)", event,
535 pcmk_strerror(rc), rc);
|
(8) Event path: |
Falling through to end of if statement. |
536 } else {
537 pcmk__warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc),
538 rc);
539 }
540
|
CID (unavailable; MK=e25f75fe95a6ce09d89a0a9b355fe1a1) (#1 of 1): Inconsistent C union access (INCONSISTENT_UNION_ACCESS): |
|
(9) Event assign_union_field: |
The union field "in" of "_pp" is written. |
|
(10) Event inconsistent_union_field_access: |
In "_pp.out", the union field used: "out" is inconsistent with the field most recently stored: "in". |
541 g_clear_pointer(&local_cib, pcmk__xml_free);
542 }
543 }
544
545 if (local_cib == NULL) {
546 if (fenced_query_cib() != pcmk_rc_ok) {
547 return;
548 }
549 need_full_refresh = true;
550 }
551
552 pcmk__refresh_node_caches_from_cib(local_cib);
553 fencing_watchdog_timeout_ms = get_fencing_watchdog_timeout(local_cib);
554
555 if (timeout_ms_saved != fencing_watchdog_timeout_ms) {
556 need_full_refresh = true;
557 }
558
559 if (need_full_refresh) {
560 fencing_topology_init();
561 cib_devices_update();
562 } else {
563 // Partial refresh
564 update_fencing_topology(event, msg);
565 update_cib_stonith_devices(patchset);
566 }
567
568 watchdog_device_update();
569 }
570
571 static void
572 init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
573 {
574 pcmk__info("Updating device list from CIB");
575 have_cib_devices = TRUE;
576 local_cib = pcmk__xml_copy(NULL, output);
577
578 pcmk__refresh_node_caches_from_cib(local_cib);
579 fencing_watchdog_timeout_ms = get_fencing_watchdog_timeout(local_cib);
580
581 fencing_topology_init();
582 cib_devices_update();
583 watchdog_device_update();
584 }
585
586 static void
587 cib_connection_destroy(gpointer user_data)
588 {
589 if (stonith_shutdown_flag) {
590 pcmk__info("Connection to the CIB manager closed");
591 return;
592 } else {
593 pcmk__crit("Lost connection to the CIB manager, shutting down");
594 }
595 if (cib_api) {
596 cib_api->cmds->signoff(cib_api);
597 }
598 stonith_shutdown(0);
599 }
600
601 /*!
602 * \internal
603 * \brief Disconnect from CIB manager
604 */
605 void
606 fenced_cib_cleanup(void)
607 {
608 if (cib_api != NULL) {
609 cib_api->cmds->del_notify_callback(cib_api, PCMK__VALUE_CIB_DIFF_NOTIFY,
610 update_cib_cache_cb);
611 cib__clean_up_connection(&cib_api);
612 }
613
614 g_clear_pointer(&local_cib, pcmk__xml_free);
615 }
616
617 void
618 setup_cib(void)
619 {
620 int rc, retries = 0;
621
622 cib_api = cib_new();
623 if (cib_api == NULL) {
624 pcmk__err("No connection to the CIB manager");
625 return;
626 }
627
628 do {
629 sleep(retries);
630 rc = cib_api->cmds->signon(cib_api, crm_system_name, cib_command);
631 } while (rc == -ENOTCONN && ++retries < 5);
632
633 if (rc != pcmk_ok) {
634 pcmk__err("Could not connect to the CIB manager: %s (%d)",
635 pcmk_strerror(rc), rc);
636 return;
637 }
638
639 rc = cib_api->cmds->add_notify_callback(cib_api,
640 PCMK__VALUE_CIB_DIFF_NOTIFY,
641 update_cib_cache_cb);
642 if (rc != pcmk_ok) {
643 pcmk__err("Could not set CIB notification callback");
644 return;
645 }
646
647 rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_none);
648 cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL,
649 "init_cib_cache_cb", init_cib_cache_cb);
650 cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
651 pcmk__info("Watching for fencing topology changes");
652 }
653