1 /*
2 * Copyright 2012-2026 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU Lesser General Public License
7 * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <errno.h> // EINVAL, ENOMEM, ENOTCONN
13 #include <stdbool.h> // true, bool, false
14 #include <stdint.h> // uint32_t, uint64_t
15 #include <stdlib.h> // NULL, free, calloc
16 #include <string.h> // strdup, strcmp, strlen
17 #include <sys/types.h> // time_t, ssize_t
18 #include <time.h> // time
19 #include <unistd.h> // close
20
21 #include <glib.h> // g_list_free_full, gpointer
22 #include <gnutls/gnutls.h> // gnutls_deinit, gnutls_bye
23 #include <libxml/parser.h> // xmlNode
24 #include <qb/qbdefs.h> // QB_MAX
25 #include <qb/qblog.h> // QB_XS
26
27 #include <crm/common/actions.h> // PCMK_DEFAULT_ACTION_TIMEOUT_MS
28 #include <crm/common/agents.h> // PCMK_RESOURCE_CLASS_STONITH
29 #include <crm/common/internal.h>
30 #include <crm/common/ipc.h> // crm_ipc_*
31 #include <crm/common/logging.h> // CRM_CHECK, CRM_LOG_ASSERT
32 #include <crm/common/mainloop.h> // mainloop_set_trigger
33 #include <crm/common/nvpair.h> // hash2smartfield, xml2list
34 #include <crm/common/options.h> // PCMK_OPT_FENCING_WATCHDOG_TIMEOUT
35 #include <crm/common/results.h> // pcmk_rc_*, pcmk_rc2legacy
36 #include <crm/common/util.h> // crm_default_remote_port
37 #include <crm/crm.h> // CRM_OP_REGISTER, CRM_SYSTEM_LRMD
38 #include <crm/fencing/internal.h> // stonith__*
39 #include <crm/lrmd.h> // lrmd_t, lrmd_s, lrmd_key_value_t
40 #include <crm/lrmd_events.h> // lrmd_event_*
41 #include <crm/lrmd_internal.h> // lrmd__init_remote_key
42 #include <crm/services.h> // services_action_free
43 #include <crm/services_internal.h> // services__copy_result
44
45 #define MAX_TLS_RECV_WAIT 10000
46
47 // GnuTLS client handshake timeout in seconds
48 #define TLS_HANDSHAKE_TIMEOUT 5
49
50 static int global_remote_msg_id = 0;
51
52 static gnutls_datum_t remote_key = { NULL, 0 };
53
54 typedef struct {
55 uint64_t type;
56 char *token;
57 mainloop_io_t *source;
58
59 /* IPC parameters */
60 crm_ipc_t *ipc;
61
62 pcmk__remote_t *remote;
63
64 /* Extra TLS parameters */
65 char *remote_nodename;
66 char *server;
67 int port;
68 pcmk__tls_t *tls;
69
70 /* while the async connection is occurring, this is the id
71 * of the connection timeout timer. */
72 int async_timer;
73 int sock;
74 /* since tls requires a round trip across the network for a
75 * request/reply, there are times where we just want to be able
76 * to send a request from the client and not wait around (or even care
77 * about) what the reply is. */
78 int expected_late_replies;
79 GList *pending_notify;
80 crm_trigger_t *process_notify;
81 crm_trigger_t *handshake_trigger;
82
83 lrmd_event_callback callback;
84
85 /* Internal IPC proxy msg passing for remote guests */
86 void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
87 void *proxy_callback_userdata;
88 char *peer_version;
89 } lrmd_private_t;
90
91 static lrmd_list_t *
92 lrmd_list_add(lrmd_list_t * head, const char *value)
93 {
94 lrmd_list_t *p, *end;
95
96 p = pcmk__assert_alloc(1, sizeof(lrmd_list_t));
97 p->val = strdup(value);
98
99 end = head;
100 while (end && end->next) {
101 end = end->next;
102 }
103
104 if (end) {
105 end->next = p;
106 } else {
107 head = p;
108 }
109
110 return head;
111 }
112
113 void
114 lrmd_list_freeall(lrmd_list_t * head)
115 {
116 lrmd_list_t *p;
117
118 while (head) {
119 char *val = (char *)head->val;
120
121 p = head->next;
122 free(val);
123 free(head);
124 head = p;
125 }
126 }
127
128 lrmd_key_value_t *
129 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
130 {
131 lrmd_key_value_t *p, *end;
132
133 p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t));
134 p->key = strdup(key);
135 p->value = strdup(value);
136
137 end = head;
138 while (end && end->next) {
139 end = end->next;
140 }
141
142 if (end) {
143 end->next = p;
144 } else {
145 head = p;
146 }
147
148 return head;
149 }
150
151 void
152 lrmd_key_value_freeall(lrmd_key_value_t * head)
153 {
154 lrmd_key_value_t *p;
155
156 while (head) {
157 p = head->next;
158 free(head->key);
159 free(head->value);
160 free(head);
161 head = p;
162 }
163 }
164
165 /*!
166 * \brief Create a new lrmd_event_data_t object
167 *
168 * \param[in] rsc_id ID of resource involved in event
169 * \param[in] task Action name
170 * \param[in] interval_ms Action interval
171 *
172 * \return Newly allocated and initialized lrmd_event_data_t
173 * \note This functions asserts on memory errors, so the return value is
174 * guaranteed to be non-NULL. The caller is responsible for freeing the
175 * result with lrmd_free_event().
176 */
177 lrmd_event_data_t *
178 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
179 {
180 lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
181
182 // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
183 event->rsc_id = pcmk__str_copy(rsc_id);
184 event->op_type = pcmk__str_copy(task);
185 event->interval_ms = interval_ms;
186 return event;
187 }
188
189 lrmd_event_data_t *
190 lrmd_copy_event(lrmd_event_data_t * event)
191 {
192 lrmd_event_data_t *copy = NULL;
193
194 copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
195
196 copy->type = event->type;
197
198 // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
199 copy->rsc_id = pcmk__str_copy(event->rsc_id);
200 copy->op_type = pcmk__str_copy(event->op_type);
201 copy->user_data = pcmk__str_copy(event->user_data);
202 copy->output = pcmk__str_copy(event->output);
203 copy->remote_nodename = pcmk__str_copy(event->remote_nodename);
204 copy->exit_reason = pcmk__str_copy(event->exit_reason);
205
206 copy->call_id = event->call_id;
207 copy->timeout = event->timeout;
208 copy->interval_ms = event->interval_ms;
209 copy->start_delay = event->start_delay;
210 copy->rsc_deleted = event->rsc_deleted;
211 copy->rc = event->rc;
212 copy->op_status = event->op_status;
213 copy->t_run = event->t_run;
214 copy->t_rcchange = event->t_rcchange;
215 copy->exec_time = event->exec_time;
216 copy->queue_time = event->queue_time;
217 copy->connection_rc = event->connection_rc;
218 copy->params = pcmk__str_table_dup(event->params);
219
220 return copy;
221 }
222
223 /*!
224 * \brief Free an executor event
225 *
226 * \param[in,out] Executor event object to free
227 */
228 void
229 lrmd_free_event(lrmd_event_data_t *event)
230 {
231 if (event == NULL) {
232 return;
233 }
234 // @TODO Why are these const char *?
235 free((void *) event->rsc_id);
236 free((void *) event->op_type);
237 free((void *) event->user_data);
238 free((void *) event->remote_nodename);
239 lrmd__reset_result(event);
240 g_clear_pointer(&event->params, g_hash_table_destroy);
241 free(event);
242 }
243
244 /* Not used with mainloop */
245 int
246 lrmd_poll(lrmd_t * lrmd, int timeout)
247 {
248 lrmd_private_t *native = lrmd->lrmd_private;
249
250 switch (native->type) {
251 case pcmk__client_ipc:
252 return crm_ipc_ready(native->ipc);
253
254 case pcmk__client_tls:
255 if (native->pending_notify) {
256 return 1;
257 } else {
258 int rc = pcmk__remote_ready(native->remote, 0);
259
260 switch (rc) {
261 case pcmk_rc_ok:
262 return 1;
263 case ETIME:
264 return 0;
265 default:
266 return pcmk_rc2legacy(rc);
267 }
268 }
269 default:
270 pcmk__err("Unsupported executor connection type (bug?): %d",
271 native->type);
272 return -EPROTONOSUPPORT;
273 }
274 }
275
276 static void
277 lrmd_dispatch_internal(gpointer data, gpointer user_data)
278 {
279 xmlNode *msg = data;
280 lrmd_t *lrmd = user_data;
281
282 const char *type;
283 const char *proxy_session = pcmk__xe_get(msg, PCMK__XA_LRMD_IPC_SESSION);
284 lrmd_private_t *native = lrmd->lrmd_private;
285 lrmd_event_data_t event = { 0, };
286
287 if (proxy_session != NULL) {
288 if (native->proxy_callback == NULL) {
289 return;
290 }
291
292 pcmk__log_xml_trace(msg, "PROXY_INBOUND");
293 native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
294 return;
295 }
296
297 if (native->callback == NULL) {
298 pcmk__trace("notify event received but client has not set callback");
299 return;
300 }
301
302 event.remote_nodename = native->remote_nodename;
303 type = pcmk__xe_get(msg, PCMK__XA_LRMD_OP);
304 pcmk__xe_get_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
305 event.rsc_id = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_ID);
306
307 if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
308 event.type = lrmd_event_register;
309 } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
310 event.type = lrmd_event_unregister;
311 } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
312 int rc = 0;
313 int exec_time = 0;
314 int queue_time = 0;
315
316 pcmk__xe_get_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
317 pcmk__xe_get_guint(msg, PCMK__XA_LRMD_RSC_INTERVAL, &event.interval_ms);
318 pcmk__xe_get_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
319 &event.start_delay);
320
321 pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc);
322 event.rc = (enum ocf_exitcode) rc;
323
324 pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS, &event.op_status);
325 pcmk__xe_get_int(msg, PCMK__XA_LRMD_RSC_DELETED, &event.rsc_deleted);
326
327 pcmk__xe_get_time(msg, PCMK__XA_LRMD_RUN_TIME, &event.t_run);
328 pcmk__xe_get_time(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &event.t_rcchange);
329
330 pcmk__xe_get_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time);
331 CRM_LOG_ASSERT(exec_time >= 0);
332 event.exec_time = QB_MAX(0, exec_time);
333
334 pcmk__xe_get_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time);
335 event.queue_time = QB_MAX(0, queue_time);
336
337 event.op_type = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_ACTION);
338 event.user_data = pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_USERDATA_STR);
339 event.type = lrmd_event_exec_complete;
340
341 /* output and exit_reason may be freed by a callback */
342 event.output = pcmk__xe_get_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
343 lrmd__set_result(&event, event.rc, event.op_status,
344 pcmk__xe_get(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));
345
346 event.params = xml2list(msg);
347 } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
348 event.type = lrmd_event_new_client;
349 } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
350 event.type = lrmd_event_poke;
351 } else {
352 return;
353 }
354
355 pcmk__trace("op %s notify event received", type);
356 native->callback(&event);
357
358 g_clear_pointer(&event.params, g_hash_table_destroy);
359 lrmd__reset_result(&event);
360 }
361
362 // \return Always 0, to indicate that IPC mainloop source should be kept
363 static int
364 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
365 {
366 lrmd_t *lrmd = userdata;
367 lrmd_private_t *native = lrmd->lrmd_private;
368
369 if (native->callback != NULL) {
370 xmlNode *msg = pcmk__xml_parse(buffer);
371
372 lrmd_dispatch_internal(msg, lrmd);
373 pcmk__xml_free(msg);
374 }
375 return 0;
376 }
377
378 static bool
379 remote_executor_connected(lrmd_t *lrmd)
380 {
381 lrmd_private_t *native = lrmd->lrmd_private;
382
383 return (native->remote->tls_session != NULL);
384 }
385
386 static void
387 lrmd_tls_disconnect(lrmd_t *lrmd)
388 {
389 lrmd_private_t *native = lrmd->lrmd_private;
390
391 if (native->remote->tls_session) {
392 gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
393 g_clear_pointer(&native->remote->tls_session, gnutls_deinit);
394 }
395
396 if (native->async_timer) {
397 g_source_remove(native->async_timer);
398 native->async_timer = 0;
399 }
400
401 if (native->source != NULL) {
402 /* Attached to mainloop */
403 g_clear_pointer(&native->source, mainloop_del_ipc_client);
404
405 } else if (native->sock >= 0) {
406 close(native->sock);
407 native->sock = -1;
408 }
409
410 g_list_free_full(native->pending_notify, (GDestroyNotify) pcmk__xml_free);
411 native->pending_notify = NULL;
412 }
413
414 static void
415 handle_ack(const xmlNode *reply)
416 {
417 int status = 0;
418
419 pcmk__log_xml_err(reply, "Bad reply");
420
421 pcmk__xe_get_int(reply, PCMK_XA_STATUS, &status);
422 pcmk__err("Received error response from executor: %s", crm_exit_str(status));
423 }
424
425 static int
426 process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native)
427 {
428 int rc = pcmk_rc_ok;
429 const char *version = pcmk__xe_get(reply, PCMK__XA_LRMD_PROTOCOL_VERSION);
430 const char *msg_type = pcmk__xe_get(reply, PCMK__XA_LRMD_OP);
431 const char *tmp_ticket = pcmk__xe_get(reply, PCMK__XA_LRMD_CLIENTID);
432 const char *start_state = pcmk__xe_get(reply, PCMK__XA_NODE_START_STATE);
433
434 /* The only reason we can receive an ACK here is because execd didn't
435 * understand the CRM_OP_REGISTER message we sent in lrmd_handshake{,_async},
436 * and the status code will always indicate some sort of error.
437 */
438 if (pcmk__xe_is(reply, PCMK__XE_ACK)) {
439 handle_ack(reply);
440 return EPROTO;
441 }
442
443 pcmk__xe_get_int(reply, PCMK__XA_LRMD_RC, &rc);
444 rc = pcmk_legacy2rc(rc);
445
446 /* The remote executor may add its uptime to the XML reply, which is useful
447 * in handling transient attributes when the connection to the remote node
448 * unexpectedly drops. If no parameter is given, just default to -1.
449 */
450 native->remote->uptime = -1;
451 pcmk__xe_get_time(reply, PCMK__XA_UPTIME, &native->remote->uptime);
452
453 if (start_state) {
454 native->remote->start_state = strdup(start_state);
455 }
456
457 if (rc == EPROTO) {
458 pcmk__err("Executor protocol version mismatch between client "
459 "(" LRMD_PROTOCOL_VERSION ") and server (%s)",
460 version);
461 pcmk__log_xml_err(reply, "Protocol Error");
462 } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
463 pcmk__err("Invalid registration message: %s", msg_type);
464 pcmk__log_xml_err(reply, "Bad reply");
465 rc = EPROTO;
466 } else if (tmp_ticket == NULL) {
467 pcmk__err("No registration token provided");
468 pcmk__log_xml_err(reply, "Bad reply");
469 rc = EPROTO;
470 } else {
471 pcmk__trace("Obtained registration token: %s", tmp_ticket);
472 native->token = strdup(tmp_ticket);
473 native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
474 rc = pcmk_rc_ok;
475 }
476
477 return rc;
478 }
479
480 static void
481 report_async_connection_result(lrmd_t *lrmd, int rc)
482 {
483 lrmd_private_t *native = lrmd->lrmd_private;
484
485 if (native->callback) {
486 lrmd_event_data_t event = { 0, };
487 event.type = lrmd_event_connect;
488 event.remote_nodename = native->remote_nodename;
489 event.connection_rc = rc;
490 native->callback(&event);
491 }
492 }
493
494 static void
495 handle_remote_msg(xmlNode *xml, lrmd_t *lrmd)
496 {
497 lrmd_private_t *native = lrmd->lrmd_private;
498 const char *msg_type = NULL;
499
500 msg_type = pcmk__xe_get(xml, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
501 if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
502 lrmd_dispatch_internal(xml, lrmd);
503 } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
504 const char *op = pcmk__xe_get(xml, PCMK__XA_LRMD_OP);
505
506 if (native->expected_late_replies > 0) {
507 native->expected_late_replies--;
508
509 /* The register op message we get as a response to lrmd_handshake_async
510 * is a reply, so we have to handle that here.
511 */
512 if (pcmk__str_eq(op, "register", pcmk__str_casei)) {
513 int rc = process_lrmd_handshake_reply(xml, native);
514 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
515 }
516 } else {
517 int reply_id = 0;
518
519 pcmk__xe_get_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
520 /* if this happens, we want to know about it */
521 pcmk__err("Got outdated Pacemaker Remote reply %d", reply_id);
522 }
523 }
524 }
525
526 /*!
527 * \internal
528 * \brief TLS dispatch function for file descriptor sources
529 *
530 * \param[in,out] userdata API connection
531 *
532 * \return -1 on error to remove the source from the mainloop, or 0 otherwise
533 * to leave it in the mainloop
534 */
535 static int
536 lrmd_tls_dispatch(gpointer userdata)
537 {
538 lrmd_t *lrmd = userdata;
539 lrmd_private_t *native = lrmd->lrmd_private;
540 xmlNode *xml = NULL;
541 int rc = pcmk_rc_ok;
542
543 if (!remote_executor_connected(lrmd)) {
544 pcmk__trace("TLS dispatch triggered after disconnect");
545 return -1;
546 }
547
548 pcmk__trace("TLS dispatch triggered");
549
550 rc = pcmk__remote_ready(native->remote, 0);
551 if (rc == pcmk_rc_ok) {
552 rc = pcmk__read_remote_message(native->remote, -1);
553 }
554
555 if (rc != pcmk_rc_ok && rc != ETIME) {
556 pcmk__info("Lost %s executor connection while reading data",
557 pcmk__s(native->remote_nodename, "local"));
558 lrmd_tls_disconnect(lrmd);
559 return -1;
560 }
561
562 /* If rc is ETIME, there was nothing to read but we may already have a
563 * full message in the buffer
564 */
565 xml = pcmk__remote_message_xml(native->remote);
566
567 if (xml == NULL) {
568 return 0;
569 }
570
571 handle_remote_msg(xml, lrmd);
572 pcmk__xml_free(xml);
573 return 0;
574 }
575
576 /* Not used with mainloop */
577 bool
578 lrmd_dispatch(lrmd_t * lrmd)
579 {
580 lrmd_private_t *private = NULL;
581
582 pcmk__assert(lrmd != NULL);
583
584 private = lrmd->lrmd_private;
585 switch (private->type) {
586 case pcmk__client_ipc:
587 while (crm_ipc_ready(private->ipc)) {
588 if (crm_ipc_read(private->ipc) > 0) {
589 const char *msg = crm_ipc_buffer(private->ipc);
590
591 lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
592 pcmk__ipc_free_client_buffer(private->ipc);
593 }
594 }
595 break;
596 case pcmk__client_tls:
597 lrmd_tls_dispatch(lrmd);
598 break;
599 default:
600 pcmk__err("Unsupported executor connection type (bug?): %d",
601 private->type);
602 }
603
604 if (!lrmd->cmds->is_connected(lrmd)) {
605 pcmk__err("Connection closed");
606 return FALSE;
607 }
608
609 return TRUE;
610 }
611
612 static xmlNode *
613 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
614 enum lrmd_call_options options)
615 {
616 xmlNode *op_msg = NULL;
617
618 CRM_CHECK(token != NULL, return NULL);
619
620 op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
621 pcmk__xe_set(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
622 pcmk__xe_set(op_msg, PCMK__XA_LRMD_OP, op);
623 pcmk__xe_set_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
624 pcmk__xe_set_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);
625
626 if (data != NULL) {
627 xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA);
628
629 pcmk__xml_copy(wrapper, data);
630 }
631
632 pcmk__trace("Created executor %s command with call options %.8lx (%d)",
633 op, (long) options, options);
634 return op_msg;
635 }
636
637 static void
638 lrmd_ipc_connection_destroy(gpointer userdata)
639 {
640 lrmd_t *lrmd = userdata;
641 lrmd_private_t *native = lrmd->lrmd_private;
642
643 switch (native->type) {
644 case pcmk__client_ipc:
645 pcmk__info("Disconnected from local executor");
646 break;
647 case pcmk__client_tls:
648 pcmk__info("Disconnected from remote executor on %s",
649 native->remote_nodename);
650 break;
651 default:
652 pcmk__err("Unsupported executor connection type %d (bug?)",
653 native->type);
654 }
655
656 /* Prevent these from being cleaned up in lrmd_api_disconnect() */
657 native->ipc = NULL;
658 native->source = NULL;
659
660 if (native->callback) {
661 lrmd_event_data_t event = { 0, };
662 event.type = lrmd_event_disconnect;
663 event.remote_nodename = native->remote_nodename;
664 native->callback(&event);
665 }
666 }
667
668 /*!
669 * \internal
670 * \brief Initialize the Pacemaker Remote authentication key
671 *
672 * Try loading the Pacemaker Remote authentication key from cache if available,
673 * otherwise from these locations, in order of preference:
674 *
675 * - The value of the PCMK_authkey_location environment variable, if set
676 * - The Pacemaker default key file location
677 *
678 * \param[out] key Where to store key
679 *
680 * \return Standard Pacemaker return code
681 */
682 int
683 lrmd__init_remote_key(gnutls_datum_t *key)
684 {
685 static const char *env_location = NULL;
686 static bool need_env = true;
687
688 int rc = pcmk_rc_ok;
689
690 if (need_env) {
691 env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
692 need_env = false;
693 }
694
695 if (remote_key.data != NULL) {
696 pcmk__copy_key(key, &remote_key);
697 return pcmk_rc_ok;
698 }
699
700 // Try location in environment variable, if set
701 if (env_location != NULL) {
702 rc = pcmk__load_key(env_location, &remote_key, true);
703
704 if (rc == pcmk_rc_ok) {
705 pcmk__copy_key(key, &remote_key);
706 return pcmk_rc_ok;
707 }
708
709 pcmk__warn("Could not read Pacemaker Remote key from %s: %s",
710 env_location, pcmk_rc_str(rc));
711 return ENOKEY;
712 }
713
714 // Try default location, if environment wasn't explicitly set to it
715 rc = pcmk__load_key(DEFAULT_REMOTE_KEY_LOCATION, &remote_key, true);
716
717 if (rc == pcmk_rc_ok) {
718 pcmk__copy_key(key, &remote_key);
719 return pcmk_rc_ok;
720 }
721
722 pcmk__warn("Could not read Pacemaker Remote key from default location "
723 DEFAULT_REMOTE_KEY_LOCATION ": %s",
724 pcmk_rc_str(rc));
725 return ENOKEY;
726 }
727
728 // \return Standard Pacemaker return code
729 int
730 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
731 const char *msg_type)
732 {
733 pcmk__xe_set_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
734 pcmk__xe_set(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
735 return pcmk__remote_send_xml(session, msg);
736 }
737
738 // \return Standard Pacemaker return code
739 static int
740 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
741 xmlNode **reply)
742 {
743 lrmd_private_t *native = lrmd->lrmd_private;
744 time_t start = time(NULL);
745 const char *msg_type = NULL;
746 int reply_id = 0;
747 int remaining_timeout = 0;
748 int rc = pcmk_rc_ok;
749
750 /* A timeout of 0 here makes no sense. We have to wait a period of time
751 * for the response to come back. If -1 or 0, default to 10 seconds. */
752 if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
753 total_timeout = MAX_TLS_RECV_WAIT;
754 }
755
756 for (*reply = NULL; *reply == NULL; ) {
757
758 *reply = pcmk__remote_message_xml(native->remote);
759 if (*reply == NULL) {
760 /* read some more off the tls buffer if we still have time left. */
761 if (remaining_timeout) {
762 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
763 } else {
764 remaining_timeout = total_timeout;
765 }
766 if (remaining_timeout <= 0) {
767 return ETIME;
768 }
769
770 rc = pcmk__read_remote_message(native->remote, remaining_timeout);
771 if (rc != pcmk_rc_ok) {
772 return rc;
773 }
774
775 *reply = pcmk__remote_message_xml(native->remote);
776 if (*reply == NULL) {
777 return ENOMSG;
778 }
779 }
780
781 pcmk__xe_get_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
782 msg_type = pcmk__xe_get(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
783
784 if (!msg_type) {
785 pcmk__err("Empty msg type received while waiting for reply");
786 g_clear_pointer(reply, pcmk__xml_free);
787
788 } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
789 /* got a notify while waiting for reply, trigger the notify to be processed later */
790 pcmk__info("queueing notify");
791 native->pending_notify = g_list_append(native->pending_notify, *reply);
792 if (native->process_notify) {
793 pcmk__info("notify trigger set");
794 mainloop_set_trigger(native->process_notify);
795 }
796 *reply = NULL;
797 } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
798 /* msg isn't a reply, make some noise */
799 pcmk__err("Expected a reply, got %s", msg_type);
800 g_clear_pointer(reply, pcmk__xml_free);
801
802 } else if (reply_id != expected_reply_id) {
803 if (native->expected_late_replies > 0) {
804 native->expected_late_replies--;
805 } else {
806 pcmk__err("Got outdated reply, expected id %d got id %d",
807 expected_reply_id, reply_id);
808 }
809
810 g_clear_pointer(reply, pcmk__xml_free);
811 }
812 }
813
814 if (native->remote->buffer && native->process_notify) {
815 mainloop_set_trigger(native->process_notify);
816 }
817
818 return rc;
819 }
820
821 // \return Standard Pacemaker return code
822 static int
823 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
824 {
825 int rc = pcmk_rc_ok;
826 lrmd_private_t *native = lrmd->lrmd_private;
827
828 global_remote_msg_id++;
829 if (global_remote_msg_id <= 0) {
830 global_remote_msg_id = 1;
831 }
832
833 rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
834 "request");
835 if (rc != pcmk_rc_ok) {
836 pcmk__err("Disconnecting because TLS message could not be sent to "
837 "Pacemaker Remote: %s",
838 pcmk_rc_str(rc));
839 lrmd_tls_disconnect(lrmd);
840 }
841 return rc;
842 }
843
844 static int
845 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
846 {
847 int rc = 0;
848 xmlNode *xml = NULL;
849
850 if (!remote_executor_connected(lrmd)) {
851 return -ENOTCONN;
852 }
853
854 rc = send_remote_message(lrmd, msg);
855 if (rc != pcmk_rc_ok) {
856 return pcmk_rc2legacy(rc);
857 }
858
859 rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
860 if (rc != pcmk_rc_ok) {
861 pcmk__err("Disconnecting remote after request %d reply not received: "
862 "%s " QB_XS " rc=%d timeout=%dms",
863 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
864 lrmd_tls_disconnect(lrmd);
865 }
866
867 if (reply) {
868 *reply = xml;
869 } else {
870 pcmk__xml_free(xml);
871 }
872
873 return pcmk_rc2legacy(rc);
874 }
875
876 static int
877 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
878 {
879 int rc = pcmk_ok;
880 lrmd_private_t *native = lrmd->lrmd_private;
881
882 switch (native->type) {
883 case pcmk__client_ipc:
884 rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
885 break;
886 case pcmk__client_tls:
887 rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
888 break;
889 default:
890 pcmk__err("Unsupported executor connection type (bug?): %d",
891 native->type);
892 rc = -EPROTONOSUPPORT;
893 }
894
895 return rc;
896 }
897
898 static int
899 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
900 {
901 int rc = pcmk_ok;
902 lrmd_private_t *native = lrmd->lrmd_private;
903
904 switch (native->type) {
905 case pcmk__client_ipc:
906 rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
907 break;
908 case pcmk__client_tls:
909 rc = send_remote_message(lrmd, msg);
910 if (rc == pcmk_rc_ok) {
911 /* we don't want to wait around for the reply, but
912 * since the request/reply protocol needs to behave the same
913 * as libqb, a reply will eventually come later anyway. */
914 native->expected_late_replies++;
915 }
916 rc = pcmk_rc2legacy(rc);
917 break;
918 default:
919 pcmk__err("Unsupported executor connection type (bug?): %d",
920 native->type);
921 rc = -EPROTONOSUPPORT;
922 }
923
924 return rc;
925 }
926
927 /*!
928 * \internal
929 * \brief Send a prepared API command to the executor
930 *
931 * \param[in,out] lrmd Existing connection to the executor
932 * \param[in] op Name of API command to send
933 * \param[in] data Command data XML to add to the sent command
934 * \param[out] output_data If expecting a reply, it will be stored here
935 * \param[in] timeout Timeout in milliseconds (if 0, defaults to
936 * a sensible value per the type of connection,
937 * standard vs. pacemaker remote);
938 * also propagated to the command XML
939 * \param[in] call_options Call options to pass to server when sending
940 * \param[in] expect_reply If true, wait for a reply from the server;
941 * must be true for IPC (as opposed to TLS) clients
942 *
943 * \return pcmk_ok on success, -errno on error
944 */
945 static int
946 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
947 xmlNode **output_data, int timeout,
948 enum lrmd_call_options options, bool expect_reply)
949 {
950 int rc = pcmk_ok;
951 lrmd_private_t *native = lrmd->lrmd_private;
952 xmlNode *op_msg = NULL;
953 xmlNode *op_reply = NULL;
954
955 if (!lrmd->cmds->is_connected(lrmd)) {
956 return -ENOTCONN;
957 }
958
959 if (op == NULL) {
960 pcmk__err("No operation specified");
961 return -EINVAL;
962 }
963
964 CRM_LOG_ASSERT(native->token != NULL);
965 pcmk__trace("Sending %s op to executor", op);
966
967 op_msg = lrmd_create_op(native->token, op, data, timeout, options);
968
969 if (op_msg == NULL) {
970 return -EINVAL;
971 }
972
973 if (expect_reply) {
974 rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
975 } else {
976 rc = lrmd_send_xml_no_reply(lrmd, op_msg);
977 goto done;
978 }
979
980 if (rc < 0) {
981 pcmk__err("Couldn't perform %s operation (timeout=%d): %d", op, timeout,
982 pcmk_strerror(rc));
983 goto done;
984
985 } else if (op_reply == NULL) {
986 rc = -ENOMSG;
987 goto done;
988 }
989
990 /* The only reason we can receive an ACK here is because we sent a request
991 * that execd didn't recognize and it responded via handle_unknown_request,
992 * and the status code will always indicate some sort of error.
993 */
994 if (pcmk__xe_is(op_reply, PCMK__XE_ACK)) {
995 handle_ack(op_reply);
996 rc = -EPROTO;
997 goto done;
998 }
999
1000 pcmk__trace("%s op reply received", op);
1001 pcmk__log_xml_trace(op_reply, "Reply");
1002
1003 rc = pcmk_ok;
1004 if (pcmk__xe_get_int(op_reply, PCMK__XA_LRMD_RC, &rc) != pcmk_rc_ok) {
1005 rc = -ENOMSG;
1006 goto done;
1007 }
1008
1009 if (output_data) {
1010 *output_data = op_reply;
1011 op_reply = NULL; /* Prevent subsequent free */
1012 }
1013
1014 done:
1015 if (!lrmd->cmds->is_connected(lrmd)) {
1016 pcmk__err("Executor disconnected");
1017 }
1018
1019 pcmk__xml_free(op_msg);
1020 pcmk__xml_free(op_reply);
1021 return rc;
1022 }
1023
1024 // \return Standard Pacemaker return code
1025 int
1026 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
1027 {
1028 int rc = pcmk_rc_ok;
1029 const char *value;
1030 lrmd_private_t *native = lrmd->lrmd_private;
1031 xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP);
1032
1033 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1034
1035 value = pcmk__cluster_option(hash, PCMK_OPT_FENCING_WATCHDOG_TIMEOUT);
1036 if ((value) &&
1037 (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
1038 pcmk__xe_set(data, PCMK__XA_LRMD_WATCHDOG, value);
1039 }
1040
1041 rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
1042 (native->type == pcmk__client_ipc));
1043 pcmk__xml_free(data);
1044 return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
1045 }
1046
1047 static int
1048 lrmd_ipc_connect(lrmd_t *lrmd, int *fd)
1049 {
1050 int rc = pcmk_ok;
1051 lrmd_private_t *native = lrmd->lrmd_private;
1052
1053 struct ipc_client_callbacks lrmd_callbacks = {
1054 .dispatch = lrmd_ipc_dispatch,
1055 .destroy = lrmd_ipc_connection_destroy
1056 };
1057
1058 pcmk__info("Connecting to executor");
1059
1060 if (fd) {
1061 /* No mainloop */
1062 native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1063 if (native->ipc != NULL) {
1064 rc = pcmk__connect_generic_ipc(native->ipc);
1065 if (rc == pcmk_rc_ok) {
1066 rc = pcmk__ipc_fd(native->ipc, fd);
1067 }
1068 if (rc != pcmk_rc_ok) {
1069 pcmk__err("Connection to executor failed: %s", pcmk_rc_str(rc));
1070 rc = -ENOTCONN;
1071 }
1072 }
1073 } else {
1074 native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1075 native->ipc = mainloop_get_ipc_client(native->source);
1076 }
1077
1078 if (native->ipc == NULL) {
1079 pcmk__debug("Could not connect to the executor API");
1080 rc = -ENOTCONN;
1081 }
1082
1083 return rc;
1084 }
1085
1086 static void
1087 lrmd_tls_connection_destroy(gpointer userdata)
1088 {
1089 lrmd_t *lrmd = userdata;
1090 lrmd_private_t *native = lrmd->lrmd_private;
1091
1092 pcmk__info("TLS connection destroyed");
1093
|
(1) Event path: |
Condition "native->remote->tls_session", taking true branch. |
1094 if (native->remote->tls_session) {
1095 gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
|
CID (unavailable; MK=bfca432f96bf3ba7d861402b70f24ecf) (#1 of 6): Inconsistent C union access (INCONSISTENT_UNION_ACCESS): |
|
(2) Event assign_union_field: |
The union field "in" of "_pp" is written. |
|
(3) Event inconsistent_union_field_access: |
In "_pp.out", the union field used: "out" is inconsistent with the field most recently stored: "in". |
1096 g_clear_pointer(&native->remote->tls_session, gnutls_deinit);
1097 }
1098
1099 g_clear_pointer(&native->tls, pcmk__free_tls);
1100
1101 if (native->sock >= 0) {
1102 close(native->sock);
1103 }
1104
1105 g_clear_pointer(&native->process_notify, mainloop_destroy_trigger);
1106
1107 g_list_free_full(native->pending_notify, (GDestroyNotify) pcmk__xml_free);
1108 native->pending_notify = NULL;
1109
1110 g_clear_pointer(&native->handshake_trigger, mainloop_destroy_trigger);
1111
1112 g_clear_pointer(&native->remote->buffer, free);
1113 g_clear_pointer(&native->remote->start_state, free);
1114 native->source = 0;
1115 native->sock = -1;
1116
1117 if (native->callback) {
1118 lrmd_event_data_t event = { 0, };
1119 event.remote_nodename = native->remote_nodename;
1120 event.type = lrmd_event_disconnect;
1121 native->callback(&event);
1122 }
1123 }
1124
1125 static void
1126 tls_handshake_failed(lrmd_t *lrmd, int tls_rc, int rc)
1127 {
1128 lrmd_private_t *native = lrmd->lrmd_private;
1129
1130 pcmk__warn("Disconnecting after TLS handshake with Pacemaker Remote server "
1131 "%s:%d failed: %s",
1132 native->server, native->port,
1133 ((rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc)));
1134 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1135
1136 g_clear_pointer(&native->remote->tls_session, gnutls_deinit);
1137 lrmd_tls_connection_destroy(lrmd);
1138 }
1139
1140 /*!
1141 * \internal
1142 * \brief Perform a TLS client handshake with a Pacemaker Remote server
1143 *
1144 * \param[in] lrmd Newly established Pacemaker Remote executor connection
1145 *
1146 * \return Standard Pacemaker return code
1147 */
1148 static int
1149 tls_client_handshake(lrmd_t *lrmd)
1150 {
1151 lrmd_private_t *native = lrmd->lrmd_private;
1152 int tls_rc = GNUTLS_E_SUCCESS;
1153 int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT,
1154 &tls_rc);
1155
1156 if (rc != pcmk_rc_ok) {
1157 tls_handshake_failed(lrmd, tls_rc, rc);
1158 }
1159
1160 return rc;
1161 }
1162
1163 /*!
1164 * \internal
1165 * \brief Notify trigger handler
1166 *
1167 * \param[in,out] userdata API connection
1168 *
1169 * \return Always return G_SOURCE_CONTINUE to leave this trigger handler in the
1170 * mainloop
1171 */
1172 static int
1173 process_pending_notifies(gpointer userdata)
1174 {
1175 lrmd_t *lrmd = userdata;
1176 lrmd_private_t *native = lrmd->lrmd_private;
1177
1178 if (native->pending_notify == NULL) {
1179 return G_SOURCE_CONTINUE;
1180 }
1181
1182 pcmk__trace("Processing pending notifies");
1183 g_list_foreach(native->pending_notify, lrmd_dispatch_internal, lrmd);
1184 g_list_free_full(native->pending_notify, (GDestroyNotify) pcmk__xml_free);
1185 native->pending_notify = NULL;
1186 return G_SOURCE_CONTINUE;
1187 }
1188
1189 static xmlNode *
1190 lrmd_handshake_hello_msg(const char *name, bool is_proxy)
1191 {
1192 xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
1193
1194 pcmk__xe_set(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
1195 pcmk__xe_set(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
1196 pcmk__xe_set(hello, PCMK__XA_LRMD_CLIENTNAME, name);
1197 pcmk__xe_set(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
1198
1199 /* advertise that we are a proxy provider */
1200 if (is_proxy) {
1201 pcmk__xe_set_bool(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
1202 }
1203
1204 return hello;
1205 }
1206
1207 static int
1208 lrmd_handshake_async(lrmd_t *lrmd, const char *name)
1209 {
1210 int rc = pcmk_rc_ok;
1211 lrmd_private_t *native = lrmd->lrmd_private;
1212 xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
1213
1214 rc = send_remote_message(lrmd, hello);
1215
1216 if (rc == pcmk_rc_ok) {
1217 native->expected_late_replies++;
1218 } else {
1219 lrmd->cmds->disconnect(lrmd);
1220 }
1221
1222 pcmk__xml_free(hello);
1223 return rc;
1224 }
1225
1226 /*!
1227 * \internal
1228 * \brief Add trigger and file descriptor mainloop sources for TLS
1229 *
1230 * \param[in,out] lrmd API connection with established TLS session
1231 * \param[in] do_api_handshake Whether to perform executor handshake
1232 *
1233 * \return Standard Pacemaker return code
1234 */
1235 static int
1236 add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake)
1237 {
1238 lrmd_private_t *native = lrmd->lrmd_private;
1239 int rc = pcmk_rc_ok;
1240
1241 char *name = pcmk__assert_asprintf("pacemaker-remote-%s:%d",
1242 native->server, native->port);
1243
1244 struct mainloop_fd_callbacks tls_fd_callbacks = {
1245 .dispatch = lrmd_tls_dispatch,
1246 .destroy = lrmd_tls_connection_destroy,
1247 };
1248
1249 native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1250 process_pending_notifies, lrmd);
1251 native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1252 &tls_fd_callbacks);
1253
1254 /* Async connections lose the client name provided by the API caller, so we
1255 * have to use our generated name here to perform the executor handshake.
1256 *
1257 * @TODO Keep track of the caller-provided name. Perhaps we should be using
1258 * that name in this function instead of generating one anyway.
1259 */
1260 if (do_api_handshake) {
1261 rc = lrmd_handshake_async(lrmd, name);
1262 }
1263 free(name);
1264 return rc;
1265 }
1266
1267 static int
1268 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
1269 {
1270 int rc = pcmk_rc_ok;
1271 lrmd_private_t *native = lrmd->lrmd_private;
1272
1273 native->sock = -1;
1274 rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1275 &(native->sock), NULL, NULL);
1276 if (rc != pcmk_rc_ok) {
1277 pcmk__warn("Pacemaker Remote connection to %s:%d failed: %s "
1278 QB_XS " rc=%d",
1279 native->server, native->port, pcmk_rc_str(rc), rc);
1280 lrmd_tls_connection_destroy(lrmd);
1281 return ENOTCONN;
1282 }
1283
1284 if (native->tls == NULL) {
1285 rc = pcmk__init_tls(&native->tls, false, true);
1286
1287 if ((rc != pcmk_rc_ok) || (native->tls == NULL)) {
1288 lrmd_tls_connection_destroy(lrmd);
1289 return rc;
1290 }
1291 }
1292
1293 if (!pcmk__x509_enabled()) {
1294 gnutls_datum_t psk_key = { NULL, 0 };
1295
1296 rc = lrmd__init_remote_key(&psk_key);
1297 if (rc != pcmk_rc_ok) {
1298 lrmd_tls_connection_destroy(lrmd);
1299 return rc;
1300 }
1301
1302 pcmk__tls_client_add_psk_key(native->tls, DEFAULT_REMOTE_USERNAME,
1303 &psk_key, true);
1304 gnutls_free(psk_key.data);
1305 }
1306
1307 native->remote->tls_session = pcmk__new_tls_session(native->tls, native->sock);
1308 if (native->remote->tls_session == NULL) {
1309 lrmd_tls_connection_destroy(lrmd);
1310 return EPROTO;
1311 }
1312
1313 if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
1314 return EKEYREJECTED;
1315 }
1316
1317 pcmk__info("Client TLS connection established with Pacemaker Remote server "
1318 "%s:%d",
1319 native->server, native->port);
1320
1321 if (fd) {
1322 *fd = native->sock;
1323 } else {
1324 rc = add_tls_to_mainloop(lrmd, false);
1325 }
1326 return rc;
1327 }
1328
1329 static int
1330 lrmd_handshake(lrmd_t *lrmd, const char *name)
1331 {
1332 int rc = pcmk_rc_ok;
1333 lrmd_private_t *native = lrmd->lrmd_private;
1334 xmlNode *reply = NULL;
1335 xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
1336
1337 rc = lrmd_send_xml(lrmd, hello, -1, &reply);
1338
1339 if (rc < 0) {
1340 pcmk__debug("Couldn't complete registration with the executor API: %s",
1341 pcmk_strerror(rc));
1342 rc = ECOMM;
1343 } else if (reply == NULL) {
1344 pcmk__err("Did not receive registration reply");
1345 rc = EPROTO;
1346 } else {
1347 rc = process_lrmd_handshake_reply(reply, native);
1348 }
1349
1350 pcmk__xml_free(reply);
1351 pcmk__xml_free(hello);
1352
1353 if (rc != pcmk_rc_ok) {
1354 lrmd->cmds->disconnect(lrmd);
1355 }
1356
1357 return rc;
1358 }
1359
1360 static int
1361 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
1362 {
1363 int rc = -ENOTCONN;
1364 lrmd_private_t *native = lrmd->lrmd_private;
1365
1366 switch (native->type) {
1367 case pcmk__client_ipc:
1368 rc = lrmd_ipc_connect(lrmd, fd);
1369 break;
1370 case pcmk__client_tls:
1371 rc = lrmd_tls_connect(lrmd, fd);
1372 rc = pcmk_rc2legacy(rc);
1373 break;
1374 default:
1375 pcmk__err("Unsupported executor connection type (bug?): %d",
1376 native->type);
1377 rc = -EPROTONOSUPPORT;
1378 }
1379
1380 if (rc == pcmk_ok) {
1381 rc = lrmd_handshake(lrmd, name);
1382 rc = pcmk_rc2legacy(rc);
1383 }
1384
1385 return rc;
1386 }
1387
1388 static void
1389 tls_handshake_succeeded(lrmd_t *lrmd)
1390 {
1391 int rc = pcmk_rc_ok;
1392 lrmd_private_t *native = lrmd->lrmd_private;
1393
1394 /* Now that the handshake is done, see if any client TLS certificate is
1395 * close to its expiration date and log if so. If a TLS certificate is not
1396 * in use, this function will just return so we don't need to check for the
1397 * session type here.
1398 */
1399 pcmk__tls_check_cert_expiration(native->remote->tls_session);
1400
1401 pcmk__info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1402 native->server, native->port);
1403 rc = add_tls_to_mainloop(lrmd, true);
1404
1405 /* If add_tls_to_mainloop failed, report that right now. Otherwise, we have
1406 * to wait until we read the async reply to report anything.
1407 */
1408 if (rc != pcmk_rc_ok) {
1409 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1410 }
1411 }
1412
1413 struct handshake_data_s {
1414 lrmd_t *lrmd;
1415 time_t start_time;
1416 int timeout_sec;
1417 };
1418
1419 static gboolean
1420 try_handshake_cb(gpointer user_data)
1421 {
1422 struct handshake_data_s *hs = user_data;
1423 lrmd_t *lrmd = hs->lrmd;
1424 lrmd_private_t *native = lrmd->lrmd_private;
1425 pcmk__remote_t *remote = native->remote;
1426
1427 int rc = pcmk_rc_ok;
1428 int tls_rc = GNUTLS_E_SUCCESS;
1429
1430 if (time(NULL) >= hs->start_time + hs->timeout_sec) {
1431 rc = ETIME;
1432
1433 tls_handshake_failed(lrmd, GNUTLS_E_TIMEDOUT, rc);
1434 free(hs);
1435 return 0;
1436 }
1437
1438 rc = pcmk__tls_client_try_handshake(remote, &tls_rc);
1439
1440 if (rc == pcmk_rc_ok) {
1441 tls_handshake_succeeded(lrmd);
1442 free(hs);
1443 return 0;
1444 } else if (rc == EAGAIN) {
1445 mainloop_set_trigger(native->handshake_trigger);
1446 return 1;
1447 } else {
1448 rc = EKEYREJECTED;
1449 tls_handshake_failed(lrmd, tls_rc, rc);
1450 free(hs);
1451 return 0;
1452 }
1453 }
1454
1455 static void
1456 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
1457 {
1458 lrmd_t *lrmd = userdata;
1459 lrmd_private_t *native = lrmd->lrmd_private;
1460 int tls_rc = GNUTLS_E_SUCCESS;
1461
1462 native->async_timer = 0;
1463
1464 if (rc != pcmk_rc_ok) {
1465 lrmd_tls_connection_destroy(lrmd);
1466 pcmk__info("Could not connect to Pacemaker Remote at %s:%d: %s "
1467 QB_XS " rc=%d",
1468 native->server, native->port, pcmk_rc_str(rc), rc);
1469 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1470 return;
1471 }
1472
1473 /* The TCP connection was successful, so establish the TLS connection. */
1474
1475 native->sock = sock;
1476
1477 if (native->tls == NULL) {
1478 rc = pcmk__init_tls(&native->tls, false, true);
1479
1480 if ((rc != pcmk_rc_ok) || (native->tls == NULL)) {
1481 lrmd_tls_connection_destroy(lrmd);
1482 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1483 return;
1484 }
1485 }
1486
1487 if (!pcmk__x509_enabled()) {
1488 gnutls_datum_t psk_key = { NULL, 0 };
1489
1490 rc = lrmd__init_remote_key(&psk_key);
1491 if (rc != pcmk_rc_ok) {
1492 pcmk__info("Could not connect to Pacemaker Remote at %s:%d: %s "
1493 QB_XS " rc=%d",
1494 native->server, native->port, pcmk_rc_str(rc), rc);
1495 lrmd_tls_connection_destroy(lrmd);
1496 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1497 return;
1498 }
1499
1500 pcmk__tls_client_add_psk_key(native->tls, DEFAULT_REMOTE_USERNAME,
1501 &psk_key, true);
1502 gnutls_free(psk_key.data);
1503 }
1504
1505 native->remote->tls_session = pcmk__new_tls_session(native->tls, sock);
1506 if (native->remote->tls_session == NULL) {
1507 lrmd_tls_connection_destroy(lrmd);
1508 report_async_connection_result(lrmd, -EPROTO);
1509 return;
1510 }
1511
1512 /* If the TLS handshake immediately succeeds or fails, we can handle that
1513 * now without having to deal with mainloops and retries. Otherwise, add a
1514 * trigger to keep trying until we get a result (or it times out).
1515 */
1516 rc = pcmk__tls_client_try_handshake(native->remote, &tls_rc);
1517 if (rc == EAGAIN) {
1518 struct handshake_data_s *hs = NULL;
1519
1520 if (native->handshake_trigger != NULL) {
1521 return;
1522 }
1523
1524 hs = pcmk__assert_alloc(1, sizeof(struct handshake_data_s));
1525 hs->lrmd = lrmd;
1526 hs->start_time = time(NULL);
1527 hs->timeout_sec = TLS_HANDSHAKE_TIMEOUT;
1528
1529 native->handshake_trigger = mainloop_add_trigger(G_PRIORITY_LOW, try_handshake_cb, hs);
1530 mainloop_set_trigger(native->handshake_trigger);
1531
1532 } else if (rc == pcmk_rc_ok) {
1533 tls_handshake_succeeded(lrmd);
1534
1535 } else {
1536 tls_handshake_failed(lrmd, tls_rc, rc);
1537 }
1538 }
1539
1540 static int
1541 lrmd_tls_connect_async(lrmd_t *lrmd, int timeout)
1542 {
1543 int rc = pcmk_rc_ok;
1544 int timer_id = 0;
1545 lrmd_private_t *native = lrmd->lrmd_private;
1546
1547 native->sock = -1;
1548 rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1549 &(native->sock), lrmd, lrmd_tcp_connect_cb);
1550 if (rc != pcmk_rc_ok) {
1551 pcmk__warn("Pacemaker Remote connection to %s:%d failed: %s "
1552 QB_XS " rc=%d",
1553 native->server, native->port, pcmk_rc_str(rc), rc);
1554 return rc;
1555 }
1556 native->async_timer = timer_id;
1557 return rc;
1558 }
1559
1560 static int
1561 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
1562 {
1563 int rc = pcmk_ok;
1564 lrmd_private_t *native = lrmd->lrmd_private;
1565
1566 CRM_CHECK(native && native->callback, return -EINVAL);
1567
1568 switch (native->type) {
1569 case pcmk__client_ipc:
1570 /* fake async connection with ipc. it should be fast
1571 * enough that we gain very little from async */
1572 rc = lrmd_api_connect(lrmd, name, NULL);
1573 if (!rc) {
1574 report_async_connection_result(lrmd, rc);
1575 }
1576 break;
1577 case pcmk__client_tls:
1578 rc = lrmd_tls_connect_async(lrmd, timeout);
1579 rc = pcmk_rc2legacy(rc);
1580 break;
1581 default:
1582 pcmk__err("Unsupported executor connection type (bug?): %d",
1583 native->type);
1584 rc = -EPROTONOSUPPORT;
1585 }
1586
1587 return rc;
1588 }
1589
1590 static int
1591 lrmd_api_is_connected(lrmd_t * lrmd)
1592 {
1593 lrmd_private_t *native = lrmd->lrmd_private;
1594
1595 switch (native->type) {
1596 case pcmk__client_ipc:
1597 return crm_ipc_connected(native->ipc);
1598 case pcmk__client_tls:
1599 return remote_executor_connected(lrmd);
1600 default:
1601 pcmk__err("Unsupported executor connection type (bug?): %d",
1602 native->type);
1603 return 0;
1604 }
1605 }
1606
1607 static int
1608 lrmd_api_poke_connection(lrmd_t *lrmd)
1609 {
1610 int rc;
1611 lrmd_private_t *native = lrmd->lrmd_private;
1612 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1613
1614 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1615 rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
1616 (native->type == pcmk__client_ipc));
1617 pcmk__xml_free(data);
1618
1619 return rc < 0 ? rc : pcmk_ok;
1620 }
1621
1622 static void
1623 lrmd_ipc_disconnect(lrmd_t *lrmd)
1624 {
1625 lrmd_private_t *native = lrmd->lrmd_private;
1626
1627 if (native->source != NULL) {
1628 /* Attached to mainloop */
1629 g_clear_pointer(&native->source, mainloop_del_ipc_client);
1630 native->ipc = NULL;
1631
1632 } else if (native->ipc) {
1633 /* Not attached to mainloop */
1634 crm_ipc_t *ipc = native->ipc;
1635
1636 native->ipc = NULL;
1637 crm_ipc_close(ipc);
1638 crm_ipc_destroy(ipc);
1639 }
1640 }
1641
1642 static int
1643 lrmd_api_disconnect(lrmd_t * lrmd)
1644 {
1645 lrmd_private_t *native = lrmd->lrmd_private;
1646 int rc = pcmk_ok;
1647
1648 switch (native->type) {
1649 case pcmk__client_ipc:
1650 pcmk__debug("Disconnecting from local executor");
1651 lrmd_ipc_disconnect(lrmd);
1652 break;
1653 case pcmk__client_tls:
1654 pcmk__debug("Disconnecting from remote executor on %s",
1655 native->remote_nodename);
1656 lrmd_tls_disconnect(lrmd);
1657 break;
1658 default:
1659 pcmk__err("Unsupported executor connection type (bug?): %d",
1660 native->type);
1661 rc = -EPROTONOSUPPORT;
1662 }
1663
1664 g_clear_pointer(&native->token, free);
1665 g_clear_pointer(&native->peer_version, free);
1666 return rc;
1667 }
1668
1669 static int
1670 lrmd_api_register_rsc(lrmd_t * lrmd,
1671 const char *rsc_id,
1672 const char *class,
1673 const char *provider, const char *type, enum lrmd_call_options options)
1674 {
1675 int rc = pcmk_ok;
1676 xmlNode *data = NULL;
1677
1678 if (!class || !type || !rsc_id) {
1679 return -EINVAL;
1680 }
1681 if (pcmk__is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1682 && (provider == NULL)) {
1683 return -EINVAL;
1684 }
1685
1686 data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1687
1688 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1689 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1690 pcmk__xe_set(data, PCMK__XA_LRMD_CLASS, class);
1691 pcmk__xe_set(data, PCMK__XA_LRMD_PROVIDER, provider);
1692 pcmk__xe_set(data, PCMK__XA_LRMD_TYPE, type);
1693 rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, true);
1694 pcmk__xml_free(data);
1695
1696 return rc;
1697 }
1698
1699 static int
1700 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
1701 {
1702 int rc = pcmk_ok;
1703 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1704
1705 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1706 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1707 rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, true);
1708 pcmk__xml_free(data);
1709
1710 return rc;
1711 }
1712
1713 lrmd_rsc_info_t *
1714 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
1715 const char *provider, const char *type)
1716 {
1717 lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t));
1718
1719 rsc_info->id = pcmk__str_copy(rsc_id);
1720 rsc_info->standard = pcmk__str_copy(standard);
1721 rsc_info->provider = pcmk__str_copy(provider);
1722 rsc_info->type = pcmk__str_copy(type);
1723 return rsc_info;
1724 }
1725
1726 lrmd_rsc_info_t *
1727 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
1728 {
1729 return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1730 rsc_info->provider, rsc_info->type);
1731 }
1732
1733 void
1734 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
1735 {
1736 if (!rsc_info) {
1737 return;
1738 }
1739 free(rsc_info->id);
1740 free(rsc_info->type);
1741 free(rsc_info->standard);
1742 free(rsc_info->provider);
1743 free(rsc_info);
1744 }
1745
1746 static lrmd_rsc_info_t *
1747 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
1748 {
1749 lrmd_rsc_info_t *rsc_info = NULL;
1750 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1751 xmlNode *output = NULL;
1752 const char *class = NULL;
1753 const char *provider = NULL;
1754 const char *type = NULL;
1755
1756 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1757 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1758 lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, true);
1759 pcmk__xml_free(data);
1760
1761 if (!output) {
1762 return NULL;
1763 }
1764
1765 class = pcmk__xe_get(output, PCMK__XA_LRMD_CLASS);
1766 provider = pcmk__xe_get(output, PCMK__XA_LRMD_PROVIDER);
1767 type = pcmk__xe_get(output, PCMK__XA_LRMD_TYPE);
1768
1769 if (!class || !type) {
1770 pcmk__xml_free(output);
1771 return NULL;
1772 } else if (pcmk__is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1773 && (provider == NULL)) {
1774 pcmk__xml_free(output);
1775 return NULL;
1776 }
1777
1778 rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1779 pcmk__xml_free(output);
1780 return rsc_info;
1781 }
1782
1783 void
1784 lrmd_free_op_info(lrmd_op_info_t *op_info)
1785 {
1786 if (op_info) {
1787 free(op_info->rsc_id);
1788 free(op_info->action);
1789 free(op_info->interval_ms_s);
1790 free(op_info->timeout_ms_s);
1791 free(op_info);
1792 }
1793 }
1794
1795 static int
1796 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
1797 enum lrmd_call_options options, GList **output)
1798 {
1799 xmlNode *data = NULL;
1800 xmlNode *output_xml = NULL;
1801 int rc = pcmk_ok;
1802
1803 if (output == NULL) {
1804 return -EINVAL;
1805 }
1806 *output = NULL;
1807
1808 // Send request
1809 if (rsc_id) {
1810 data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1811 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1812 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1813 }
1814 rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1815 timeout_ms, options, true);
1816 if (data) {
1817 pcmk__xml_free(data);
1818 }
1819
1820 // Process reply
1821 if ((rc != pcmk_ok) || (output_xml == NULL)) {
1822 return rc;
1823 }
1824 for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml,
1825 PCMK__XE_LRMD_RSC, NULL,
1826 NULL);
1827 (rsc_xml != NULL) && (rc == pcmk_ok);
1828 rsc_xml = pcmk__xe_next(rsc_xml, PCMK__XE_LRMD_RSC)) {
1829
1830 rsc_id = pcmk__xe_get(rsc_xml, PCMK__XA_LRMD_RSC_ID);
1831 if (rsc_id == NULL) {
1832 pcmk__err("Could not parse recurring operation information from "
1833 "executor");
1834 continue;
1835 }
1836 for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml,
1837 PCMK__XE_LRMD_RSC_OP,
1838 NULL, NULL);
1839 op_xml != NULL;
1840 op_xml = pcmk__xe_next(op_xml, PCMK__XE_LRMD_RSC_OP)) {
1841
1842 lrmd_op_info_t *op_info =
1843 pcmk__assert_alloc(1, sizeof(lrmd_op_info_t));
1844
1845 op_info->rsc_id = pcmk__str_copy(rsc_id);
1846 op_info->action = pcmk__xe_get_copy(op_xml,
1847 PCMK__XA_LRMD_RSC_ACTION);
1848 op_info->interval_ms_s =
1849 pcmk__xe_get_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
1850 op_info->timeout_ms_s = pcmk__xe_get_copy(op_xml,
1851 PCMK__XA_LRMD_TIMEOUT);
1852 *output = g_list_prepend(*output, op_info);
1853 }
1854 }
1855 pcmk__xml_free(output_xml);
1856 return rc;
1857 }
1858
1859
1860 static void
1861 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
1862 {
1863 lrmd_private_t *native = lrmd->lrmd_private;
1864
1865 native->callback = callback;
1866 }
1867
1868 static int
1869 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
1870 const char *type, char **output,
1871 enum lrmd_call_options options)
1872 {
1873 return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1874 output, options, NULL);
1875 }
1876
1877 static int
1878 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
1879 const char *userdata, guint interval_ms,
1880 int timeout, /* ms */
1881 int start_delay, /* ms */
1882 enum lrmd_call_options options, lrmd_key_value_t * params)
1883 {
1884 int rc = pcmk_ok;
1885 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1886 xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
1887 lrmd_key_value_t *tmp = NULL;
1888
1889 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1890 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1891 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ACTION, action);
1892 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
1893 pcmk__xe_set_guint(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
1894 pcmk__xe_set_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
1895 pcmk__xe_set_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);
1896
1897 for (tmp = params; tmp; tmp = tmp->next) {
1898 hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
1899 }
1900
1901 rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, true);
1902 pcmk__xml_free(data);
1903
1904 lrmd_key_value_freeall(params);
1905 return rc;
1906 }
1907
1908 static int
1909 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
1910 guint interval_ms)
1911 {
1912 int rc = pcmk_ok;
1913 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1914
1915 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
1916 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ACTION, action);
1917 pcmk__xe_set(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1918 pcmk__xe_set_guint(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
1919 rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, true);
1920 pcmk__xml_free(data);
1921 return rc;
1922 }
1923
1924 static int
1925 list_stonith_agents(lrmd_list_t **resources)
1926 {
1927 int rc = 0;
1928 stonith_t *stonith_api = stonith__api_new();
1929 stonith_key_value_t *stonith_resources = NULL;
1930 stonith_key_value_t *dIter = NULL;
1931
1932 if (stonith_api == NULL) {
1933 pcmk__err("Could not list fence agents: API memory allocation failed");
1934 return -ENOMEM;
1935 }
1936 stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
1937 &stonith_resources, 0);
1938 stonith_api->cmds->free(stonith_api);
1939
1940 for (dIter = stonith_resources; dIter; dIter = dIter->next) {
1941 rc++;
1942 if (resources) {
1943 *resources = lrmd_list_add(*resources, dIter->value);
1944 }
1945 }
1946
1947 stonith__key_value_freeall(stonith_resources, true, false);
1948 return rc;
1949 }
1950
1951 static int
1952 lrmd_api_list_agents(lrmd_t *lrmd, lrmd_list_t **resources, const char *class,
1953 const char *provider)
1954 {
1955 int rc = 0;
1956 int stonith_count = 0; // Initially, whether to include stonith devices
1957
1958 if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1959 stonith_count = 1;
1960
1961 } else {
1962 GList *gIter = NULL;
1963 GList *agents = resources_list_agents(class, provider);
1964
1965 for (gIter = agents; gIter != NULL; gIter = gIter->next) {
1966 *resources = lrmd_list_add(*resources, (const char *)gIter->data);
1967 rc++;
1968 }
1969 g_list_free_full(agents, free);
1970
1971 if (!class) {
1972 stonith_count = 1;
1973 }
1974 }
1975
1976 if (stonith_count) {
1977 // Now, if stonith devices are included, how many there are
1978 stonith_count = list_stonith_agents(resources);
1979 if (stonith_count > 0) {
1980 rc += stonith_count;
1981 }
1982 }
1983 if (rc == 0) {
1984 pcmk__notice("No agents found for class %s", class);
1985 rc = -EPROTONOSUPPORT;
1986 }
1987 return rc;
1988 }
1989
1990 static bool
1991 does_provider_have_agent(const char *agent, const char *provider,
1992 const char *class)
1993 {
1994 bool found = false;
1995 GList *agents = NULL;
1996 GList *gIter2 = NULL;
1997
1998 agents = resources_list_agents(class, provider);
1999 for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2000 if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2001 found = true;
2002 }
2003 }
2004 g_list_free_full(agents, free);
2005 return found;
2006 }
2007
2008 static int
2009 lrmd_api_list_ocf_providers(lrmd_t *lrmd, const char *agent,
2010 lrmd_list_t **providers)
2011 {
2012 int rc = pcmk_ok;
2013 char *provider = NULL;
2014 GList *ocf_providers = NULL;
2015 GList *gIter = NULL;
2016
2017 ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2018
2019 for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2020 provider = gIter->data;
2021 if (!agent || does_provider_have_agent(agent, provider,
2022 PCMK_RESOURCE_CLASS_OCF)) {
2023 *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2024 rc++;
2025 }
2026 }
2027
2028 g_list_free_full(ocf_providers, free);
2029 return rc;
2030 }
2031
2032 static int
2033 lrmd_api_list_standards(lrmd_t *lrmd, lrmd_list_t **supported)
2034 {
2035 int rc = 0;
2036 GList *standards = NULL;
2037 GList *gIter = NULL;
2038
2039 standards = resources_list_standards();
2040
2041 for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2042 *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2043 rc++;
2044 }
2045
2046 if (list_stonith_agents(NULL) > 0) {
2047 *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2048 rc++;
2049 }
2050
2051 g_list_free_full(standards, free);
2052 return rc;
2053 }
2054
2055 /* timeout is in ms */
2056 static int
2057 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
2058 int timeout, lrmd_key_value_t *params)
2059 {
2060 int rc = pcmk_ok;
2061 xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT);
2062 xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2063 lrmd_key_value_t *tmp = NULL;
2064
2065 pcmk__xe_set(data, PCMK__XA_LRMD_ORIGIN, __func__);
2066 pcmk__xe_set(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
2067 pcmk__xe_set(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
2068 pcmk__xe_set_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2069
2070 for (tmp = params; tmp; tmp = tmp->next) {
2071 hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2072 }
2073
2074 rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2075 lrmd_opt_notify_orig_only, true);
2076 pcmk__xml_free(data);
2077
2078 lrmd_key_value_freeall(params);
2079 return rc;
2080 }
2081
2082 static int
2083 stonith_get_metadata(const char *type, char **output)
2084 {
2085 int rc = pcmk_ok;
2086 stonith_t *stonith_api = stonith__api_new();
2087
2088 if (stonith_api == NULL) {
2089 pcmk__err("Could not get fence agent meta-data: API memory allocation "
2090 "failed");
2091 return -ENOMEM;
2092 }
2093
2094 rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, NULL,
2095 output, 0);
2096 if ((rc == pcmk_ok) && (*output == NULL)) {
2097 rc = -EIO;
2098 }
2099 stonith_api->cmds->free(stonith_api);
2100 return rc;
2101 }
2102
2103 static int
2104 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
2105 const char *provider, const char *type,
2106 char **output, enum lrmd_call_options options,
2107 lrmd_key_value_t *params)
2108 {
2109 svc_action_t *action = NULL;
2110 GHashTable *params_table = NULL;
2111
2112 if (!standard || !type) {
2113 lrmd_key_value_freeall(params);
2114 return -EINVAL;
2115 }
2116
2117 if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2118 lrmd_key_value_freeall(params);
2119
2120 // stonith-class resources don't support a provider
2121 return stonith_get_metadata(type, output);
2122 }
2123
2124 params_table = pcmk__strkey_table(free, free);
2125 for (const lrmd_key_value_t *param = params; param; param = param->next) {
2126 pcmk__insert_dup(params_table, param->key, param->value);
2127 }
2128 action = services__create_resource_action(type, standard, provider, type,
2129 PCMK_ACTION_META_DATA, 0,
2130 PCMK_DEFAULT_ACTION_TIMEOUT_MS,
2131 params_table, 0);
2132 lrmd_key_value_freeall(params);
2133
2134 if (action == NULL) {
2135 return -ENOMEM;
2136 }
2137 if (action->rc != PCMK_OCF_UNKNOWN) {
2138 services_action_free(action);
2139 return -EINVAL;
2140 }
2141
2142 if (!services_action_sync(action)) {
2143 pcmk__err("Failed to retrieve meta-data for %s:%s:%s", standard,
2144 provider, type);
2145 services_action_free(action);
2146 return -EIO;
2147 }
2148
2149 if (!action->stdout_data) {
2150 pcmk__err("Failed to receive meta-data for %s:%s:%s", standard,
2151 provider, type);
2152 services_action_free(action);
2153 return -EIO;
2154 }
2155
2156 *output = strdup(action->stdout_data);
2157 services_action_free(action);
2158
2159 return pcmk_ok;
2160 }
2161
2162 /*!
2163 * \internal
2164 * \brief Create an executor API object
2165 *
2166 * \param[in] nodename If the object will be used for a remote connection,
2167 * the node name to use in cluster for remote executor
2168 * \param[in] server If the object will be used for a remote connection,
2169 * the resolvable host name to connect to
2170 * \param[in] port If the object will be used for a remote connection,
2171 * port number on \p server to connect to
2172 *
2173 * \return Newly allocated API object
2174 *
2175 * \note The caller is responsible for freeing the return value using
2176 * \c lrmd_api_delete().
2177 * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2178 * value will be used for both. If the caller leaves both NULL, an API
2179 * object will be created for a local executor connection.
2180 */
2181 static lrmd_t *
2182 new_lrmd_api(const char *nodename, const char *server, int port)
2183 {
2184 lrmd_t *api = pcmk__assert_alloc(1, sizeof(lrmd_t));
2185 lrmd_private_t *api_priv = pcmk__assert_alloc(1, sizeof(lrmd_private_t));
2186
2187 api->lrmd_private = api_priv;
2188
2189 // @TODO Do we need to do this for local connections?
2190 api_priv->remote = pcmk__assert_alloc(1, sizeof(pcmk__remote_t));
2191
2192 api->cmds = pcmk__assert_alloc(1, sizeof(lrmd_api_operations_t));
2193
2194 // Set methods
2195 api->cmds->connect = lrmd_api_connect;
2196 api->cmds->connect_async = lrmd_api_connect_async;
2197 api->cmds->is_connected = lrmd_api_is_connected;
2198 api->cmds->poke_connection = lrmd_api_poke_connection;
2199 api->cmds->disconnect = lrmd_api_disconnect;
2200 api->cmds->register_rsc = lrmd_api_register_rsc;
2201 api->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2202 api->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2203 api->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2204 api->cmds->set_callback = lrmd_api_set_callback;
2205 api->cmds->get_metadata = lrmd_api_get_metadata;
2206 api->cmds->exec = lrmd_api_exec;
2207 api->cmds->cancel = lrmd_api_cancel;
2208 api->cmds->list_agents = lrmd_api_list_agents;
2209 api->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2210 api->cmds->list_standards = lrmd_api_list_standards;
2211 api->cmds->exec_alert = lrmd_api_exec_alert;
2212 api->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2213
2214 if ((nodename == NULL) && (server == NULL)) {
2215 api_priv->type = pcmk__client_ipc;
2216 return api;
2217 }
2218
2219 if (nodename == NULL) {
2220 nodename = server;
2221 } else if (server == NULL) {
2222 server = nodename;
2223 }
2224
2225 api_priv->type = pcmk__client_tls;
2226 api_priv->remote_nodename = pcmk__str_copy(nodename);
2227 api_priv->server = pcmk__str_copy(server);
2228 api_priv->port = (port != 0)? port : crm_default_remote_port();
2229 return api;
2230 }
2231
2232 lrmd_t *
2233 lrmd_api_new(void)
2234 {
2235 return new_lrmd_api(NULL, NULL, 0);
2236 }
2237
2238 lrmd_t *
2239 lrmd_remote_api_new(const char *nodename, const char *server, int port)
2240 {
2241 return new_lrmd_api(nodename, server, port);
2242 }
2243
2244 void
2245 lrmd_api_delete(lrmd_t * lrmd)
2246 {
2247 if (lrmd == NULL) {
2248 return;
2249 }
2250 if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2251 if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2252 lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2253 }
2254 free(lrmd->cmds);
2255 }
2256 if (lrmd->lrmd_private != NULL) {
2257 lrmd_private_t *native = lrmd->lrmd_private;
2258
2259 free(native->server);
2260 free(native->remote_nodename);
2261 free(native->remote);
2262 free(native->token);
2263 free(native->peer_version);
2264 free(lrmd->lrmd_private);
2265 }
2266 free(lrmd);
2267 }
2268
2269 struct metadata_cb {
2270 void (*callback)(int pid, const pcmk__action_result_t *result,
2271 void *user_data);
2272 void *user_data;
2273 };
2274
2275 /*!
2276 * \internal
2277 * \brief Process asynchronous metadata completion
2278 *
2279 * \param[in,out] action Metadata action that completed
2280 */
2281 static void
2282 metadata_complete(svc_action_t *action)
2283 {
2284 struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2285 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2286
2287 services__copy_result(action, &result);
2288 pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2289
2290 metadata_cb->callback(0, &result, metadata_cb->user_data);
2291 result.action_stdout = NULL; // Prevent free, because action owns it
2292 result.action_stderr = NULL; // Prevent free, because action owns it
2293 pcmk__reset_result(&result);
2294 free(metadata_cb);
2295 }
2296
2297 /*!
2298 * \internal
2299 * \brief Retrieve agent metadata asynchronously
2300 *
2301 * \param[in] rsc Resource agent specification
2302 * \param[in] callback Function to call with result (this will always be
2303 * called, whether by this function directly or later
2304 * via the main loop, and on success the metadata will
2305 * be in its result argument's action_stdout)
2306 * \param[in,out] user_data User data to pass to callback
2307 *
2308 * \return Standard Pacemaker return code
2309 * \note This function is not a lrmd_api_operations_t method because it does not
2310 * need an lrmd_t object and does not go through the executor, but
2311 * executes the agent directly.
2312 */
2313 int
2314 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
2315 void (*callback)(int pid,
2316 const pcmk__action_result_t *result,
2317 void *user_data),
2318 void *user_data)
2319 {
2320 svc_action_t *action = NULL;
2321 struct metadata_cb *metadata_cb = NULL;
2322 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2323
2324 CRM_CHECK(callback != NULL, return EINVAL);
2325
2326 if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2327 pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2328 PCMK_EXEC_ERROR_FATAL,
2329 "Invalid resource specification");
2330 callback(0, &result, user_data);
2331 pcmk__reset_result(&result);
2332 return EINVAL;
2333 }
2334
2335 if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2336 return stonith__metadata_async(rsc->type,
2337 pcmk__timeout_ms2s(PCMK_DEFAULT_ACTION_TIMEOUT_MS),
2338 callback, user_data);
2339 }
2340
2341 action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2342 rsc->standard, rsc->provider,
2343 rsc->type,
2344 PCMK_ACTION_META_DATA, 0,
2345 PCMK_DEFAULT_ACTION_TIMEOUT_MS,
2346 NULL, 0);
2347 if (action == NULL) {
2348 pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2349 "Out of memory");
2350 callback(0, &result, user_data);
2351 pcmk__reset_result(&result);
2352 return ENOMEM;
2353 }
2354 if (action->rc != PCMK_OCF_UNKNOWN) {
2355 services__copy_result(action, &result);
2356 callback(0, &result, user_data);
2357 pcmk__reset_result(&result);
2358 services_action_free(action);
2359 return EINVAL;
2360 }
2361
2362 action->cb_data = calloc(1, sizeof(struct metadata_cb));
2363 if (action->cb_data == NULL) {
2364 services_action_free(action);
2365 pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2366 "Out of memory");
2367 callback(0, &result, user_data);
2368 pcmk__reset_result(&result);
2369 return ENOMEM;
2370 }
2371
2372 metadata_cb = (struct metadata_cb *) action->cb_data;
2373 metadata_cb->callback = callback;
2374 metadata_cb->user_data = user_data;
2375 if (!services_action_async(action, metadata_complete)) {
2376 services_action_free(action);
2377 return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2378 }
2379
2380 // The services library has taken responsibility for action
2381 return pcmk_rc_ok;
2382 }
2383
2384 /*!
2385 * \internal
2386 * \brief Set the result of an executor event
2387 *
2388 * \param[in,out] event Executor event to set
2389 * \param[in] rc OCF exit status of event
2390 * \param[in] op_status Executor status of event
2391 * \param[in] exit_reason Human-friendly description of event
2392 */
2393 void
2394 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
2395 const char *exit_reason)
2396 {
2397 if (event == NULL) {
2398 return;
2399 }
2400
2401 event->rc = rc;
2402 event->op_status = op_status;
2403
2404 // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
2405 pcmk__str_update((char **) &event->exit_reason, exit_reason);
2406 }
2407
2408 /*!
2409 * \internal
2410 * \brief Clear an executor event's exit reason, output, and error output
2411 *
2412 * \param[in,out] event Executor event to reset
2413 */
2414 void
2415 lrmd__reset_result(lrmd_event_data_t *event)
2416 {
2417 if (event == NULL) {
2418 return;
2419 }
2420
2421 g_clear_pointer(&event->exit_reason, free);
2422 g_clear_pointer(&event->output, free);
2423 }
2424
2425 /*!
2426 * \internal
2427 * \brief Get the uptime of a remote resource connection
2428 *
2429 * When the cluster connects to a remote resource, part of that resource's
2430 * handshake includes the uptime of the remote resource's connection. This
2431 * uptime is stored in the lrmd_t object.
2432 *
2433 * \return The connection's uptime, or -1 if unknown
2434 */
2435 time_t
2436 lrmd__uptime(lrmd_t *lrmd)
2437 {
2438 lrmd_private_t *native = lrmd->lrmd_private;
2439
2440 if (native->remote == NULL) {
2441 return -1;
2442 } else {
2443 return native->remote->uptime;
2444 }
2445 }
2446
2447 const char *
2448 lrmd__node_start_state(lrmd_t *lrmd)
2449 {
2450 lrmd_private_t *native = lrmd->lrmd_private;
2451
2452 if (native->remote == NULL) {
2453 return NULL;
2454 } else {
2455 return native->remote->start_state;
2456 }
2457 }
2458
2459 void
2460 lrmd__proxy_set_callback(lrmd_t *lrmd, void *user_data,
2461 void (*cb)(lrmd_t *, void *, xmlNode *))
2462 {
2463 lrmd_private_t *native = lrmd->lrmd_private;
2464
2465 native->proxy_callback = cb;
2466 native->proxy_callback_userdata = user_data;
2467 }
2468
2469 int
2470 lrmd__proxy_send(lrmd_t *lrmd, xmlNode *msg)
2471 {
2472 CRM_CHECK(msg != NULL, return -EINVAL);
2473
2474 if (lrmd == NULL) {
2475 return -ENOTCONN;
2476 }
2477
2478 pcmk__xe_set(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);
2479 pcmk__log_xml_trace(msg, "PROXY_OUTBOUND");
2480
2481 return lrmd_send_xml_no_reply(lrmd, msg);
2482 }
2483