1    	/*
2    	 * Copyright 2008-2026 the Pacemaker project contributors
3    	 *
4    	 * The version control history for this file may have further details.
5    	 *
6    	 * This source code is licensed under the GNU Lesser General Public License
7    	 * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
8    	 */
9    	
10   	#include <crm_internal.h>
11   	#include <crm/crm.h>
12   	
13   	#include <sys/param.h>
14   	#include <stdio.h>
15   	#include <sys/types.h>
16   	#include <sys/stat.h>
17   	#include <unistd.h>
18   	#include <sys/socket.h>
19   	#include <arpa/inet.h>
20   	#include <netinet/in.h>
21   	#include <netinet/ip.h>
22   	#include <netinet/tcp.h>
23   	#include <netdb.h>
24   	#include <stdlib.h>
25   	#include <errno.h>
26   	#include <inttypes.h>   // PRIu32, PRIx32
27   	
28   	#include <glib.h>
29   	#include <bzlib.h>
30   	
31   	#include <crm/common/ipc_internal.h>
32   	#include <crm/common/xml.h>
33   	#include <crm/common/mainloop.h>
34   	#include <crm/common/remote_internal.h>
35   	
36   	#ifdef HAVE_GNUTLS_GNUTLS_H
37   	#  include <gnutls/gnutls.h>
38   	#endif
39   	
40   	/* Swab macros from linux/swab.h */
41   	#ifdef HAVE_LINUX_SWAB_H
42   	#  include <linux/swab.h>
43   	#else
44   	/*
45   	 * casts are necessary for constants, because we never know how for sure
46   	 * how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
47   	 */
48   	#define __swab16(x) ((uint16_t)(                                      \
49   	        (((uint16_t)(x) & (uint16_t)0x00ffU) << 8) |                  \
50   	        (((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))
51   	
52   	#define __swab32(x) ((uint32_t)(                                      \
53   	        (((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) |            \
54   	        (((uint32_t)(x) & (uint32_t)0x0000ff00UL) <<  8) |            \
55   	        (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >>  8) |            \
56   	        (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))
57   	
58   	#define __swab64(x) ((uint64_t)(                                      \
59   	        (((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) |   \
60   	        (((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) |   \
61   	        (((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) |   \
62   	        (((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) <<  8) |   \
63   	        (((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >>  8) |   \
64   	        (((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) |   \
65   	        (((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) |   \
66   	        (((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
67   	#endif
68   	
69   	#define REMOTE_MSG_VERSION 1
70   	#define ENDIAN_LOCAL 0xBADADBBD
71   	
72   	struct remote_header_v0 {
73   	    uint32_t endian;    /* Detect messages from hosts with different endian-ness */
74   	    uint32_t version;
75   	    uint64_t id;
76   	    uint64_t flags;
77   	    uint32_t size_total;
78   	    uint32_t payload_offset;
79   	    uint32_t payload_compressed;
80   	    uint32_t payload_uncompressed;
81   	
82   	        /* New fields get added here */
83   	
84   	} __attribute__ ((packed));
85   	
86   	/*!
87   	 * \internal
88   	 * \brief Retrieve remote message header, in local endianness
89   	 *
90   	 * Return a pointer to the header portion of a remote connection's message
91   	 * buffer, converting the header to local endianness if needed.
92   	 *
93   	 * \param[in,out] remote  Remote connection with new message
94   	 *
95   	 * \return Pointer to message header, localized if necessary
96   	 */
97   	static struct remote_header_v0 *
98   	localized_remote_header(pcmk__remote_t *remote)
99   	{
100  	    struct remote_header_v0 *header = NULL;
101  	    size_t expected_size = 0;
102  	
(1) Event path: Condition "remote == NULL", taking false branch.
(2) Event path: Condition "remote->buffer == NULL", taking false branch.
(3) Event path: Condition "remote->buffer_offset < 40UL /* sizeof (struct remote_header_v0) */", taking false branch.
103  	    if ((remote == NULL) || (remote->buffer == NULL)
104  	        || (remote->buffer_offset < sizeof(struct remote_header_v0))) {
105  	
106  	        // Caller error or we haven't received the full header yet
107  	        return NULL;
108  	    }
109  	
110  	    header = (struct remote_header_v0 *) remote->buffer;
(4) Event path: Condition "header->endian != 3134905277U", taking true branch.
111  	    if (header->endian != ENDIAN_LOCAL) {
112  	        uint32_t endian = __swab32(header->endian);
113  	
(5) Event path: Condition "!(endian == 3134905277U)", taking false branch.
114  	        CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
(6) Event path: Condition "endian != 3134905277U", taking false branch.
115  	        if(endian != ENDIAN_LOCAL) {
116  	            crm_err("Invalid message detected, endian mismatch: %" PRIx32
117  	                    " is neither %" PRIx32 " nor the swab'd %" PRIx32,
118  	                    ENDIAN_LOCAL, header->endian, endian);
119  	            return NULL;
120  	        }
121  	
(7) Event path: Condition "0", taking false branch.
122  	        header->id = __swab64(header->id);
(8) Event path: Condition "0", taking false branch.
123  	        header->flags = __swab64(header->flags);
(9) Event path: Condition "0", taking false branch.
124  	        header->endian = __swab32(header->endian);
125  	
(10) Event path: Condition "0", taking false branch.
126  	        header->version = __swab32(header->version);
(11) Event path: Condition "0", taking false branch.
127  	        header->size_total = __swab32(header->size_total);
(12) Event path: Condition "0", taking false branch.
128  	        header->payload_offset = __swab32(header->payload_offset);
(13) Event path: Condition "0", taking false branch.
129  	        header->payload_compressed = __swab32(header->payload_compressed);
(14) Event path: Condition "0", taking false branch.
(15) Event tainted_data_return: "__fswab32" returns tainted data. [details]
(16) Event tainted_data_transitive: Calling function "__fswab32" with tainted argument "header->payload_uncompressed" results in tainted data. [details]
(17) Event var_assign_alias: Assigning: "header->payload_uncompressed" = "(__u32)(0 ? (__u32)((((__u32)header->payload_uncompressed & 0xffU) << 24) | (((__u32)header->payload_uncompressed & 0xff00U) << 8) | (((__u32)header->payload_uncompressed & 0xff0000U) >> 8) | (((__u32)header->payload_uncompressed & 0xff000000U) >> 24)) : __fswab32(header->payload_uncompressed))", which taints "header->payload_uncompressed".
Also see events: [return_tainted_data]
130  	        header->payload_uncompressed = __swab32(header->payload_uncompressed);
131  	    }
132  	
133  	    // Sanity checks
(18) Event path: Condition "header->payload_offset != 40UL /* sizeof (struct remote_header_v0) */", taking false branch.
134  	    if (header->payload_offset != sizeof(struct remote_header_v0)) {
135  	        crm_err("Header payload offset %" PRIu32 " does not have expected "
136  	                "size %zu", header->payload_offset,
137  	                sizeof(struct remote_header_v0));
138  	        return NULL;
139  	    }
140  	
(19) Event path: Condition "header->payload_compressed != 0", taking true branch.
141  	    if (header->payload_compressed != 0) {
(20) Event path: Condition "header->payload_compressed > 18446744073709551615UL - header->payload_offset", taking false branch.
142  	        if (header->payload_compressed > (SIZE_MAX - header->payload_offset)) {
143  	            crm_err("Header compressed size %" PRIu32 " is too large",
144  	                    header->payload_compressed);
145  	            return NULL;
146  	        }
147  	
148  	        expected_size = (size_t) header->payload_offset
149  	                        + header->payload_compressed;
150  	
(21) Event path: Falling through to end of if statement.
151  	    } else {
152  	        if (header->payload_uncompressed > (SIZE_MAX - header->payload_offset)) {
153  	            crm_err("Header uncompressed size %" PRIu32 " is too large",
154  	                    header->payload_uncompressed);
155  	            return NULL;
156  	        }
157  	
158  	        expected_size = (size_t) header->payload_offset
159  	                        + header->payload_uncompressed;
160  	    }
161  	
(22) Event path: Condition "expected_size != header->size_total", taking false branch.
162  	    if (expected_size != header->size_total) {
163  	        crm_err("Header total size %" PRIu32 " does not match calculated "
164  	                "size %zu", header->size_total, expected_size);
165  	        return NULL;
166  	    }
167  	
(23) Event return_tainted_data: Returning tainted data "header".
Also see events: [tainted_data_return][tainted_data_transitive][var_assign_alias]
168  	    return header;
169  	}
170  	
171  	#ifdef HAVE_GNUTLS_GNUTLS_H
172  	
173  	int
174  	pcmk__tls_client_try_handshake(pcmk__remote_t *remote, int *gnutls_rc)
175  	{
176  	    int rc = pcmk_rc_ok;
177  	
178  	    if (gnutls_rc != NULL) {
179  	        *gnutls_rc = GNUTLS_E_SUCCESS;
180  	    }
181  	
182  	    rc = gnutls_handshake(*remote->tls_session);
183  	
184  	    switch (rc) {
185  	        case GNUTLS_E_SUCCESS:
186  	            rc = pcmk_rc_ok;
187  	            break;
188  	
189  	        case GNUTLS_E_INTERRUPTED:
190  	        case GNUTLS_E_AGAIN:
191  	            rc = EAGAIN;
192  	            break;
193  	
194  	        default:
195  	            if (gnutls_rc != NULL) {
196  	                *gnutls_rc = rc;
197  	            }
198  	
199  	            rc = EPROTO;
200  	            break;
201  	    }
202  	
203  	    return rc;
204  	}
205  	
206  	int pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_sec,
207  	                               int *gnutls_rc)
208  	{
209  	    const time_t time_limit = time(NULL) + timeout_sec;
210  	
211  	    do {
212  	        int rc = pcmk__tls_client_try_handshake(remote, gnutls_rc);
213  	
214  	        if (rc != EAGAIN) {
215  	            return rc;
216  	        }
217  	    } while (time(NULL) < time_limit);
218  	
219  	    return ETIME;
220  	}
221  	
222  	/*!
223  	 * \internal
224  	 * \brief Set minimum prime size required by TLS client
225  	 *
226  	 * \param[in] session  TLS session to affect
227  	 */
228  	static void
229  	set_minimum_dh_bits(const gnutls_session_t *session)
230  	{
231  	    int dh_min_bits;
232  	
233  	    pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
234  	                       0);
235  	
236  	    /* This function is deprecated since GnuTLS 3.1.7, in favor of letting
237  	     * the priority string imply the DH requirements, but this is the only
238  	     * way to give the user control over compatibility with older servers.
239  	     */
240  	    if (dh_min_bits > 0) {
241  	        crm_info("Requiring server use a Diffie-Hellman prime of at least %d bits",
242  	                 dh_min_bits);
243  	        crm_warn("Support for the " PCMK__ENV_DH_MIN_BITS " "
244  	                 "environment variable is deprecated and will be removed "
245  	                 "in a future release");
246  	        gnutls_dh_set_prime_bits(*session, dh_min_bits);
247  	    }
248  	}
249  	
250  	static unsigned int
251  	get_bound_dh_bits(unsigned int dh_bits)
252  	{
253  	    int dh_min_bits;
254  	    int dh_max_bits;
255  	
256  	    pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
257  	                       0);
258  	    pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MAX_BITS), &dh_max_bits,
259  	                       0);
260  	
261  	    if ((dh_max_bits > 0) && (dh_max_bits < dh_min_bits)) {
262  	        crm_warn("Ignoring PCMK_dh_max_bits less than PCMK_dh_min_bits");
263  	        dh_max_bits = 0;
264  	    }
265  	    if ((dh_min_bits > 0) && (dh_bits < dh_min_bits)) {
266  	        return dh_min_bits;
267  	    }
268  	    if ((dh_max_bits > 0) && (dh_bits > dh_max_bits)) {
269  	        return dh_max_bits;
270  	    }
271  	    return dh_bits;
272  	}
273  	
274  	/*!
275  	 * \internal
276  	 * \brief Initialize a new TLS session
277  	 *
278  	 * \param[in] csock       Connected socket for TLS session
279  	 * \param[in] conn_type   GNUTLS_SERVER or GNUTLS_CLIENT
280  	 * \param[in] cred_type   GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
281  	 * \param[in] credentials TLS session credentials
282  	 *
283  	 * \return Pointer to newly created session object, or NULL on error
284  	 */
285  	gnutls_session_t *
286  	pcmk__new_tls_session(int csock, unsigned int conn_type,
287  	                      gnutls_credentials_type_t cred_type, void *credentials)
288  	{
289  	    int rc = GNUTLS_E_SUCCESS;
290  	    const char *prio_base = NULL;
291  	    char *prio = NULL;
292  	    gnutls_session_t *session = NULL;
293  	
294  	    /* Determine list of acceptable ciphers, etc. Pacemaker always adds the
295  	     * values required for its functionality.
296  	     *
297  	     * For an example of anonymous authentication, see:
298  	     * http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
299  	     */
300  	
301  	    prio_base = pcmk__env_option(PCMK__ENV_TLS_PRIORITIES);
302  	    if (prio_base == NULL) {
303  	        prio_base = PCMK_GNUTLS_PRIORITIES;
304  	    }
305  	    prio = crm_strdup_printf("%s:%s", prio_base,
306  	                             (cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK");
307  	
308  	    session = gnutls_malloc(sizeof(gnutls_session_t));
309  	    if (session == NULL) {
310  	        rc = GNUTLS_E_MEMORY_ERROR;
311  	        goto error;
312  	    }
313  	
314  	    rc = gnutls_init(session, conn_type);
315  	    if (rc != GNUTLS_E_SUCCESS) {
316  	        goto error;
317  	    }
318  	
319  	    /* @TODO On the server side, it would be more efficient to cache the
320  	     * priority with gnutls_priority_init2() and set it with
321  	     * gnutls_priority_set() for all sessions.
322  	     */
323  	    rc = gnutls_priority_set_direct(*session, prio, NULL);
324  	    if (rc != GNUTLS_E_SUCCESS) {
325  	        goto error;
326  	    }
327  	    if (conn_type == GNUTLS_CLIENT) {
328  	        set_minimum_dh_bits(session);
329  	    }
330  	
331  	    gnutls_transport_set_ptr(*session,
332  	                             (gnutls_transport_ptr_t) GINT_TO_POINTER(csock));
333  	
334  	    rc = gnutls_credentials_set(*session, cred_type, credentials);
335  	    if (rc != GNUTLS_E_SUCCESS) {
336  	        goto error;
337  	    }
338  	    free(prio);
339  	    return session;
340  	
341  	error:
342  	    crm_err("Could not initialize %s TLS %s session: %s "
343  	            CRM_XS " rc=%d priority='%s'",
344  	            (cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
345  	            (conn_type == GNUTLS_SERVER)? "server" : "client",
346  	            gnutls_strerror(rc), rc, prio);
347  	    free(prio);
348  	    if (session != NULL) {
349  	        gnutls_free(session);
350  	    }
351  	    return NULL;
352  	}
353  	
354  	/*!
355  	 * \internal
356  	 * \brief Initialize Diffie-Hellman parameters for a TLS server
357  	 *
358  	 * \param[out] dh_params  Parameter object to initialize
359  	 *
360  	 * \return Standard Pacemaker return code
361  	 * \todo The current best practice is to allow the client and server to
362  	 *       negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
363  	 *       However, we have to support both older versions of GnuTLS (<3.6) that
364  	 *       don't support the extension on our side, and older Pacemaker versions
365  	 *       that don't support the extension on the other side. The next best
366  	 *       practice would be to use a known good prime (see RFC 5114 section 2.2),
367  	 *       possibly stored in a file distributed with Pacemaker.
368  	 */
369  	int
370  	pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
371  	{
372  	    int rc = GNUTLS_E_SUCCESS;
373  	    unsigned int dh_bits = 0;
374  	
375  	    rc = gnutls_dh_params_init(dh_params);
376  	    if (rc != GNUTLS_E_SUCCESS) {
377  	        goto error;
378  	    }
379  	
380  	    dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
381  	                                          GNUTLS_SEC_PARAM_NORMAL);
382  	    if (dh_bits == 0) {
383  	        rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
384  	        goto error;
385  	    }
386  	    dh_bits = get_bound_dh_bits(dh_bits);
387  	
388  	    crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
389  	             dh_bits);
390  	    rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
391  	    if (rc != GNUTLS_E_SUCCESS) {
392  	        goto error;
393  	    }
394  	
395  	    return pcmk_rc_ok;
396  	
397  	error:
398  	    crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
399  	            CRM_XS " rc=%d", gnutls_strerror(rc), rc);
400  	    return EPROTO;
401  	}
402  	
403  	/*!
404  	 * \internal
405  	 * \brief Process handshake data from TLS client
406  	 *
407  	 * Read as much TLS handshake data as is available.
408  	 *
409  	 * \param[in] client  Client connection
410  	 *
411  	 * \return Standard Pacemaker return code (of particular interest, EAGAIN
412  	 *         if some data was successfully read but more data is needed)
413  	 */
414  	int
415  	pcmk__read_handshake_data(const pcmk__client_t *client)
416  	{
417  	    int rc = 0;
418  	
419  	    pcmk__assert((client != NULL) && (client->remote != NULL)
420  	                 && (client->remote->tls_session != NULL));
421  	
422  	    do {
423  	        rc = gnutls_handshake(*client->remote->tls_session);
424  	    } while (rc == GNUTLS_E_INTERRUPTED);
425  	
426  	    if (rc == GNUTLS_E_AGAIN) {
427  	        /* No more data is available at the moment. This function should be
428  	         * invoked again once the client sends more.
429  	         */
430  	        return EAGAIN;
431  	    } else if (rc != GNUTLS_E_SUCCESS) {
432  	        crm_err("TLS handshake with remote client failed: %s "
433  	                CRM_XS " rc=%d", gnutls_strerror(rc), rc);
434  	        return EPROTO;
435  	    }
436  	    return pcmk_rc_ok;
437  	}
438  	
439  	// \return Standard Pacemaker return code
440  	static int
441  	send_tls(gnutls_session_t *session, struct iovec *iov)
442  	{
443  	    const char *unsent = iov->iov_base;
444  	    size_t unsent_len = iov->iov_len;
445  	    ssize_t gnutls_rc;
446  	
447  	    if (unsent == NULL) {
448  	        return EINVAL;
449  	    }
450  	
451  	    crm_trace("Sending TLS message of %llu bytes",
452  	              (unsigned long long) unsent_len);
453  	    while (true) {
454  	        gnutls_rc = gnutls_record_send(*session, unsent, unsent_len);
455  	
456  	        if (gnutls_rc == GNUTLS_E_INTERRUPTED || gnutls_rc == GNUTLS_E_AGAIN) {
457  	            crm_trace("Retrying to send %llu bytes remaining",
458  	                      (unsigned long long) unsent_len);
459  	
460  	        } else if (gnutls_rc < 0) {
461  	            // Caller can log as error if necessary
462  	            crm_info("TLS connection terminated: %s " CRM_XS " rc=%lld",
463  	                     gnutls_strerror((int) gnutls_rc),
464  	                     (long long) gnutls_rc);
465  	            return ECONNABORTED;
466  	
467  	        } else if (gnutls_rc < unsent_len) {
468  	            crm_trace("Sent %lld of %llu bytes remaining",
469  	                      (long long) gnutls_rc, (unsigned long long) unsent_len);
470  	            unsent_len -= gnutls_rc;
471  	            unsent += gnutls_rc;
472  	        } else {
473  	            crm_trace("Sent all %lld bytes remaining", (long long) gnutls_rc);
474  	            break;
475  	        }
476  	    }
477  	    return pcmk_rc_ok;
478  	}
479  	#endif
480  	
481  	// \return Standard Pacemaker return code
482  	static int
483  	send_plaintext(int sock, struct iovec *iov)
484  	{
485  	    const char *unsent = iov->iov_base;
486  	    size_t unsent_len = iov->iov_len;
487  	    ssize_t write_rc;
488  	
489  	    if (unsent == NULL) {
490  	        return EINVAL;
491  	    }
492  	
493  	    crm_debug("Sending plaintext message of %llu bytes to socket %d",
494  	              (unsigned long long) unsent_len, sock);
495  	    while (true) {
496  	        write_rc = write(sock, unsent, unsent_len);
497  	        if (write_rc < 0) {
498  	            int rc = errno;
499  	
500  	            if ((errno == EINTR) || (errno == EAGAIN)) {
501  	                crm_trace("Retrying to send %llu bytes remaining to socket %d",
502  	                          (unsigned long long) unsent_len, sock);
503  	                continue;
504  	            }
505  	
506  	            // Caller can log as error if necessary
507  	            crm_info("Could not send message: %s " CRM_XS " rc=%d socket=%d",
508  	                     pcmk_rc_str(rc), rc, sock);
509  	            return rc;
510  	
511  	        } else if (write_rc < unsent_len) {
512  	            crm_trace("Sent %lld of %llu bytes remaining",
513  	                      (long long) write_rc, (unsigned long long) unsent_len);
514  	            unsent += write_rc;
515  	            unsent_len -= write_rc;
516  	            continue;
517  	
518  	        } else {
519  	            crm_trace("Sent all %lld bytes remaining: %.100s",
520  	                      (long long) write_rc, (char *) (iov->iov_base));
521  	            break;
522  	        }
523  	    }
524  	    return pcmk_rc_ok;
525  	}
526  	
527  	// \return Standard Pacemaker return code
528  	static int
529  	remote_send_iovs(pcmk__remote_t *remote, struct iovec *iov, int iovs)
530  	{
531  	    int rc = pcmk_rc_ok;
532  	
533  	    for (int lpc = 0; (lpc < iovs) && (rc == pcmk_rc_ok); lpc++) {
534  	#ifdef HAVE_GNUTLS_GNUTLS_H
535  	        if (remote->tls_session) {
536  	            rc = send_tls(remote->tls_session, &(iov[lpc]));
537  	            continue;
538  	        }
539  	#endif
540  	        if (remote->tcp_socket) {
541  	            rc = send_plaintext(remote->tcp_socket, &(iov[lpc]));
542  	        } else {
543  	            rc = ESOCKTNOSUPPORT;
544  	        }
545  	    }
546  	    return rc;
547  	}
548  	
549  	/*!
550  	 * \internal
551  	 * \brief Send an XML message over a Pacemaker Remote connection
552  	 *
553  	 * \param[in,out] remote  Pacemaker Remote connection to use
554  	 * \param[in]     msg     XML to send
555  	 *
556  	 * \return Standard Pacemaker return code
557  	 */
558  	int
559  	pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg)
560  	{
561  	    int rc = pcmk_rc_ok;
562  	    static uint64_t id = 0;
563  	    GString *xml_text = NULL;
564  	
565  	    struct iovec iov[2];
566  	    struct remote_header_v0 *header;
567  	
568  	    CRM_CHECK((remote != NULL) && (msg != NULL), return EINVAL);
569  	
570  	    xml_text = g_string_sized_new(1024);
571  	    pcmk__xml_string(msg, 0, xml_text, 0);
572  	    CRM_CHECK(xml_text->len > 0,
573  	              g_string_free(xml_text, TRUE); return EINVAL);
574  	
575  	    header = pcmk__assert_alloc(1, sizeof(struct remote_header_v0));
576  	
577  	    iov[0].iov_base = header;
578  	    iov[0].iov_len = sizeof(struct remote_header_v0);
579  	
580  	    iov[1].iov_len = 1 + xml_text->len;
581  	    iov[1].iov_base = g_string_free(xml_text, FALSE);
582  	
583  	    id++;
584  	    header->id = id;
585  	    header->endian = ENDIAN_LOCAL;
586  	    header->version = REMOTE_MSG_VERSION;
587  	    header->payload_offset = iov[0].iov_len;
588  	    header->payload_uncompressed = iov[1].iov_len;
589  	
590  	    if ((UINT32_MAX - iov[0].iov_len) < iov[1].iov_len) {
591  	        crm_err("Remote message size %zu + %zu exceeds maximum of %" PRIu32,
592  	                iov[0].iov_len, iov[1].iov_len, UINT32_MAX);
593  	        goto done;
594  	    }
595  	
596  	    header->size_total = iov[0].iov_len + iov[1].iov_len;
597  	
598  	    rc = remote_send_iovs(remote, iov, 2);
599  	    if (rc != pcmk_rc_ok) {
600  	        crm_err("Could not send remote message: %s " CRM_XS " rc=%d",
601  	                pcmk_rc_str(rc), rc);
602  	    }
603  	
604  	done:
605  	    free(iov[0].iov_base);
606  	    g_free((gchar *) iov[1].iov_base);
607  	    return rc;
608  	}
609  	
610  	/*!
611  	 * \internal
612  	 * \brief Obtain the XML from the currently buffered remote connection message
613  	 *
614  	 * \param[in,out] remote  Remote connection possibly with message available
615  	 *
616  	 * \return Newly allocated XML object corresponding to message data, or NULL
617  	 * \note This effectively removes the message from the connection buffer.
618  	 */
619  	xmlNode *
620  	pcmk__remote_message_xml(pcmk__remote_t *remote)
621  	{
622  	    xmlNode *xml = NULL;
623  	    struct remote_header_v0 *header = localized_remote_header(remote);
624  	
(1) Event path: Condition "header == NULL", taking false branch.
625  	    if (header == NULL) {
626  	        return NULL;
627  	    }
628  	
629  	    /* Support compression on the receiving end now, in case we ever want to add it later */
(2) Event path: Condition "header->payload_compressed != 0", taking true branch.
630  	    if (header->payload_compressed != 0) {
631  	        int rc = 0;
632  	        unsigned int size_u = 0;
633  	        char *uncompressed = NULL;
634  	        size_t buffer_size = 0;
635  	
636  	#if (UINT32_MAX < UINT_MAX)
637  	        if (header->payload_uncompressed >= UINT_MAX) {
638  	            crm_err("Couldn't decompress message because uncompressed "
639  	                    "payload size (%" PRIu32 ") is greater than UINT_MAX "
640  	                    "(%u)", header->payload_uncompressed, UINT_MAX);
641  	            return NULL;
642  	        }
643  	#endif
644  	
645  	        /* @TODO Is the extra byte for the null terminator?
646  	         * pcmk__remote_send_xml() also adds one byte to the iov length.
647  	         * (However, we do need to account for the possibility of receiving a
648  	         * message from an untrusted sender.)
649  	         */
650  	        size_u = 1 + header->payload_uncompressed;
651  	
652  	        /* Header and uncompressed payload must fit in the destination buffer.
653  	         * We do not need to separately check the header size here since
654  	         * localized_remote_header will return NULL if it's incorrect.
655  	         */
656  	#if (UINT_MAX >= SIZE_MAX)
657  	        if ((size_u >= SIZE_MAX)
658  	            || (header->payload_offset > (SIZE_MAX - size_u))) {
659  	#else
(3) Event path: Condition "header->payload_offset > 18446744073709551615UL - size_u", taking false branch.
660  	        if (header->payload_offset > (SIZE_MAX - size_u)) {
661  	#endif
662  	            crm_err("Couldn't decompress message because the required buffer "
663  	                    "size (%" PRIu32 " + %u) is greater than SIZE_MAX (%zu)",
664  	                    header->payload_offset, size_u, SIZE_MAX);
665  	            return NULL;
666  	        }
667  	
668  	        buffer_size = (size_t) header->payload_offset + size_u;
(4) Event path: Condition "buffer_size > 20971520UL /* 20 * 1024 * 1024 */", taking false branch.
669  	        if (buffer_size > PCMK__REMOTE_MSG_MAX_SIZE) {
670  	            crm_err("Message size %zu is larger than max allowed %u bytes",
671  	                    buffer_size, PCMK__REMOTE_MSG_MAX_SIZE);
672  	            return NULL;
673  	        }
674  	
(5) Event path: Switch case default.
(6) Event path: Condition "trace_cs == NULL", taking true branch.
(7) Event path: Condition "crm_is_callsite_active(trace_cs, _level, 0)", taking false branch.
(8) Event path: Breaking from switch.
675  	        crm_trace("Decompressing message data %" PRIu32 " bytes into %u "
676  	                  "bytes", header->payload_compressed, size_u);
677  	
678  	        uncompressed = pcmk__assert_alloc(buffer_size, sizeof(char));
679  	
680  	        rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset,
681  	                                        &size_u,
682  	                                        remote->buffer + header->payload_offset,
683  	                                        header->payload_compressed, 1, 0);
684  	        rc = pcmk__bzlib2rc(rc);
685  	
(9) Event path: Condition "rc != pcmk_rc_ok", taking false branch.
686  	        if (rc != pcmk_rc_ok && header->version > REMOTE_MSG_VERSION) {
687  	            crm_warn("Couldn't decompress v%d message, we only understand v%d",
688  	                     header->version, REMOTE_MSG_VERSION);
689  	            free(uncompressed);
690  	            return NULL;
691  	
(10) Event path: Condition "rc != pcmk_rc_ok", taking false branch.
692  	        } else if (rc != pcmk_rc_ok) {
693  	            crm_err("Decompression failed: %s " CRM_XS " rc=%d",
694  	                    pcmk_rc_str(rc), rc);
695  	            free(uncompressed);
696  	            return NULL;
697  	        }
698  	
(11) Event path: Condition "!(size_u == header->payload_uncompressed)", taking false branch.
699  	        pcmk__assert(size_u == header->payload_uncompressed);
700  	
701  	        memcpy(uncompressed, remote->buffer, header->payload_offset);       /* Preserve the header */
702  	        remote->buffer_size = header->payload_offset + size_u;
703  	
704  	        free(remote->buffer);
705  	        remote->buffer = uncompressed;
(12) Event tainted_data_return: "localized_remote_header" returns tainted data. [details]
(13) Event tainted_data_transitive: Calling function "localized_remote_header" with tainted argument "*remote->buffer" taints "localized_remote_header(remote)->payload_uncompressed". [details]
(14) Event var_assign: Assigning: "header" = "localized_remote_header(remote)", which taints "header->payload_uncompressed".
Also see events: [data_index]
706  	        header = localized_remote_header(remote);
707  	    }
708  	
709  	    /* take ownership of the buffer */
710  	    remote->buffer_offset = 0;
711  	
(15) Event data_index: Using tainted expression "40UL + header->payload_uncompressed - 1UL" as an index to pointer "remote->buffer".
Also see events: [tainted_data_return][tainted_data_transitive][var_assign]
712  	    CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0);
713  	
714  	    xml = pcmk__xml_parse(remote->buffer + header->payload_offset);
715  	    if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
716  	        crm_warn("Couldn't parse v%d message, we only understand v%d",
717  	                 header->version, REMOTE_MSG_VERSION);
718  	
719  	    } else if (xml == NULL) {
720  	        crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
721  	    }
722  	
723  	    crm_log_xml_trace(xml, "[remote msg]");
724  	    return xml;
725  	}
726  	
727  	static int
728  	get_remote_socket(const pcmk__remote_t *remote)
729  	{
730  	#ifdef HAVE_GNUTLS_GNUTLS_H
731  	    if (remote->tls_session) {
732  	        void *sock_ptr = gnutls_transport_get_ptr(*remote->tls_session);
733  	
734  	        return GPOINTER_TO_INT(sock_ptr);
735  	    }
736  	#endif
737  	
738  	    if (remote->tcp_socket) {
739  	        return remote->tcp_socket;
740  	    }
741  	
742  	    crm_err("Remote connection type undetermined (bug?)");
743  	    return -1;
744  	}
745  	
746  	/*!
747  	 * \internal
748  	 * \brief Wait for a remote session to have data to read
749  	 *
750  	 * \param[in] remote      Connection to check
751  	 * \param[in] timeout_ms  Maximum time (in ms) to wait
752  	 *
753  	 * \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
754  	 *         there is data ready to be read, and ETIME if there is no data within
755  	 *         the specified timeout)
756  	 */
757  	int
758  	pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms)
759  	{
760  	    struct pollfd fds = { 0, };
761  	    int sock = 0;
762  	    int rc = 0;
763  	    time_t start;
764  	    int timeout = timeout_ms;
765  	
766  	    sock = get_remote_socket(remote);
767  	    if (sock <= 0) {
768  	        crm_trace("No longer connected");
769  	        return ENOTCONN;
770  	    }
771  	
772  	    start = time(NULL);
773  	    errno = 0;
774  	    do {
775  	        fds.fd = sock;
776  	        fds.events = POLLIN;
777  	
778  	        /* If we got an EINTR while polling, and we have a
779  	         * specific timeout we are trying to honor, attempt
780  	         * to adjust the timeout to the closest second. */
781  	        if (errno == EINTR && (timeout > 0)) {
782  	            timeout = timeout_ms - ((time(NULL) - start) * 1000);
783  	            if (timeout < 1000) {
784  	                timeout = 1000;
785  	            }
786  	        }
787  	
788  	        rc = poll(&fds, 1, timeout);
789  	    } while (rc < 0 && errno == EINTR);
790  	
791  	    if (rc < 0) {
792  	        return errno;
793  	    }
794  	    return (rc == 0)? ETIME : pcmk_rc_ok;
795  	}
796  	
797  	/*!
798  	 * \internal
799  	 * \brief Read bytes from non-blocking remote connection
800  	 *
801  	 * \param[in,out] remote  Remote connection to read
802  	 *
803  	 * \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
804  	 *         a full message has been received, or EAGAIN for a partial message)
805  	 * \note Use only with non-blocking sockets after polling the socket.
806  	 * \note This function will return when the socket read buffer is empty or an
807  	 *       error is encountered.
808  	 */
809  	int
810  	pcmk__read_available_remote_data(pcmk__remote_t *remote)
811  	{
812  	    int rc = pcmk_rc_ok;
813  	    size_t read_len = sizeof(struct remote_header_v0);
814  	    struct remote_header_v0 *header = localized_remote_header(remote);
815  	    bool received = false;
816  	    ssize_t read_rc;
817  	
(1) Event path: Condition "header", taking false branch.
818  	    if(header) {
819  	        /* Stop at the end of the current message */
820  	        read_len = header->size_total;
821  	    }
822  	
(2) Event path: Condition "read_len > 20971520UL /* 20 * 1024 * 1024 */", taking false branch.
823  	    if (read_len > PCMK__REMOTE_MSG_MAX_SIZE) {
824  	        crm_err("Message size %zu is larger than max allowed %u bytes",
825  	                read_len, PCMK__REMOTE_MSG_MAX_SIZE);
826  	        return EINVAL;
827  	    }
828  	
829  	    /* automatically grow the buffer when needed */
(3) Event path: Condition "remote->buffer_size < read_len", taking true branch.
830  	    if(remote->buffer_size < read_len) {
831  	        remote->buffer_size = 2 * read_len;
(4) Event path: Switch case default.
(5) Event path: Condition "trace_cs == NULL", taking true branch.
(6) Event path: Condition "crm_is_callsite_active(trace_cs, _level, 0)", taking false branch.
(7) Event path: Breaking from switch.
832  	        crm_trace("Expanding buffer to %llu bytes",
833  	                  (unsigned long long) remote->buffer_size);
834  	        remote->buffer = pcmk__realloc(remote->buffer, remote->buffer_size + 1);
835  	    }
836  	
837  	#ifdef HAVE_GNUTLS_GNUTLS_H
(8) Event path: Condition "!received", taking true branch.
(9) Event path: Condition "remote->tls_session", taking false branch.
838  	    if (!received && remote->tls_session) {
839  	        read_rc = gnutls_record_recv(*(remote->tls_session),
840  	                                     remote->buffer + remote->buffer_offset,
841  	                                     remote->buffer_size - remote->buffer_offset);
842  	        if (read_rc == GNUTLS_E_INTERRUPTED) {
843  	            rc = EINTR;
844  	        } else if (read_rc == GNUTLS_E_AGAIN) {
845  	            rc = EAGAIN;
846  	        } else if (read_rc < 0) {
847  	            crm_debug("TLS receive failed: %s (%lld)",
848  	                      gnutls_strerror(read_rc), (long long) read_rc);
849  	            rc = EIO;
850  	        }
851  	        received = true;
852  	    }
853  	#endif
854  	
(10) Event path: Condition "!received", taking true branch.
(11) Event path: Condition "remote->tcp_socket", taking true branch.
855  	    if (!received && remote->tcp_socket) {
(12) Event tainted_data_argument: Calling function "read" taints parameter "remote->buffer[remote->buffer_offset]".
856  	        read_rc = read(remote->tcp_socket,
857  	                       remote->buffer + remote->buffer_offset,
858  	                       remote->buffer_size - remote->buffer_offset);
(13) Event path: Condition "read_rc < 0", taking false branch.
859  	        if (read_rc < 0) {
860  	            rc = errno;
861  	        }
862  	        received = true;
863  	    }
864  	
(14) Event path: Condition "!received", taking false branch.
865  	    if (!received) {
866  	        crm_err("Remote connection type undetermined (bug?)");
867  	        return ESOCKTNOSUPPORT;
868  	    }
869  	
870  	    /* process any errors. */
(15) Event path: Condition "read_rc > 0", taking true branch.
871  	    if (read_rc > 0) {
872  	        remote->buffer_offset += read_rc;
873  	        /* always null terminate buffer, the +1 to alloc always allows for this. */
874  	        remote->buffer[remote->buffer_offset] = '\0';
(16) Event path: Switch case default.
(17) Event path: Condition "trace_cs == NULL", taking true branch.
(18) Event path: Condition "crm_is_callsite_active(trace_cs, _level, 0)", taking false branch.
(19) Event path: Breaking from switch.
875  	        crm_trace("Received %lld more bytes (%llu total)",
876  	                  (long long) read_rc,
877  	                  (unsigned long long) remote->buffer_offset);
878  	
(20) Event path: Falling through to end of if statement.
879  	    } else if ((rc == EINTR) || (rc == EAGAIN)) {
880  	        crm_trace("No data available for non-blocking remote read: %s (%d)",
881  	                  pcmk_rc_str(rc), rc);
882  	
883  	    } else if (read_rc == 0) {
884  	        crm_debug("End of remote data encountered after %llu bytes",
885  	                  (unsigned long long) remote->buffer_offset);
886  	        return ENOTCONN;
887  	
888  	    } else {
889  	        crm_debug("Error receiving remote data after %llu bytes: %s (%d)",
890  	                  (unsigned long long) remote->buffer_offset,
891  	                  pcmk_rc_str(rc), rc);
892  	        return ENOTCONN;
893  	    }
894  	
895  	    header = localized_remote_header(remote);
(21) Event path: Condition "header", taking true branch.
896  	    if(header) {
(22) Event path: Condition "remote->buffer_offset < header->size_total", taking true branch.
897  	        if(remote->buffer_offset < header->size_total) {
(23) Event path: Switch case default.
(24) Event path: Condition "trace_cs == NULL", taking true branch.
(25) Event path: Condition "crm_is_callsite_active(trace_cs, _level, 0)", taking false branch.
(26) Event path: Breaking from switch.
898  	            crm_trace("Read partial remote message (%llu of %u bytes)",
899  	                      (unsigned long long) remote->buffer_offset,
900  	                      header->size_total);
(27) Event path: Falling through to end of if statement.
901  	        } else {
902  	            crm_trace("Read full remote message of %llu bytes",
903  	                      (unsigned long long) remote->buffer_offset);
904  	            return pcmk_rc_ok;
905  	        }
906  	    }
907  	
908  	    return EAGAIN;
909  	}
910  	
911  	/*!
912  	 * \internal
913  	 * \brief Read one message from a remote connection
914  	 *
915  	 * \param[in,out] remote      Remote connection to read
916  	 * \param[in]     timeout_ms  Fail if message not read in this many milliseconds
917  	 *                            (10s will be used if 0, and 60s if negative)
918  	 *
919  	 * \return Standard Pacemaker return code
920  	 */
921  	int
922  	pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms)
923  	{
924  	    int rc = pcmk_rc_ok;
925  	    time_t start = time(NULL);
926  	    int remaining_timeout = 0;
927  	
928  	    if (timeout_ms == 0) {
929  	        timeout_ms = 10000;
930  	    } else if (timeout_ms < 0) {
931  	        timeout_ms = 60000;
932  	    }
933  	
934  	    remaining_timeout = timeout_ms;
935  	    while (remaining_timeout > 0) {
936  	
937  	        crm_trace("Waiting for remote data (%d ms of %d ms timeout remaining)",
938  	                  remaining_timeout, timeout_ms);
939  	        rc = pcmk__remote_ready(remote, remaining_timeout);
940  	
941  	        if (rc == ETIME) {
942  	            crm_err("Timed out (%d ms) while waiting for remote data",
943  	                    remaining_timeout);
944  	            return rc;
945  	
946  	        } else if (rc != pcmk_rc_ok) {
947  	            crm_debug("Wait for remote data aborted (will retry): %s "
948  	                      CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
949  	
950  	        } else {
951  	            rc = pcmk__read_available_remote_data(remote);
952  	            if (rc == pcmk_rc_ok) {
953  	                return rc;
954  	            } else if (rc == EAGAIN) {
955  	                crm_trace("Waiting for more remote data");
956  	            } else {
957  	                crm_debug("Could not receive remote data: %s " CRM_XS " rc=%d",
958  	                          pcmk_rc_str(rc), rc);
959  	            }
960  	        }
961  	
962  	        // Don't waste time retrying after fatal errors
963  	        if ((rc == ENOTCONN) || (rc == ESOCKTNOSUPPORT)) {
964  	            return rc;
965  	        }
966  	
967  	        remaining_timeout = timeout_ms - ((time(NULL) - start) * 1000);
968  	    }
969  	    return ETIME;
970  	}
971  	
972  	struct tcp_async_cb_data {
973  	    int sock;
974  	    int timeout_ms;
975  	    time_t start;
976  	    void *userdata;
977  	    void (*callback) (void *userdata, int rc, int sock);
978  	};
979  	
980  	// \return TRUE if timer should be rescheduled, FALSE otherwise
981  	static gboolean
982  	check_connect_finished(gpointer userdata)
983  	{
984  	    struct tcp_async_cb_data *cb_data = userdata;
985  	    int rc;
986  	
987  	    fd_set rset, wset;
988  	    struct timeval ts = { 0, };
989  	
990  	    if (cb_data->start == 0) {
991  	        // Last connect() returned success immediately
992  	        rc = pcmk_rc_ok;
993  	        goto dispatch_done;
994  	    }
995  	
996  	    // If the socket is ready for reading or writing, the connect succeeded
997  	    FD_ZERO(&rset);
998  	    FD_SET(cb_data->sock, &rset);
999  	    wset = rset;
1000 	    rc = select(cb_data->sock + 1, &rset, &wset, NULL, &ts);
1001 	
1002 	    if (rc < 0) { // select() error
1003 	        rc = errno;
1004 	        if ((rc == EINPROGRESS) || (rc == EAGAIN)) {
1005 	            if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
1006 	                return TRUE; // There is time left, so reschedule timer
1007 	            } else {
1008 	                rc = ETIMEDOUT;
1009 	            }
1010 	        }
1011 	        crm_trace("Could not check socket %d for connection success: %s (%d)",
1012 	                  cb_data->sock, pcmk_rc_str(rc), rc);
1013 	
1014 	    } else if (rc == 0) { // select() timeout
1015 	        if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
1016 	            return TRUE; // There is time left, so reschedule timer
1017 	        }
1018 	        crm_debug("Timed out while waiting for socket %d connection success",
1019 	                  cb_data->sock);
1020 	        rc = ETIMEDOUT;
1021 	
1022 	    // select() returned number of file descriptors that are ready
1023 	
1024 	    } else if (FD_ISSET(cb_data->sock, &rset)
1025 	               || FD_ISSET(cb_data->sock, &wset)) {
1026 	
1027 	        // The socket is ready; check it for connection errors
1028 	        int error = 0;
1029 	        socklen_t len = sizeof(error);
1030 	
1031 	        if (getsockopt(cb_data->sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
1032 	            rc = errno;
1033 	            crm_trace("Couldn't check socket %d for connection errors: %s (%d)",
1034 	                      cb_data->sock, pcmk_rc_str(rc), rc);
1035 	        } else if (error != 0) {
1036 	            rc = error;
1037 	            crm_trace("Socket %d connected with error: %s (%d)",
1038 	                      cb_data->sock, pcmk_rc_str(rc), rc);
1039 	        } else {
1040 	            rc = pcmk_rc_ok;
1041 	        }
1042 	
1043 	    } else { // Should not be possible
1044 	        crm_trace("select() succeeded, but socket %d not in resulting "
1045 	                  "read/write sets", cb_data->sock);
1046 	        rc = EAGAIN;
1047 	    }
1048 	
1049 	  dispatch_done:
1050 	    if (rc == pcmk_rc_ok) {
1051 	        crm_trace("Socket %d is connected", cb_data->sock);
1052 	    } else {
1053 	        close(cb_data->sock);
1054 	        cb_data->sock = -1;
1055 	    }
1056 	
1057 	    if (cb_data->callback) {
1058 	        cb_data->callback(cb_data->userdata, rc, cb_data->sock);
1059 	    }
1060 	    free(cb_data);
1061 	    return FALSE; // Do not reschedule timer
1062 	}
1063 	
1064 	/*!
1065 	 * \internal
1066 	 * \brief Attempt to connect socket, calling callback when done
1067 	 *
1068 	 * Set a given socket non-blocking, then attempt to connect to it,
1069 	 * retrying periodically until success or a timeout is reached.
1070 	 * Call a caller-supplied callback function when completed.
1071 	 *
1072 	 * \param[in]  sock        Newly created socket
1073 	 * \param[in]  addr        Socket address information for connect
1074 	 * \param[in]  addrlen     Size of socket address information in bytes
1075 	 * \param[in]  timeout_ms  Fail if not connected within this much time
1076 	 * \param[out] timer_id    If not NULL, store retry timer ID here
1077 	 * \param[in]  userdata    User data to pass to callback
1078 	 * \param[in]  callback    Function to call when connection attempt completes
1079 	 *
1080 	 * \return Standard Pacemaker return code
1081 	 */
1082 	static int
1083 	connect_socket_retry(int sock, const struct sockaddr *addr, socklen_t addrlen,
1084 	                     int timeout_ms, int *timer_id, void *userdata,
1085 	                     void (*callback) (void *userdata, int rc, int sock))
1086 	{
1087 	    int rc = 0;
1088 	    int interval = 500;
1089 	    int timer;
1090 	    struct tcp_async_cb_data *cb_data = NULL;
1091 	
1092 	    rc = pcmk__set_nonblocking(sock);
1093 	    if (rc != pcmk_rc_ok) {
1094 	        crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
1095 	                 pcmk_rc_str(rc), rc);
1096 	        return rc;
1097 	    }
1098 	
1099 	    rc = connect(sock, addr, addrlen);
1100 	    if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
1101 	        rc = errno;
1102 	        crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
1103 	                 pcmk_rc_str(rc), rc);
1104 	        return rc;
1105 	    }
1106 	
1107 	    cb_data = pcmk__assert_alloc(1, sizeof(struct tcp_async_cb_data));
1108 	    cb_data->userdata = userdata;
1109 	    cb_data->callback = callback;
1110 	    cb_data->sock = sock;
1111 	    cb_data->timeout_ms = timeout_ms;
1112 	
1113 	    if (rc == 0) {
1114 	        /* The connect was successful immediately, we still return to mainloop
1115 	         * and let this callback get called later. This avoids the user of this api
1116 	         * to have to account for the fact the callback could be invoked within this
1117 	         * function before returning. */
1118 	        cb_data->start = 0;
1119 	        interval = 1;
1120 	    } else {
1121 	        cb_data->start = time(NULL);
1122 	    }
1123 	
1124 	    /* This timer function does a non-blocking poll on the socket to see if we
1125 	     * can use it. Once we can, the connect has completed. This method allows us
1126 	     * to connect without blocking the mainloop.
1127 	     *
1128 	     * @TODO Use a mainloop fd callback for this instead of polling. Something
1129 	     *       about the way mainloop is currently polling prevents this from
1130 	     *       working at the moment though. (See connect(2) regarding EINPROGRESS
1131 	     *       for possible new handling needed.)
1132 	     */
1133 	    crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
1134 	              interval, sock);
1135 	    timer = g_timeout_add(interval, check_connect_finished, cb_data);
1136 	    if (timer_id) {
1137 	        *timer_id = timer;
1138 	    }
1139 	
1140 	    // timer callback should be taking care of cb_data
1141 	    // cppcheck-suppress memleak
1142 	    return pcmk_rc_ok;
1143 	}
1144 	
1145 	/*!
1146 	 * \internal
1147 	 * \brief Attempt once to connect socket and set it non-blocking
1148 	 *
1149 	 * \param[in]  sock        Newly created socket
1150 	 * \param[in]  addr        Socket address information for connect
1151 	 * \param[in]  addrlen     Size of socket address information in bytes
1152 	 *
1153 	 * \return Standard Pacemaker return code
1154 	 */
1155 	static int
1156 	connect_socket_once(int sock, const struct sockaddr *addr, socklen_t addrlen)
1157 	{
1158 	    int rc = connect(sock, addr, addrlen);
1159 	
1160 	    if (rc < 0) {
1161 	        rc = errno;
1162 	        crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
1163 	                 pcmk_rc_str(rc), rc);
1164 	        return rc;
1165 	    }
1166 	
1167 	    rc = pcmk__set_nonblocking(sock);
1168 	    if (rc != pcmk_rc_ok) {
1169 	        crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
1170 	                 pcmk_rc_str(rc), rc);
1171 	        return rc;
1172 	    }
1173 	
1174 	    return pcmk_ok;
1175 	}
1176 	
1177 	/*!
1178 	 * \internal
1179 	 * \brief Connect to server at specified TCP port
1180 	 *
1181 	 * \param[in]  host        Name of server to connect to
1182 	 * \param[in]  port        Server port to connect to
1183 	 * \param[in]  timeout_ms  If asynchronous, fail if not connected in this time
1184 	 * \param[out] timer_id    If asynchronous and this is non-NULL, retry timer ID
1185 	 *                         will be put here (for ease of cancelling by caller)
1186 	 * \param[out] sock_fd     Where to store socket file descriptor
1187 	 * \param[in]  userdata    If asynchronous, data to pass to callback
1188 	 * \param[in]  callback    If NULL, attempt a single synchronous connection,
1189 	 *                         otherwise retry asynchronously then call this
1190 	 *
1191 	 * \return Standard Pacemaker return code
1192 	 */
1193 	int
1194 	pcmk__connect_remote(const char *host, int port, int timeout, int *timer_id,
1195 	                     int *sock_fd, void *userdata,
1196 	                     void (*callback) (void *userdata, int rc, int sock))
1197 	{
1198 	    char buffer[INET6_ADDRSTRLEN];
1199 	    struct addrinfo *res = NULL;
1200 	    struct addrinfo *rp = NULL;
1201 	    struct addrinfo hints;
1202 	    const char *server = host;
1203 	    int rc;
1204 	    int sock = -1;
1205 	
1206 	    CRM_CHECK((host != NULL) && (sock_fd != NULL), return EINVAL);
1207 	
1208 	    // Get host's IP address(es)
1209 	    memset(&hints, 0, sizeof(struct addrinfo));
1210 	    hints.ai_family = AF_UNSPEC;        /* Allow IPv4 or IPv6 */
1211 	    hints.ai_socktype = SOCK_STREAM;
1212 	    hints.ai_flags = AI_CANONNAME;
1213 	
1214 	    rc = getaddrinfo(server, NULL, &hints, &res);
1215 	    rc = pcmk__gaierror2rc(rc);
1216 	
1217 	    if (rc != pcmk_rc_ok) {
1218 	        crm_err("Unable to get IP address info for %s: %s",
1219 	                server, pcmk_rc_str(rc));
1220 	        goto async_cleanup;
1221 	    }
1222 	
1223 	    if (!res || !res->ai_addr) {
1224 	        crm_err("Unable to get IP address info for %s: no result", server);
1225 	        rc = ENOTCONN;
1226 	        goto async_cleanup;
1227 	    }
1228 	
1229 	    // getaddrinfo() returns a list of host's addresses, try them in order
1230 	    for (rp = res; rp != NULL; rp = rp->ai_next) {
1231 	        struct sockaddr *addr = rp->ai_addr;
1232 	
1233 	        if (!addr) {
1234 	            continue;
1235 	        }
1236 	
1237 	        if (rp->ai_canonname) {
1238 	            server = res->ai_canonname;
1239 	        }
1240 	        crm_debug("Got canonical name %s for %s", server, host);
1241 	
1242 	        sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
1243 	        if (sock == -1) {
1244 	            rc = errno;
1245 	            crm_warn("Could not create socket for remote connection to %s:%d: "
1246 	                     "%s " CRM_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
1247 	            continue;
1248 	        }
1249 	
1250 	        /* Set port appropriately for address family */
1251 	        /* (void*) casts avoid false-positive compiler alignment warnings */
1252 	        if (addr->sa_family == AF_INET6) {
1253 	            ((struct sockaddr_in6 *)(void*)addr)->sin6_port = htons(port);
1254 	        } else {
1255 	            ((struct sockaddr_in *)(void*)addr)->sin_port = htons(port);
1256 	        }
1257 	
1258 	        memset(buffer, 0, PCMK__NELEM(buffer));
1259 	        pcmk__sockaddr2str(addr, buffer);
1260 	        crm_info("Attempting remote connection to %s:%d", buffer, port);
1261 	
1262 	        if (callback) {
1263 	            if (connect_socket_retry(sock, rp->ai_addr, rp->ai_addrlen, timeout,
1264 	                                     timer_id, userdata, callback) == pcmk_rc_ok) {
1265 	                goto async_cleanup; /* Success for now, we'll hear back later in the callback */
1266 	            }
1267 	
1268 	        } else if (connect_socket_once(sock, rp->ai_addr,
1269 	                                       rp->ai_addrlen) == pcmk_rc_ok) {
1270 	            break;          /* Success */
1271 	        }
1272 	
1273 	        // Connect failed
1274 	        close(sock);
1275 	        sock = -1;
1276 	        rc = ENOTCONN;
1277 	    }
1278 	
1279 	async_cleanup:
1280 	
1281 	    if (res) {
1282 	        freeaddrinfo(res);
1283 	    }
1284 	    *sock_fd = sock;
1285 	    return rc;
1286 	}
1287 	
1288 	/*!
1289 	 * \internal
1290 	 * \brief Convert an IP address (IPv4 or IPv6) to a string for logging
1291 	 *
1292 	 * \param[in]  sa  Socket address for IP
1293 	 * \param[out] s   Storage for at least INET6_ADDRSTRLEN bytes
1294 	 *
1295 	 * \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
1296 	 *          struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
1297 	 *          as long as its sa_family member is set correctly.
1298 	 */
1299 	void
1300 	pcmk__sockaddr2str(const void *sa, char *s)
1301 	{
1302 	    switch (((const struct sockaddr *) sa)->sa_family) {
1303 	        case AF_INET:
1304 	            inet_ntop(AF_INET, &(((const struct sockaddr_in *) sa)->sin_addr),
1305 	                      s, INET6_ADDRSTRLEN);
1306 	            break;
1307 	
1308 	        case AF_INET6:
1309 	            inet_ntop(AF_INET6,
1310 	                      &(((const struct sockaddr_in6 *) sa)->sin6_addr),
1311 	                      s, INET6_ADDRSTRLEN);
1312 	            break;
1313 	
1314 	        default:
1315 	            strcpy(s, "<invalid>");
1316 	    }
1317 	}
1318 	
1319 	/*!
1320 	 * \internal
1321 	 * \brief Accept a client connection on a remote server socket
1322 	 *
1323 	 * \param[in]  ssock  Server socket file descriptor being listened on
1324 	 * \param[out] csock  Where to put new client socket's file descriptor
1325 	 *
1326 	 * \return Standard Pacemaker return code
1327 	 */
1328 	int
1329 	pcmk__accept_remote_connection(int ssock, int *csock)
1330 	{
1331 	    int rc;
1332 	    struct sockaddr_storage addr;
1333 	    socklen_t laddr = sizeof(addr);
1334 	    char addr_str[INET6_ADDRSTRLEN];
1335 	#ifdef TCP_USER_TIMEOUT
1336 	    long sbd_timeout = 0;
1337 	#endif
1338 	
1339 	    /* accept the connection */
1340 	    memset(&addr, 0, sizeof(addr));
1341 	    *csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
1342 	    if (*csock == -1) {
1343 	        rc = errno;
1344 	        crm_err("Could not accept remote client connection: %s "
1345 	                CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
1346 	        return rc;
1347 	    }
1348 	    pcmk__sockaddr2str(&addr, addr_str);
1349 	    crm_info("Accepted new remote client connection from %s", addr_str);
1350 	
1351 	    rc = pcmk__set_nonblocking(*csock);
1352 	    if (rc != pcmk_rc_ok) {
1353 	        crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
1354 	                pcmk_rc_str(rc), rc);
1355 	        close(*csock);
1356 	        *csock = -1;
1357 	        return rc;
1358 	    }
1359 	
1360 	#ifdef TCP_USER_TIMEOUT
1361 	    sbd_timeout = pcmk__get_sbd_watchdog_timeout();
1362 	    if (sbd_timeout > 0) {
1363 	        // Time to fail and retry before watchdog
1364 	        long half = sbd_timeout / 2;
1365 	        unsigned int optval = (half <= UINT_MAX)? half : UINT_MAX;
1366 	
1367 	        rc = setsockopt(*csock, SOL_TCP, TCP_USER_TIMEOUT,
1368 	                        &optval, sizeof(optval));
1369 	        if (rc < 0) {
1370 	            rc = errno;
1371 	            crm_err("Could not set TCP timeout to %d ms on remote connection: "
1372 	                    "%s " CRM_XS " rc=%d", optval, pcmk_rc_str(rc), rc);
1373 	            close(*csock);
1374 	            *csock = -1;
1375 	            return rc;
1376 	        }
1377 	    }
1378 	#endif
1379 	
1380 	    return rc;
1381 	}
1382 	
1383 	/*!
1384 	 * \brief Get the default remote connection TCP port on this host
1385 	 *
1386 	 * \return Remote connection TCP port number
1387 	 */
1388 	int
1389 	crm_default_remote_port(void)
1390 	{
1391 	    static int port = 0;
1392 	
1393 	    if (port == 0) {
1394 	        const char *env = pcmk__env_option(PCMK__ENV_REMOTE_PORT);
1395 	
1396 	        if (env) {
1397 	            errno = 0;
1398 	            port = strtol(env, NULL, 10);
1399 	            if (errno || (port < 1) || (port > 65535)) {
1400 	                crm_warn("Environment variable PCMK_" PCMK__ENV_REMOTE_PORT
1401 	                         " has invalid value '%s', using %d instead",
1402 	                         env, DEFAULT_REMOTE_PORT);
1403 	                port = DEFAULT_REMOTE_PORT;
1404 	            }
1405 	        } else {
1406 	            port = DEFAULT_REMOTE_PORT;
1407 	        }
1408 	    }
1409 	    return port;
1410 	}
1411