1 /*
2 * Copyright 2004-2026 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU Lesser General Public License
7 * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <stdbool.h>
13 #include <sys/time.h>
14 #include <sys/resource.h>
15
16 #include <crm/crm.h>
17 #include <crm/common/mainloop.h>
18 #include <crm/common/xml.h>
19
20 #include <crm/cluster/internal.h>
21 #include <crm/cluster/election_internal.h>
22 #include "crmcluster_private.h"
23
24 #define STORM_INTERVAL 2 /* in seconds */
25
26 struct pcmk__election {
27 enum election_result state; // Current state of election
28 unsigned int count; // How many times local node has voted
29 void (*cb)(pcmk_cluster_t *); // Function to call if election is won
30 GHashTable *voted; // Key = node name, value = how node voted
31 pcmk__main_loop_timer_t *timeout; // When to abort if all votes not received
32 int election_wins; // Track wins, for storm detection
33 bool wrote_blackbox; // Write a storm blackbox at most once
34 time_t expires; // When storm detection period ends
35 time_t last_election_loss; // When dampening period ends
36 };
37
38 static void
39 election_complete(pcmk_cluster_t *cluster)
40 {
41 pcmk__assert((cluster != NULL) && (cluster->priv->election != NULL));
42 cluster->priv->election->state = election_won;
43 if (cluster->priv->election->cb != NULL) {
44 cluster->priv->election->cb(cluster);
45 }
46 election_reset(cluster);
47 }
48
49 static gboolean
50 election_timer_cb(void *user_data)
51 {
52 pcmk_cluster_t *cluster = user_data;
53
54 pcmk__info("Declaring local node as winner after election timed out");
55 election_complete(cluster);
56 return G_SOURCE_REMOVE;
57 }
58
59 /*!
60 * \internal
61 * \brief Get current state of an election
62 *
63 * \param[in] cluster Cluster with election
64 *
65 * \return Current state of \e
66 */
67 enum election_result
68 election_state(const pcmk_cluster_t *cluster)
69 {
70 if ((cluster == NULL) || (cluster->priv->election == NULL)) {
71 return election_error;
72 }
73 return cluster->priv->election->state;
74 }
75
76 /* The local node will be declared the winner if missing votes are not received
77 * within this time. The value is chosen to be the same as the default for the
78 * election-timeout cluster option.
79 */
80 #define ELECTION_TIMEOUT_MS 120000
81
82 /*!
83 * \internal
84 * \brief Track election state in a cluster
85 *
86 * Every node that wishes to participate in an election must initialize the
87 * election once, typically at start-up.
88 *
89 * \param[in] cluster Cluster that election is for
90 * \param[in] cb Function to call if local node wins election
91 */
92 void
93 election_init(pcmk_cluster_t *cluster, void (*cb)(pcmk_cluster_t *))
94 {
95 const char *name = pcmk__s(crm_system_name, "election");
96
97 CRM_CHECK(cluster->priv->election == NULL, return);
98
|
CID (unavailable; MK=bf77ad6f79598a6146f5533ef1330eae) (#1 of 1): Resource not released (INCOMPLETE_DEALLOCATOR): |
|
(1) Event allocation: |
Memory is allocated. [details] |
|
(2) Event allocation: |
The field "cluster->priv->election" is allocated, but not released in the identified deallocator. |
| Also see events: |
[deallocator] |
99 cluster->priv->election = pcmk__assert_alloc(1, sizeof(pcmk__election_t));
100 cluster->priv->election->cb = cb;
101 cluster->priv->election->timeout =
102 pcmk__main_loop_timer_new(name, ELECTION_TIMEOUT_MS, election_timer_cb,
103 cluster);
104 }
105
106 /*!
107 * \internal
108 * \brief Disregard any previous vote by specified peer
109 *
110 * This discards any recorded vote from a specified peer. Election users should
111 * call this whenever a voting peer becomes inactive.
112 *
113 * \param[in,out] cluster Cluster with election
114 * \param[in] uname Name of peer to disregard
115 */
116 void
117 election_remove(pcmk_cluster_t *cluster, const char *uname)
118 {
119 if ((cluster != NULL) && (cluster->priv->election != NULL)
120 && (uname != NULL) && (cluster->priv->election->voted != NULL)) {
121 pcmk__trace("Discarding (no-)vote from lost peer %s", uname);
122 g_hash_table_remove(cluster->priv->election->voted, uname);
123 }
124 }
125
126 /*!
127 * \internal
128 * \brief Stop election timer and disregard all votes
129 *
130 * \param[in,out] cluster Cluster with election
131 */
132 void
133 election_reset(pcmk_cluster_t *cluster)
134 {
135 if ((cluster == NULL) || (cluster->priv->election == NULL)) {
136 return;
137 }
138
139 pcmk__trace("Resetting election");
140
141 if (cluster->priv->election->timeout != NULL) {
142 pcmk__main_loop_timer_stop(cluster->priv->election->timeout);
143 }
144
145 g_clear_pointer(&cluster->priv->election->voted, g_hash_table_destroy);
146 }
147
148 /*!
149 * \internal
150 * \brief Free an election object
151 *
152 * Free all memory associated with an election object, stopping its
153 * election timer (if running).
154 *
155 * \param[in,out] cluster Cluster with election
156 */
157 void
|
(3) Event deallocator: |
Deallocator for "struct pcmk__cluster". |
| Also see events: |
[allocation][allocation] |
158 election_fini(pcmk_cluster_t *cluster)
159 {
160 if ((cluster != NULL) && (cluster->priv->election != NULL)) {
161 election_reset(cluster);
162 pcmk__trace("Destroying election");
163 pcmk__main_loop_timer_free(cluster->priv->election->timeout);
164 g_clear_pointer(&cluster->priv->election, free);
165 }
166 }
167
168 static void
169 election_timeout_start(pcmk_cluster_t *cluster)
170 {
171 pcmk__main_loop_timer_start(cluster->priv->election->timeout);
172 }
173
174 /*!
175 * \internal
176 * \brief Stop an election's timer, if running
177 *
178 * \param[in,out] cluster Cluster with election
179 */
180 void
181 election_timeout_stop(pcmk_cluster_t *cluster)
182 {
183 if ((cluster == NULL) || (cluster->priv->election == NULL)
184 || (cluster->priv->election->timeout == NULL)) {
185
186 return;
187 }
188
189 pcmk__main_loop_timer_stop(cluster->priv->election->timeout);
190 }
191
192 /*!
193 * \internal
194 * \brief Change an election's timeout (restarting timer if running)
195 *
196 * \param[in,out] cluster Cluster with election
197 * \param[in] interval_ms New timer interval in milliseconds
198 */
199 void
200 election_timeout_set_interval(pcmk_cluster_t *cluster, unsigned int interval_ms)
201 {
202 pcmk__main_loop_timer_t *timer = NULL;
203
204 CRM_CHECK((cluster != NULL)
205 && (cluster->priv->election != NULL)
206 && (cluster->priv->election->timeout != NULL),
207 return);
208
209 timer = cluster->priv->election->timeout;
210
211 if (timer->interval_ms == interval_ms) {
212 return;
213 }
214
215 timer->interval_ms = interval_ms;
216
217 if (pcmk__main_loop_timer_running(timer)) {
218 // Restart the timer using the new interval if it changed
219 pcmk__main_loop_timer_start(timer);
220 }
221 }
222
223 static int
224 get_uptime(struct timeval *output)
225 {
226 static time_t expires = 0;
227 static struct rusage info;
228
229 time_t tm_now = time(NULL);
230
231 if (expires < tm_now) {
232 int rc = 0;
233
234 output->tv_sec = 0;
235 output->tv_usec = 0;
236
237 info.ru_utime.tv_sec = 0;
238 info.ru_utime.tv_usec = 0;
239
240 rc = getrusage(RUSAGE_SELF, &info);
241 if (rc < 0) {
242 pcmk__err("Could not calculate the current uptime: %s",
243 strerror(errno));
244 expires = 0;
245 return -1;
246 }
247
248 pcmk__debug("Current CPU usage is: %llds, %lldus",
249 (long long) info.ru_utime.tv_sec,
250 (long long) info.ru_utime.tv_usec);
251 }
252
253 expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */
254 output->tv_sec = info.ru_utime.tv_sec;
255 output->tv_usec = info.ru_utime.tv_usec;
256
257 return 1;
258 }
259
260 static int
261 compare_age(struct timeval your_age)
262 {
263 struct timeval our_age;
264
265 get_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */
266
267 if (our_age.tv_sec > your_age.tv_sec) {
268 pcmk__debug("Win: %lld vs %lld (seconds)",
269 (long long) our_age.tv_sec, (long long) your_age.tv_sec);
270 return 1;
271 } else if (our_age.tv_sec < your_age.tv_sec) {
272 pcmk__debug("Lose: %lld vs %lld (seconds)",
273 (long long) our_age.tv_sec, (long long) your_age.tv_sec);
274 return -1;
275 } else if (our_age.tv_usec > your_age.tv_usec) {
276 pcmk__debug("Win: %lld.%06lld vs %lld.%06lld (usec)",
277 (long long) our_age.tv_sec, (long long) our_age.tv_usec,
278 (long long) your_age.tv_sec, (long long) your_age.tv_usec);
279 return 1;
280 } else if (our_age.tv_usec < your_age.tv_usec) {
281 pcmk__debug("Lose: %lld.%06lld vs %lld.%06lld (usec)",
282 (long long) our_age.tv_sec, (long long) our_age.tv_usec,
283 (long long) your_age.tv_sec, (long long) your_age.tv_usec);
284 return -1;
285 }
286
287 return 0;
288 }
289
290 /*!
291 * \internal
292 * \brief Start a new election by offering local node's candidacy
293 *
294 * Broadcast a "vote" election message containing the local node's ID,
295 * (incremented) election counter, and uptime, and start the election timer.
296 *
297 * \param[in,out] cluster Cluster with election
298 *
299 * \note Any nodes agreeing to the candidacy will send a "no-vote" reply, and if
300 * all active peers do so, or if the election times out, the local node
301 * wins the election. (If we lose to any peer vote, we will stop the
302 * timer, so a timeout means we did not lose -- either some peer did not
303 * vote, or we did not call election_check() in time.)
304 */
305 void
306 election_vote(pcmk_cluster_t *cluster)
307 {
308 struct timeval age;
309 xmlNode *vote = NULL;
310 pcmk__node_status_t *our_node = NULL;
311 const char *message_type = NULL;
312
313 CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return);
314
315 if (cluster->priv->node_name == NULL) {
316 pcmk__err("Cannot start an election: Local node name unknown");
317 return;
318 }
319
320 our_node = pcmk__get_node(0, cluster->priv->node_name, NULL,
321 pcmk__node_search_cluster_member);
322 if (!pcmk__cluster_is_node_active(our_node)) {
323 pcmk__trace("Cannot vote yet: local node not connected to cluster");
324 return;
325 }
326
327 election_reset(cluster);
328 cluster->priv->election->state = election_in_progress;
329 message_type = pcmk__server_message_type(cluster->priv->server);
330
331 /* @COMPAT We use message_type as the sender and recipient system for
332 * backward compatibility (see T566).
333 */
334 vote = pcmk__new_request(cluster->priv->server, message_type,
335 NULL, message_type, CRM_OP_VOTE, NULL);
336
337 cluster->priv->election->count++;
338 pcmk__xe_set(vote, PCMK__XA_ELECTION_OWNER,
339 pcmk__cluster_get_xml_id(our_node));
340 pcmk__xe_set_int(vote, PCMK__XA_ELECTION_ID,
341 cluster->priv->election->count);
342
343 // Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is actually microseconds
344 get_uptime(&age);
345 pcmk__xe_set_timeval(vote, PCMK__XA_ELECTION_AGE_SEC,
346 PCMK__XA_ELECTION_AGE_NANO_SEC, &age);
347
348 pcmk__cluster_send_message(NULL, cluster->priv->server, vote);
349 pcmk__xml_free(vote);
350
351 pcmk__debug("Started election round %u", cluster->priv->election->count);
352 election_timeout_start(cluster);
353 }
354
355 /*!
356 * \internal
357 * \brief Check whether local node has won an election
358 *
359 * If all known peers have sent no-vote messages, stop the election timer, set
360 * the election state to won, and call any registered win callback.
361 *
362 * \param[in,out] cluster Cluster with election
363 *
364 * \return TRUE if local node has won, FALSE otherwise
365 * \note If all known peers have sent no-vote messages, but the election owner
366 * does not call this function, the election will not be won (and the
367 * callback will not be called) until the election times out.
368 * \note This should be called when election_count_vote() returns
369 * \c election_in_progress.
370 */
371 bool
372 election_check(pcmk_cluster_t *cluster)
373 {
374 int voted_size = 0;
375 int num_members = 0;
376
377 CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL),
378 return false);
379
380 if (cluster->priv->election->voted == NULL) {
381 pcmk__trace("Election check requested, but no votes received yet");
382 return FALSE;
383 }
384
385 voted_size = g_hash_table_size(cluster->priv->election->voted);
386 num_members = pcmk__cluster_num_active_nodes();
387
388 /* in the case of #voted > #members, it is better to
389 * wait for the timeout and give the cluster time to
390 * stabilize
391 */
392 if (voted_size >= num_members) {
393 /* we won and everyone has voted */
394 election_timeout_stop(cluster);
395 if (voted_size > num_members) {
396 GHashTableIter gIter;
397 const pcmk__node_status_t *node = NULL;
398 char *key = NULL;
399
400 pcmk__warn("Received too many votes in election");
401 g_hash_table_iter_init(&gIter, pcmk__peer_cache);
402 while (g_hash_table_iter_next(&gIter, NULL, (void **) &node)) {
403 if (pcmk__cluster_is_node_active(node)) {
404 pcmk__warn("* expected vote: %s", node->name);
405 }
406 }
407
408 g_hash_table_iter_init(&gIter, cluster->priv->election->voted);
409 while (g_hash_table_iter_next(&gIter, (void **) &key, NULL)) {
410 pcmk__warn("* actual vote: %s", key);
411 }
412
413 }
414
415 pcmk__info("Election won by local node");
416 election_complete(cluster);
417 return TRUE;
418
419 } else {
420 pcmk__debug("Election still waiting on %d of %d vote%s",
421 (num_members - voted_size), num_members,
422 pcmk__plural_s(num_members));
423 }
424
425 return FALSE;
426 }
427
428 #define LOSS_DAMPEN 2 /* in seconds */
429
430 struct vote {
431 const char *op;
432 const char *from;
433 const char *version;
434 const char *election_owner;
435 int election_id;
436 struct timeval age;
437 };
438
439 /*!
440 * \internal
441 * \brief Unpack an election message
442 *
443 * \param[in] message Election message XML
444 * \param[out] vote Parsed fields from message
445 *
446 * \return TRUE if election message and election are valid, FALSE otherwise
447 * \note The parsed struct's pointer members are valid only for the lifetime of
448 * the message argument.
449 */
450 static bool
451 parse_election_message(const xmlNode *message, struct vote *vote)
452 {
453 CRM_CHECK(message && vote, return FALSE);
454
455 vote->election_id = -1;
456 vote->age.tv_sec = -1;
457 vote->age.tv_usec = -1;
458
459 vote->op = pcmk__xe_get(message, PCMK__XA_CRM_TASK);
460 vote->from = pcmk__xe_get(message, PCMK__XA_SRC);
461 vote->version = pcmk__xe_get(message, PCMK_XA_VERSION);
462 vote->election_owner = pcmk__xe_get(message, PCMK__XA_ELECTION_OWNER);
463
464 pcmk__xe_get_int(message, PCMK__XA_ELECTION_ID, &vote->election_id);
465
466 if ((vote->op == NULL) || (vote->from == NULL) || (vote->version == NULL)
467 || (vote->election_owner == NULL) || (vote->election_id < 0)) {
468
469 pcmk__warn("Invalid %s message from %s", pcmk__s(vote->op, "election"),
470 pcmk__s(vote->from, "unspecified node"));
471 pcmk__log_xml_trace(message, "bad-vote");
472 return FALSE;
473 }
474
475 // Op-specific validation
476
477 if (pcmk__str_eq(vote->op, CRM_OP_VOTE, pcmk__str_none)) {
478 /* Only vote ops have uptime.
479 Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is in microseconds.
480 */
481 if ((pcmk__xe_get_timeval(message, PCMK__XA_ELECTION_AGE_SEC,
482 PCMK__XA_ELECTION_AGE_NANO_SEC,
483 &vote->age) != pcmk_rc_ok)
484 || (vote->age.tv_sec < 0) || (vote->age.tv_usec < 0)) {
485
486 pcmk__warn("Cannot count election %s from %s because uptime is "
487 "missing or invalid",
488 vote->op, vote->from);
489 return FALSE;
490 }
491
492 } else if (!pcmk__str_eq(vote->op, CRM_OP_NOVOTE, pcmk__str_none)) {
493 pcmk__info("Cannot process election message from %s because %s is not "
494 "a known election op",
495 vote->from, vote->op);
496 return FALSE;
497 }
498
499 /* If the membership cache is NULL, we REALLY shouldn't be voting --
500 * the question is how we managed to get here.
501 */
502 if (pcmk__peer_cache == NULL) {
503 pcmk__info("Cannot count election %s from %s becasue no peer "
504 "information available",
505 vote->op, vote->from);
506 return FALSE;
507 }
508 return TRUE;
509 }
510
511 static void
512 record_vote(pcmk_cluster_t *cluster, struct vote *vote)
513 {
514 pcmk__assert((vote->from != NULL) && (vote->op != NULL));
515
516 if (cluster->priv->election->voted == NULL) {
517 cluster->priv->election->voted = pcmk__strkey_table(free, free);
518 }
519 pcmk__insert_dup(cluster->priv->election->voted, vote->from, vote->op);
520 }
521
522 static void
523 send_no_vote(pcmk_cluster_t *cluster, pcmk__node_status_t *peer,
524 struct vote *vote)
525 {
526 const char *message_type = NULL;
527 xmlNode *novote = NULL;
528
529 message_type = pcmk__server_message_type(cluster->priv->server);
530 novote = pcmk__new_request(cluster->priv->server, message_type,
531 vote->from, message_type, CRM_OP_NOVOTE, NULL);
532 pcmk__xe_set(novote, PCMK__XA_ELECTION_OWNER, vote->election_owner);
533 pcmk__xe_set_int(novote, PCMK__XA_ELECTION_ID, vote->election_id);
534
535 pcmk__cluster_send_message(peer, cluster->priv->server, novote);
536 pcmk__xml_free(novote);
537 }
538
539 /*!
540 * \internal
541 * \brief Process an election message (vote or no-vote) from a peer
542 *
543 * \param[in,out] cluster Cluster with election
544 * \param[in] message Election message XML from peer
545 * \param[in] can_win Whether local node is eligible to win
546 *
547 * \return Election state after new vote is considered
548 * \note If the peer message is a vote, and we prefer the peer to win, this will
549 * send a no-vote reply to the peer.
550 * \note The situations "we lost to this vote" from "this is a late no-vote
551 * after we've already lost" both return election_lost. If a caller needs
552 * to distinguish them, it should save the current state before calling
553 * this function, and then compare the result.
554 */
555 enum election_result
556 election_count_vote(pcmk_cluster_t *cluster, const xmlNode *message,
557 bool can_win)
558 {
559 int log_level = LOG_INFO;
560 gboolean done = FALSE;
561 gboolean we_lose = FALSE;
562 const char *reason = NULL;
563 bool we_are_owner = FALSE;
564 pcmk__node_status_t *our_node = NULL;
565 pcmk__node_status_t *your_node = NULL;
566 time_t tm_now = time(NULL);
567 struct vote vote;
568
569 CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL)
570 && (message != NULL) && (cluster->priv->node_name != NULL),
571 return election_error);
572
573 if (!parse_election_message(message, &vote)) {
574 return election_error;
575 }
576
577 your_node = pcmk__get_node(0, vote.from, NULL,
578 pcmk__node_search_cluster_member);
579 our_node = pcmk__get_node(0, cluster->priv->node_name, NULL,
580 pcmk__node_search_cluster_member);
581 we_are_owner = (our_node != NULL)
582 && pcmk__str_eq(pcmk__cluster_get_xml_id(our_node),
583 vote.election_owner, pcmk__str_none);
584
585 if (!can_win) {
586 reason = "Not eligible";
587 we_lose = TRUE;
588
589 } else if (!pcmk__cluster_is_node_active(our_node)) {
590 reason = "We are not part of the cluster";
591 log_level = LOG_ERR;
592 we_lose = TRUE;
593
594 } else if (we_are_owner
595 && (vote.election_id != cluster->priv->election->count)) {
596 log_level = LOG_TRACE;
597 reason = "Superseded";
598 done = TRUE;
599
600 } else if (!pcmk__cluster_is_node_active(your_node)) {
601 /* Possibly we cached the message in the FSA queue at a point that it wasn't */
602 reason = "Peer is not part of our cluster";
603 log_level = LOG_WARNING;
604 done = TRUE;
605
606 } else if (pcmk__str_eq(vote.op, CRM_OP_NOVOTE, pcmk__str_none)
607 || pcmk__str_eq(vote.from, cluster->priv->node_name,
608 pcmk__str_casei)) {
609 /* Receiving our own broadcast vote, or a no-vote from peer, is a vote
610 * for us to win
611 */
612 if (!we_are_owner) {
613 pcmk__warn("Cannot count election round %d %s from %s because we "
614 "did not start election (node ID %s did)",
615 vote.election_id, vote.op, vote.from,
616 vote.election_owner);
617 return election_error;
618 }
619 if (cluster->priv->election->state != election_in_progress) {
620 // Should only happen if we already lost
621 pcmk__debug("Not counting election round %d %s from %s because no "
622 "election in progress",
623 vote.election_id, vote.op, vote.from);
624 return cluster->priv->election->state;
625 }
626 record_vote(cluster, &vote);
627 reason = "Recorded";
628 done = TRUE;
629
630 } else {
631 // A peer vote requires a comparison to determine which node is better
632 int age_result = compare_age(vote.age);
633 int version_result = pcmk__compare_versions(vote.version,
634 CRM_FEATURE_SET);
635
636 if (version_result < 0) {
637 reason = "Version";
638 we_lose = TRUE;
639
640 } else if (version_result > 0) {
641 reason = "Version";
642
643 } else if (age_result < 0) {
644 reason = "Uptime";
645 we_lose = TRUE;
646
647 } else if (age_result > 0) {
648 reason = "Uptime";
649
650 } else if (strcasecmp(cluster->priv->node_name, vote.from) > 0) {
651 reason = "Host name";
652 we_lose = TRUE;
653
654 } else {
655 reason = "Host name";
656 }
657 }
658
659 if (cluster->priv->election->expires < tm_now) {
660 cluster->priv->election->election_wins = 0;
661 cluster->priv->election->expires = tm_now + STORM_INTERVAL;
662
663 } else if (done == FALSE && we_lose == FALSE) {
664 int peers = 1 + g_hash_table_size(pcmk__peer_cache);
665
666 /* If every node has to vote down every other node, thats N*(N-1) total elections
667 * Allow some leeway before _really_ complaining
668 */
669 cluster->priv->election->election_wins++;
670 if (cluster->priv->election->election_wins > (peers * peers)) {
671 pcmk__warn("Election storm detected: %d wins in %d seconds",
672 cluster->priv->election->election_wins, STORM_INTERVAL);
673 cluster->priv->election->election_wins = 0;
674 cluster->priv->election->expires = tm_now + STORM_INTERVAL;
675 if (!(cluster->priv->election->wrote_blackbox)) {
676 /* It's questionable whether a black box (from every node in the
677 * cluster) would be truly helpful in diagnosing an election
678 * storm. It's also highly doubtful a production environment
679 * would get multiple election storms from distinct causes, so
680 * saving one blackbox per process lifetime should be
681 * sufficient. Alternatives would be to save a timestamp of the
682 * last blackbox write instead of a boolean, and write a new one
683 * if some amount of time has passed; or to save a storm count,
684 * write a blackbox on every Nth occurrence.
685 */
686 crm_write_blackbox(0, NULL);
687 cluster->priv->election->wrote_blackbox = true;
688 }
689 }
690 }
691
692 if (done) {
693 do_crm_log(log_level + 1,
694 "Processed election round %u %s (current round %d) "
695 "from %s (%s)",
696 vote.election_id, vote.op, cluster->priv->election->count,
697 vote.from, reason);
698 return cluster->priv->election->state;
699
700 } else if (we_lose == FALSE) {
701 /* We track the time of the last election loss to implement an election
702 * dampening period, reducing the likelihood of an election storm. If
703 * this node has lost within the dampening period, don't start a new
704 * election, even if we win against a peer's vote -- the peer we lost to
705 * should win again.
706 *
707 * @TODO This has a problem case: if an election winner immediately
708 * leaves the cluster, and a new election is immediately called, all
709 * nodes could lose, with no new winner elected. The ideal solution
710 * would be to tie the election structure with the peer caches, which
711 * would allow us to clear the dampening when the previous winner
712 * leaves (and would allow other improvements as well).
713 */
714 if ((cluster->priv->election->last_election_loss == 0)
715 || ((tm_now - cluster->priv->election->last_election_loss)
716 > (time_t) LOSS_DAMPEN)) {
717
718 do_crm_log(log_level,
719 "Election round %d (started by node ID %s) pass: "
720 "%s from %s (%s)",
721 vote.election_id, vote.election_owner, vote.op,
722 vote.from, reason);
723
724 cluster->priv->election->last_election_loss = 0;
725 election_timeout_stop(cluster);
726
727 /* Start a new election by voting down this, and other, peers */
728 cluster->priv->election->state = election_start;
729 return cluster->priv->election->state;
730 } else {
731 char *loss_time = NULL;
732
733 loss_time = ctime(&cluster->priv->election->last_election_loss);
734 if (loss_time) {
735 // Show only HH:MM:SS
736 loss_time += 11;
737 loss_time[8] = '\0';
738 }
739 pcmk__info("Ignoring election round %d (started by node ID %s) "
740 "pass vs %s because we lost less than %ds ago at %s",
741 vote.election_id, vote.election_owner, vote.from,
742 LOSS_DAMPEN, pcmk__s(loss_time, "unknown"));
743 }
744 }
745
746 cluster->priv->election->last_election_loss = tm_now;
747
748 do_crm_log(log_level,
749 "Election round %d (started by node ID %s) lost: "
750 "%s from %s (%s)",
751 vote.election_id, vote.election_owner, vote.op,
752 vote.from, reason);
753
754 election_reset(cluster);
755 send_no_vote(cluster, your_node, &vote);
756 cluster->priv->election->state = election_lost;
757 return cluster->priv->election->state;
758 }
759
760 /*!
761 * \internal
762 * \brief Reset any election dampening currently in effect
763 *
764 * \param[in,out] cluster Cluster with election
765 */
766 void
767 election_clear_dampening(pcmk_cluster_t *cluster)
768 {
769 if ((cluster != NULL) && (cluster->priv->election != NULL)) {
770 cluster->priv->election->last_election_loss = 0;
771 }
772 }
773