corosync 3.1.5
totemknet.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2016-2020 Red Hat, Inc.
3 *
4 * All rights reserved.
5 *
6 * Author: Christine Caulfield (ccaulfie@redhat.com)
7
8 * This software licensed under BSD license, the text of which follows:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the MontaVista Software, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#include <config.h>
36
37#include <assert.h>
38#include <sys/mman.h>
39#include <sys/types.h>
40#include <sys/stat.h>
41#include <sys/socket.h>
42#include <netdb.h>
43#include <sys/un.h>
44#include <sys/ioctl.h>
45#include <sys/param.h>
46#include <netinet/in.h>
47#include <net/ethernet.h>
48#include <arpa/inet.h>
49#include <unistd.h>
50#include <fcntl.h>
51#include <stdlib.h>
52#include <stdio.h>
53#include <errno.h>
54#include <pthread.h>
55#include <sched.h>
56#include <time.h>
57#include <sys/time.h>
58#include <sys/poll.h>
59#include <sys/uio.h>
60#include <limits.h>
61
62#include <qb/qbdefs.h>
63#include <qb/qbloop.h>
64#ifdef HAVE_LIBNOZZLE
65#include <libgen.h>
66#include <libnozzle.h>
67#endif
68
69#include <corosync/sq.h>
70#include <corosync/swab.h>
71#include <corosync/logsys.h>
72#include <corosync/icmap.h>
74#include "totemknet.h"
75
76#include "main.h"
77#include "util.h"
78
79#include <libknet.h>
81
82#ifndef MSG_NOSIGNAL
83#define MSG_NOSIGNAL 0
84#endif
85
86#ifdef HAVE_LIBNOZZLE
87static int setup_nozzle(void *knet_context);
88#endif
89
90/* Should match that used by cfg */
91#define CFG_INTERFACE_STATUS_MAX_LEN 512
92
94 struct crypto_instance *crypto_inst;
95
96 qb_loop_t *poll_handle;
97
98 knet_handle_t knet_handle;
99
101
102 void *context;
103
105 void *context,
106 const void *msg,
107 unsigned int msg_len,
108 const struct sockaddr_storage *system_from);
109
111 void *context,
112 const struct totem_ip_address *iface_address,
113 unsigned int link_no);
114
116 void *context,
117 int net_mtu);
118
120
121 /*
122 * Function and data used to log messages
123 */
125
127
129
131
133
135
137
139 int level,
140 int subsys,
141 const char *function,
142 const char *file,
143 int line,
144 const char *format,
145 ...)__attribute__((format(printf, 6, 7)));
146
148
149 char iov_buffer[KNET_MAX_PACKET_SIZE];
150
152
154
156
158
160
162
164
165 qb_loop_timer_handle timer_netif_check_timeout;
166
167 qb_loop_timer_handle timer_merge_detect_timeout;
168
170
172
173 int logpipes[2];
175
176 pthread_mutex_t log_mutex;
177#ifdef HAVE_LIBNOZZLE
178 char *nozzle_name;
179 char *nozzle_ipaddr;
180 char *nozzle_prefix;
181 char *nozzle_macaddr;
182 nozzle_t nozzle_handle;
183#endif
184};
185
186/* Awkward. But needed to get stats from knet */
188
189struct work_item {
190 const void *msg;
191 unsigned int msg_len;
193};
194
196 void *knet_context);
197
198
199static int totemknet_configure_compression (
200 struct totemknet_instance *instance,
201 struct totem_config *totem_config);
202
203static void totemknet_start_merge_detect_timeout(
204 void *knet_context);
205
206static void totemknet_stop_merge_detect_timeout(
207 void *knet_context);
208
209static void log_flush_messages (
210 void *knet_context);
211
212static void totemknet_instance_initialize (struct totemknet_instance *instance)
213{
214 int res;
215
216 memset (instance, 0, sizeof (struct totemknet_instance));
217 res = pthread_mutex_init(&instance->log_mutex, NULL);
218 /*
219 * There is not too much else what can be done.
220 */
221 assert(res == 0);
222}
223
224#define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
225do { \
226 (void)pthread_mutex_lock(&instance->log_mutex); \
227 instance->totemknet_log_printf ( \
228 level, subsys, function, file, line, \
229 (const char *)format, ##args); \
230 (void)pthread_mutex_unlock(&instance->log_mutex); \
231} while (0);
232
233#define knet_log_printf(level, format, args...) \
234do { \
235 knet_log_printf_lock ( \
236 level, instance->totemknet_subsys_id, \
237 __FUNCTION__, __FILE__, __LINE__, \
238 (const char *)format, ##args); \
239} while (0);
240
241#define libknet_log_printf(level, format, args...) \
242do { \
243 knet_log_printf_lock ( \
244 level, instance->knet_subsys_id, \
245 __FUNCTION__, "libknet.h", __LINE__, \
246 (const char *)format, ##args); \
247} while (0);
248
249#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
250do { \
251 char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
252 const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
253 instance->totemknet_log_printf ( \
254 level, instance->totemknet_subsys_id, \
255 __FUNCTION__, __FILE__, __LINE__, \
256 fmt ": %s (%d)", ##args, _error_ptr, err_num); \
257 } while(0)
258
259
260#ifdef HAVE_LIBNOZZLE
261static inline int is_ether_addr_multicast(const uint8_t *addr)
262{
263 return (addr[0] & 0x01);
264}
265static inline int is_ether_addr_zero(const uint8_t *addr)
266{
267 return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
268}
269
270static int ether_host_filter_fn(void *private_data,
271 const unsigned char *outdata,
272 ssize_t outdata_len,
273 uint8_t tx_rx,
274 knet_node_id_t this_host_id,
275 knet_node_id_t src_host_id,
276 int8_t *channel,
277 knet_node_id_t *dst_host_ids,
278 size_t *dst_host_ids_entries)
279{
280 struct ether_header *eth_h = (struct ether_header *)outdata;
281 uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
282 uint16_t dst_host_id;
283
284 if (is_ether_addr_zero(dst_mac))
285 return -1;
286
287 if (is_ether_addr_multicast(dst_mac)) {
288 return 1;
289 }
290
291 memmove(&dst_host_id, &dst_mac[4], 2);
292
293 dst_host_ids[0] = ntohs(dst_host_id);
294 *dst_host_ids_entries = 1;
295
296 return 0;
297}
298#endif
299
300static int dst_host_filter_callback_fn(void *private_data,
301 const unsigned char *outdata,
302 ssize_t outdata_len,
303 uint8_t tx_rx,
304 knet_node_id_t this_host_id,
305 knet_node_id_t src_host_id,
306 int8_t *channel,
307 knet_node_id_t *dst_host_ids,
308 size_t *dst_host_ids_entries)
309{
310 struct totem_message_header *header = (struct totem_message_header *)outdata;
311 int res;
312
313#ifdef HAVE_LIBNOZZLE
314 if (*channel != 0) {
315 return ether_host_filter_fn(private_data,
316 outdata, outdata_len,
317 tx_rx,
318 this_host_id, src_host_id,
319 channel,
320 dst_host_ids,
321 dst_host_ids_entries);
322 }
323#endif
324 if (header->target_nodeid) {
325 dst_host_ids[0] = header->target_nodeid;
326 *dst_host_ids_entries = 1;
327 res = 0; /* unicast message */
328 }
329 else {
330 *dst_host_ids_entries = 0;
331 res = 1; /* multicast message */
332 }
333 return res;
334}
335
336static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
337{
338 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
339
340 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
341 if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
342 knet_handle_remove_datafd(instance->knet_handle, datafd);
343 }
344}
345
346static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
347{
348 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
349
350 // TODO: what? if anything.
351 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
352}
353
354static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
355{
356 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
357 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
358
359 /* We don't need to tell corosync the actual knet MTU */
360// instance->totemknet_mtu_changed(instance->context, data_mtu);
361}
362
364 void *knet_context,
365 const char *cipher_type,
366 const char *hash_type)
367{
368 return (0);
369}
370
371
372static inline void ucast_sendmsg (
373 struct totemknet_instance *instance,
374 struct totem_ip_address *system_to,
375 const void *msg,
376 unsigned int msg_len)
377{
378 int res = 0;
379 struct totem_message_header *header = (struct totem_message_header *)msg;
380 struct msghdr msg_ucast;
381 struct iovec iovec;
382
383 header->target_nodeid = system_to->nodeid;
384
385 iovec.iov_base = (void *)msg;
386 iovec.iov_len = msg_len;
387
388 /*
389 * Build unicast message
390 */
391 memset(&msg_ucast, 0, sizeof(msg_ucast));
392 msg_ucast.msg_iov = (void *)&iovec;
393 msg_ucast.msg_iovlen = 1;
394#ifdef HAVE_MSGHDR_CONTROL
395 msg_ucast.msg_control = 0;
396#endif
397#ifdef HAVE_MSGHDR_CONTROLLEN
398 msg_ucast.msg_controllen = 0;
399#endif
400#ifdef HAVE_MSGHDR_FLAGS
401 msg_ucast.msg_flags = 0;
402#endif
403#ifdef HAVE_MSGHDR_ACCRIGHTS
404 msg_ucast.msg_accrights = NULL;
405#endif
406#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
407 msg_ucast.msg_accrightslen = 0;
408#endif
409
410 /*
411 * Transmit unicast message
412 * An error here is recovered by totemsrp
413 */
414
415 res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
416 if (res < 0) {
418 "sendmsg(ucast) failed (non-critical)");
419 }
420}
421
422static inline void mcast_sendmsg (
423 struct totemknet_instance *instance,
424 const void *msg,
425 unsigned int msg_len,
426 int only_active)
427{
428 int res;
429 struct totem_message_header *header = (struct totem_message_header *)msg;
430 struct msghdr msg_mcast;
431 struct iovec iovec;
432
433 iovec.iov_base = (void *)msg;
434 iovec.iov_len = msg_len;
435
437
438 /*
439 * Build multicast message
440 */
441 memset(&msg_mcast, 0, sizeof(msg_mcast));
442 msg_mcast.msg_iov = (void *)&iovec;
443 msg_mcast.msg_iovlen = 1;
444#ifdef HAVE_MSGHDR_CONTROL
445 msg_mcast.msg_control = 0;
446#endif
447#ifdef HAVE_MSGHDR_CONTROLLEN
448 msg_mcast.msg_controllen = 0;
449#endif
450#ifdef HAVE_MSGHDR_FLAGS
451 msg_mcast.msg_flags = 0;
452#endif
453#ifdef HAVE_MSGHDR_ACCRIGHTS
454 msg_mcast.msg_accrights = NULL;
455#endif
456#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
457 msg_mcast.msg_accrightslen = 0;
458#endif
459
460
461// log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
462
463 res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
464 if (res < msg_len) {
465 knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
466 }
467
468 if (!only_active || instance->send_merge_detect_message) {
469 /*
470 * Current message was sent to all nodes
471 */
473 instance->send_merge_detect_message = 0;
474 }
475}
476
477static int node_compare(const void *aptr, const void *bptr)
478{
479 uint16_t a,b;
480
481 a = *(uint16_t *)aptr;
482 b = *(uint16_t *)bptr;
483
484 return a > b;
485}
486
487#ifndef OWN_INDEX_NONE
488#define OWN_INDEX_NONE -1
489#endif
490
492 void *knet_context,
493 unsigned int nodeid,
494 struct totem_node_status *node_status)
495{
496 int i;
497 int res = 0;
498 struct knet_link_status link_status;
499 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
500 struct knet_host_status knet_host_status;
501 uint8_t link_list[KNET_MAX_LINK];
502 size_t num_links;
503
504 if (!instance->knet_handle) {
505 return CS_ERR_NOT_EXIST; /* Not using knet */
506 }
507
508 if (!node_status) {
510 }
511
512 res = knet_host_get_status(instance->knet_handle,
513 nodeid,
514 &knet_host_status);
515 if (res) {
516 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
517 return (-1);
518 }
519 node_status->nodeid = nodeid;
520 node_status->reachable = knet_host_status.reachable;
521 node_status->remote = knet_host_status.remote;
522 node_status->external = knet_host_status.external;
523
524#ifdef HAVE_KNET_ONWIRE_VER
525 res = knet_handle_get_onwire_ver(instance->knet_handle,
526 nodeid,
527 &node_status->onwire_min,
528 &node_status->onwire_max,
529 &node_status->onwire_ver);
530 if (res) {
531 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
532 return (-1);
533 }
534#endif
535 /* Get link info */
536 res = knet_link_get_link_list(instance->knet_handle,
537 nodeid, link_list, &num_links);
538 if (res) {
539 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
540 return (-1);
541 }
542
543 /* node_status[] has been zeroed for us in totempg.c */
544 for (i=0; i < num_links; i++) {
545 if (!instance->totem_config->interfaces[link_list[i]].configured) {
546 continue;
547 }
548 res = knet_link_get_status(instance->knet_handle,
549 nodeid,
550 link_list[i],
551 &link_status,
552 sizeof(link_status));
553 if (res == 0) {
554 node_status->link_status[link_list[i]].enabled = link_status.enabled;
555 node_status->link_status[link_list[i]].connected = link_status.connected;
556 node_status->link_status[link_list[i]].dynconnected = link_status.dynconnected;
557 node_status->link_status[link_list[i]].mtu = link_status.mtu;
558 memcpy(node_status->link_status[link_list[i]].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
559 memcpy(node_status->link_status[link_list[i]].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
560 } else {
561 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
562 }
563 }
564 return res;
565}
566
567
568
569int totemknet_ifaces_get (void *knet_context,
570 char ***status,
571 unsigned int *iface_count)
572{
573 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
574 struct knet_link_status link_status;
575 knet_node_id_t host_list[KNET_MAX_HOST];
576 uint8_t link_list[KNET_MAX_LINK];
577 size_t num_hosts;
578 size_t num_links;
579 size_t link_idx;
580 int i,j;
581 char *ptr;
582 int res = 0;
583
584 /*
585 * Don't do the whole 'link_info' bit if the caller just wants
586 * a count of interfaces.
587 */
588 if (status) {
589 int own_idx = OWN_INDEX_NONE;
590
591 res = knet_host_get_host_list(instance->knet_handle,
592 host_list, &num_hosts);
593 if (res) {
594 return (-1);
595 }
596 qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
597
598 for (j=0; j<num_hosts; j++) {
599 if (host_list[j] == instance->our_nodeid) {
600 own_idx = j;
601 break;
602 }
603 }
604
605 for (i=0; i<INTERFACE_MAX; i++) {
606 memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
607 if (own_idx != OWN_INDEX_NONE) {
608 instance->link_status[i][own_idx] = 'n';
609 }
610 instance->link_status[i][num_hosts] = '\0';
611 }
612
613 /* This is all a bit "inside-out" because "status" is a set of strings per link
614 * and knet orders things by host
615 */
616 for (j=0; j<num_hosts; j++) {
617 if (own_idx != OWN_INDEX_NONE && j == own_idx) {
618 continue ;
619 }
620
621 res = knet_link_get_link_list(instance->knet_handle,
622 host_list[j], link_list, &num_links);
623 if (res) {
624 return (-1);
625 }
626
627 link_idx = 0;
628 for (i=0; i < num_links; i++) {
629 /*
630 * Skip over links that are unconfigured to corosync. This is basically
631 * link0 if corosync isn't using it for comms, as we will still
632 * have it set up for loopback.
633 */
634 if (!instance->totem_config->interfaces[link_list[i]].configured) {
635 continue;
636 }
637 ptr = instance->link_status[link_idx++];
638
639 res = knet_link_get_status(instance->knet_handle,
640 host_list[j],
641 link_list[i],
642 &link_status,
643 sizeof(link_status));
644 if (res == 0) {
645 ptr[j] = '0' + (link_status.enabled |
646 link_status.connected<<1 |
647 link_status.dynconnected<<2);
648 }
649 else {
651 "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
652 ptr[j] = '?';
653 }
654 }
655 }
656 *status = instance->link_status;
657 }
658
659 *iface_count = INTERFACE_MAX;
660
661 return (res);
662}
663
665 void *knet_context)
666{
667 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
668 int res = 0;
669 int i,j;
670 static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
671 uint8_t links[KNET_MAX_LINK];
672 size_t num_nodes;
673 size_t num_links;
674
675 knet_log_printf(LOG_DEBUG, "totemknet: finalize");
676
677 qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
678 qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
679
680 /*
681 * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
682 */
683 res = knet_handle_setfwd(instance->knet_handle, 0);
684 if (res) {
685 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
686 }
687
688 res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
689 if (res) {
690 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
691 /* Crash out anyway */
692 goto finalise_error;
693 }
694
695 /* Tidily shut down all nodes & links. */
696 for (i=0; i<num_nodes; i++) {
697
698 res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
699 if (res) {
700 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
701 goto finalise_error;
702 }
703 for (j=0; j<num_links; j++) {
704 res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
705 if (res) {
706 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
707 }
708 res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
709 if (res) {
710 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
711 }
712 }
713 res = knet_host_remove(instance->knet_handle, nodes[i]);
714 if (res) {
715 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
716 }
717 }
718
719finalise_error:
720 res = knet_handle_free(instance->knet_handle);
721 if (res) {
722 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
723 }
724
725 totemknet_stop_merge_detect_timeout(instance);
726
727 log_flush_messages(instance);
728
729 /*
730 * Error is deliberately ignored
731 */
732 (void)pthread_mutex_destroy(&instance->log_mutex);
733
734 return (res);
735}
736
737static int log_deliver_fn (
738 int fd,
739 int revents,
740 void *data)
741{
742 struct totemknet_instance *instance = (struct totemknet_instance *)data;
743 char buffer[sizeof(struct knet_log_msg)*4];
744 char *bufptr = buffer;
745 int done = 0;
746 int len;
747
748 len = read(fd, buffer, sizeof(buffer));
749 while (done < len) {
750 struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
751 switch (msg->msglevel) {
752 case KNET_LOG_ERR:
754 knet_log_get_subsystem_name(msg->subsystem),
755 msg->msg);
756 break;
757 case KNET_LOG_WARN:
759 knet_log_get_subsystem_name(msg->subsystem),
760 msg->msg);
761 break;
762 case KNET_LOG_INFO:
764 knet_log_get_subsystem_name(msg->subsystem),
765 msg->msg);
766 break;
767 case KNET_LOG_DEBUG:
769 knet_log_get_subsystem_name(msg->subsystem),
770 msg->msg);
771 break;
772 }
773 bufptr += sizeof(struct knet_log_msg);
774 done += sizeof(struct knet_log_msg);
775 }
776 return 0;
777}
778
779static int data_deliver_fn (
780 int fd,
781 int revents,
782 void *data)
783{
784 struct totemknet_instance *instance = (struct totemknet_instance *)data;
785 struct msghdr msg_hdr;
786 struct iovec iov_recv;
787 struct sockaddr_storage system_from;
788 ssize_t msg_len;
789 int truncated_packet;
790
791 iov_recv.iov_base = instance->iov_buffer;
792 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
793
794 msg_hdr.msg_name = &system_from;
795 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
796 msg_hdr.msg_iov = &iov_recv;
797 msg_hdr.msg_iovlen = 1;
798#ifdef HAVE_MSGHDR_CONTROL
799 msg_hdr.msg_control = 0;
800#endif
801#ifdef HAVE_MSGHDR_CONTROLLEN
802 msg_hdr.msg_controllen = 0;
803#endif
804#ifdef HAVE_MSGHDR_FLAGS
805 msg_hdr.msg_flags = 0;
806#endif
807#ifdef HAVE_MSGHDR_ACCRIGHTS
808 msg_hdr.msg_accrights = NULL;
809#endif
810#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
811 msg_hdr.msg_accrightslen = 0;
812#endif
813
814 msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
815 if (msg_len <= 0) {
816 return (0);
817 }
818
819 truncated_packet = 0;
820
821#ifdef HAVE_MSGHDR_FLAGS
822 if (msg_hdr.msg_flags & MSG_TRUNC) {
823 truncated_packet = 1;
824 }
825#else
826 /*
827 * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
828 * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
829 */
830 if (bytes_received == KNET_MAX_PACKET_SIZE) {
831 truncated_packet = 1;
832 }
833#endif
834
835 if (truncated_packet) {
837 "Received too big message. This may be because something bad is happening"
838 "on the network (attack?), or you tried join more nodes than corosync is"
839 "compiled with (%u) or bug in the code (bad estimation of "
840 "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
841 return (0);
842 }
843
844 /*
845 * Handle incoming message
846 */
847 instance->totemknet_deliver_fn (
848 instance->context,
849 instance->iov_buffer,
850 msg_len,
851 &system_from);
852
853 return (0);
854}
855
856static void timer_function_netif_check_timeout (
857 void *data)
858{
859 struct totemknet_instance *instance = (struct totemknet_instance *)data;
860 int i;
861
862 for (i=0; i < INTERFACE_MAX; i++) {
863 if (!instance->totem_config->interfaces[i].configured) {
864 continue;
865 }
866 instance->totemknet_iface_change_fn (instance->context,
867 &instance->my_ids[i],
868 i);
869 }
870}
871
872static void knet_set_access_list_config(struct totemknet_instance *instance)
873{
874#ifdef HAVE_KNET_ACCESS_LIST
875 uint32_t value;
876 cs_error_t err;
877
879 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
880
881 err = knet_handle_enable_access_lists(instance->knet_handle, value);
882 if (err) {
883 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
884 }
885#endif
886}
887
888
889/* NOTE: this relies on the fact that totem_reload_notify() is called first */
890static void totemknet_refresh_config(
891 int32_t event,
892 const char *key_name,
893 struct icmap_notify_value new_val,
894 struct icmap_notify_value old_val,
895 void *user_data)
896{
897 uint8_t reloading;
898 uint32_t value;
899 uint32_t link_no;
900 size_t num_nodes;
901 knet_node_id_t host_ids[KNET_MAX_HOST];
902 int i;
903 int err;
904 struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
905
906 ENTER();
907
908 /*
909 * If a full reload is in progress then don't do anything until it's done and
910 * can reconfigure it all atomically
911 */
912 if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
913 return;
914 }
915
916 knet_set_access_list_config(instance);
917
918 if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
919
921 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
922 err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
923 if (err) {
924 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
925 }
926 }
927
928 /* Configure link parameters for each node */
929 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
930 if (err != 0) {
931 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
932 }
933
934 for (i=0; i<num_nodes; i++) {
935 for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
936 if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
937 continue;
938 }
939
940 err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
941 instance->totem_config->interfaces[link_no].knet_ping_interval,
942 instance->totem_config->interfaces[link_no].knet_ping_timeout,
943 instance->totem_config->interfaces[link_no].knet_ping_precision);
944 if (err) {
945 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
946 }
947 err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
948 instance->totem_config->interfaces[link_no].knet_pong_count);
949 if (err) {
950 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
951 }
952 err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
953 instance->totem_config->interfaces[link_no].knet_link_priority);
954 if (err) {
955 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
956 }
957
958 }
959 }
960
961 LEAVE();
962}
963
964static void totemknet_add_config_notifications(struct totemknet_instance *instance)
965{
966 icmap_track_t icmap_track_totem = NULL;
967 icmap_track_t icmap_track_reload = NULL;
968
969 ENTER();
970
971 icmap_track_add("totem.",
973 totemknet_refresh_config,
974 instance,
975 &icmap_track_totem);
976
977 icmap_track_add("config.totemconfig_reload_in_progress",
979 totemknet_refresh_config,
980 instance,
981 &icmap_track_reload);
982
983 LEAVE();
984}
985
986static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
987{
988
989 return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
990 strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
991
992}
993
994static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
995{
996 struct knet_handle_crypto_cfg crypto_cfg;
997 int res;
998
999 /* These have already been validated */
1000 memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
1001 memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
1002 memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
1003 memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1004 crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1005
1006#ifdef HAVE_KNET_CRYPTO_RECONF
1007
1008 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
1009 crypto_cfg.crypto_model,
1010 crypto_cfg.crypto_cipher_type,
1011 crypto_cfg.crypto_hash_type,
1012 instance->totem_config->crypto_index
1013 );
1014
1015 /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
1016 if (!totemknet_is_crypto_enabled(instance)) {
1017 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1018 if (res) {
1019 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
1020 }
1021 }
1022
1023 /* use_config will be called later when all nodes are synced */
1024 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
1025 if (res == -1) {
1026 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
1027 goto exit_error;
1028 }
1029 if (res == -2) {
1030 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
1031 goto exit_error;
1032 }
1033#else
1034 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
1035 crypto_cfg.crypto_model,
1036 crypto_cfg.crypto_cipher_type,
1037 crypto_cfg.crypto_hash_type
1038 );
1039
1040 res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1041 if (res == -1) {
1042 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1043 goto exit_error;
1044 }
1045 if (res == -2) {
1046 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1047 goto exit_error;
1048 }
1049#endif
1050
1051
1052exit_error:
1053 return res;
1054}
1055
1056/*
1057 * Create an instance
1058 */
1060 qb_loop_t *poll_handle,
1061 void **knet_context,
1062 struct totem_config *totem_config,
1063 totemsrp_stats_t *stats,
1064 void *context,
1065
1066 void (*deliver_fn) (
1067 void *context,
1068 const void *msg,
1069 unsigned int msg_len,
1070 const struct sockaddr_storage *system_from),
1071
1072 void (*iface_change_fn) (
1073 void *context,
1074 const struct totem_ip_address *iface_address,
1075 unsigned int link_no),
1076
1077 void (*mtu_changed) (
1078 void *context,
1079 int net_mtu),
1080
1081 void (*target_set_completed) (
1082 void *context))
1083{
1084 struct totemknet_instance *instance;
1085 char *tmp_str;
1086 int8_t channel=0;
1087 int allow_knet_handle_fallback=0;
1088 int res;
1089 int i;
1090
1091 instance = malloc (sizeof (struct totemknet_instance));
1092 if (instance == NULL) {
1093 return (-1);
1094 }
1095
1096 totemknet_instance_initialize (instance);
1097
1098 instance->totem_config = totem_config;
1099
1100 /*
1101 * Configure logging
1102 */
1103 instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1110
1111 instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1112
1113 /*
1114 * Initialize local variables for totemknet
1115 */
1116
1117 instance->our_nodeid = instance->totem_config->node_id;
1118
1119 for (i=0; i< INTERFACE_MAX; i++) {
1120 totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1121 instance->my_ids[i].nodeid = instance->our_nodeid;
1122 instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1123
1124 /* Needed for totemsrp */
1126 }
1127
1128 instance->poll_handle = poll_handle;
1129
1130 instance->context = context;
1131 instance->totemknet_deliver_fn = deliver_fn;
1132
1133 instance->totemknet_iface_change_fn = iface_change_fn;
1134
1135 instance->totemknet_mtu_changed = mtu_changed;
1136
1137 instance->totemknet_target_set_completed = target_set_completed;
1138
1139 instance->loopback_link = 0;
1140
1141 res = pipe(instance->logpipes);
1142 if (res == -1) {
1143 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1144 goto exit_error;
1145 }
1146 if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1147 fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1148 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1149 goto exit_error;
1150 }
1151
1152 if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
1153 if (strcmp(tmp_str, "yes") == 0) {
1154 allow_knet_handle_fallback = 1;
1155 }
1156 free(tmp_str);
1157 }
1158
1159#if defined(KNET_API_VER) && (KNET_API_VER == 2)
1160 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1161#else
1162 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1163#endif
1164
1165 if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
1166 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
1167#if defined(KNET_API_VER) && (KNET_API_VER == 2)
1168 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1169#else
1170 instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1171#endif
1172 }
1173
1174 if (!instance->knet_handle) {
1175 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1176 goto exit_error;
1177 }
1178
1179 knet_set_access_list_config(instance);
1180
1181 res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1182 if (res) {
1183 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1184 }
1185 res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1186 if (res) {
1187 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1188 }
1189 res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1190 if (res) {
1191 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1192 }
1193 res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1194 if (res) {
1195 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1196 }
1197 res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1198 if (res) {
1199 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1200 }
1201 global_instance = instance;
1202
1203 /* Get an fd into knet */
1204 instance->knet_fd = 0;
1205 res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1206 if (res) {
1207 knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1208 goto exit_error;
1209 }
1210
1211 /* Enable crypto if requested */
1212#ifdef HAVE_KNET_CRYPTO_RECONF
1213 if (totemknet_is_crypto_enabled(instance)) {
1214 res = totemknet_set_knet_crypto(instance);
1215 if (res == 0) {
1216 res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1217 if (res) {
1218 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1219 goto exit_error;
1220 }
1221 } else {
1222 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1223 goto exit_error;
1224 }
1225 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1226 if (res) {
1227 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1228 goto exit_error;
1229 }
1230
1231 } else {
1232 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1233 if (res) {
1234 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1235 goto exit_error;
1236 }
1237 }
1238#else
1239 if (totemknet_is_crypto_enabled(instance)) {
1240 res = totemknet_set_knet_crypto(instance);
1241 if (res) {
1242 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1243 goto exit_error;
1244 }
1245 }
1246#endif
1247
1248 /* Set up compression */
1249 if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1250 /* Not fatal, but will log */
1251 (void)totemknet_configure_compression(instance, totem_config);
1252 }
1253
1254 knet_handle_setfwd(instance->knet_handle, 1);
1255
1256 instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1257 if (strcmp(instance->totem_config->link_mode, "active")==0) {
1258 instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1259 }
1260 if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1261 instance->link_mode = KNET_LINK_POLICY_RR;
1262 }
1263
1264 for (i=0; i<INTERFACE_MAX; i++) {
1265 instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1266 if (!instance->link_status[i]) {
1267 goto exit_error;
1268 }
1269 }
1270
1271 qb_loop_poll_add (instance->poll_handle,
1272 QB_LOOP_MED,
1273 instance->logpipes[0],
1274 POLLIN, instance, log_deliver_fn);
1275
1276 qb_loop_poll_add (instance->poll_handle,
1277 QB_LOOP_HIGH,
1278 instance->knet_fd,
1279 POLLIN, instance, data_deliver_fn);
1280
1281 /*
1282 * Upper layer isn't ready to receive message because it hasn't
1283 * initialized yet. Add short timer to check the interfaces.
1284 */
1285 qb_loop_timer_add (instance->poll_handle,
1286 QB_LOOP_MED,
1287 100*QB_TIME_NS_IN_MSEC,
1288 (void *)instance,
1289 timer_function_netif_check_timeout,
1290 &instance->timer_netif_check_timeout);
1291
1292 totemknet_start_merge_detect_timeout(instance);
1293
1294 /* Start listening for config changes */
1295 totemknet_add_config_notifications(instance);
1296
1297 /* Add stats keys to icmap */
1299
1300 knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1301 *knet_context = instance;
1302
1303 return (0);
1304
1305exit_error:
1306 log_flush_messages(instance);
1307 free(instance);
1308 return (-1);
1309}
1310
1312{
1313 /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1314 return malloc(KNET_MAX_PACKET_SIZE + 512);
1315}
1316
1318{
1319 return free (ptr);
1320}
1321
1323 void *knet_context,
1324 int processor_count)
1325{
1326 return (0);
1327}
1328
1330{
1331 return (0);
1332}
1333
1335{
1336 return (0);
1337}
1338
1340 void *knet_context,
1341 const void *msg,
1342 unsigned int msg_len)
1343{
1344 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1345 int res = 0;
1346
1347 ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1348
1349 return (res);
1350}
1352 void *knet_context,
1353 const void *msg,
1354 unsigned int msg_len)
1355{
1356 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1357 int res = 0;
1358
1359 mcast_sendmsg (instance, msg, msg_len, 0);
1360
1361 return (res);
1362}
1363
1365 void *knet_context,
1366 const void *msg,
1367 unsigned int msg_len)
1368{
1369 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1370 int res = 0;
1371
1372 mcast_sendmsg (instance, msg, msg_len, 1);
1373
1374 return (res);
1375}
1376
1377
1379{
1380 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1381 int res = 0;
1382
1383 knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1384
1385 return (res);
1386}
1387
1389{
1390 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1391
1392 knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1393}
1394
1396 void *knet_context,
1397 unsigned int nodeid)
1398{
1399 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1400 int res = 0;
1401
1402 instance->token_target.nodeid = nodeid;
1403
1404 instance->totemknet_target_set_completed (instance->context);
1405
1406 return (res);
1407}
1408
1410 void *knet_context)
1411{
1412 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1413 unsigned int res;
1414 struct sockaddr_storage system_from;
1415 struct msghdr msg_hdr;
1416 struct iovec iov_recv;
1417 struct pollfd ufd;
1418 int nfds;
1419 int msg_processed = 0;
1420
1421 iov_recv.iov_base = instance->iov_buffer;
1422 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1423
1424 msg_hdr.msg_name = &system_from;
1425 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1426 msg_hdr.msg_iov = &iov_recv;
1427 msg_hdr.msg_iovlen = 1;
1428#ifdef HAVE_MSGHDR_CONTROL
1429 msg_hdr.msg_control = 0;
1430#endif
1431#ifdef HAVE_MSGHDR_CONTROLLEN
1432 msg_hdr.msg_controllen = 0;
1433#endif
1434#ifdef HAVE_MSGHDR_FLAGS
1435 msg_hdr.msg_flags = 0;
1436#endif
1437#ifdef HAVE_MSGHDR_ACCRIGHTS
1438 msg_msg_hdr.msg_accrights = NULL;
1439#endif
1440#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1441 msg_msg_hdr.msg_accrightslen = 0;
1442#endif
1443
1444 do {
1445 ufd.fd = instance->knet_fd;
1446 ufd.events = POLLIN;
1447 nfds = poll (&ufd, 1, 0);
1448 if (nfds == 1 && ufd.revents & POLLIN) {
1449 res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1450 if (res != -1) {
1451 msg_processed = 1;
1452 } else {
1453 msg_processed = -1;
1454 }
1455 }
1456 } while (nfds == 1);
1457
1458 return (msg_processed);
1459}
1460
1461int totemknet_iface_set (void *knet_context,
1462 const struct totem_ip_address *local_addr,
1463 unsigned short ip_port,
1464 unsigned int iface_no)
1465{
1466 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1467
1468 totemip_copy(&instance->my_ids[iface_no], local_addr);
1469
1470 knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1471
1472 instance->ip_port[iface_no] = ip_port;
1473
1474 return 0;
1475}
1476
1477
1479 void *knet_context,
1480 const struct totem_ip_address *local,
1481 const struct totem_ip_address *member,
1482 int link_no)
1483{
1484 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1485 int err;
1486 int port = instance->ip_port[link_no];
1487 struct sockaddr_storage remote_ss;
1488 struct sockaddr_storage local_ss;
1489 int addrlen;
1490 int i;
1491 int host_found = 0;
1492 knet_node_id_t host_ids[KNET_MAX_HOST];
1493 size_t num_host_ids;
1494
1495 /* Only create 1 loopback link and use link 0 */
1496 if (member->nodeid == instance->our_nodeid) {
1497 if (!instance->loopback_link) {
1498 link_no = 0;
1499 instance->loopback_link = 1;
1500 } else {
1501 /* Already done */
1502 return 0;
1503 }
1504 }
1505
1506 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1507 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1508
1509
1510 /* Only add the host if it doesn't already exist in knet */
1511 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1512 if (err) {
1513 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1514 return -1;
1515 }
1516 for (i=0; i<num_host_ids; i++) {
1517 if (host_ids[i] == member->nodeid) {
1518 host_found = 1;
1519 }
1520 }
1521
1522 if (!host_found) {
1523 err = knet_host_add(instance->knet_handle, member->nodeid);
1524 if (err != 0 && errno != EEXIST) {
1525 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1526 return -1;
1527 }
1528 } else {
1529 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1530 }
1531
1532
1533 if (err == 0) {
1534 if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1535 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1536 return -1;
1537 }
1538 }
1539
1540 memset(&local_ss, 0, sizeof(local_ss));
1541 memset(&remote_ss, 0, sizeof(remote_ss));
1542 /* Casts to remove const */
1543 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1544 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1545
1546 if (member->nodeid == instance->our_nodeid) {
1547 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1548
1549 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1550 KNET_TRANSPORT_LOOPBACK,
1551 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1552 }
1553 else {
1554 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1555 instance->totem_config->interfaces[link_no].knet_transport,
1556 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1557 }
1558 if (err) {
1559 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1560 return -1;
1561 }
1562
1563 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1564 instance->totem_config->interfaces[link_no].knet_link_priority);
1565
1566 err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1567 instance->totem_config->interfaces[link_no].knet_link_priority);
1568 if (err) {
1569 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1570 }
1571
1572 /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1573 get done in totemknet_refresh_config */
1574 if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1575 err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1576 instance->totem_config->interfaces[link_no].knet_ping_interval,
1577 instance->totem_config->interfaces[link_no].knet_ping_timeout,
1578 instance->totem_config->interfaces[link_no].knet_ping_precision);
1579 if (err) {
1580 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1581 }
1582 err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1583 instance->totem_config->interfaces[link_no].knet_pong_count);
1584 if (err) {
1585 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1586 }
1587 }
1588
1589 err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1590 if (err) {
1591 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1592 return -1;
1593 }
1594
1595 /* register stats */
1596 stats_knet_add_member(member->nodeid, link_no);
1597 return (0);
1598}
1599
1601 void *knet_context,
1602 const struct totem_ip_address *token_target,
1603 int link_no)
1604{
1605 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1606 int res;
1607 uint8_t link_list[KNET_MAX_LINK];
1608 size_t num_links;
1609
1610 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1611
1612 /* Don't remove the link with the loopback on it until we shut down */
1613 if (token_target->nodeid == instance->our_nodeid) {
1614 return 0;
1615 }
1616
1617 /* Tidy stats */
1619
1620 /* Remove the link first */
1621 res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1622 if (res != 0) {
1623 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1624 return res;
1625 }
1626
1627 res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1628 if (res != 0) {
1629 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1630 return res;
1631 }
1632
1633 /* If this is the last link, then remove the node */
1634 res = knet_link_get_link_list(instance->knet_handle,
1635 token_target->nodeid, link_list, &num_links);
1636 if (res) {
1637 return (0); /* not really failure */
1638 }
1639
1640 if (num_links == 0) {
1641 res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1642 }
1643 return res;
1644}
1645
1647 void *knet_context)
1648{
1649 return (0);
1650}
1651
1652
1653static int totemknet_configure_compression (
1654 struct totemknet_instance *instance,
1655 struct totem_config *totem_config)
1656{
1657 struct knet_handle_compress_cfg compress_cfg;
1658 int res = 0;
1659
1660 assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1661 strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1662
1663 compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1664 compress_cfg.compress_level = totem_config->knet_compression_level;
1665
1666 res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1667 if (res) {
1668 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1669 }
1670 return res;
1671}
1672
1674 void *knet_context,
1675 struct totem_config *totem_config)
1676{
1677 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1678 int res = 0;
1679
1680 (void)totemknet_configure_compression(instance, totem_config);
1681
1682#ifdef HAVE_LIBNOZZLE
1683 /* Set up nozzle device(s). Return code is ignored, because inability
1684 * configure nozzle is not fatal problem, errors are logged and
1685 * there is not much else we can do */
1686 (void)setup_nozzle(instance);
1687#endif
1688
1690 /* Flip crypto_index */
1692 res = totemknet_set_knet_crypto(instance);
1693
1694 knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1698 }
1699 return (res);
1700}
1701
1702
1704 void *knet_context,
1705 struct totem_config *totem_config,
1707{
1708#ifdef HAVE_KNET_CRYPTO_RECONF
1709 int res;
1710 int config_to_use;
1711 int config_to_clear;
1712 struct knet_handle_crypto_cfg crypto_cfg;
1713 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1714
1715 knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1716
1717 switch (phase) {
1719 config_to_use = totem_config->crypto_index;
1720 if (!totemknet_is_crypto_enabled(instance)) {
1721 config_to_use = 0; /* we are clearing it */
1722 }
1723
1724 /* Enable the new config on this node */
1725 res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1726 if (res == -1) {
1727 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1728 }
1729 break;
1730
1732 /*
1733 * All nodes should now have the new config. clear the old one out
1734 * OR disable crypto entirely if that's what the new config insists on.
1735 */
1736 config_to_clear = 3-totem_config->crypto_index;
1737 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1738
1739 strcpy(crypto_cfg.crypto_model, "none");
1740 strcpy(crypto_cfg.crypto_cipher_type, "none");
1741 strcpy(crypto_cfg.crypto_hash_type, "none");
1742 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1743 if (res == -1) {
1744 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1745 }
1746 if (res == -2) {
1747 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1748 }
1749
1750 /* If crypto is enabled then disable all cleartext reception */
1751 if (totemknet_is_crypto_enabled(instance)) {
1752 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1753 if (res) {
1754 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1755 }
1756 }
1757 }
1758#endif
1759 return 0;
1760}
1761
1763 void *knet_context)
1764{
1765 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1766
1767 (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1768}
1769
1770/* For the stats module */
1772 knet_node_id_t node, uint8_t link_no,
1773 struct knet_link_status *status)
1774{
1775 int res;
1776 int ret = CS_OK;
1777
1778 /* We are probably not using knet */
1779 if (!global_instance) {
1780 return CS_ERR_NOT_EXIST;
1781 }
1782
1783 if (link_no >= INTERFACE_MAX) {
1784 return CS_ERR_NOT_EXIST; /* Invalid link number */
1785 }
1786
1787 res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1788 if (res) {
1789 switch (errno) {
1790 case EINVAL:
1792 break;
1793 case EBUSY:
1794 ret = CS_ERR_BUSY;
1795 break;
1796 case EDEADLK:
1797 ret = CS_ERR_TRY_AGAIN;
1798 break;
1799 default:
1800 ret = CS_ERR_LIBRARY;
1801 break;
1802 }
1803 }
1804
1805 return (ret);
1806}
1807
1809 struct knet_handle_stats *stats)
1810{
1811 int res;
1812
1813 /* We are probably not using knet */
1814 if (!global_instance) {
1815 return CS_ERR_NOT_EXIST;
1816 }
1817
1818 res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1819 if (res != 0) {
1820 return (qb_to_cs_error(-errno));
1821 }
1822
1823 return CS_OK;
1824}
1825
1826static void timer_function_merge_detect_timeout (
1827 void *data)
1828{
1829 struct totemknet_instance *instance = (struct totemknet_instance *)data;
1830
1831 if (instance->merge_detect_messages_sent_before_timeout == 0) {
1832 instance->send_merge_detect_message = 1;
1833 }
1834
1836
1837 totemknet_start_merge_detect_timeout(instance);
1838}
1839
1840static void totemknet_start_merge_detect_timeout(
1841 void *knet_context)
1842{
1843 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1844
1845 qb_loop_timer_add(instance->poll_handle,
1846 QB_LOOP_MED,
1847 instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1848 (void *)instance,
1849 timer_function_merge_detect_timeout,
1850 &instance->timer_merge_detect_timeout);
1851
1852}
1853
1854static void totemknet_stop_merge_detect_timeout(
1855 void *knet_context)
1856{
1857 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1858
1859 qb_loop_timer_del(instance->poll_handle,
1860 instance->timer_merge_detect_timeout);
1861}
1862
1863static void log_flush_messages (void *knet_context)
1864{
1865 struct pollfd pfd;
1866 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1867 int cont;
1868
1869 cont = 1;
1870
1871 while (cont) {
1872 pfd.fd = instance->logpipes[0];
1873 pfd.events = POLLIN;
1874 pfd.revents = 0;
1875
1876 if ((poll(&pfd, 1, 0) > 0) &&
1877 (pfd.revents & POLLIN) &&
1878 (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1879 cont = 1;
1880 } else {
1881 cont = 0;
1882 }
1883 }
1884}
1885
1886
1887#ifdef HAVE_LIBNOZZLE
1888#define NOZZLE_NAME "nozzle.name"
1889#define NOZZLE_IPADDR "nozzle.ipaddr"
1890#define NOZZLE_PREFIX "nozzle.ipprefix"
1891#define NOZZLE_MACADDR "nozzle.macaddr"
1892
1893#define NOZZLE_CHANNEL 1
1894
1895
1896static char *get_nozzle_script_dir(void *knet_context)
1897{
1898 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1899 char filename[PATH_MAX + FILENAME_MAX + 1];
1900 static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1901 int res;
1902 const char *dirname_res;
1903
1904 /*
1905 * Build script directory based on corosync.conf file location
1906 */
1907 res = snprintf(filename, sizeof(filename), "%s",
1909 if (res >= sizeof(filename)) {
1910 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1911 return NULL;
1912 }
1913
1914 dirname_res = dirname(filename);
1915
1916 res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1917 dirname_res, "updown.d");
1918 if (res >= sizeof(updown_dirname)) {
1919 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1920 return NULL;
1921 }
1922 return updown_dirname;
1923}
1924
1925/*
1926 * Deliberately doesn't return the status as caller doesn't care.
1927 * The result will be logged though
1928 */
1929static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1930{
1931 int res;
1932 char *exec_string;
1933
1934 res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1935 if (res == -1 && errno != ENOENT) {
1936 knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1937 } else if (res == -2) {
1938 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1939 knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1940 }
1941}
1942
1943/*
1944 * Reparse IP address to add in our node ID
1945 * IPv6 addresses must end in '::'
1946 * IPv4 addresses must just be valid
1947 * '/xx' lengths are optional for IPv6, mandatory for IPv4
1948 *
1949 * Returns the modified IP address as a string to pass into libnozzle
1950 */
1951static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1952 const char *input_addr,
1953 const char *prefix, int nodeid,
1954 char *output_addr, size_t output_len)
1955{
1956 char *coloncolon;
1957 int bits;
1958 int max_prefix = 64;
1959 uint32_t nodeid_mask;
1960 uint32_t addr_mask;
1961 uint32_t masked_nodeid;
1962 struct in_addr *addr;
1963 struct totem_ip_address totemip;
1964
1965 coloncolon = strstr(input_addr, "::");
1966 if (!coloncolon) {
1967 max_prefix = 30;
1968 }
1969
1970 bits = atoi(prefix);
1971 if (bits < 8 || bits > max_prefix) {
1972 knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1973 return -1;
1974 }
1975
1976 /* IPv6 is easy */
1977 if (coloncolon) {
1978 memcpy(output_addr, input_addr, coloncolon-input_addr);
1979 sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1980 return 0;
1981 }
1982
1983 /* For IPv4 we need to parse the address into binary, mask off the required bits,
1984 * add in the masked_nodeid and 'print' it out again
1985 */
1986 nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1987 addr_mask = UINT32_MAX ^ nodeid_mask;
1988 masked_nodeid = nodeid & nodeid_mask;
1989
1990 if (totemip_parse(&totemip, input_addr, AF_INET)) {
1991 knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1992 return -1;
1993 }
1994 addr = (struct in_addr *)&totemip.addr;
1995 addr->s_addr &= htonl(addr_mask);
1996 addr->s_addr |= htonl(masked_nodeid);
1997
1998 inet_ntop(AF_INET, addr, output_addr, output_len);
1999 return 0;
2000}
2001
2002static int create_nozzle_device(void *knet_context, const char *name,
2003 const char *ipaddr, const char *prefix,
2004 const char *macaddr)
2005{
2006 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2007 char device_name[IFNAMSIZ+1];
2008 size_t size = IFNAMSIZ;
2009 int8_t channel = NOZZLE_CHANNEL;
2010 nozzle_t nozzle_dev;
2011 int nozzle_fd;
2012 int res;
2013 char *updown_dir;
2014 char parsed_ipaddr[INET6_ADDRSTRLEN];
2015 char mac[19];
2016
2017 memset(device_name, 0, size);
2018 memset(&mac, 0, sizeof(mac));
2019 strncpy(device_name, name, size);
2020
2021 updown_dir = get_nozzle_script_dir(knet_context);
2022 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
2023
2024 nozzle_dev = nozzle_open(device_name, size, updown_dir);
2025 if (!nozzle_dev) {
2026 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
2027 return -1;
2028 }
2029 instance->nozzle_handle = nozzle_dev;
2030
2031 if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
2032 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
2033 goto out_clean;
2034 }
2035
2036 if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
2037 /* Prints its own errors */
2038 goto out_clean;
2039 }
2040 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
2041 if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
2042 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
2043 goto out_clean;
2044 }
2045
2046 nozzle_fd = nozzle_get_fd(nozzle_dev);
2047 knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
2048
2049 res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
2050 if (res != 0) {
2051 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
2052 goto out_clean;
2053 }
2054
2055 run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
2056
2057 res = nozzle_set_up(nozzle_dev);
2058 if (res != 0) {
2059 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
2060 goto out_clean;
2061 }
2062 run_nozzle_script(instance, NOZZLE_UP, "up");
2063
2064 return 0;
2065
2066out_clean:
2067 nozzle_close(nozzle_dev);
2068 return -1;
2069}
2070
2071static int remove_nozzle_device(void *knet_context)
2072{
2073 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2074 int res;
2075 int datafd;
2076
2077 res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
2078 if (res != 0) {
2079 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2080 return -1;
2081 }
2082
2083 res = knet_handle_remove_datafd(instance->knet_handle, datafd);
2084 if (res != 0) {
2085 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2086 return -1;
2087 }
2088
2089 run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
2090 res = nozzle_set_down(instance->nozzle_handle);
2091 if (res != 0) {
2092 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
2093 return -1;
2094 }
2095 run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
2096
2097 res = nozzle_close(instance->nozzle_handle);
2098 if (res != 0) {
2099 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
2100 return -1;
2101 }
2102 knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2103 return 0;
2104}
2105
2106static void free_nozzle(struct totemknet_instance *instance)
2107{
2108 free(instance->nozzle_name);
2109 free(instance->nozzle_ipaddr);
2110 free(instance->nozzle_prefix);
2111 free(instance->nozzle_macaddr);
2112
2113 instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2114 instance->nozzle_macaddr = NULL;
2115}
2116
2117static int setup_nozzle(void *knet_context)
2118{
2119 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2120 char *ipaddr_str = NULL;
2121 char *name_str = NULL;
2122 char *prefix_str = NULL;
2123 char *macaddr_str = NULL;
2124 char mac[32];
2125 int name_res;
2126 int macaddr_res;
2127 int res = -1;
2128
2129 /*
2130 * Return value ignored on purpose. icmap_get_string changes
2131 * ipaddr_str/prefix_str only on success.
2132 */
2133 (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2134 (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2135 macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2136 name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2137
2138 /* Is is being removed? */
2139 if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2140 remove_nozzle_device(instance);
2141 free_nozzle(instance);
2142 goto out_free;
2143 }
2144
2145 if (!name_str) {
2146 /* no nozzle */
2147 goto out_free;
2148 }
2149
2150 if (!ipaddr_str) {
2151 knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2152 goto out_free;
2153 }
2154
2155 if (!prefix_str) {
2156 knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2157 goto out_free;
2158 }
2159
2160 if (macaddr_str && strlen(macaddr_str) != 17) {
2161 knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2162 goto out_free;
2163 }
2164 if (!macaddr_str) {
2165 macaddr_str = (char*)"54:54:01:00:00:00";
2166 }
2167
2168 if (instance->nozzle_name &&
2169 (strcmp(name_str, instance->nozzle_name) == 0) &&
2170 (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2171 (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2172 (instance->nozzle_macaddr == NULL ||
2173 strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2174 /* Nothing has changed */
2175 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2176 goto out_free;
2177 }
2178
2179 /* Add nodeid into MAC address */
2180 memcpy(mac, macaddr_str, 12);
2181 snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2182 instance->our_nodeid >> 8,
2183 instance->our_nodeid & 0xFF);
2184 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2185
2186 if (name_res == CS_OK && name_str) {
2187 /* Reconfigure */
2188 if (instance->nozzle_name) {
2189 remove_nozzle_device(instance);
2190 free_nozzle(instance);
2191 }
2192
2193 res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2194 mac);
2195
2196 instance->nozzle_name = strdup(name_str);
2197 instance->nozzle_ipaddr = strdup(ipaddr_str);
2198 instance->nozzle_prefix = strdup(prefix_str);
2199 instance->nozzle_macaddr = strdup(macaddr_str);
2200 if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2201 !instance->nozzle_prefix) {
2202 knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2203 /*
2204 * This 'free' will cause a complete reconfigure of the device next time we reload
2205 * but will also let the the current device keep working until then.
2206 * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2207 */
2208 free_nozzle(instance);
2209 }
2210 }
2211
2212out_free:
2213 free(name_str);
2214 free(ipaddr_str);
2215 free(prefix_str);
2216 if (macaddr_res == CS_OK) {
2217 free(macaddr_str);
2218 }
2219
2220 return res;
2221}
2222#endif // HAVE_LIBNOZZLE
#define INTERFACE_MAX
Definition: coroapi.h:88
unsigned int nodeid
Definition: coroapi.h:0
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:2
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:98
@ CS_ERR_BUSY
Definition: corotypes.h:108
@ CS_ERR_TRY_AGAIN
Definition: corotypes.h:104
@ CS_OK
Definition: corotypes.h:99
@ CS_ERR_INVALID_PARAM
Definition: corotypes.h:105
@ CS_ERR_LIBRARY
Definition: corotypes.h:100
@ CS_ERR_NOT_EXIST
Definition: corotypes.h:110
uint32_t value
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:868
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:892
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
#define LEAVE
Definition: logsys.h:325
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
#define ENTER
Definition: logsys.h:324
const char * corosync_get_config_file(void)
Definition: main.c:212
void * user_data
Definition: sam.c:127
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:223
unsigned int private_key_len
Definition: totem.h:176
unsigned int node_id
Definition: totem.h:167
uint32_t knet_compression_threshold
Definition: totem.h:235
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:207
struct totem_interface * interfaces
Definition: totem.h:165
int crypto_changed
Definition: totem.h:231
unsigned int merge_timeout
Definition: totem.h:197
int knet_compression_level
Definition: totem.h:237
unsigned int net_mtu
Definition: totem.h:209
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:233
unsigned int block_unlisted_ips
Definition: totem.h:245
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:174
int crypto_index
Definition: totem.h:229
unsigned int knet_pmtud_interval
Definition: totem.h:169
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:225
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:205
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:227
int knet_ping_timeout
Definition: totem.h:93
int knet_link_priority
Definition: totem.h:91
struct totem_ip_address boundto
Definition: totem.h:84
uint16_t ip_port
Definition: totem.h:87
int knet_ping_interval
Definition: totem.h:92
uint8_t configured
Definition: totem.h:89
int knet_ping_precision
Definition: totem.h:94
int knet_pong_count
Definition: totem.h:95
int knet_transport
Definition: totem.h:96
struct totem_ip_address bindnet
Definition: totem.h:83
The totem_ip_address struct.
Definition: coroapi.h:111
unsigned int nodeid
Definition: coroapi.h:112
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:101
unsigned int target_nodeid
Definition: totem.h:132
uint8_t onwire_max
Definition: totem.h:271
uint8_t reachable
Definition: totem.h:267
uint8_t remote
Definition: totem.h:268
unsigned int nodeid
Definition: totem.h:266
uint8_t onwire_min
Definition: totem.h:270
uint8_t onwire_ver
Definition: totem.h:272
uint8_t external
Definition: totem.h:269
struct knet_link_status link_status[KNET_MAX_LINK]
Definition: totem.h:273
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
pthread_mutex_t log_mutex
Definition: totemknet.c:176
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
struct totem_config * totem_config
Definition: totemknet.c:161
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
knet_handle_t knet_handle
Definition: totemknet.c:98
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
int totemknet_log_level_debug
Definition: totemknet.c:132
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
struct totem_ip_address token_target
Definition: totemknet.c:163
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
void(*) void knet_context)
Definition: totemknet.c:147
int totemknet_log_level_warning
Definition: totemknet.c:128
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
int send_merge_detect_message
Definition: totemknet.c:169
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
int totemknet_log_level_error
Definition: totemknet.c:126
int totemknet_log_level_security
Definition: totemknet.c:124
qb_loop_t * poll_handle
Definition: totemknet.c:96
int totemknet_log_level_notice
Definition: totemknet.c:130
const void * msg
Definition: totemknet.c:190
unsigned int msg_len
Definition: totemknet.c:191
struct totemknet_instance * instance
Definition: totemknet.c:192
typedef __attribute__
cfg_message_crypto_reconfig_phase_t
Definition: totem.h:154
@ CRYPTO_RECONFIG_PHASE_CLEANUP
Definition: totem.h:156
@ CRYPTO_RECONFIG_PHASE_ACTIVATE
Definition: totem.h:155
char type
Definition: totem.h:2
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:306
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:264
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:256
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1351
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1461
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:664
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1329
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1646
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1311
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1322
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1364
struct totemknet_instance * global_instance
Definition: totemknet.c:187
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1317
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:569
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1478
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:363
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1600
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:1059
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1339
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1771
#define OWN_INDEX_NONE
Definition: totemknet.c:488
int totemknet_nodestatus_get(void *knet_context, unsigned int nodeid, struct totem_node_status *node_status)
Definition: totemknet.c:491
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1808
#define MSG_NOSIGNAL
Definition: totemknet.c:83
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1762
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1334
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1388
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:233
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1395
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:249
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1673
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition: totemknet.c:1703
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:241
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1409
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1378
struct totem_message_header header
Definition: totemsrp.c:0
struct srp_addr system_from
Definition: totemsrp.c:1
void stats_knet_add_handle(void)
Definition: stats.c:753
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:740
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:730