corosync  3.0.0
totemknet.c
Go to the documentation of this file.
1 
2 /*
3  * Copyright (c) 2016-2018 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Christine Caulfield (ccaulfie@redhat.com)
8 
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <sys/mman.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <sys/socket.h>
43 #include <netdb.h>
44 #include <sys/un.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 #include <sys/time.h>
57 #include <sys/poll.h>
58 #include <sys/uio.h>
59 #include <limits.h>
60 
61 #include <qb/qbdefs.h>
62 #include <qb/qbloop.h>
63 
64 #include <corosync/sq.h>
65 #include <corosync/swab.h>
66 #include <corosync/logsys.h>
67 #include <corosync/icmap.h>
68 #include <corosync/totem/totemip.h>
69 #include "totemknet.h"
70 
71 #include "util.h"
72 
73 #include <libknet.h>
75 
76 #ifndef MSG_NOSIGNAL
77 #define MSG_NOSIGNAL 0
78 #endif
79 
80 /* Should match that used by cfg */
81 #define CFG_INTERFACE_STATUS_MAX_LEN 512
82 
84  struct crypto_instance *crypto_inst;
85 
86  qb_loop_t *poll_handle;
87 
88  knet_handle_t knet_handle;
89 
90  int link_mode;
91 
92  void *context;
93 
95  void *context,
96  const void *msg,
97  unsigned int msg_len,
98  const struct sockaddr_storage *system_from);
99 
101  void *context,
102  const struct totem_ip_address *iface_address,
103  unsigned int link_no);
104 
106  void *context,
107  int net_mtu);
108 
110 
111  /*
112  * Function and data used to log messages
113  */
115 
117 
119 
121 
123 
125 
127 
129  int level,
130  int subsys,
131  const char *function,
132  const char *file,
133  int line,
134  const char *format,
135  ...)__attribute__((format(printf, 6, 7)));
136 
137  void *knet_context;
138 
139  char iov_buffer[KNET_MAX_PACKET_SIZE];
140 
142 
144 
146 
148 
150 
152 
154 
155  qb_loop_timer_handle timer_netif_check_timeout;
156 
157  qb_loop_timer_handle timer_merge_detect_timeout;
158 
160 
162 
163  int logpipes[2];
164  int knet_fd;
165 };
166 
167 /* Awkward. But needed to get stats from knet */
169 
170 struct work_item {
171  const void *msg;
172  unsigned int msg_len;
174 };
175 
177  void *knet_context);
178 
179 static void totemknet_start_merge_detect_timeout(
180  void *knet_context);
181 
182 static void totemknet_stop_merge_detect_timeout(
183  void *knet_context);
184 
185 static void log_flush_messages (
186  void *knet_context);
187 
188 static void totemknet_instance_initialize (struct totemknet_instance *instance)
189 {
190  memset (instance, 0, sizeof (struct totemknet_instance));
191 }
192 
193 #define knet_log_printf(level, format, args...) \
194 do { \
195  instance->totemknet_log_printf ( \
196  level, instance->totemknet_subsys_id, \
197  __FUNCTION__, __FILE__, __LINE__, \
198  (const char *)format, ##args); \
199 } while (0);
200 
201 #define libknet_log_printf(level, format, args...) \
202 do { \
203  instance->totemknet_log_printf ( \
204  level, instance->knet_subsys_id, \
205  __FUNCTION__, "libknet.h", __LINE__, \
206  (const char *)format, ##args); \
207 } while (0);
208 
209 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
210 do { \
211  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
212  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
213  instance->totemknet_log_printf ( \
214  level, instance->totemknet_subsys_id, \
215  __FUNCTION__, __FILE__, __LINE__, \
216  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
217  } while(0)
218 
219 
220 static int dst_host_filter_callback_fn(void *private_data,
221  const unsigned char *outdata,
222  ssize_t outdata_len,
223  uint8_t tx_rx,
224  knet_node_id_t this_host_id,
225  knet_node_id_t src_host_id,
226  int8_t *channel,
227  knet_node_id_t *dst_host_ids,
228  size_t *dst_host_ids_entries)
229 {
230  struct totem_message_header *header = (struct totem_message_header *)outdata;
231  int res;
232 
233  *channel = 0;
234  if (header->target_nodeid) {
235  dst_host_ids[0] = header->target_nodeid;
236  *dst_host_ids_entries = 1;
237  res = 0; /* unicast message */
238  }
239  else {
240  *dst_host_ids_entries = 0;
241  res = 1; /* multicast message */
242  }
243  return res;
244 }
245 
246 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
247 {
248  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
249 
250  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
251  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
252  knet_handle_remove_datafd(instance->knet_handle, datafd);
253  }
254 }
255 
256 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
257 {
258  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
259 
260  // TODO: what? if anything.
261  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: %d reachable: %d", host_id, reachable);
262 }
263 
264 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
265 {
266  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
267  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
268 
269  /* We don't need to tell corosync the actual knet MTU */
270 // instance->totemknet_mtu_changed(instance->context, data_mtu);
271 }
272 
274  void *knet_context,
275  const char *cipher_type,
276  const char *hash_type)
277 {
278  return (0);
279 }
280 
281 
282 static inline void ucast_sendmsg (
283  struct totemknet_instance *instance,
284  struct totem_ip_address *system_to,
285  const void *msg,
286  unsigned int msg_len)
287 {
288  int res = 0;
289  struct totem_message_header *header = (struct totem_message_header *)msg;
290  struct msghdr msg_ucast;
291  struct iovec iovec;
292 
293  header->target_nodeid = system_to->nodeid;
294 
295  iovec.iov_base = (void *)msg;
296  iovec.iov_len = msg_len;
297 
298  /*
299  * Build unicast message
300  */
301  memset(&msg_ucast, 0, sizeof(msg_ucast));
302  msg_ucast.msg_iov = (void *)&iovec;
303  msg_ucast.msg_iovlen = 1;
304 #ifdef HAVE_MSGHDR_CONTROL
305  msg_ucast.msg_control = 0;
306 #endif
307 #ifdef HAVE_MSGHDR_CONTROLLEN
308  msg_ucast.msg_controllen = 0;
309 #endif
310 #ifdef HAVE_MSGHDR_FLAGS
311  msg_ucast.msg_flags = 0;
312 #endif
313 #ifdef HAVE_MSGHDR_ACCRIGHTS
314  msg_ucast.msg_accrights = NULL;
315 #endif
316 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
317  msg_ucast.msg_accrightslen = 0;
318 #endif
319 
320  /*
321  * Transmit unicast message
322  * An error here is recovered by totemsrp
323  */
324 
325  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
326  if (res < 0) {
328  "sendmsg(ucast) failed (non-critical)");
329  }
330 }
331 
332 static inline void mcast_sendmsg (
333  struct totemknet_instance *instance,
334  const void *msg,
335  unsigned int msg_len,
336  int only_active)
337 {
338  int res;
339  struct totem_message_header *header = (struct totem_message_header *)msg;
340  struct msghdr msg_mcast;
341  struct iovec iovec;
342 
343  iovec.iov_base = (void *)msg;
344  iovec.iov_len = msg_len;
345 
346  header->target_nodeid = 0;
347 
348  /*
349  * Build multicast message
350  */
351  memset(&msg_mcast, 0, sizeof(msg_mcast));
352  msg_mcast.msg_iov = (void *)&iovec;
353  msg_mcast.msg_iovlen = 1;
354 #ifdef HAVE_MSGHDR_CONTROL
355  msg_mcast.msg_control = 0;
356 #endif
357 #ifdef HAVE_MSGHDR_CONTROLLEN
358  msg_mcast.msg_controllen = 0;
359 #endif
360 #ifdef HAVE_MSGHDR_FLAGS
361  msg_mcast.msg_flags = 0;
362 #endif
363 #ifdef HAVE_MSGHDR_ACCRIGHTS
364  msg_mcast.msg_accrights = NULL;
365 #endif
366 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
367  msg_mcast.msg_accrightslen = 0;
368 #endif
369 
370 
371 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
372 
373  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
374  if (res < msg_len) {
375  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
376  }
377 
378  if (!only_active || instance->send_merge_detect_message) {
379  /*
380  * Current message was sent to all nodes
381  */
383  instance->send_merge_detect_message = 0;
384  }
385 }
386 
387 static int node_compare(const void *aptr, const void *bptr)
388 {
389  uint16_t a,b;
390 
391  a = *(uint16_t *)aptr;
392  b = *(uint16_t *)bptr;
393 
394  return a > b;
395 }
396 
398  char ***status,
399  unsigned int *iface_count)
400 {
401  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
402  struct knet_link_status link_status;
403  knet_node_id_t host_list[KNET_MAX_HOST];
404  uint8_t link_list[KNET_MAX_LINK];
405  size_t num_hosts;
406  size_t num_links;
407  size_t link_idx;
408  int i,j;
409  char *ptr;
410  int res = 0;
411 
412  /*
413  * Don't do the whole 'link_info' bit if the caller just wants
414  * a count of interfaces.
415  */
416  if (status) {
417 
418  res = knet_host_get_host_list(instance->knet_handle,
419  host_list, &num_hosts);
420  if (res) {
421  return (-1);
422  }
423  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
424 
425  for (i=0; i<INTERFACE_MAX; i++) {
426  memset(instance->link_status[i], 'n', CFG_INTERFACE_STATUS_MAX_LEN-1);
427  instance->link_status[i][num_hosts] = '\0';
428  }
429 
430  /* This is all a bit "inside-out" because "status" is a set of strings per link
431  * and knet orders things by host
432  */
433  for (j=0; j<num_hosts; j++) {
434  res = knet_link_get_link_list(instance->knet_handle,
435  host_list[j], link_list, &num_links);
436  if (res) {
437  return (-1);
438  }
439 
440  link_idx = 0;
441  for (i=0; i < num_links; i++) {
442  /*
443  * Skip over links that are unconfigured to corosync. This is basically
444  * link0 if corosync isn't using it for comms, as we will still
445  * have it set up for loopback.
446  */
447  if (!instance->totem_config->interfaces[link_list[i]].configured) {
448  continue;
449  }
450  ptr = instance->link_status[link_idx++];
451 
452  res = knet_link_get_status(instance->knet_handle,
453  host_list[j],
454  link_list[i],
455  &link_status,
456  sizeof(link_status));
457  if (res == 0) {
458  ptr[j] = '0' + (link_status.enabled |
459  link_status.connected<<1 |
460  link_status.dynconnected<<2);
461  }
462  else {
463  ptr[j] = '?';
464  }
465  }
466  }
467  *status = instance->link_status;
468  }
469 
470  *iface_count = INTERFACE_MAX;
471 
472  return (res);
473 }
474 
476  void *knet_context)
477 {
478  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
479  int res = 0;
480  int i,j;
481  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
482  uint8_t links[KNET_MAX_LINK];
483  size_t num_nodes;
484  size_t num_links;
485 
486  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
487 
488  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
489  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
490 
491  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
492  if (res) {
493  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
494  /* Crash out anyway */
495  goto finalise_error;
496  }
497 
498  /* Tidily shut down all nodes & links. This ensures that the LEAVE message will be sent */
499  for (i=0; i<num_nodes; i++) {
500 
501  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
502  if (res) {
503  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node %d: %s", nodes[i], strerror(errno));
504  goto finalise_error;
505  }
506  for (j=0; j<num_links; j++) {
507  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
508  if (res) {
509  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node %d, link %d) failed: %s", nodes[i], links[j], strerror(errno));
510  }
511  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
512  if (res) {
513  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node %d, link %d) failed: %s", nodes[i], links[j], strerror(errno));
514  }
515  }
516  res = knet_host_remove(instance->knet_handle, nodes[i]);
517  if (res) {
518  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node %d) failed: %s", nodes[i], strerror(errno));
519  }
520  }
521 
522 finalise_error:
523  res = knet_handle_setfwd(instance->knet_handle, 0);
524  if (res) {
525  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
526  }
527  res = knet_handle_free(instance->knet_handle);
528  if (res) {
529  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
530  }
531 
532  totemknet_stop_merge_detect_timeout(instance);
533 
534  log_flush_messages(instance);
535 
536  return (res);
537 }
538 
539 static int log_deliver_fn (
540  int fd,
541  int revents,
542  void *data)
543 {
544  struct totemknet_instance *instance = (struct totemknet_instance *)data;
545  char buffer[sizeof(struct knet_log_msg)*4];
546  char *bufptr = buffer;
547  int done = 0;
548  int len;
549 
550  len = read(fd, buffer, sizeof(buffer));
551  while (done < len) {
552  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
553  switch (msg->msglevel) {
554  case KNET_LOG_ERR:
556  knet_log_get_subsystem_name(msg->subsystem),
557  msg->msg);
558  break;
559  case KNET_LOG_WARN:
561  knet_log_get_subsystem_name(msg->subsystem),
562  msg->msg);
563  break;
564  case KNET_LOG_INFO:
566  knet_log_get_subsystem_name(msg->subsystem),
567  msg->msg);
568  break;
569  case KNET_LOG_DEBUG:
571  knet_log_get_subsystem_name(msg->subsystem),
572  msg->msg);
573  break;
574  }
575  bufptr += sizeof(struct knet_log_msg);
576  done += sizeof(struct knet_log_msg);
577  }
578  return 0;
579 }
580 
581 static int data_deliver_fn (
582  int fd,
583  int revents,
584  void *data)
585 {
586  struct totemknet_instance *instance = (struct totemknet_instance *)data;
587  struct msghdr msg_hdr;
588  struct iovec iov_recv;
589  struct sockaddr_storage system_from;
590  ssize_t msg_len;
591  int truncated_packet;
592 
593  iov_recv.iov_base = instance->iov_buffer;
594  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
595 
596  msg_hdr.msg_name = &system_from;
597  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
598  msg_hdr.msg_iov = &iov_recv;
599  msg_hdr.msg_iovlen = 1;
600 #ifdef HAVE_MSGHDR_CONTROL
601  msg_hdr.msg_control = 0;
602 #endif
603 #ifdef HAVE_MSGHDR_CONTROLLEN
604  msg_hdr.msg_controllen = 0;
605 #endif
606 #ifdef HAVE_MSGHDR_FLAGS
607  msg_hdr.msg_flags = 0;
608 #endif
609 #ifdef HAVE_MSGHDR_ACCRIGHTS
610  msg_hdr.msg_accrights = NULL;
611 #endif
612 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
613  msg_hdr.msg_accrightslen = 0;
614 #endif
615 
616  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
617  if (msg_len <= 0) {
618  return (0);
619  }
620 
621  truncated_packet = 0;
622 
623 #ifdef HAVE_MSGHDR_FLAGS
624  if (msg_hdr.msg_flags & MSG_TRUNC) {
625  truncated_packet = 1;
626  }
627 #else
628  /*
629  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
630  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
631  */
632  if (bytes_received == KNET_MAX_PACKET_SIZE) {
633  truncated_packet = 1;
634  }
635 #endif
636 
637  if (truncated_packet) {
639  "Received too big message. This may be because something bad is happening"
640  "on the network (attack?), or you tried join more nodes than corosync is"
641  "compiled with (%u) or bug in the code (bad estimation of "
642  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
643  return (0);
644  }
645 
646  /*
647  * Handle incoming message
648  */
649  instance->totemknet_deliver_fn (
650  instance->context,
651  instance->iov_buffer,
652  msg_len,
653  &system_from);
654 
655  return (0);
656 }
657 
658 static void timer_function_netif_check_timeout (
659  void *data)
660 {
661  struct totemknet_instance *instance = (struct totemknet_instance *)data;
662  int i;
663 
664  for (i=0; i < INTERFACE_MAX; i++) {
665  if (!instance->totem_config->interfaces[i].configured) {
666  continue;
667  }
668  instance->totemknet_iface_change_fn (instance->context,
669  &instance->my_ids[i],
670  i);
671  }
672 }
673 
674 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
675 static void totemknet_refresh_config(
676  int32_t event,
677  const char *key_name,
678  struct icmap_notify_value new_val,
679  struct icmap_notify_value old_val,
680  void *user_data)
681 {
682  uint8_t reloading;
683  uint32_t value;
684  uint32_t link_no;
685  size_t num_nodes;
686  knet_node_id_t host_ids[KNET_MAX_HOST];
687  int i;
688  int err;
689  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
690 
691  ENTER();
692 
693  /*
694  * If a full reload is in progress then don't do anything until it's done and
695  * can reconfigure it all atomically
696  */
697  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
698  return;
699  }
700 
701  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
702 
704  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
705  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
706  if (err) {
707  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
708  }
709  }
710 
711  /* Configure link parameters for each node */
712  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
713  if (err != 0) {
714  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
715  }
716 
717  for (i=0; i<num_nodes; i++) {
718  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
719  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
720  continue;
721  }
722 
723  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
724  instance->totem_config->interfaces[link_no].knet_ping_interval,
725  instance->totem_config->interfaces[link_no].knet_ping_timeout,
726  instance->totem_config->interfaces[link_no].knet_ping_precision);
727  if (err) {
728  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node %d link %d failed", host_ids[i], link_no);
729  }
730  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
731  instance->totem_config->interfaces[link_no].knet_pong_count);
732  if (err) {
733  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node %d link %d failed",host_ids[i], link_no);
734  }
735  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
736  instance->totem_config->interfaces[link_no].knet_link_priority);
737  if (err) {
738  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node %d link %d failed", host_ids[i], link_no);
739  }
740 
741  }
742  }
743 
744  LEAVE();
745 }
746 
747 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
748 {
749  icmap_track_t icmap_track_totem = NULL;
750  icmap_track_t icmap_track_reload = NULL;
751 
752  ENTER();
753 
754  icmap_track_add("totem.",
756  totemknet_refresh_config,
757  instance,
758  &icmap_track_totem);
759 
760  icmap_track_add("config.totemconfig_reload_in_progress",
762  totemknet_refresh_config,
763  instance,
764  &icmap_track_reload);
765 
766  LEAVE();
767 }
768 
769 /*
770  * Create an instance
771  */
773  qb_loop_t *poll_handle,
774  void **knet_context,
775  struct totem_config *totem_config,
776  totemsrp_stats_t *stats,
777  void *context,
778 
779  void (*deliver_fn) (
780  void *context,
781  const void *msg,
782  unsigned int msg_len,
783  const struct sockaddr_storage *system_from),
784 
785  void (*iface_change_fn) (
786  void *context,
787  const struct totem_ip_address *iface_address,
788  unsigned int link_no),
789 
790  void (*mtu_changed) (
791  void *context,
792  int net_mtu),
793 
794  void (*target_set_completed) (
795  void *context))
796 {
797  struct totemknet_instance *instance;
798  int8_t channel=0;
799  int res;
800  int i;
801 
802  instance = malloc (sizeof (struct totemknet_instance));
803  if (instance == NULL) {
804  return (-1);
805  }
806 
807  totemknet_instance_initialize (instance);
808 
809  instance->totem_config = totem_config;
810 
811  /*
812  * Configure logging
813  */
814  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
821 
822  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
823 
824  /*
825  * Initialize local variables for totemknet
826  */
827 
828  instance->our_nodeid = instance->totem_config->node_id;
829 
830  for (i=0; i< INTERFACE_MAX; i++) {
831  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
832  instance->my_ids[i].nodeid = instance->our_nodeid;
833  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
834 
835  /* Needed for totemsrp */
836  totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
837  }
838 
839  instance->poll_handle = poll_handle;
840 
841  instance->context = context;
842  instance->totemknet_deliver_fn = deliver_fn;
843 
844  instance->totemknet_iface_change_fn = iface_change_fn;
845 
846  instance->totemknet_mtu_changed = mtu_changed;
847 
848  instance->totemknet_target_set_completed = target_set_completed;
849 
850  instance->loopback_link = 0;
851 
852  res = pipe(instance->logpipes);
853  if (res == -1) {
854  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
855  goto exit_error;
856  }
857  fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK);
858  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK);
859 
860  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
861 
862  if (!instance->knet_handle) {
863  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
864  goto exit_error;
865  }
866  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
867  if (res) {
868  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
869  }
870  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
871  if (res) {
872  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
873  }
874  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
875  if (res) {
876  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
877  }
878  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
879  if (res) {
880  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
881  }
882  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
883  if (res) {
884  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
885  }
886  global_instance = instance;
887 
888  /* Get an fd into knet */
889  instance->knet_fd = 0;
890  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
891  if (res) {
892  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
893  goto exit_error;
894  }
895 
896  /* Enable crypto if requested */
897  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
898  struct knet_handle_crypto_cfg crypto_cfg;
899 
900  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
901  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
902  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
903  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
904  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
905 
906  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
907  if (res == -1) {
908  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
909  goto exit_error;
910  }
911  if (res == -2) {
912  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
913  goto exit_error;
914  }
915  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
916  }
917 
918  /* Set up compression */
919  totemknet_reconfigure(instance, instance->totem_config);
920 
921  knet_handle_setfwd(instance->knet_handle, 1);
922 
923  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
924  if (strcmp(instance->totem_config->link_mode, "active")==0) {
925  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
926  }
927  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
928  instance->link_mode = KNET_LINK_POLICY_RR;
929  }
930 
931  for (i=0; i<INTERFACE_MAX; i++) {
932  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
933  if (!instance->link_status[i]) {
934  goto exit_error;
935  }
936  }
937 
938  qb_loop_poll_add (instance->poll_handle,
939  QB_LOOP_MED,
940  instance->logpipes[0],
941  POLLIN, instance, log_deliver_fn);
942 
943  qb_loop_poll_add (instance->poll_handle,
944  QB_LOOP_HIGH,
945  instance->knet_fd,
946  POLLIN, instance, data_deliver_fn);
947 
948  /*
949  * Upper layer isn't ready to receive message because it hasn't
950  * initialized yet. Add short timer to check the interfaces.
951  */
952  qb_loop_timer_add (instance->poll_handle,
953  QB_LOOP_MED,
954  100*QB_TIME_NS_IN_MSEC,
955  (void *)instance,
956  timer_function_netif_check_timeout,
957  &instance->timer_netif_check_timeout);
958 
959  totemknet_start_merge_detect_timeout(instance);
960 
961  /* Start listening for config changes */
962  totemknet_add_config_notifications(instance);
963 
964  /* Add stats keys to icmap */
966 
967  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
968  *knet_context = instance;
969 
970  return (0);
971 
972 exit_error:
973  log_flush_messages(instance);
974  free(instance);
975  return (-1);
976 }
977 
979 {
980  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
981  return malloc(KNET_MAX_PACKET_SIZE + 512);
982 }
983 
984 void totemknet_buffer_release (void *ptr)
985 {
986  return free (ptr);
987 }
988 
990  void *knet_context,
991  int processor_count)
992 {
993  return (0);
994 }
995 
996 int totemknet_recv_flush (void *knet_context)
997 {
998  return (0);
999 }
1000 
1001 int totemknet_send_flush (void *knet_context)
1002 {
1003  return (0);
1004 }
1005 
1007  void *knet_context,
1008  const void *msg,
1009  unsigned int msg_len)
1010 {
1011  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1012  int res = 0;
1013 
1014  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1015 
1016  return (res);
1017 }
1019  void *knet_context,
1020  const void *msg,
1021  unsigned int msg_len)
1022 {
1023  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1024  int res = 0;
1025 
1026  mcast_sendmsg (instance, msg, msg_len, 0);
1027 
1028  return (res);
1029 }
1030 
1032  void *knet_context,
1033  const void *msg,
1034  unsigned int msg_len)
1035 {
1036  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1037  int res = 0;
1038 
1039  mcast_sendmsg (instance, msg, msg_len, 1);
1040 
1041  return (res);
1042 }
1043 
1044 
1045 extern int totemknet_iface_check (void *knet_context)
1046 {
1047  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1048  int res = 0;
1049 
1050  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1051 
1052  return (res);
1053 }
1054 
1055 extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1056 {
1057  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1058 
1059  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1060 }
1061 
1063  void *knet_context,
1064  unsigned int nodeid)
1065 {
1066  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1067  int res = 0;
1068 
1069  instance->token_target.nodeid = nodeid;
1070 
1071  instance->totemknet_target_set_completed (instance->context);
1072 
1073  return (res);
1074 }
1075 
1077  void *knet_context)
1078 {
1079  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1080  unsigned int res;
1081  struct sockaddr_storage system_from;
1082  struct msghdr msg_hdr;
1083  struct iovec iov_recv;
1084  struct pollfd ufd;
1085  int nfds;
1086  int msg_processed = 0;
1087 
1088  iov_recv.iov_base = instance->iov_buffer;
1089  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1090 
1091  msg_hdr.msg_name = &system_from;
1092  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1093  msg_hdr.msg_iov = &iov_recv;
1094  msg_hdr.msg_iovlen = 1;
1095 #ifdef HAVE_MSGHDR_CONTROL
1096  msg_hdr.msg_control = 0;
1097 #endif
1098 #ifdef HAVE_MSGHDR_CONTROLLEN
1099  msg_hdr.msg_controllen = 0;
1100 #endif
1101 #ifdef HAVE_MSGHDR_FLAGS
1102  msg_hdr.msg_flags = 0;
1103 #endif
1104 #ifdef HAVE_MSGHDR_ACCRIGHTS
1105  msg_msg_hdr.msg_accrights = NULL;
1106 #endif
1107 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1108  msg_msg_hdr.msg_accrightslen = 0;
1109 #endif
1110 
1111  do {
1112  ufd.fd = instance->knet_fd;
1113  ufd.events = POLLIN;
1114  nfds = poll (&ufd, 1, 0);
1115  if (nfds == 1 && ufd.revents & POLLIN) {
1116  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1117  if (res != -1) {
1118  msg_processed = 1;
1119  } else {
1120  msg_processed = -1;
1121  }
1122  }
1123  } while (nfds == 1);
1124 
1125  return (msg_processed);
1126 }
1127 
1128 int totemknet_iface_set (void *knet_context,
1129  const struct totem_ip_address *local_addr,
1130  unsigned short ip_port,
1131  unsigned int iface_no)
1132 {
1133  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1134 
1135  totemip_copy(&instance->my_ids[iface_no], local_addr);
1136 
1137  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1138 
1139  instance->ip_port[iface_no] = ip_port;
1140 
1141  return 0;
1142 }
1143 
1144 
1146  void *knet_context,
1147  const struct totem_ip_address *local,
1148  const struct totem_ip_address *member,
1149  int link_no)
1150 {
1151  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1152  int err;
1153  int port = instance->ip_port[link_no];
1154  struct sockaddr_storage remote_ss;
1155  struct sockaddr_storage local_ss;
1156  int addrlen;
1157  int i;
1158  int host_found = 0;
1159  knet_node_id_t host_ids[KNET_MAX_HOST];
1160  size_t num_host_ids;
1161 
1162  /* Only create 1 loopback link and use link 0 */
1163  if (member->nodeid == instance->our_nodeid) {
1164  if (!instance->loopback_link) {
1165  link_no = 0;
1166  instance->loopback_link = 1;
1167  } else {
1168  /* Already done */
1169  return 0;
1170  }
1171  }
1172 
1173  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: %d (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1174  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: %d (%s)", local->nodeid, totemip_print(local));
1175 
1176 
1177  /* Only add the host if it doesn't already exist in knet */
1178  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1179  if (err) {
1180  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1181  return -1;
1182  }
1183  for (i=0; i<num_host_ids; i++) {
1184  if (host_ids[i] == member->nodeid) {
1185  host_found = 1;
1186  }
1187  }
1188 
1189  if (!host_found) {
1190  err = knet_host_add(instance->knet_handle, member->nodeid);
1191  if (err != 0 && errno != EEXIST) {
1192  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1193  return -1;
1194  }
1195  } else {
1196  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid %d already added", member->nodeid);
1197  }
1198 
1199 
1200  if (err == 0) {
1201  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1202  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1203  return -1;
1204  }
1205  }
1206 
1207  memset(&local_ss, 0, sizeof(local_ss));
1208  /* Casts to remove const */
1209  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1210  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1211 
1212  if (member->nodeid == instance->our_nodeid) {
1213  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1214 
1215  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1216  KNET_TRANSPORT_LOOPBACK,
1217  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1218  }
1219  else {
1220  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1221  instance->totem_config->interfaces[link_no].knet_transport,
1222  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1223  }
1224  if (err) {
1225  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1226  return -1;
1227  }
1228 
1229  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1230  instance->totem_config->interfaces[link_no].knet_link_priority);
1231 
1232  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1233  instance->totem_config->interfaces[link_no].knet_link_priority);
1234  if (err) {
1235  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid %d, link %d failed", member->nodeid, link_no);
1236  }
1237 
1238  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1239  instance->totem_config->interfaces[link_no].knet_ping_interval,
1240  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1241  instance->totem_config->interfaces[link_no].knet_ping_precision);
1242  if (err) {
1243  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid %d, link %d failed", member->nodeid, link_no);
1244  }
1245  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1246  instance->totem_config->interfaces[link_no].knet_pong_count);
1247  if (err) {
1248  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid %d, link %d failed", member->nodeid, link_no);
1249  }
1250 
1251  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1252  if (err) {
1253  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid %d, link %d failed", member->nodeid, link_no);
1254  return -1;
1255  }
1256 
1257  /* register stats */
1258  stats_knet_add_member(member->nodeid, link_no);
1259  return (0);
1260 }
1261 
1263  void *knet_context,
1264  const struct totem_ip_address *token_target,
1265  int link_no)
1266 {
1267  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1268  int res;
1269  uint8_t link_list[KNET_MAX_LINK];
1270  size_t num_links;
1271 
1272  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: %d, link=%d", token_target->nodeid, link_no);
1273 
1274  /* Don't remove the link with the loopback on it until we shut down */
1275  if (token_target->nodeid == instance->our_nodeid) {
1276  return 0;
1277  }
1278 
1279  /* Tidy stats */
1280  stats_knet_del_member(token_target->nodeid, link_no);
1281 
1282  /* Remove the link first */
1283  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1284  if (res != 0) {
1285  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid %d, link %d failed", token_target->nodeid, link_no);
1286  return res;
1287  }
1288 
1289  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1290  if (res != 0) {
1291  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid %d, link %d failed", token_target->nodeid, link_no);
1292  return res;
1293  }
1294 
1295  /* If this is the last link, then remove the node */
1296  res = knet_link_get_link_list(instance->knet_handle,
1297  token_target->nodeid, link_list, &num_links);
1298  if (res) {
1299  return (0); /* not really failure */
1300  }
1301 
1302  if (num_links == 0) {
1303  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1304  }
1305  return res;
1306 }
1307 
1309  void *knet_context)
1310 {
1311  return (0);
1312 }
1313 
1315  void *knet_context,
1316  struct totem_config *totem_config)
1317 {
1318  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1319  struct knet_handle_compress_cfg compress_cfg;
1320  int res = 0;
1321 
1322  if (totem_config->knet_compression_model) {
1323  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1324  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1325  compress_cfg.compress_level = totem_config->knet_compression_level;
1326 
1327  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1328  if (res) {
1329  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1330  }
1331  }
1332  return (res);
1333 }
1334 
1335 
1337  void *knet_context)
1338 {
1339  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1340 
1341  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1342 }
1343 
1344 /* For the stats module */
1346  knet_node_id_t node, uint8_t link_no,
1347  struct knet_link_status *status)
1348 {
1349  int res;
1350  int ret = CS_OK;
1351 
1352  /* We are probably not using knet */
1353  if (!global_instance) {
1354  return CS_ERR_NOT_EXIST;
1355  }
1356 
1357  if (link_no >= INTERFACE_MAX) {
1358  return CS_ERR_NOT_EXIST; /* Invalid link number */
1359  }
1360 
1361  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1362  if (res) {
1363  switch (errno) {
1364  case EINVAL:
1365  ret = CS_ERR_INVALID_PARAM;
1366  break;
1367  case EBUSY:
1368  ret = CS_ERR_BUSY;
1369  break;
1370  case EDEADLK:
1371  ret = CS_ERR_TRY_AGAIN;
1372  break;
1373  default:
1374  ret = CS_ERR_LIBRARY;
1375  break;
1376  }
1377  }
1378 
1379  return (ret);
1380 }
1381 
1383  struct knet_handle_stats *stats)
1384 {
1385  /* We are probably not using knet */
1386  if (!global_instance) {
1387  return CS_ERR_NOT_EXIST;
1388  }
1389 
1390  return knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1391 }
1392 
1393 static void timer_function_merge_detect_timeout (
1394  void *data)
1395 {
1396  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1397 
1398  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1399  instance->send_merge_detect_message = 1;
1400  }
1401 
1403 
1404  totemknet_start_merge_detect_timeout(instance);
1405 }
1406 
1407 static void totemknet_start_merge_detect_timeout(
1408  void *knet_context)
1409 {
1410  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1411 
1412  qb_loop_timer_add(instance->poll_handle,
1413  QB_LOOP_MED,
1414  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1415  (void *)instance,
1416  timer_function_merge_detect_timeout,
1417  &instance->timer_merge_detect_timeout);
1418 
1419 }
1420 
1421 static void totemknet_stop_merge_detect_timeout(
1422  void *knet_context)
1423 {
1424  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1425 
1426  qb_loop_timer_del(instance->poll_handle,
1427  instance->timer_merge_detect_timeout);
1428 }
1429 
1430 static void log_flush_messages (void *knet_context)
1431 {
1432  struct pollfd pfd;
1433  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1434  int cont;
1435 
1436  cont = 1;
1437 
1438  while (cont) {
1439  pfd.fd = instance->logpipes[0];
1440  pfd.events = POLLIN;
1441  pfd.revents = 0;
1442 
1443  if ((poll(&pfd, 1, 0) > 0) &&
1444  (pfd.revents & POLLIN) &&
1445  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1446  cont = 1;
1447  } else {
1448  cont = 0;
1449  }
1450  }
1451 }
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:145
int knet_ping_precision
Definition: totem.h:92
int totemknet_log_level_security
Definition: totemknet.c:114
qb_loop_t * poll_handle
Definition: totemknet.c:86
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:141
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1262
#define MSG_NOSIGNAL
Definition: totemknet.c:77
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
int knet_link_priority
Definition: totem.h:89
uint32_t value
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:157
struct totem_interface * interfaces
Definition: totem.h:158
void stats_knet_add_handle(void)
Definition: stats.c:637
int totemknet_log_level_error
Definition: totemknet.c:116
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:161
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:201
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:143
The totem_ip_address struct.
Definition: coroapi.h:111
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:81
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:237
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:193
int send_merge_detect_message
Definition: totemknet.c:159
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:475
uint32_t knet_compression_threshold
Definition: totem.h:226
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:100
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
unsigned int knet_pmtud_interval
Definition: totem.h:162
char * crypto_model
Definition: totem.h:218
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1345
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:984
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1336
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:209
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:96
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:624
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
unsigned int private_key_len
Definition: totem.h:169
int totemknet_log_level_notice
Definition: totemknet.c:120
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
#define INTERFACE_MAX
Definition: coroapi.h:88
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:273
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:155
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1018
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1045
void(*) void knet_context)
Definition: totemknet.c:135
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:826
unsigned int node_id
Definition: totem.h:160
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
uint8_t configured
Definition: totem.h:87
int totemknet_log_level_warning
Definition: totemknet.c:118
char * knet_compression_model
Definition: totem.h:224
void * user_data
Definition: sam.c:127
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1062
struct totem_config * totem_config
Definition: totemknet.c:151
unsigned int nodeid
Definition: coroapi.h:112
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1314
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1128
int knet_transport
Definition: totem.h:94
char * crypto_hash_type
Definition: totem.h:222
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:989
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:996
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1001
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
struct totem_ip_address boundto
Definition: totem.h:83
typedef __attribute__
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:850
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:139
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1382
struct totemknet_instance * global_instance
Definition: totemknet.c:168
struct totem_message_header header
Definition: totemsrp.c:260
uint16_t ip_port
Definition: totem.h:85
int knet_compression_level
Definition: totem.h:228
struct crypto_instance * crypto_inst
Definition: totemknet.c:84
#define ENTER
Definition: logsys.h:324
unsigned int net_mtu
Definition: totem.h:202
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:109
struct totem_ip_address token_target
Definition: totemknet.c:153
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1145
int knet_pong_count
Definition: totem.h:93
struct totemknet_instance * instance
Definition: totemknet.c:173
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
int knet_ping_interval
Definition: totem.h:90
const void * msg
Definition: totemknet.c:171
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:94
int knet_ping_timeout
Definition: totem.h:91
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:245
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:128
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:105
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:614
struct srp_addr system_from
Definition: totemsrp.c:261
char * crypto_cipher_type
Definition: totem.h:220
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1055
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1308
unsigned int merge_timeout
Definition: totem.h:190
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1031
int totemknet_log_level_debug
Definition: totemknet.c:122
unsigned int target_nodeid
Definition: totem.h:130
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1006
struct totem_ip_address bindnet
Definition: totem.h:82
unsigned int nodeid
Definition: coroapi.h:75
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1076
unsigned int msg_len
Definition: totemknet.c:172
#define LEAVE
Definition: logsys.h:325
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:772
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:397
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1151
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem.nodeid", "totem.version", ...
Definition: icmap.h:85
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:978
knet_handle_t knet_handle
Definition: totemknet.c:88