Did some fixes to the opposite hosts table and added some nice features
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2011 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "cdefs.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20 #include "opt.h"
21 #include "str.h"
22
23 #include <arpa/inet.h> /* inet_aton() */
24 #include <netdb.h> /* struct addrinfo */
25 #include <assert.h>
26 #include <errno.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h> /* memset(), strcmp() */
30 #include <unistd.h>
31
32 int hosts_db_show_macs = 0;
33
34 /* FIXME: specify somewhere more sane/tunable */
35 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
36
37 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
38 typedef void (free_func_t)(struct bucket *);
39 typedef const void * (key_func_t)(const struct bucket *);
40 typedef int (find_func_t)(const struct bucket *, const void *);
41 typedef struct bucket * (make_func_t)(const void *);
42 typedef void (format_cols_func_t)(struct str *, const unsigned int, const enum sort_dir, const int);
43 typedef void (format_row_func_t)(struct str *, const struct bucket *,
44 const char *, const struct bucket *);
45
46 struct hashtable {
47 uint8_t bits; /* size of hashtable in bits */
48 uint32_t size, mask;
49 uint32_t count, count_max, count_keep; /* items in table */
50 uint32_t coeff; /* coefficient for Fibonacci hashing */
51 struct bucket **table;
52
53 struct {
54 uint64_t inserts, searches, deletions, rehashes;
55 } stats;
56
57 hash_func_t *hash_func;
58 /* returns hash value of given key (passed as void*) */
59
60 free_func_t *free_func;
61 /* free of bucket payload */
62
63 key_func_t *key_func;
64 /* returns pointer to key of bucket (to pass to hash_func) */
65
66 find_func_t *find_func;
67 /* returns true if given bucket matches key (passed as void*) */
68
69 make_func_t *make_func;
70 /* returns bucket containing new record with key (passed as void*) */
71
72 format_cols_func_t *format_cols_func;
73 /* append table columns to str */
74
75 format_row_func_t *format_row_func;
76 /* format record and append to str */
77 };
78
79 static void hashtable_reduce(struct hashtable *ht);
80 static void hashtable_free(struct hashtable *h);
81
82 #define HOST_BITS 1 /* initial size of hosts table */
83 #define HOST_OPP_BITS 1 /* initial size of hosts table */
84 #define PORT_BITS 1 /* initial size of ports tables */
85 #define PROTO_BITS 1 /* initial size of proto table */
86
87 /* We only use one hosts_db hashtable and this is it. */
88 static struct hashtable *hosts_db = NULL;
89
90 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
91 static const double phi_1 =
92 0.61803398874989490252573887119069695472717285156250;
93
94 /* Co-prime of u, using phi^-1 */
95 inline static uint32_t
96 coprime(const uint32_t u)
97 {
98 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
99 }
100
101 /*
102 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
103 * src/sys/netinet/tcp_hostcache.c 1.1
104 */
105 inline static uint32_t
106 ipv4_hash(const struct addr *const a)
107 {
108 uint32_t ip = a->ip.v4;
109 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
110 }
111
112 #ifndef s6_addr32
113 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
114 * taken care of GNU/Linux and GNU/kfreebsd. */
115 # define s6_addr32 __u6_addr.__u6_addr32
116 #endif
117
118 /*
119 * This is the IPv6 hash function used by FreeBSD in the same file as above,
120 * svn rev 122922.
121 */
122 inline static uint32_t
123 ipv6_hash(const struct addr *const a)
124 {
125 const struct in6_addr *const ip6 = &(a->ip.v6);
126 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
127 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
128 }
129
130 /* ---------------------------------------------------------------------------
131 * hash_func collection
132 */
133 static uint32_t
134 hash_func_host(const struct hashtable *h _unused_, const void *key)
135 {
136 const struct addr *a = key;
137 if (a->family == IPv4)
138 return (ipv4_hash(a));
139 else {
140 assert(a->family == IPv6);
141 return (ipv6_hash(a));
142 }
143 }
144
145 #define CASTKEY(type) (*((const type *)key))
146
147 static uint32_t
148 hash_func_short(const struct hashtable *h, const void *key)
149 {
150 return (CASTKEY(uint16_t) * h->coeff);
151 }
152
153 static uint32_t
154 hash_func_byte(const struct hashtable *h, const void *key)
155 {
156 return (CASTKEY(uint8_t) * h->coeff);
157 }
158
159 /* ---------------------------------------------------------------------------
160 * key_func collection
161 */
162
163 static const void *
164 key_func_host(const struct bucket *b)
165 {
166 return &(b->u.host.addr);
167 }
168
169 static const void *
170 key_func_host_opp(const struct bucket *b)
171 {
172 return &(b->u.host_opp.addr);
173 }
174
175 static const void *
176 key_func_port_tcp(const struct bucket *b)
177 {
178 return &(b->u.port_tcp.port);
179 }
180
181 static const void *
182 key_func_port_udp(const struct bucket *b)
183 {
184 return &(b->u.port_udp.port);
185 }
186
187 static const void *
188 key_func_ip_proto(const struct bucket *b)
189 {
190 return &(b->u.ip_proto.proto);
191 }
192
193 /* ---------------------------------------------------------------------------
194 * find_func collection
195 */
196
197 static int
198 find_func_host(const struct bucket *b, const void *key)
199 {
200 return (addr_equal(key, &(b->u.host.addr)));
201 }
202
203 static int
204 find_func_host_opp(const struct bucket *b, const void *key)
205 {
206 return (addr_equal(key, &(b->u.host_opp.addr)));
207 }
208
209 static int
210 find_func_port_tcp(const struct bucket *b, const void *key)
211 {
212 return (b->u.port_tcp.port == CASTKEY(uint16_t));
213 }
214
215 static int
216 find_func_port_udp(const struct bucket *b, const void *key)
217 {
218 return (b->u.port_udp.port == CASTKEY(uint16_t));
219 }
220
221 static int
222 find_func_ip_proto(const struct bucket *b, const void *key)
223 {
224 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
225 }
226
227 /* ---------------------------------------------------------------------------
228 * make_func collection
229 */
230
231 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
232 struct bucket *next; \
233 uint64_t in, out, total; \
234 union { struct type t; } u; } _custom_bucket; \
235 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
236 struct type *name_content = &(name_bucket->u.type); \
237 name_bucket->next = NULL; \
238 name_bucket->in = name_bucket->out = name_bucket->total = 0;
239
240 static struct bucket *
241 make_func_host(const void *key)
242 {
243 MAKE_BUCKET(b, h, host);
244 h->addr = CASTKEY(struct addr);
245 h->dns = NULL;
246 h->last_seen = now;
247 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
248 h->ports_tcp = NULL;
249 h->ports_udp = NULL;
250 h->ip_protos = NULL;
251 h->hosts_opp = NULL;
252 return (b);
253 }
254
255 static void
256 free_func_host(struct bucket *b)
257 {
258 struct host *h = &(b->u.host);
259 if (h->dns != NULL) free(h->dns);
260 hashtable_free(h->ports_tcp);
261 hashtable_free(h->ports_udp);
262 hashtable_free(h->ip_protos);
263 hashtable_free(h->hosts_opp);
264 }
265
266 static struct bucket *
267 make_func_host_opp(const void *key)
268 {
269 MAKE_BUCKET(b, ho, host_opp);
270 ho->addr = CASTKEY(struct addr);
271 ho->last_seen = now;
272 return (b);
273 }
274
275 static struct bucket *
276 make_func_port_tcp(const void *key)
277 {
278 MAKE_BUCKET(b, p, port_tcp);
279 p->port = CASTKEY(uint16_t);
280 p->syn = 0;
281 return (b);
282 }
283
284 static struct bucket *
285 make_func_port_udp(const void *key)
286 {
287 MAKE_BUCKET(b, p, port_udp);
288 p->port = CASTKEY(uint16_t);
289 return (b);
290 }
291
292 static struct bucket *
293 make_func_ip_proto(const void *key)
294 {
295 MAKE_BUCKET(b, p, ip_proto);
296 p->proto = CASTKEY(uint8_t);
297 return (b);
298 }
299
300 static void
301 free_func_simple(struct bucket *b _unused_)
302 {
303 /* nop */
304 }
305
306 /* ---------------------------------------------------------------------------
307 * format_func collection (ordered by struct)
308 */
309
310 static void
311 format_cols_host(struct str *buf, const unsigned int start,
312 const enum sort_dir sort, const int full)
313 {
314 char *qs;
315 int result;
316
317 result=asprintf(&qs, (full==1)?"?full=yes&sort=":"?start=%d&sort=", start);
318 assert(result>=0);
319
320 str_appendf(buf,
321 "<table>\n"
322 "<tr>\n"
323 " <th><a href=\"%s%sip\">IP</a></th>\n"
324 " <th><a href=\"%s%shostname\">Hostname</a></th>\n",
325 qs,(sort==IP)?"rev":"",
326 qs,(sort==HOSTNAME)?"rev":""
327 );
328 if (hosts_db_show_macs) str_appendf(buf,
329 " <th><a href=\"%s%smac\">MAC Address</a></th>\n",
330 qs,(sort==MAC)?"rev":""
331 );
332 str_appendf(buf,
333 " <th><a href=\"%s%sin\">In</a></th>\n"
334 " <th><a href=\"%s%sout\">Out</a></th>\n"
335 " <th><a href=\"%s%stotal\">Total</a></th>\n",
336 qs,(sort==IN)?"rev":"",
337 qs,(sort==OUT)?"rev":"",
338 qs,(sort==TOTAL)?"rev":""
339 );
340 if (opt_want_lastseen) str_appendf(buf,
341 " <th><a href=\"%s%slastseen\">Last seen</a></th>\n",
342 qs,(sort==LASTSEEN)?"rev":""
343 );
344 str_append(buf,
345 "</tr>\n");
346
347 //I think qs is freed when buffer is freed but not sure
348 }
349
350 static void
351 format_row_host(struct str *buf, const struct bucket *b,
352 const char *css_class, const struct bucket *host)
353 {
354 struct dns_request request;
355 const char *ip = addr_to_str(&(b->u.host.addr));
356
357 str_appendf(buf,
358 "<tr class=\"%s\">\n"
359 " <td><a href=\"./%s/\">%s</a></td>\n"
360 " <td>%s</td>\n",
361 css_class,
362 ip, ip,
363 (b->u.host.dns == NULL) ? "(resolving...)" : b->u.host.dns);
364
365 if (hosts_db_show_macs)
366 str_appendf(buf,
367 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
368 b->u.host.mac_addr[0],
369 b->u.host.mac_addr[1],
370 b->u.host.mac_addr[2],
371 b->u.host.mac_addr[3],
372 b->u.host.mac_addr[4],
373 b->u.host.mac_addr[5]);
374
375 str_appendf(buf,
376 " <td class=\"num\">%'qu</td>\n"
377 " <td class=\"num\">%'qu</td>\n"
378 " <td class=\"num\">%'qu</td>\n",
379 b->in, b->out, b->total);
380
381 if (opt_want_lastseen) {
382 time_t last_t = b->u.host.last_seen;
383 struct str *lastseen = NULL;
384
385 if (now >= last_t)
386 lastseen = length_of_time(now - last_t);
387
388 str_append(buf,
389 " <td class=\"num\">");
390 if (lastseen == NULL)
391 str_append(buf, "(clock error)");
392 else {
393 str_appendstr(buf, lastseen);
394 str_free(lastseen);
395 }
396 str_append(buf,
397 "</td>");
398 }
399
400 str_appendf(buf,
401 "</tr>\n");
402
403 /* Only resolve hosts "on demand" */
404 if (b->u.host.dns == NULL) {
405 memcpy(&(request.addr), &(b->u.host.addr), sizeof(request.addr));
406 //false
407 request.has_parent = (1==2);
408 dns_queue(&request);
409 }
410 }
411
412 static void
413 format_cols_host_opp(struct str *buf, const unsigned int start,
414 const enum sort_dir sort, const int full)
415 {
416 char *qs;
417 int result;
418
419 result=asprintf(&qs, (full==1)?"?full=yes&sort=":"?start=%d&sort=", start);
420 assert(result>=0);
421
422 str_appendf(buf,
423 "<table>\n"
424 "<tr>\n"
425 " <th><a href=\"%s%sip\">IP</a></th>\n"
426 " <th><a href=\"%s%shostname\">Hostname</a></th>\n",
427 qs,(sort==IPO)?"rev":"",
428 qs,(sort==HOSTNAMEO)?"rev":""
429 );
430 str_appendf(buf,
431 " <th><a href=\"%s%sin\">In</a></th>\n"
432 " <th><a href=\"%s%sout\">Out</a></th>\n"
433 " <th><a href=\"%s%stotal\">Total</a></th>\n",
434 qs,(sort==IN)?"rev":"",
435 qs,(sort==OUT)?"rev":"",
436 qs,(sort==TOTAL)?"rev":""
437 );
438 if (opt_want_lastseen) str_appendf(buf,
439 " <th><a href=\"%s%slastseen\">Last seen</a></th>\n",
440 qs,(sort==LASTSEENO)?"rev":""
441 );
442 str_append(buf,
443 "</tr>\n");
444 }
445
446 static void
447 format_row_host_opp(struct str *buf, const struct bucket *b,
448 const char *css_class, const struct bucket *host)
449 {
450 struct dns_request request;
451 const struct host_opp *ho = &(b->u.host_opp);
452 const char *ip = addr_to_str(&(ho->addr));
453
454 str_appendf(buf,
455 "<tr class=\"%s\">\n"
456 " <td><a href=\"../%s/\">%s</a></td>\n"
457 " <td>%s</td>\n"
458 " <td class=\"num\">%'qu</td>\n"
459 " <td class=\"num\">%'qu</td>\n"
460 " <td class=\"num\">%'qu</td>\n",
461 css_class,
462 ip,ip,
463 ((ho->dns == NULL) ? "(resolving...)" : ho->dns),
464 b->in, b->out, b->total
465 );
466 if (opt_want_lastseen) {
467 time_t last_t = ho->last_seen;
468 struct str *lastseen = NULL;
469
470 if (now >= last_t)
471 lastseen = length_of_time(now - last_t);
472
473 str_append(buf,
474 " <td class=\"num\">");
475 if (lastseen == NULL)
476 str_append(buf, "(clock error)");
477 else {
478 str_appendstr(buf, lastseen);
479 str_free(lastseen);
480 }
481 str_append(buf,
482 "</td>");
483 }
484
485 str_appendf(buf,
486 "</tr>\n");
487
488
489 /* Only resolve hosts "on demand" */
490 if (ho->dns == NULL) {
491 memcpy(&(request.addr), &(ho->addr), sizeof(request.addr));
492 memcpy(&(request.parent_addr), &(host->u.host.addr), sizeof(request.parent_addr));
493 //true
494 request.has_parent = (1==1);
495 verbosef("Queueing dns request for opp. host %s on parent %s",
496 addr_to_str(&request.addr),
497 addr_to_str(&request.parent_addr));
498 dns_queue(&request);
499 }
500 }
501
502
503 static void
504 format_cols_port_tcp(struct str *buf, const unsigned int start,
505 const enum sort_dir sort, const int full)
506 {
507 str_append(buf,
508 "<table>\n"
509 "<tr>\n"
510 " <th>Port</td>\n"
511 " <th>Service</td>\n"
512 " <th>In</td>\n"
513 " <th>Out</td>\n"
514 " <th>Total</td>\n"
515 " <th>SYNs</td>\n"
516 "</tr>\n"
517 );
518 }
519
520 static void
521 format_row_port_tcp(struct str *buf, const struct bucket *b,
522 const char *css_class, const struct bucket *host)
523 {
524 const struct port_tcp *p = &(b->u.port_tcp);
525
526 str_appendf(buf,
527 "<tr class=\"%s\">\n"
528 " <td class=\"num\">%u</td>\n"
529 " <td>%s</td>\n"
530 " <td class=\"num\">%'qu</td>\n"
531 " <td class=\"num\">%'qu</td>\n"
532 " <td class=\"num\">%'qu</td>\n"
533 " <td class=\"num\">%'qu</td>\n"
534 "</tr>\n",
535 css_class,
536 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
537 );
538 }
539
540 static void
541 format_cols_port_udp(struct str *buf, const unsigned int start,
542 const enum sort_dir sort, const int full)
543 {
544 str_append(buf,
545 "<table>\n"
546 "<tr>\n"
547 " <th>Port</td>\n"
548 " <th>Service</td>\n"
549 " <th>In</td>\n"
550 " <th>Out</td>\n"
551 " <th>Total</td>\n"
552 "</tr>\n"
553 );
554 }
555
556 static void
557 format_row_port_udp(struct str *buf, const struct bucket *b,
558 const char *css_class, const struct bucket *host)
559 {
560 const struct port_udp *p = &(b->u.port_udp);
561
562 str_appendf(buf,
563 "<tr class=\"%s\">\n"
564 " <td class=\"num\">%u</td>\n"
565 " <td>%s</td>\n"
566 " <td class=\"num\">%'qu</td>\n"
567 " <td class=\"num\">%'qu</td>\n"
568 " <td class=\"num\">%'qu</td>\n"
569 "</tr>\n",
570 css_class,
571 p->port, getservudp(p->port), b->in, b->out, b->total
572 );
573 }
574
575 static void
576 format_cols_ip_proto(struct str *buf, const unsigned int start,
577 const enum sort_dir sort, const int full)
578 {
579 str_append(buf,
580 "<table>\n"
581 "<tr>\n"
582 " <th>#</td>\n"
583 " <th>Protocol</td>\n"
584 " <th>In</td>\n"
585 " <th>Out</td>\n"
586 " <th>Total</td>\n"
587 "</tr>\n"
588 );
589 }
590
591 static void
592 format_row_ip_proto(struct str *buf, const struct bucket *b,
593 const char *css_class, const struct bucket *host)
594 {
595 const struct ip_proto *p = &(b->u.ip_proto);
596
597 str_appendf(buf,
598 "<tr class=\"%s\">\n"
599 " <td class=\"num\">%u</td>\n"
600 " <td>%s</td>\n"
601 " <td class=\"num\">%'qu</td>\n"
602 " <td class=\"num\">%'qu</td>\n"
603 " <td class=\"num\">%'qu</td>\n"
604 "</tr>\n",
605 css_class,
606 p->proto, getproto(p->proto),
607 b->in, b->out, b->total
608 );
609 }
610
611 /* ---------------------------------------------------------------------------
612 * Initialise a hashtable.
613 */
614 static struct hashtable *
615 hashtable_make(const uint8_t bits,
616 const unsigned int count_max,
617 const unsigned int count_keep,
618 hash_func_t *hash_func,
619 free_func_t *free_func,
620 key_func_t *key_func,
621 find_func_t *find_func,
622 make_func_t *make_func,
623 format_cols_func_t *format_cols_func,
624 format_row_func_t *format_row_func)
625 {
626 struct hashtable *hash;
627 assert(bits > 0);
628
629 hash = xmalloc(sizeof(*hash));
630 hash->bits = bits;
631 hash->count_max = count_max;
632 hash->count_keep = count_keep;
633 hash->size = 1U << bits;
634 hash->mask = hash->size - 1;
635 hash->coeff = coprime(hash->size);
636 hash->hash_func = hash_func;
637 hash->free_func = free_func;
638 hash->key_func = key_func;
639 hash->find_func = find_func;
640 hash->make_func = make_func;
641 hash->format_cols_func = format_cols_func;
642 hash->format_row_func = format_row_func;
643 hash->count = 0;
644 hash->table = xcalloc(hash->size, sizeof(*hash->table));
645 memset(&(hash->stats), 0, sizeof(hash->stats));
646 return (hash);
647 }
648
649 /* ---------------------------------------------------------------------------
650 * Initialise global hosts_db.
651 */
652 void
653 hosts_db_init(void)
654 {
655 assert(hosts_db == NULL);
656 hosts_db = hashtable_make(HOST_BITS, opt_hosts_max, opt_hosts_keep,
657 hash_func_host, free_func_host, key_func_host, find_func_host,
658 make_func_host, format_cols_host, format_row_host);
659 }
660
661 static void
662 hashtable_rehash(struct hashtable *h, const uint8_t bits)
663 {
664 struct bucket **old_table, **new_table;
665 uint32_t i, old_size;
666 assert(h != NULL);
667 assert(bits > 0);
668
669 h->stats.rehashes++;
670 old_size = h->size;
671 old_table = h->table;
672
673 h->bits = bits;
674 h->size = 1U << bits;
675 h->mask = h->size - 1;
676 h->coeff = coprime(h->size);
677 new_table = xcalloc(h->size, sizeof(*new_table));
678
679 for (i=0; i<old_size; i++) {
680 struct bucket *next, *b = old_table[i];
681 while (b != NULL) {
682 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
683 next = b->next;
684 b->next = new_table[pos];
685 new_table[pos] = b;
686 b = next;
687 }
688 }
689
690 free(h->table);
691 h->table = new_table;
692 }
693
694 static void
695 hashtable_insert(struct hashtable *h, struct bucket *b)
696 {
697 uint32_t pos;
698 assert(h != NULL);
699 assert(b != NULL);
700 assert(b->next == NULL);
701
702 /* Rehash on 80% occupancy */
703 if ((h->count > h->size) ||
704 ((h->size - h->count) < h->size / 5))
705 hashtable_rehash(h, h->bits+1);
706
707 pos = h->hash_func(h, h->key_func(b)) & h->mask;
708 if (h->table[pos] == NULL)
709 h->table[pos] = b;
710 else {
711 /* Insert at top of chain. */
712 b->next = h->table[pos];
713 h->table[pos] = b;
714 }
715 h->count++;
716 h->stats.inserts++;
717 }
718
719 /* Return bucket matching key, or NULL if no such entry. */
720 static struct bucket *
721 hashtable_search(struct hashtable *h, const void *key)
722 {
723 uint32_t pos;
724 struct bucket *b;
725 h->stats.searches++;
726 pos = h->hash_func(h, key) & h->mask;
727 b = h->table[pos];
728 while (b != NULL) {
729 if (h->find_func(b, key))
730 return (b);
731 else
732 b = b->next;
733 }
734 return (NULL);
735 }
736
737 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
738 /* Search for a key. If it's not there, make and insert a bucket for it. */
739 static struct bucket *
740 hashtable_find_or_insert(struct hashtable *h, const void *key,
741 const reduce_bool allow_reduce)
742 {
743 struct bucket *b = hashtable_search(h, key);
744 if (b == NULL) {
745 /* Not found, so insert after checking occupancy. */
746 if (allow_reduce && (h->count >= h->count_max))
747 hashtable_reduce(h);
748 b = h->make_func(key);
749 hashtable_insert(h, b);
750 }
751 return (b);
752 }
753
754 /*
755 * Frees the hashtable and the buckets. The contents are assumed to be
756 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
757 * the bucket.
758 */
759 static void
760 hashtable_free(struct hashtable *h)
761 {
762 uint32_t i;
763
764 if (h == NULL)
765 return;
766 for (i=0; i<h->size; i++) {
767 struct bucket *tmp, *b = h->table[i];
768 while (b != NULL) {
769 tmp = b;
770 b = b->next;
771 h->free_func(tmp);
772 free(tmp);
773 }
774 }
775 free(h->table);
776 free(h);
777 }
778
779 /* ---------------------------------------------------------------------------
780 * Return existing host or insert a new one.
781 */
782 struct bucket *
783 host_get(const struct addr *const a)
784 {
785 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
786 }
787
788 /* ---------------------------------------------------------------------------
789 * Find host, returns NULL if not in DB.
790 */
791 struct bucket *
792 host_find(const struct addr *const a)
793 {
794 return (hashtable_search(hosts_db, a));
795 }
796
797 /* ---------------------------------------------------------------------------
798 * Find host, returns NULL if not in DB.
799 */
800 struct bucket *
801 host_find_host_opp(struct bucket *host, const struct addr *const a)
802 {
803 struct host *h = &host->u.host;
804 assert(h != NULL);
805 if (h->hosts_opp != NULL) {
806 return (hashtable_search(h->hosts_opp, a));
807 }
808 else {
809 verbosef("Nothing to find as opp. hosts table empty");
810 return (NULL);
811 }
812 }
813
814 /* ---------------------------------------------------------------------------
815 * Find host, returns NULL if not in DB.
816 */
817 static struct bucket *
818 host_search(const char *ipstr)
819 {
820 struct addr a;
821 struct addrinfo hints, *ai;
822
823 memset(&hints, 0, sizeof(hints));
824 hints.ai_family = AF_UNSPEC;
825 hints.ai_flags = AI_NUMERICHOST;
826
827 if (getaddrinfo(ipstr, NULL, &hints, &ai))
828 return (NULL); /* invalid addr */
829
830 if (ai->ai_family == AF_INET) {
831 a.family = IPv4;
832 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
833 }
834 else if (ai->ai_family == AF_INET6) {
835 a.family = IPv6;
836 memcpy(&(a.ip.v6),
837 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
838 sizeof(a.ip.v6));
839 } else {
840 freeaddrinfo(ai);
841 return (NULL); /* unknown family */
842 }
843 freeaddrinfo(ai);
844
845 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
846 return (hashtable_search(hosts_db, &a));
847 }
848
849 /* ---------------------------------------------------------------------------
850 * Reduce a hashtable to the top <keep> entries.
851 */
852 static void
853 hashtable_reduce(struct hashtable *ht)
854 {
855 uint32_t i, pos, rmd;
856 const struct bucket **table;
857 uint64_t cutoff;
858
859 assert(ht->count_keep < ht->count);
860
861 /* Fill table with pointers to buckets in hashtable. */
862 table = xcalloc(ht->count, sizeof(*table));
863 for (pos=0, i=0; i<ht->size; i++) {
864 struct bucket *b = ht->table[i];
865 while (b != NULL) {
866 table[pos++] = b;
867 b = b->next;
868 }
869 }
870 assert(pos == ht->count);
871 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
872 cutoff = table[ht->count_keep]->total;
873 free(table);
874
875 /* Remove all elements with total <= cutoff. */
876 rmd = 0;
877 for (i=0; i<ht->size; i++) {
878 struct bucket *last = NULL, *next, *b = ht->table[i];
879 while (b != NULL) {
880 next = b->next;
881 if (b->total <= cutoff) {
882 /* Remove this one. */
883 ht->free_func(b);
884 free(b);
885 if (last == NULL)
886 ht->table[i] = next;
887 else
888 last->next = next;
889 rmd++;
890 ht->count--;
891 } else {
892 last = b;
893 }
894 b = next;
895 }
896 }
897 verbosef("hashtable_reduce: removed %u buckets, left %u",
898 rmd, ht->count);
899 hashtable_rehash(ht, ht->bits); /* is this needed? */
900 }
901
902 /* Reduce hosts_db if needed. */
903 void hosts_db_reduce(void)
904 {
905 if (hosts_db->count >= hosts_db->count_max)
906 hashtable_reduce(hosts_db);
907 }
908
909 /* ---------------------------------------------------------------------------
910 * Reset hosts_db to empty.
911 */
912 void
913 hosts_db_reset(void)
914 {
915 unsigned int i;
916
917 for (i=0; i<hosts_db->size; i++) {
918 struct bucket *next, *b = hosts_db->table[i];
919 while (b != NULL) {
920 next = b->next;
921 hosts_db->free_func(b);
922 free(b);
923 b = next;
924 }
925 hosts_db->table[i] = NULL;
926 }
927 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
928 hosts_db->count = 0;
929 }
930
931 /* ---------------------------------------------------------------------------
932 * Deallocate hosts_db.
933 */
934 void hosts_db_free(void)
935 {
936 uint32_t i;
937
938 assert(hosts_db != NULL);
939 for (i=0; i<hosts_db->size; i++) {
940 struct bucket *tmp, *b = hosts_db->table[i];
941 while (b != NULL) {
942 tmp = b;
943 b = b->next;
944 hosts_db->free_func(tmp);
945 free(tmp);
946 }
947 }
948 free(hosts_db->table);
949 free(hosts_db);
950 hosts_db = NULL;
951 }
952
953 /* ---------------------------------------------------------------------------
954 * Find or create a host_opp inside a host.
955 */
956 struct bucket *
957 host_get_host_opp(struct bucket *host, const struct addr *const a)
958 {
959 struct host *h = &host->u.host;
960 assert(h != NULL);
961 if (h->hosts_opp == NULL) {
962 h->hosts_opp = hashtable_make(HOST_OPP_BITS, opt_hosts_max, opt_hosts_keep,
963 hash_func_host, free_func_simple, key_func_host_opp,
964 find_func_host_opp, make_func_host_opp,
965 format_cols_host_opp, format_row_host_opp);
966 }
967 return (hashtable_find_or_insert(h->hosts_opp, a, ALLOW_REDUCE));
968 }
969
970 /* ---------------------------------------------------------------------------
971 * Find or create a port_tcp inside a host.
972 */
973 struct bucket *
974 host_get_port_tcp(struct bucket *host, const uint16_t port)
975 {
976 struct host *h = &host->u.host;
977 assert(h != NULL);
978 if (h->ports_tcp == NULL)
979 h->ports_tcp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
980 hash_func_short, free_func_simple, key_func_port_tcp,
981 find_func_port_tcp, make_func_port_tcp,
982 format_cols_port_tcp, format_row_port_tcp);
983 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
984 }
985
986 /* ---------------------------------------------------------------------------
987 * Find or create a port_udp inside a host.
988 */
989 struct bucket *
990 host_get_port_udp(struct bucket *host, const uint16_t port)
991 {
992 struct host *h = &host->u.host;
993 assert(h != NULL);
994 if (h->ports_udp == NULL)
995 h->ports_udp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
996 hash_func_short, free_func_simple, key_func_port_udp,
997 find_func_port_udp, make_func_port_udp,
998 format_cols_port_udp, format_row_port_udp);
999 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
1000 }
1001
1002 /* ---------------------------------------------------------------------------
1003 * Find or create an ip_proto inside a host.
1004 */
1005 struct bucket *
1006 host_get_ip_proto(struct bucket *host, const uint8_t proto)
1007 {
1008 struct host *h = &host->u.host;
1009 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
1010 assert(h != NULL);
1011 if (h->ip_protos == NULL)
1012 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
1013 hash_func_byte, free_func_simple, key_func_ip_proto,
1014 find_func_ip_proto, make_func_ip_proto,
1015 format_cols_ip_proto, format_row_ip_proto);
1016 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
1017 }
1018
1019 static struct str *html_hosts_main(const char *qs);
1020 static struct str *html_hosts_detail(const char *ip, const char *qs);
1021
1022 /* ---------------------------------------------------------------------------
1023 * Web interface: delegate the /hosts/ space.
1024 */
1025 struct str *
1026 html_hosts(const char *uri, const char *query)
1027 {
1028 unsigned int i, num_elems;
1029 char **elem = split('/', uri, &num_elems);
1030 struct str *buf = NULL;
1031
1032 assert(num_elems >= 1);
1033 assert(strcmp(elem[0], "hosts") == 0);
1034
1035 if (num_elems == 1)
1036 /* /hosts/ */
1037 buf = html_hosts_main(query);
1038 else if (num_elems == 2)
1039 /* /hosts/<IP of host>/ */
1040 buf = html_hosts_detail(elem[1],query);
1041
1042 for (i=0; i<num_elems; i++)
1043 free(elem[i]);
1044 free(elem);
1045 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
1046 other codes to be possible */
1047 }
1048
1049 /* ---------------------------------------------------------------------------
1050 * Format hashtable into HTML.
1051 */
1052 static void
1053 format_table(struct str *buf, struct hashtable *ht, unsigned int start,
1054 const enum sort_dir sort, const int full, const struct bucket *host,
1055 const char *sortstr, int shownavbar)
1056 {
1057 const struct bucket **table;
1058 unsigned int i, pos, end;
1059 int alt = 0;
1060
1061 if ((ht == NULL) || (ht->count == 0)) {
1062 str_append(buf, "<p>The table is empty.</p>\n");
1063 return;
1064 }
1065
1066 /* Fill table with pointers to buckets in hashtable. */
1067 table = xcalloc(ht->count, sizeof(*table));
1068 for (pos=0, i=0; i<ht->size; i++) {
1069 struct bucket *b = ht->table[i];
1070 while (b != NULL) {
1071 table[pos++] = b;
1072 b = b->next;
1073 }
1074 }
1075 assert(pos == ht->count);
1076
1077 if (full) {
1078 /* full report overrides start and end */
1079 start = 0;
1080 end = ht->count;
1081 }
1082 else
1083 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
1084
1085 str_appendf(buf, "(%u-%u of %u)<br>\n", start+1, end, ht->count);
1086 qsort_buckets(table, ht->count, start, end, sort);
1087 ht->format_cols_func(buf,start,sort,full);
1088
1089 for (i=start; i<end; i++) {
1090 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2", host);
1091 alt = !alt; /* alternate class for table rows */
1092 }
1093 free(table);
1094 str_append(buf, "</table>\n");
1095
1096 if (shownavbar) {
1097 int last = (int)ht->count - ((int)ht->count % MAX_ENTRIES);
1098
1099 #define FRST "&lt;&lt; first page"
1100 #define PREV "&lt; prev page"
1101 #define FULL "full table"
1102 #define NEXT "next page &gt;"
1103 #define LAST "last page &gt;&gt;"
1104
1105 /* <<first <prev | full | next> last>> */
1106 if (sortstr == NULL) sortstr = "total";
1107 //FIRST
1108 if (start == 0)
1109 str_append(buf, FRST " | ");
1110 else
1111 str_appendf(buf, "<a href=\"?start=0&sort=%s\">" FRST "</a> | ",
1112 sortstr);
1113 //PREV
1114 if (start > 0) {
1115 int prev = start - MAX_ENTRIES;
1116 if (prev < 0)
1117 prev = 0;
1118 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
1119 prev, sortstr);
1120 } else
1121 str_append(buf, PREV);
1122 //FULL
1123 if (full)
1124 str_append(buf, " | " FULL);
1125 else
1126 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
1127 sortstr);
1128 //NEXT
1129 if (start+MAX_ENTRIES < (int)ht->count)
1130 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
1131 start+MAX_ENTRIES, sortstr);
1132 else
1133 str_append(buf, " | " NEXT);
1134 //LAST
1135 if (last == (int)ht->count) last = (int)ht->count - MAX_ENTRIES;
1136 if (last < 0) last = 0;
1137 if (last == 0)
1138 str_append(buf, " | " LAST);
1139 else
1140 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" LAST "</a>",
1141 last,sortstr);
1142
1143 str_append(buf, "<br>\n");
1144 #undef FRST
1145 #undef PREV
1146 #undef FULL
1147 #undef NEXT
1148 #undef LAST
1149 }
1150
1151 }
1152
1153 /* ---------------------------------------------------------------------------
1154 * Web interface: sorted table of hosts.
1155 */
1156 static struct str *
1157 html_hosts_main(const char *qs)
1158 {
1159 struct str *buf = str_make();
1160 char *qs_start, *qs_sort, *qs_full, *ep;
1161 int start, full = 0;
1162 enum sort_dir sort;
1163
1164 /* parse query string */
1165 qs_start = qs_get(qs, "start");
1166 qs_sort = qs_get(qs, "sort");
1167 qs_full = qs_get(qs, "full");
1168 if (qs_full != NULL) {
1169 full = 1;
1170 free(qs_full);
1171 }
1172
1173 /* validate sort */
1174 if (qs_sort == NULL) sort = TOTAL;
1175 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
1176 else if (strcmp(qs_sort, "revtotal") == 0) sort = REVTOTAL;
1177 else if (strcmp(qs_sort, "in") == 0) sort = IN;
1178 else if (strcmp(qs_sort, "revin") == 0) sort = REVIN;
1179 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
1180 else if (strcmp(qs_sort, "revout") == 0) sort = REVOUT;
1181 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
1182 else if (strcmp(qs_sort, "revlastseen") == 0) sort = REVLASTSEEN;
1183 else if (strcmp(qs_sort, "ip") == 0) sort = IP;
1184 else if (strcmp(qs_sort, "revip") == 0) sort = REVIP;
1185 else if (strcmp(qs_sort, "hostname") == 0) sort = HOSTNAME;
1186 else if (strcmp(qs_sort, "revhostname") == 0) sort = REVHOSTNAME;
1187 else if (strcmp(qs_sort, "mac") == 0) sort = MAC;
1188 else if (strcmp(qs_sort, "revmac") == 0) sort = REVMAC;
1189 else {
1190 str_append(buf, "Error: invalid value for \"sort\".\n");
1191 goto done;
1192 }
1193
1194 /* parse start */
1195 if (qs_start == NULL)
1196 start = 0;
1197 else {
1198 start = (int)strtoul(qs_start, &ep, 10);
1199 if (*ep != '\0') {
1200 str_append(buf, "Error: \"start\" is not a number.\n");
1201 goto done;
1202 }
1203 if ((errno == ERANGE) ||
1204 (start < 0) || (start >= (int)hosts_db->count)) {
1205 str_append(buf, "Error: \"start\" is out of bounds.\n");
1206 goto done;
1207 }
1208 }
1209
1210 html_open(buf, "Hosts", /*path_depth=*/1, /*want_graph_js=*/0);
1211 format_table(buf, hosts_db, start, sort, full, NULL, qs_sort, 1);
1212 html_close(buf);
1213 done:
1214 if (qs_start != NULL) free(qs_start);
1215 if (qs_sort != NULL) free(qs_sort);
1216 return buf;
1217 }
1218
1219 /* ---------------------------------------------------------------------------
1220 * Web interface: detailed view of a single host.
1221 */
1222 static struct str *
1223 html_hosts_detail(const char *ip, const char *qs)
1224 {
1225 struct bucket *h;
1226 struct str *buf, *ls_len;
1227 char ls_when[100];
1228 const char *canonical;
1229 time_t ls;
1230 char *qs_dns, *qs_start, *qs_sort, *qs_full, *ep;
1231 int start, full = 0;
1232 struct dns_request request;
1233 enum sort_dir sort;
1234
1235 h = host_search(ip);
1236 if (h == NULL)
1237 return (NULL); /* no such host */
1238
1239 /* parse query string */
1240 qs_start = qs_get(qs, "start");
1241 qs_full = qs_get(qs, "full");
1242 if (qs_full != NULL) {
1243 full = 1;
1244 free(qs_full);
1245 }
1246 qs_sort = qs_get(qs, "sort");
1247 qs_dns = qs_get(qs, "resetdns");
1248 if (qs_dns != NULL) if (strcmp(qs_dns, "true") == 0) {
1249 verbosef("Checking if we need to reset DNS");
1250 if (h->u.host.dns != NULL) {
1251 verbosef("Resetting DNS");
1252 free(h->u.host.dns);
1253 h->u.host.dns = NULL;
1254 }
1255 }
1256
1257 /* parse start */
1258 if (qs_start == NULL)
1259 start = 0;
1260 else {
1261 start = (int)strtoul(qs_start, &ep, 10);
1262 if (*ep != '\0') {
1263 str_append(buf, "Error: \"start\" is not a number.\n");
1264 goto done;
1265 }
1266 if ((errno == ERANGE) ||
1267 (start < 0) || (start >= (int)h->u.host.hosts_opp->count)) {
1268 str_append(buf, "Error: \"start\" is out of bounds.\n");
1269 goto done;
1270 }
1271 }
1272
1273 /* validate sort */
1274 if (qs_sort == NULL) sort = TOTAL;
1275 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
1276 else if (strcmp(qs_sort, "revtotal") == 0) sort = REVTOTAL;
1277 else if (strcmp(qs_sort, "in") == 0) sort = IN;
1278 else if (strcmp(qs_sort, "revin") == 0) sort = REVIN;
1279 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
1280 else if (strcmp(qs_sort, "revout") == 0) sort = REVOUT;
1281 else if (strcmp(qs_sort, "ip") == 0) sort = IPO;
1282 else if (strcmp(qs_sort, "revip") == 0) sort = REVIPO;
1283 else if (strcmp(qs_sort, "hostname") == 0) sort = HOSTNAMEO;
1284 else if (strcmp(qs_sort, "revhostname") == 0) sort = REVHOSTNAMEO;
1285 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEENO;
1286 else if (strcmp(qs_sort, "revlastseen") == 0) sort = REVLASTSEENO;
1287 else {
1288 str_append(buf, "Error: invalid value for \"sort\".\n");
1289 goto done;
1290 }
1291
1292
1293 canonical = addr_to_str(&(h->u.host.addr));
1294
1295 /* Overview. */
1296 buf = str_make();
1297 html_open(buf, ip, /*path_depth=*/2, /*want_graph_js=*/0);
1298 if (strcmp(ip, canonical) != 0)
1299 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1300 str_appendf(buf,
1301 "<p>\n"
1302 "<b>Hostname:</b> <a href=\"?resetdns=%s\">%s</a><br>\n",
1303 (h->u.host.dns == NULL)?"false":"true",
1304 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1305
1306 /* Resolve host "on demand" */
1307 if (h->u.host.dns == NULL) {
1308 memcpy(&(request.addr), &(h->u.host.addr), sizeof(request.addr));
1309 //false
1310 request.has_parent = (1==2);
1311 dns_queue(&request);
1312 }
1313
1314 if (hosts_db_show_macs)
1315 str_appendf(buf,
1316 "<b>MAC Address:</b> "
1317 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1318 h->u.host.mac_addr[0],
1319 h->u.host.mac_addr[1],
1320 h->u.host.mac_addr[2],
1321 h->u.host.mac_addr[3],
1322 h->u.host.mac_addr[4],
1323 h->u.host.mac_addr[5]);
1324
1325 str_append(buf,
1326 "</p>\n"
1327 "<p>\n"
1328 "<b>Last seen:</b> ");
1329
1330 ls = h->u.host.last_seen;
1331 if (strftime(ls_when, sizeof(ls_when),
1332 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1333 str_append(buf, ls_when);
1334
1335 if (h->u.host.last_seen <= now) {
1336 ls_len = length_of_time(now - h->u.host.last_seen);
1337 str_append(buf, " (");
1338 str_appendstr(buf, ls_len);
1339 str_free(ls_len);
1340 str_append(buf, " ago)");
1341 } else {
1342 str_append(buf, " (in the future, possible clock problem)");
1343 }
1344
1345 str_appendf(buf,
1346 "</p>\n"
1347 "<p>\n"
1348 " <b>In:</b> %'qu<br>\n"
1349 " <b>Out:</b> %'qu<br>\n"
1350 " <b>Total:</b> %'qu<br>\n"
1351 "</p>\n",
1352 h->in, h->out, h->total);
1353
1354 str_append(buf, "<h3>TCP ports</h3>\n");
1355 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,1,h,qs_sort,0);
1356
1357 str_append(buf, "<h3>UDP ports</h3>\n");
1358 format_table(buf, h->u.host.ports_udp, 0,TOTAL,1,h,qs_sort,0);
1359
1360 str_append(buf, "<h3>IP protocols</h3>\n");
1361 format_table(buf, h->u.host.ip_protos, 0,TOTAL,1,h,qs_sort,0);
1362
1363 str_append(buf, "<h3>Opposite hosts</h3>\n");
1364 format_table(buf, h->u.host.hosts_opp, start, sort, full, h, qs_sort, 1);
1365
1366 done:
1367 if (qs_start != NULL) free(qs_start);
1368 if (qs_dns != NULL) free(qs_dns);
1369 if (qs_sort != NULL) free(qs_sort);
1370 html_close(buf);
1371 return (buf);
1372 }
1373
1374 /* ---------------------------------------------------------------------------
1375 * Database import and export code:
1376 * Initially written and contributed by Ben Stewart.
1377 * copyright (c) 2007-2011 Ben Stewart, Emil Mikulic.
1378 */
1379 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1380 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1381 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1382 static int hosts_db_export_opp(const struct hashtable *h, const int fd);
1383
1384 static const char
1385 export_proto_ip = 'P',
1386 export_proto_tcp = 'T',
1387 export_proto_udp = 'U',
1388 export_host_opp = 'O';
1389
1390 static const unsigned char
1391 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1392 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1393 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1394
1395 /* ---------------------------------------------------------------------------
1396 * Load a host's ip_proto table from a file.
1397 * Returns 0 on failure, 1 on success.
1398 */
1399 static int
1400 hosts_db_import_ip(const int fd, struct bucket *host)
1401 {
1402 uint8_t count, i;
1403
1404 if (!expect8(fd, export_proto_ip)) return 0;
1405 if (!read8(fd, &count)) return 0;
1406
1407 for (i=0; i<count; i++) {
1408 struct bucket *b;
1409 uint8_t proto;
1410 uint64_t in, out;
1411
1412 if (!read8(fd, &proto)) return 0;
1413 if (!read64(fd, &in)) return 0;
1414 if (!read64(fd, &out)) return 0;
1415
1416 /* Store data */
1417 b = host_get_ip_proto(host, proto);
1418 b->in = in;
1419 b->out = out;
1420 b->total = in + out;
1421 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1422 }
1423 return 1;
1424 }
1425
1426 /* ---------------------------------------------------------------------------
1427 * Load a host's port_tcp table from a file.
1428 * Returns 0 on failure, 1 on success.
1429 */
1430 static int
1431 hosts_db_import_tcp(const int fd, struct bucket *host)
1432 {
1433 uint16_t count, i;
1434
1435 if (!expect8(fd, export_proto_tcp)) return 0;
1436 if (!read16(fd, &count)) return 0;
1437
1438 for (i=0; i<count; i++) {
1439 struct bucket *b;
1440 uint16_t port;
1441 uint64_t in, out, syn;
1442
1443 if (!read16(fd, &port)) return 0;
1444 if (!read64(fd, &syn)) return 0;
1445 if (!read64(fd, &in)) return 0;
1446 if (!read64(fd, &out)) return 0;
1447
1448 /* Store data */
1449 b = host_get_port_tcp(host, port);
1450 b->in = in;
1451 b->out = out;
1452 b->total = in + out;
1453 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1454 b->u.port_tcp.syn = syn;
1455 }
1456 return 1;
1457 }
1458
1459 /* ---------------------------------------------------------------------------
1460 * Load a host's port_tcp table from a file.
1461 * Returns 0 on failure, 1 on success.
1462 */
1463 static int
1464 hosts_db_import_udp(const int fd, struct bucket *host)
1465 {
1466 uint16_t count, i;
1467
1468 if (!expect8(fd, export_proto_udp)) return 0;
1469 if (!read16(fd, &count)) return 0;
1470
1471 for (i=0; i<count; i++) {
1472 struct bucket *b;
1473 uint16_t port;
1474 uint64_t in, out;
1475
1476 if (!read16(fd, &port)) return 0;
1477 if (!read64(fd, &in)) return 0;
1478 if (!read64(fd, &out)) return 0;
1479
1480 /* Store data */
1481 b = host_get_port_udp(host, port);
1482 b->in = in;
1483 b->out = out;
1484 b->total = in + out;
1485 assert(b->u.port_udp.port == port); /* done by make_func */
1486 }
1487 return 1;
1488 }
1489
1490 /* ---------------------------------------------------------------------------
1491 * Load a host's host_opp table from a file.
1492 * Returns 0 on failure, 1 on success.
1493 */
1494 static int
1495 hosts_db_import_opp(const int fd, struct bucket *host)
1496 {
1497 uint32_t count, i;
1498
1499 if (!expect8(fd, export_host_opp)) return 0;
1500 if (!read32(fd, &count)) return 0;
1501
1502 for (i=0; i<count; i++) {
1503 struct bucket *b;
1504 struct addr a;
1505 uint64_t in, out;
1506
1507 if (!readaddr(fd, &a)) return 0;
1508 if (!read64(fd, &in)) return 0;
1509 if (!read64(fd, &out)) return 0;
1510
1511 /* Store data */
1512 b = host_get_host_opp(host, &a);
1513 b->in = in;
1514 b->out = out;
1515 b->total = in + out;
1516 assert(addr_equal(&(b->u.host_opp.addr),&a)); /* done by make_func */
1517 }
1518 return 1;
1519 }
1520
1521 /* ---------------------------------------------------------------------------
1522 * Load all hosts from a file.
1523 * Returns 0 on failure, 1 on success.
1524 */
1525 static int
1526 hosts_db_import_host(const int fd)
1527 {
1528 struct bucket *host;
1529 struct addr a;
1530 uint8_t hostname_len;
1531 uint64_t in, out;
1532 unsigned int pos = xtell(fd);
1533 char hdr[4];
1534 int ver = 0;
1535
1536 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1537 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1538 ver = 3;
1539 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1540 ver = 2;
1541 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1542 ver = 1;
1543 else {
1544 warnx("bad host header: %02x%02x%02x%02x",
1545 hdr[0], hdr[1], hdr[2], hdr[3]);
1546 return 0;
1547 }
1548
1549 if (ver == 3) {
1550 if (!readaddr(fd, &a))
1551 return 0;
1552 } else {
1553 assert((ver == 1) || (ver == 2));
1554 if (!readaddr_ipv4(fd, &a))
1555 return 0;
1556 }
1557 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1558 host = host_get(&a);
1559 assert(addr_equal(&(host->u.host.addr), &a));
1560
1561 if (ver > 1) {
1562 uint64_t t;
1563 if (!read64(fd, &t)) return 0;
1564 host->u.host.last_seen = (time_t)t;
1565 }
1566
1567 assert(sizeof(host->u.host.mac_addr) == 6);
1568 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1569 return 0;
1570
1571 /* HOSTNAME */
1572 assert(host->u.host.dns == NULL); /* make fn? */
1573 if (!read8(fd, &hostname_len)) return 0;
1574 if (hostname_len > 0) {
1575 host->u.host.dns = xmalloc(hostname_len + 1);
1576 host->u.host.dns[0] = '\0';
1577
1578 /* At this point, the hostname is attached to a host which is in our
1579 * hosts_db, so if we bail out due to an import error, this pointer
1580 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1581 */
1582
1583 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1584 host->u.host.dns[hostname_len] = '\0';
1585 }
1586
1587 if (!read64(fd, &in)) return 0;
1588 if (!read64(fd, &out)) return 0;
1589
1590 host->in = in;
1591 host->out = out;
1592 host->total = in + out;
1593
1594 /* Host's port and proto subtables: */
1595 if (!hosts_db_import_ip(fd, host)) return 0;
1596 if (!hosts_db_import_tcp(fd, host)) return 0;
1597 if (!hosts_db_import_udp(fd, host)) return 0;
1598 if (!hosts_db_import_opp(fd, host)) return 0;
1599 return 1;
1600 }
1601
1602 /* ---------------------------------------------------------------------------
1603 * Database Import: Grab hosts_db from a file provided by the caller.
1604 *
1605 * This function will retrieve the data sans the header. We expect the caller
1606 * to have validated the header of the hosts_db segment, and left the file
1607 * sitting at the start of the data.
1608 */
1609 int hosts_db_import(const int fd)
1610 {
1611 uint32_t host_count, i;
1612
1613 if (!read32(fd, &host_count)) return 0;
1614
1615 for (i=0; i<host_count; i++)
1616 if (!hosts_db_import_host(fd)) return 0;
1617
1618 return 1;
1619 }
1620
1621 /* ---------------------------------------------------------------------------
1622 * Database Export: Dump hosts_db into a file provided by the caller.
1623 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1624 */
1625 int hosts_db_export(const int fd)
1626 {
1627 uint32_t i;
1628 struct bucket *b;
1629
1630 if (!write32(fd, hosts_db->count)) return 0;
1631
1632 for (i = 0; i<hosts_db->size; i++)
1633 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1634 /* For each host: */
1635 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1636 return 0;
1637
1638 if (!writeaddr(fd, &(b->u.host.addr))) return 0;
1639
1640 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1641
1642 assert(sizeof(b->u.host.mac_addr) == 6);
1643 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1644 return 0;
1645
1646 /* HOSTNAME */
1647 if (b->u.host.dns == NULL) {
1648 if (!write8(fd, 0)) return 0;
1649 } else {
1650 int dnslen = strlen(b->u.host.dns);
1651
1652 if (dnslen > 255) {
1653 warnx("found a very long hostname: \"%s\"\n"
1654 "wasn't expecting one longer than 255 chars (this one is %d)",
1655 b->u.host.dns, dnslen);
1656 dnslen = 255;
1657 }
1658
1659 if (!write8(fd, (uint8_t)dnslen)) return 0;
1660 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1661 }
1662
1663 if (!write64(fd, b->in)) return 0;
1664 if (!write64(fd, b->out)) return 0;
1665
1666 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1667 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1668 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1669 if (!hosts_db_export_opp(b->u.host.hosts_opp, fd)) return 0;
1670 }
1671 return 1;
1672 }
1673
1674 /* ---------------------------------------------------------------------------
1675 * Dump the ip_proto table of a host.
1676 */
1677 static int
1678 hosts_db_export_ip(const struct hashtable *h, const int fd)
1679 {
1680 uint32_t i, written = 0;
1681 struct bucket *b;
1682
1683 /* IP DATA */
1684 if (!write8(fd, export_proto_ip)) return 0;
1685
1686 /* If no data, write a IP Proto count of 0 and we're done. */
1687 if (h == NULL) {
1688 if (!write8(fd, 0)) return 0;
1689 return 1;
1690 }
1691
1692 assert(h->count < 256);
1693 if (!write8(fd, (uint8_t)h->count)) return 0;
1694
1695 for (i = 0; i<h->size; i++)
1696 for (b = h->table[i]; b != NULL; b = b->next) {
1697 /* For each ip_proto bucket: */
1698
1699 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1700 if (!write64(fd, b->in)) return 0;
1701 if (!write64(fd, b->out)) return 0;
1702 written++;
1703 }
1704 assert(written == h->count);
1705 return 1;
1706 }
1707
1708 /* ---------------------------------------------------------------------------
1709 * Dump the port_tcp table of a host.
1710 */
1711 static int
1712 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1713 {
1714 struct bucket *b;
1715 uint32_t i, written = 0;
1716
1717 /* TCP DATA */
1718 if (!write8(fd, export_proto_tcp)) return 0;
1719
1720 /* If no data, write a count of 0 and we're done. */
1721 if (h == NULL) {
1722 if (!write16(fd, 0)) return 0;
1723 return 1;
1724 }
1725
1726 assert(h->count < 65536);
1727 if (!write16(fd, (uint16_t)h->count)) return 0;
1728
1729 for (i = 0; i<h->size; i++)
1730 for (b = h->table[i]; b != NULL; b = b->next) {
1731 if (!write16(fd, b->u.port_tcp.port)) return 0;
1732 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1733 if (!write64(fd, b->in)) return 0;
1734 if (!write64(fd, b->out)) return 0;
1735 written++;
1736 }
1737 assert(written == h->count);
1738 return 1;
1739 }
1740
1741 /* ---------------------------------------------------------------------------
1742 * Dump the port_udp table of a host.
1743 */
1744 static int
1745 hosts_db_export_udp(const struct hashtable *h, const int fd)
1746 {
1747 struct bucket *b;
1748 uint32_t i, written = 0;
1749
1750 /* UDP DATA */
1751 if (!write8(fd, export_proto_udp)) return 0;
1752
1753 /* If no data, write a count of 0 and we're done. */
1754 if (h == NULL) {
1755 if (!write16(fd, 0)) return 0;
1756 return 1;
1757 }
1758
1759 assert(h->count < 65536);
1760 if (!write16(fd, (uint16_t)h->count)) return 0;
1761
1762 for (i = 0; i<h->size; i++)
1763 for (b = h->table[i]; b != NULL; b = b->next) {
1764 if (!write16(fd, b->u.port_udp.port)) return 0;
1765 if (!write64(fd, b->in)) return 0;
1766 if (!write64(fd, b->out)) return 0;
1767 written++;
1768 }
1769 assert(written == h->count);
1770 return 1;
1771 }
1772
1773 /* ---------------------------------------------------------------------------
1774 * Dump the host_opp table of a host.
1775 */
1776 static int
1777 hosts_db_export_opp(const struct hashtable *h, const int fd)
1778 {
1779 struct bucket *b;
1780 uint32_t i, written = 0;
1781
1782 /* OPP HOSTS DATA */
1783 if (!write8(fd, export_host_opp)) return 0;
1784
1785 /* If no data, write a count of 0 and we're done. */
1786 if (h == NULL) {
1787 if (!write16(fd, 0)) return 0;
1788 return 1;
1789 }
1790
1791 if (!write32(fd, h->count)) return 0;
1792
1793 for (i = 0; i<h->size; i++)
1794 for (b = h->table[i]; b != NULL; b = b->next) {
1795 if (!writeaddr(fd, &(b->u.host_opp.addr))) return 0;
1796 if (!write64(fd, b->in)) return 0;
1797 if (!write64(fd, b->out)) return 0;
1798 written++;
1799 }
1800 assert(written == h->count);
1801 return 1;
1802 }
1803
1804 /* vim:set ts=3 sw=3 tw=78 expandtab: */