Update ChangeLog for 3.0.717.
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2011 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "cdefs.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20 #include "opt.h"
21 #include "str.h"
22
23 #include <netdb.h> /* struct addrinfo */
24 #include <assert.h>
25 #include <errno.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h> /* memset(), strcmp() */
29 #include <time.h>
30 #include <unistd.h>
31
32 int hosts_db_show_macs = 0;
33
34 /* FIXME: specify somewhere more sane/tunable */
35 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
36
37 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
38 typedef void (free_func_t)(struct bucket *);
39 typedef const void * (key_func_t)(const struct bucket *);
40 typedef int (find_func_t)(const struct bucket *, const void *);
41 typedef struct bucket * (make_func_t)(const void *);
42 typedef void (format_cols_func_t)(struct str *);
43 typedef void (format_row_func_t)(struct str *, const struct bucket *,
44 const char *);
45
46 struct hashtable {
47 uint8_t bits; /* size of hashtable in bits */
48 uint32_t size, mask;
49 uint32_t count, count_max, count_keep; /* items in table */
50 uint32_t coeff; /* coefficient for Fibonacci hashing */
51 struct bucket **table;
52
53 struct {
54 uint64_t inserts, searches, deletions, rehashes;
55 } stats;
56
57 hash_func_t *hash_func;
58 /* returns hash value of given key (passed as void*) */
59
60 free_func_t *free_func;
61 /* free of bucket payload */
62
63 key_func_t *key_func;
64 /* returns pointer to key of bucket (to pass to hash_func) */
65
66 find_func_t *find_func;
67 /* returns true if given bucket matches key (passed as void*) */
68
69 make_func_t *make_func;
70 /* returns bucket containing new record with key (passed as void*) */
71
72 format_cols_func_t *format_cols_func;
73 /* append table columns to str */
74
75 format_row_func_t *format_row_func;
76 /* format record and append to str */
77 };
78
79 static void hashtable_reduce(struct hashtable *ht);
80 static void hashtable_free(struct hashtable *h);
81
82 #define HOST_BITS 1 /* initial size of hosts table */
83 #define PORT_BITS 1 /* initial size of ports tables */
84 #define PROTO_BITS 1 /* initial size of proto table */
85
86 /* We only use one hosts_db hashtable and this is it. */
87 static struct hashtable *hosts_db = NULL;
88
89 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
90 static const double phi_1 =
91 0.61803398874989490252573887119069695472717285156250;
92
93 /* Co-prime of u, using phi^-1 */
94 inline static uint32_t
95 coprime(const uint32_t u)
96 {
97 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
98 }
99
100 /*
101 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
102 * src/sys/netinet/tcp_hostcache.c 1.1
103 */
104 inline static uint32_t
105 ipv4_hash(const struct addr *const a)
106 {
107 uint32_t ip = a->ip.v4;
108 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
109 }
110
111 #ifndef s6_addr32
112 # ifdef sun
113 /*
114 * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/netinet/in.h#130
115 */
116 # define s6_addr32 _S6_un._S6_u32
117 # else
118 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
119 * taken care of GNU/Linux and GNU/kfreebsd. */
120 # define s6_addr32 __u6_addr.__u6_addr32
121 # endif
122 #endif
123
124 /*
125 * This is the IPv6 hash function used by FreeBSD in the same file as above,
126 * svn rev 122922.
127 */
128 inline static uint32_t
129 ipv6_hash(const struct addr *const a)
130 {
131 const struct in6_addr *const ip6 = &(a->ip.v6);
132 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
133 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
134 }
135
136 /* ---------------------------------------------------------------------------
137 * hash_func collection
138 */
139 static uint32_t
140 hash_func_host(const struct hashtable *h _unused_, const void *key)
141 {
142 const struct addr *a = key;
143 if (a->family == IPv4)
144 return (ipv4_hash(a));
145 else {
146 assert(a->family == IPv6);
147 return (ipv6_hash(a));
148 }
149 }
150
151 #define CASTKEY(type) (*((const type *)key))
152
153 static uint32_t
154 hash_func_short(const struct hashtable *h, const void *key)
155 {
156 return (CASTKEY(uint16_t) * h->coeff);
157 }
158
159 static uint32_t
160 hash_func_byte(const struct hashtable *h, const void *key)
161 {
162 return (CASTKEY(uint8_t) * h->coeff);
163 }
164
165 /* ---------------------------------------------------------------------------
166 * key_func collection
167 */
168
169 static const void *
170 key_func_host(const struct bucket *b)
171 {
172 return &(b->u.host.addr);
173 }
174
175 static const void *
176 key_func_port_tcp(const struct bucket *b)
177 {
178 return &(b->u.port_tcp.port);
179 }
180
181 static const void *
182 key_func_port_udp(const struct bucket *b)
183 {
184 return &(b->u.port_udp.port);
185 }
186
187 static const void *
188 key_func_ip_proto(const struct bucket *b)
189 {
190 return &(b->u.ip_proto.proto);
191 }
192
193 /* ---------------------------------------------------------------------------
194 * find_func collection
195 */
196
197 static int
198 find_func_host(const struct bucket *b, const void *key)
199 {
200 return (addr_equal(key, &(b->u.host.addr)));
201 }
202
203 static int
204 find_func_port_tcp(const struct bucket *b, const void *key)
205 {
206 return (b->u.port_tcp.port == CASTKEY(uint16_t));
207 }
208
209 static int
210 find_func_port_udp(const struct bucket *b, const void *key)
211 {
212 return (b->u.port_udp.port == CASTKEY(uint16_t));
213 }
214
215 static int
216 find_func_ip_proto(const struct bucket *b, const void *key)
217 {
218 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
219 }
220
221 /* ---------------------------------------------------------------------------
222 * make_func collection
223 */
224
225 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
226 struct bucket *next; \
227 uint64_t in, out, total; \
228 union { struct type t; } u; } _custom_bucket; \
229 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
230 struct type *name_content = &(name_bucket->u.type); \
231 name_bucket->next = NULL; \
232 name_bucket->in = name_bucket->out = name_bucket->total = 0;
233
234 static struct bucket *
235 make_func_host(const void *key)
236 {
237 MAKE_BUCKET(b, h, host);
238 h->addr = CASTKEY(struct addr);
239 h->dns = NULL;
240 h->last_seen_mono = 0;
241 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
242 h->ports_tcp = NULL;
243 h->ports_udp = NULL;
244 h->ip_protos = NULL;
245 return (b);
246 }
247
248 static void
249 free_func_host(struct bucket *b)
250 {
251 struct host *h = &(b->u.host);
252 if (h->dns != NULL) free(h->dns);
253 hashtable_free(h->ports_tcp);
254 hashtable_free(h->ports_udp);
255 hashtable_free(h->ip_protos);
256 }
257
258 static struct bucket *
259 make_func_port_tcp(const void *key)
260 {
261 MAKE_BUCKET(b, p, port_tcp);
262 p->port = CASTKEY(uint16_t);
263 p->syn = 0;
264 return (b);
265 }
266
267 static struct bucket *
268 make_func_port_udp(const void *key)
269 {
270 MAKE_BUCKET(b, p, port_udp);
271 p->port = CASTKEY(uint16_t);
272 return (b);
273 }
274
275 static struct bucket *
276 make_func_ip_proto(const void *key)
277 {
278 MAKE_BUCKET(b, p, ip_proto);
279 p->proto = CASTKEY(uint8_t);
280 return (b);
281 }
282
283 static void
284 free_func_simple(struct bucket *b _unused_)
285 {
286 /* nop */
287 }
288
289 /* ---------------------------------------------------------------------------
290 * format_func collection (ordered by struct)
291 */
292
293 static void
294 format_cols_host(struct str *buf)
295 {
296 /* FIXME: don't clobber parts of the query string
297 * specifically "full" and "start"
298 * when setting sort direction
299 */
300 str_append(buf,
301 "<table>\n"
302 "<tr>\n"
303 " <th>IP</th>\n"
304 " <th>Hostname</th>\n");
305 if (hosts_db_show_macs) str_append(buf,
306 " <th>MAC Address</th>\n");
307 str_append(buf,
308 " <th><a href=\"?sort=in\">In</a></th>\n"
309 " <th><a href=\"?sort=out\">Out</a></th>\n"
310 " <th><a href=\"?sort=total\">Total</a></th>\n");
311 if (opt_want_lastseen) str_append(buf,
312 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
313 str_append(buf,
314 "</tr>\n");
315 }
316
317 static void
318 format_row_host(struct str *buf, const struct bucket *b,
319 const char *css_class)
320 {
321 const char *ip = addr_to_str(&(b->u.host.addr));
322
323 str_appendf(buf,
324 "<tr class=\"%s\">\n"
325 " <td><a href=\"./%s/\">%s</a></td>\n"
326 " <td>%s</td>\n",
327 css_class,
328 ip, ip,
329 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
330
331 if (hosts_db_show_macs)
332 str_appendf(buf,
333 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
334 b->u.host.mac_addr[0],
335 b->u.host.mac_addr[1],
336 b->u.host.mac_addr[2],
337 b->u.host.mac_addr[3],
338 b->u.host.mac_addr[4],
339 b->u.host.mac_addr[5]);
340
341 str_appendf(buf,
342 " <td class=\"num\">%'qu</td>\n"
343 " <td class=\"num\">%'qu</td>\n"
344 " <td class=\"num\">%'qu</td>\n",
345 b->in, b->out, b->total);
346
347 if (opt_want_lastseen) {
348 long last = b->u.host.last_seen_mono;
349 struct str *last_str = NULL;
350
351 if ((now_mono() >= last) && (last > 0))
352 last_str = length_of_time(now_mono() - last);
353
354 str_append(buf, " <td class=\"num\">");
355 if (last_str == NULL) {
356 if (last == 0)
357 str_append(buf, "(never)");
358 else
359 str_append(buf, "(clock error)");
360 } else {
361 str_appendstr(buf, last_str);
362 str_free(last_str);
363 }
364 str_append(buf, "</td>");
365 }
366
367 str_appendf(buf, "</tr>\n");
368
369 /* Only resolve hosts "on demand" */
370 if (b->u.host.dns == NULL)
371 dns_queue(&(b->u.host.addr));
372 }
373
374 static void
375 format_cols_port_tcp(struct str *buf)
376 {
377 str_append(buf,
378 "<table>\n"
379 "<tr>\n"
380 " <th>Port</td>\n"
381 " <th>Service</td>\n"
382 " <th>In</td>\n"
383 " <th>Out</td>\n"
384 " <th>Total</td>\n"
385 " <th>SYNs</td>\n"
386 "</tr>\n"
387 );
388 }
389
390 static void
391 format_row_port_tcp(struct str *buf, const struct bucket *b,
392 const char *css_class)
393 {
394 const struct port_tcp *p = &(b->u.port_tcp);
395
396 str_appendf(buf,
397 "<tr class=\"%s\">\n"
398 " <td class=\"num\">%u</td>\n"
399 " <td>%s</td>\n"
400 " <td class=\"num\">%'qu</td>\n"
401 " <td class=\"num\">%'qu</td>\n"
402 " <td class=\"num\">%'qu</td>\n"
403 " <td class=\"num\">%'qu</td>\n"
404 "</tr>\n",
405 css_class,
406 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
407 );
408 }
409
410 static void
411 format_cols_port_udp(struct str *buf)
412 {
413 str_append(buf,
414 "<table>\n"
415 "<tr>\n"
416 " <th>Port</td>\n"
417 " <th>Service</td>\n"
418 " <th>In</td>\n"
419 " <th>Out</td>\n"
420 " <th>Total</td>\n"
421 "</tr>\n"
422 );
423 }
424
425 static void
426 format_row_port_udp(struct str *buf, const struct bucket *b,
427 const char *css_class)
428 {
429 const struct port_udp *p = &(b->u.port_udp);
430
431 str_appendf(buf,
432 "<tr class=\"%s\">\n"
433 " <td class=\"num\">%u</td>\n"
434 " <td>%s</td>\n"
435 " <td class=\"num\">%'qu</td>\n"
436 " <td class=\"num\">%'qu</td>\n"
437 " <td class=\"num\">%'qu</td>\n"
438 "</tr>\n",
439 css_class,
440 p->port, getservudp(p->port), b->in, b->out, b->total
441 );
442 }
443
444 static void
445 format_cols_ip_proto(struct str *buf)
446 {
447 str_append(buf,
448 "<table>\n"
449 "<tr>\n"
450 " <th>#</td>\n"
451 " <th>Protocol</td>\n"
452 " <th>In</td>\n"
453 " <th>Out</td>\n"
454 " <th>Total</td>\n"
455 "</tr>\n"
456 );
457 }
458
459 static void
460 format_row_ip_proto(struct str *buf, const struct bucket *b,
461 const char *css_class)
462 {
463 const struct ip_proto *p = &(b->u.ip_proto);
464
465 str_appendf(buf,
466 "<tr class=\"%s\">\n"
467 " <td class=\"num\">%u</td>\n"
468 " <td>%s</td>\n"
469 " <td class=\"num\">%'qu</td>\n"
470 " <td class=\"num\">%'qu</td>\n"
471 " <td class=\"num\">%'qu</td>\n"
472 "</tr>\n",
473 css_class,
474 p->proto, getproto(p->proto),
475 b->in, b->out, b->total
476 );
477 }
478
479 /* ---------------------------------------------------------------------------
480 * Initialise a hashtable.
481 */
482 static struct hashtable *
483 hashtable_make(const uint8_t bits,
484 const unsigned int count_max,
485 const unsigned int count_keep,
486 hash_func_t *hash_func,
487 free_func_t *free_func,
488 key_func_t *key_func,
489 find_func_t *find_func,
490 make_func_t *make_func,
491 format_cols_func_t *format_cols_func,
492 format_row_func_t *format_row_func)
493 {
494 struct hashtable *hash;
495 assert(bits > 0);
496
497 hash = xmalloc(sizeof(*hash));
498 hash->bits = bits;
499 hash->count_max = count_max;
500 hash->count_keep = count_keep;
501 hash->size = 1U << bits;
502 hash->mask = hash->size - 1;
503 hash->coeff = coprime(hash->size);
504 hash->hash_func = hash_func;
505 hash->free_func = free_func;
506 hash->key_func = key_func;
507 hash->find_func = find_func;
508 hash->make_func = make_func;
509 hash->format_cols_func = format_cols_func;
510 hash->format_row_func = format_row_func;
511 hash->count = 0;
512 hash->table = xcalloc(hash->size, sizeof(*hash->table));
513 memset(&(hash->stats), 0, sizeof(hash->stats));
514 return (hash);
515 }
516
517 /* ---------------------------------------------------------------------------
518 * Initialise global hosts_db.
519 */
520 void
521 hosts_db_init(void)
522 {
523 assert(hosts_db == NULL);
524 hosts_db = hashtable_make(HOST_BITS, opt_hosts_max, opt_hosts_keep,
525 hash_func_host, free_func_host, key_func_host, find_func_host,
526 make_func_host, format_cols_host, format_row_host);
527 }
528
529 static void
530 hashtable_rehash(struct hashtable *h, const uint8_t bits)
531 {
532 struct bucket **old_table, **new_table;
533 uint32_t i, old_size;
534 assert(h != NULL);
535 assert(bits > 0);
536
537 h->stats.rehashes++;
538 old_size = h->size;
539 old_table = h->table;
540
541 h->bits = bits;
542 h->size = 1U << bits;
543 h->mask = h->size - 1;
544 h->coeff = coprime(h->size);
545 new_table = xcalloc(h->size, sizeof(*new_table));
546
547 for (i=0; i<old_size; i++) {
548 struct bucket *next, *b = old_table[i];
549 while (b != NULL) {
550 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
551 next = b->next;
552 b->next = new_table[pos];
553 new_table[pos] = b;
554 b = next;
555 }
556 }
557
558 free(h->table);
559 h->table = new_table;
560 }
561
562 static void
563 hashtable_insert(struct hashtable *h, struct bucket *b)
564 {
565 uint32_t pos;
566 assert(h != NULL);
567 assert(b != NULL);
568 assert(b->next == NULL);
569
570 /* Rehash on 80% occupancy */
571 if ((h->count > h->size) ||
572 ((h->size - h->count) < h->size / 5))
573 hashtable_rehash(h, h->bits+1);
574
575 pos = h->hash_func(h, h->key_func(b)) & h->mask;
576 if (h->table[pos] == NULL)
577 h->table[pos] = b;
578 else {
579 /* Insert at top of chain. */
580 b->next = h->table[pos];
581 h->table[pos] = b;
582 }
583 h->count++;
584 h->stats.inserts++;
585 }
586
587 /* Return bucket matching key, or NULL if no such entry. */
588 static struct bucket *
589 hashtable_search(struct hashtable *h, const void *key)
590 {
591 uint32_t pos;
592 struct bucket *b;
593
594 h->stats.searches++;
595 pos = h->hash_func(h, key) & h->mask;
596 b = h->table[pos];
597 while (b != NULL) {
598 if (h->find_func(b, key))
599 return (b);
600 else
601 b = b->next;
602 }
603 return (NULL);
604 }
605
606 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
607 /* Search for a key. If it's not there, make and insert a bucket for it. */
608 static struct bucket *
609 hashtable_find_or_insert(struct hashtable *h, const void *key,
610 const reduce_bool allow_reduce)
611 {
612 struct bucket *b = hashtable_search(h, key);
613
614 if (b == NULL) {
615 /* Not found, so insert after checking occupancy. */
616 if (allow_reduce && (h->count >= h->count_max))
617 hashtable_reduce(h);
618 b = h->make_func(key);
619 hashtable_insert(h, b);
620 }
621 return (b);
622 }
623
624 /*
625 * Frees the hashtable and the buckets. The contents are assumed to be
626 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
627 * the bucket.
628 */
629 static void
630 hashtable_free(struct hashtable *h)
631 {
632 uint32_t i;
633
634 if (h == NULL)
635 return;
636 for (i=0; i<h->size; i++) {
637 struct bucket *tmp, *b = h->table[i];
638 while (b != NULL) {
639 tmp = b;
640 b = b->next;
641 h->free_func(tmp);
642 free(tmp);
643 }
644 }
645 free(h->table);
646 free(h);
647 }
648
649 /* ---------------------------------------------------------------------------
650 * Return existing host or insert a new one.
651 */
652 struct bucket *
653 host_get(const struct addr *const a)
654 {
655 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
656 }
657
658 /* ---------------------------------------------------------------------------
659 * Find host, returns NULL if not in DB.
660 */
661 struct bucket *
662 host_find(const struct addr *const a)
663 {
664 return (hashtable_search(hosts_db, a));
665 }
666
667 /* ---------------------------------------------------------------------------
668 * Find host, returns NULL if not in DB.
669 */
670 static struct bucket *
671 host_search(const char *ipstr)
672 {
673 struct addr a;
674 struct addrinfo hints, *ai;
675
676 memset(&hints, 0, sizeof(hints));
677 hints.ai_family = AF_UNSPEC;
678 hints.ai_flags = AI_NUMERICHOST;
679
680 if (getaddrinfo(ipstr, NULL, &hints, &ai))
681 return (NULL); /* invalid addr */
682
683 if (ai->ai_family == AF_INET) {
684 a.family = IPv4;
685 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
686 }
687 else if (ai->ai_family == AF_INET6) {
688 a.family = IPv6;
689 memcpy(&(a.ip.v6),
690 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
691 sizeof(a.ip.v6));
692 } else {
693 freeaddrinfo(ai);
694 return (NULL); /* unknown family */
695 }
696 freeaddrinfo(ai);
697
698 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
699 return (hashtable_search(hosts_db, &a));
700 }
701
702 /* ---------------------------------------------------------------------------
703 * Reduce a hashtable to the top <keep> entries.
704 */
705 static void
706 hashtable_reduce(struct hashtable *ht)
707 {
708 uint32_t i, pos, rmd;
709 const struct bucket **table;
710 uint64_t cutoff;
711
712 assert(ht->count_keep < ht->count);
713
714 /* Fill table with pointers to buckets in hashtable. */
715 table = xcalloc(ht->count, sizeof(*table));
716 for (pos=0, i=0; i<ht->size; i++) {
717 struct bucket *b = ht->table[i];
718 while (b != NULL) {
719 table[pos++] = b;
720 b = b->next;
721 }
722 }
723 assert(pos == ht->count);
724 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
725 cutoff = table[ht->count_keep]->total;
726 free(table);
727
728 /* Remove all elements with total <= cutoff. */
729 rmd = 0;
730 for (i=0; i<ht->size; i++) {
731 struct bucket *last = NULL, *next, *b = ht->table[i];
732 while (b != NULL) {
733 next = b->next;
734 if (b->total <= cutoff) {
735 /* Remove this one. */
736 ht->free_func(b);
737 free(b);
738 if (last == NULL)
739 ht->table[i] = next;
740 else
741 last->next = next;
742 rmd++;
743 ht->count--;
744 } else {
745 last = b;
746 }
747 b = next;
748 }
749 }
750 verbosef("hashtable_reduce: removed %u buckets, left %u",
751 rmd, ht->count);
752 hashtable_rehash(ht, ht->bits); /* is this needed? */
753 }
754
755 /* Reduce hosts_db if needed. */
756 void hosts_db_reduce(void)
757 {
758 if (hosts_db->count >= hosts_db->count_max)
759 hashtable_reduce(hosts_db);
760 }
761
762 /* ---------------------------------------------------------------------------
763 * Reset hosts_db to empty.
764 */
765 void
766 hosts_db_reset(void)
767 {
768 unsigned int i;
769
770 for (i=0; i<hosts_db->size; i++) {
771 struct bucket *next, *b = hosts_db->table[i];
772 while (b != NULL) {
773 next = b->next;
774 hosts_db->free_func(b);
775 free(b);
776 b = next;
777 }
778 hosts_db->table[i] = NULL;
779 }
780 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
781 hosts_db->count = 0;
782 }
783
784 /* ---------------------------------------------------------------------------
785 * Deallocate hosts_db.
786 */
787 void hosts_db_free(void)
788 {
789 uint32_t i;
790
791 assert(hosts_db != NULL);
792 for (i=0; i<hosts_db->size; i++) {
793 struct bucket *tmp, *b = hosts_db->table[i];
794 while (b != NULL) {
795 tmp = b;
796 b = b->next;
797 hosts_db->free_func(tmp);
798 free(tmp);
799 }
800 }
801 free(hosts_db->table);
802 free(hosts_db);
803 hosts_db = NULL;
804 }
805
806 /* ---------------------------------------------------------------------------
807 * Find or create a port_tcp inside a host.
808 */
809 struct bucket *
810 host_get_port_tcp(struct bucket *host, const uint16_t port)
811 {
812 struct host *h = &host->u.host;
813 assert(h != NULL);
814 if (h->ports_tcp == NULL)
815 h->ports_tcp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
816 hash_func_short, free_func_simple, key_func_port_tcp,
817 find_func_port_tcp, make_func_port_tcp,
818 format_cols_port_tcp, format_row_port_tcp);
819 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
820 }
821
822 /* ---------------------------------------------------------------------------
823 * Find or create a port_udp inside a host.
824 */
825 struct bucket *
826 host_get_port_udp(struct bucket *host, const uint16_t port)
827 {
828 struct host *h = &host->u.host;
829 assert(h != NULL);
830 if (h->ports_udp == NULL)
831 h->ports_udp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
832 hash_func_short, free_func_simple, key_func_port_udp,
833 find_func_port_udp, make_func_port_udp,
834 format_cols_port_udp, format_row_port_udp);
835 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
836 }
837
838 /* ---------------------------------------------------------------------------
839 * Find or create an ip_proto inside a host.
840 */
841 struct bucket *
842 host_get_ip_proto(struct bucket *host, const uint8_t proto)
843 {
844 struct host *h = &host->u.host;
845 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
846 assert(h != NULL);
847 if (h->ip_protos == NULL)
848 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
849 hash_func_byte, free_func_simple, key_func_ip_proto,
850 find_func_ip_proto, make_func_ip_proto,
851 format_cols_ip_proto, format_row_ip_proto);
852 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
853 }
854
855 static struct str *html_hosts_main(const char *qs);
856 static struct str *html_hosts_detail(const char *ip);
857
858 /* ---------------------------------------------------------------------------
859 * Web interface: delegate the /hosts/ space.
860 */
861 struct str *
862 html_hosts(const char *uri, const char *query)
863 {
864 unsigned int i, num_elems;
865 char **elem = split('/', uri, &num_elems);
866 struct str *buf = NULL;
867
868 assert(num_elems >= 1);
869 assert(strcmp(elem[0], "hosts") == 0);
870
871 if (num_elems == 1)
872 /* /hosts/ */
873 buf = html_hosts_main(query);
874 else if (num_elems == 2)
875 /* /hosts/<IP of host>/ */
876 buf = html_hosts_detail(elem[1]);
877
878 for (i=0; i<num_elems; i++)
879 free(elem[i]);
880 free(elem);
881 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
882 other codes to be possible */
883 }
884
885 /* ---------------------------------------------------------------------------
886 * Format hashtable into HTML.
887 */
888 static void
889 format_table(struct str *buf, struct hashtable *ht, unsigned int start,
890 const enum sort_dir sort, const int full)
891 {
892 const struct bucket **table;
893 unsigned int i, pos, end;
894 int alt = 0;
895
896 if ((ht == NULL) || (ht->count == 0)) {
897 str_append(buf, "<p>The table is empty.</p>\n");
898 return;
899 }
900
901 /* Fill table with pointers to buckets in hashtable. */
902 table = xcalloc(ht->count, sizeof(*table));
903 for (pos=0, i=0; i<ht->size; i++) {
904 struct bucket *b = ht->table[i];
905 while (b != NULL) {
906 table[pos++] = b;
907 b = b->next;
908 }
909 }
910 assert(pos == ht->count);
911
912 if (full) {
913 /* full report overrides start and end */
914 start = 0;
915 end = ht->count;
916 } else
917 end = MIN(ht->count, (uint32_t)start+MAX_ENTRIES);
918
919 str_appendf(buf, "(%u-%u of %u)<br>\n", start+1, end, ht->count);
920 qsort_buckets(table, ht->count, start, end, sort);
921 ht->format_cols_func(buf);
922
923 for (i=start; i<end; i++) {
924 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
925 alt = !alt; /* alternate class for table rows */
926 }
927 free(table);
928 str_append(buf, "</table>\n");
929 }
930
931 /* ---------------------------------------------------------------------------
932 * Web interface: sorted table of hosts.
933 */
934 static struct str *
935 html_hosts_main(const char *qs)
936 {
937 struct str *buf = str_make();
938 char *qs_start, *qs_sort, *qs_full, *ep;
939 const char *sortstr;
940 int start, full = 0;
941 enum sort_dir sort;
942
943 /* parse query string */
944 qs_start = qs_get(qs, "start");
945 qs_sort = qs_get(qs, "sort");
946 qs_full = qs_get(qs, "full");
947 if (qs_full != NULL) {
948 full = 1;
949 free(qs_full);
950 }
951
952 /* validate sort */
953 if (qs_sort == NULL) sort = TOTAL;
954 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
955 else if (strcmp(qs_sort, "in") == 0) sort = IN;
956 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
957 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
958 else {
959 str_append(buf, "Error: invalid value for \"sort\".\n");
960 goto done;
961 }
962
963 /* parse start */
964 if (qs_start == NULL)
965 start = 0;
966 else {
967 start = (int)strtoul(qs_start, &ep, 10);
968 if (*ep != '\0') {
969 str_append(buf, "Error: \"start\" is not a number.\n");
970 goto done;
971 }
972 if ((errno == ERANGE) ||
973 (start < 0) || (start >= (int)hosts_db->count)) {
974 str_append(buf, "Error: \"start\" is out of bounds.\n");
975 goto done;
976 }
977 }
978
979 #define PREV "&lt;&lt;&lt; prev page"
980 #define NEXT "next page &gt;&gt;&gt;"
981 #define FULL "full table"
982
983 html_open(buf, "Hosts", /*path_depth=*/1, /*want_graph_js=*/0);
984 format_table(buf, hosts_db, start, sort, full);
985
986 /* <prev | full | stats | next> */
987 sortstr = qs_sort;
988 if (sortstr == NULL) sortstr = "total";
989 if (start > 0) {
990 int prev = start - MAX_ENTRIES;
991 if (prev < 0)
992 prev = 0;
993 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
994 prev, sortstr);
995 } else
996 str_append(buf, PREV);
997
998 if (full)
999 str_append(buf, " | " FULL);
1000 else
1001 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
1002 sortstr);
1003
1004 if (start+MAX_ENTRIES < (int)hosts_db->count)
1005 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
1006 start+MAX_ENTRIES, sortstr);
1007 else
1008 str_append(buf, " | " NEXT);
1009
1010 str_append(buf, "<br>\n");
1011
1012 html_close(buf);
1013 done:
1014 if (qs_start != NULL) free(qs_start);
1015 if (qs_sort != NULL) free(qs_sort);
1016 return buf;
1017 #undef PREV
1018 #undef NEXT
1019 #undef FULL
1020 }
1021
1022 /* ---------------------------------------------------------------------------
1023 * Web interface: detailed view of a single host.
1024 */
1025 static struct str *html_hosts_detail(const char *ip) {
1026 struct bucket *h;
1027 struct str *buf, *ls_len;
1028 char ls_when[100];
1029 const char *canonical;
1030 time_t last_real;
1031
1032 h = host_search(ip);
1033 if (h == NULL)
1034 return (NULL); /* no such host */
1035
1036 canonical = addr_to_str(&(h->u.host.addr));
1037
1038 /* Overview. */
1039 buf = str_make();
1040 html_open(buf, ip, /*path_depth=*/2, /*want_graph_js=*/0);
1041 if (strcmp(ip, canonical) != 0)
1042 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1043 str_appendf(buf,
1044 "<p>\n"
1045 "<b>Hostname:</b> %s<br>\n",
1046 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1047
1048 /* Resolve host "on demand" */
1049 if (h->u.host.dns == NULL)
1050 dns_queue(&(h->u.host.addr));
1051
1052 if (hosts_db_show_macs)
1053 str_appendf(buf,
1054 "<b>MAC Address:</b> "
1055 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1056 h->u.host.mac_addr[0],
1057 h->u.host.mac_addr[1],
1058 h->u.host.mac_addr[2],
1059 h->u.host.mac_addr[3],
1060 h->u.host.mac_addr[4],
1061 h->u.host.mac_addr[5]);
1062
1063 str_append(buf,
1064 "</p>\n"
1065 "<p>\n"
1066 "<b>Last seen:</b> ");
1067
1068 last_real = mono_to_real(h->u.host.last_seen_mono);
1069 if (strftime(ls_when, sizeof(ls_when),
1070 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&last_real)) != 0)
1071 str_append(buf, ls_when);
1072
1073 if (h->u.host.last_seen_mono <= now_mono()) {
1074 ls_len = length_of_time(now_mono() - h->u.host.last_seen_mono);
1075 str_append(buf, " (");
1076 str_appendstr(buf, ls_len);
1077 str_free(ls_len);
1078 str_append(buf, " ago)");
1079 } else {
1080 str_append(buf, " (in the future, possible clock problem)");
1081 }
1082
1083 str_appendf(buf,
1084 "</p>\n"
1085 "<p>\n"
1086 " <b>In:</b> %'qu<br>\n"
1087 " <b>Out:</b> %'qu<br>\n"
1088 " <b>Total:</b> %'qu<br>\n"
1089 "</p>\n",
1090 h->in, h->out, h->total);
1091
1092 str_append(buf, "<h3>TCP ports</h3>\n");
1093 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1094
1095 str_append(buf, "<h3>UDP ports</h3>\n");
1096 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1097
1098 str_append(buf, "<h3>IP protocols</h3>\n");
1099 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1100
1101 html_close(buf);
1102 return buf;
1103 }
1104
1105 /* ---------------------------------------------------------------------------
1106 * Database import and export code:
1107 * Initially written and contributed by Ben Stewart.
1108 * copyright (c) 2007-2011 Ben Stewart, Emil Mikulic.
1109 */
1110 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1111 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1112 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1113
1114 static const char
1115 export_proto_ip = 'P',
1116 export_proto_tcp = 'T',
1117 export_proto_udp = 'U';
1118
1119 static const unsigned char
1120 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1121 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1122 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1123
1124 /* ---------------------------------------------------------------------------
1125 * Load a host's ip_proto table from a file.
1126 * Returns 0 on failure, 1 on success.
1127 */
1128 static int
1129 hosts_db_import_ip(const int fd, struct bucket *host)
1130 {
1131 uint8_t count, i;
1132
1133 if (!expect8(fd, export_proto_ip)) return 0;
1134 if (!read8(fd, &count)) return 0;
1135
1136 for (i=0; i<count; i++) {
1137 struct bucket *b;
1138 uint8_t proto;
1139 uint64_t in, out;
1140
1141 if (!read8(fd, &proto)) return 0;
1142 if (!read64(fd, &in)) return 0;
1143 if (!read64(fd, &out)) return 0;
1144
1145 /* Store data */
1146 b = host_get_ip_proto(host, proto);
1147 b->in = in;
1148 b->out = out;
1149 b->total = in + out;
1150 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1151 }
1152 return 1;
1153 }
1154
1155 /* ---------------------------------------------------------------------------
1156 * Load a host's port_tcp table from a file.
1157 * Returns 0 on failure, 1 on success.
1158 */
1159 static int
1160 hosts_db_import_tcp(const int fd, struct bucket *host)
1161 {
1162 uint16_t count, i;
1163
1164 if (!expect8(fd, export_proto_tcp)) return 0;
1165 if (!read16(fd, &count)) return 0;
1166
1167 for (i=0; i<count; i++) {
1168 struct bucket *b;
1169 uint16_t port;
1170 uint64_t in, out, syn;
1171
1172 if (!read16(fd, &port)) return 0;
1173 if (!read64(fd, &syn)) return 0;
1174 if (!read64(fd, &in)) return 0;
1175 if (!read64(fd, &out)) return 0;
1176
1177 /* Store data */
1178 b = host_get_port_tcp(host, port);
1179 b->in = in;
1180 b->out = out;
1181 b->total = in + out;
1182 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1183 b->u.port_tcp.syn = syn;
1184 }
1185 return 1;
1186 }
1187
1188 /* ---------------------------------------------------------------------------
1189 * Load a host's port_tcp table from a file.
1190 * Returns 0 on failure, 1 on success.
1191 */
1192 static int
1193 hosts_db_import_udp(const int fd, struct bucket *host)
1194 {
1195 uint16_t count, i;
1196
1197 if (!expect8(fd, export_proto_udp)) return 0;
1198 if (!read16(fd, &count)) return 0;
1199
1200 for (i=0; i<count; i++) {
1201 struct bucket *b;
1202 uint16_t port;
1203 uint64_t in, out;
1204
1205 if (!read16(fd, &port)) return 0;
1206 if (!read64(fd, &in)) return 0;
1207 if (!read64(fd, &out)) return 0;
1208
1209 /* Store data */
1210 b = host_get_port_udp(host, port);
1211 b->in = in;
1212 b->out = out;
1213 b->total = in + out;
1214 assert(b->u.port_udp.port == port); /* done by make_func */
1215 }
1216 return 1;
1217 }
1218
1219 /* ---------------------------------------------------------------------------
1220 * Load all hosts from a file.
1221 * Returns 0 on failure, 1 on success.
1222 */
1223 static int
1224 hosts_db_import_host(const int fd)
1225 {
1226 struct bucket *host;
1227 struct addr a;
1228 uint8_t hostname_len;
1229 uint64_t in, out;
1230 unsigned int pos = xtell(fd);
1231 char hdr[4];
1232 int ver = 0;
1233
1234 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1235 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1236 ver = 3;
1237 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1238 ver = 2;
1239 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1240 ver = 1;
1241 else {
1242 warnx("bad host header: %02x%02x%02x%02x",
1243 hdr[0], hdr[1], hdr[2], hdr[3]);
1244 return 0;
1245 }
1246
1247 if (ver == 3) {
1248 if (!readaddr(fd, &a))
1249 return 0;
1250 } else {
1251 assert((ver == 1) || (ver == 2));
1252 if (!readaddr_ipv4(fd, &a))
1253 return 0;
1254 }
1255 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1256 host = host_get(&a);
1257 assert(addr_equal(&(host->u.host.addr), &a));
1258
1259 if (ver > 1) {
1260 uint64_t t;
1261 if (!read64(fd, &t)) return 0;
1262 host->u.host.last_seen_mono = real_to_mono(t);
1263 }
1264
1265 assert(sizeof(host->u.host.mac_addr) == 6);
1266 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1267 return 0;
1268
1269 /* HOSTNAME */
1270 assert(host->u.host.dns == NULL); /* make fn? */
1271 if (!read8(fd, &hostname_len)) return 0;
1272 if (hostname_len > 0) {
1273 host->u.host.dns = xmalloc(hostname_len + 1);
1274 host->u.host.dns[0] = '\0';
1275
1276 /* At this point, the hostname is attached to a host which is in our
1277 * hosts_db, so if we bail out due to an import error, this pointer
1278 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1279 */
1280
1281 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1282 host->u.host.dns[hostname_len] = '\0';
1283 }
1284
1285 if (!read64(fd, &in)) return 0;
1286 if (!read64(fd, &out)) return 0;
1287
1288 host->in = in;
1289 host->out = out;
1290 host->total = in + out;
1291
1292 /* Host's port and proto subtables: */
1293 if (!hosts_db_import_ip(fd, host)) return 0;
1294 if (!hosts_db_import_tcp(fd, host)) return 0;
1295 if (!hosts_db_import_udp(fd, host)) return 0;
1296 return 1;
1297 }
1298
1299 /* ---------------------------------------------------------------------------
1300 * Database Import: Grab hosts_db from a file provided by the caller.
1301 *
1302 * This function will retrieve the data sans the header. We expect the caller
1303 * to have validated the header of the hosts_db segment, and left the file
1304 * sitting at the start of the data.
1305 */
1306 int hosts_db_import(const int fd)
1307 {
1308 uint32_t host_count, i;
1309
1310 if (!read32(fd, &host_count)) return 0;
1311
1312 for (i=0; i<host_count; i++)
1313 if (!hosts_db_import_host(fd)) return 0;
1314
1315 return 1;
1316 }
1317
1318 /* ---------------------------------------------------------------------------
1319 * Database Export: Dump hosts_db into a file provided by the caller.
1320 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1321 */
1322 int hosts_db_export(const int fd)
1323 {
1324 uint32_t i;
1325 struct bucket *b;
1326
1327 if (!write32(fd, hosts_db->count)) return 0;
1328
1329 for (i = 0; i<hosts_db->size; i++)
1330 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1331 /* For each host: */
1332 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1333 return 0;
1334
1335 if (!writeaddr(fd, &(b->u.host.addr)))
1336 return 0;
1337
1338 if (!write64(fd, (uint64_t)mono_to_real(b->u.host.last_seen_mono)))
1339 return 0;
1340
1341 assert(sizeof(b->u.host.mac_addr) == 6);
1342 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1343 return 0;
1344
1345 /* HOSTNAME */
1346 if (b->u.host.dns == NULL) {
1347 if (!write8(fd, 0)) return 0;
1348 } else {
1349 int dnslen = strlen(b->u.host.dns);
1350
1351 if (dnslen > 255) {
1352 warnx("found a very long hostname: \"%s\"\n"
1353 "wasn't expecting one longer than 255 chars (this one is %d)",
1354 b->u.host.dns, dnslen);
1355 dnslen = 255;
1356 }
1357
1358 if (!write8(fd, (uint8_t)dnslen)) return 0;
1359 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1360 }
1361
1362 if (!write64(fd, b->in)) return 0;
1363 if (!write64(fd, b->out)) return 0;
1364
1365 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1366 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1367 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1368 }
1369 return 1;
1370 }
1371
1372 /* ---------------------------------------------------------------------------
1373 * Dump the ip_proto table of a host.
1374 */
1375 static int
1376 hosts_db_export_ip(const struct hashtable *h, const int fd)
1377 {
1378 uint32_t i, written = 0;
1379 struct bucket *b;
1380
1381 /* IP DATA */
1382 if (!write8(fd, export_proto_ip)) return 0;
1383
1384 /* If no data, write a IP Proto count of 0 and we're done. */
1385 if (h == NULL) {
1386 if (!write8(fd, 0)) return 0;
1387 return 1;
1388 }
1389
1390 assert(h->count < 256);
1391 if (!write8(fd, (uint8_t)h->count)) return 0;
1392
1393 for (i = 0; i<h->size; i++)
1394 for (b = h->table[i]; b != NULL; b = b->next) {
1395 /* For each ip_proto bucket: */
1396
1397 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1398 if (!write64(fd, b->in)) return 0;
1399 if (!write64(fd, b->out)) return 0;
1400 written++;
1401 }
1402 assert(written == h->count);
1403 return 1;
1404 }
1405
1406 /* ---------------------------------------------------------------------------
1407 * Dump the port_tcp table of a host.
1408 */
1409 static int
1410 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1411 {
1412 struct bucket *b;
1413 uint32_t i, written = 0;
1414
1415 /* TCP DATA */
1416 if (!write8(fd, export_proto_tcp)) return 0;
1417
1418 /* If no data, write a count of 0 and we're done. */
1419 if (h == NULL) {
1420 if (!write16(fd, 0)) return 0;
1421 return 1;
1422 }
1423
1424 assert(h->count < 65536);
1425 if (!write16(fd, (uint16_t)h->count)) return 0;
1426
1427 for (i = 0; i<h->size; i++)
1428 for (b = h->table[i]; b != NULL; b = b->next) {
1429 if (!write16(fd, b->u.port_tcp.port)) return 0;
1430 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1431 if (!write64(fd, b->in)) return 0;
1432 if (!write64(fd, b->out)) return 0;
1433 written++;
1434 }
1435 assert(written == h->count);
1436 return 1;
1437 }
1438
1439 /* ---------------------------------------------------------------------------
1440 * Dump the port_udp table of a host.
1441 */
1442 static int
1443 hosts_db_export_udp(const struct hashtable *h, const int fd)
1444 {
1445 struct bucket *b;
1446 uint32_t i, written = 0;
1447
1448 /* UDP DATA */
1449 if (!write8(fd, export_proto_udp)) return 0;
1450
1451 /* If no data, write a count of 0 and we're done. */
1452 if (h == NULL) {
1453 if (!write16(fd, 0)) return 0;
1454 return 1;
1455 }
1456
1457 assert(h->count < 65536);
1458 if (!write16(fd, (uint16_t)h->count)) return 0;
1459
1460 for (i = 0; i<h->size; i++)
1461 for (b = h->table[i]; b != NULL; b = b->next) {
1462 if (!write16(fd, b->u.port_udp.port)) return 0;
1463 if (!write64(fd, b->in)) return 0;
1464 if (!write64(fd, b->out)) return 0;
1465 written++;
1466 }
1467 assert(written == h->count);
1468 return 1;
1469 }
1470
1471 /* vim:set ts=3 sw=3 tw=78 expandtab: */