We don't use inet_aton anymore, drop the <arpa/inet.h> includes.
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2011 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "cdefs.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20 #include "opt.h"
21 #include "str.h"
22
23 #include <netdb.h> /* struct addrinfo */
24 #include <assert.h>
25 #include <errno.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h> /* memset(), strcmp() */
29 #include <unistd.h>
30
31 int hosts_db_show_macs = 0;
32
33 /* FIXME: specify somewhere more sane/tunable */
34 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
35
36 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
37 typedef void (free_func_t)(struct bucket *);
38 typedef const void * (key_func_t)(const struct bucket *);
39 typedef int (find_func_t)(const struct bucket *, const void *);
40 typedef struct bucket * (make_func_t)(const void *);
41 typedef void (format_cols_func_t)(struct str *);
42 typedef void (format_row_func_t)(struct str *, const struct bucket *,
43 const char *);
44
45 struct hashtable {
46 uint8_t bits; /* size of hashtable in bits */
47 uint32_t size, mask;
48 uint32_t count, count_max, count_keep; /* items in table */
49 uint32_t coeff; /* coefficient for Fibonacci hashing */
50 struct bucket **table;
51
52 struct {
53 uint64_t inserts, searches, deletions, rehashes;
54 } stats;
55
56 hash_func_t *hash_func;
57 /* returns hash value of given key (passed as void*) */
58
59 free_func_t *free_func;
60 /* free of bucket payload */
61
62 key_func_t *key_func;
63 /* returns pointer to key of bucket (to pass to hash_func) */
64
65 find_func_t *find_func;
66 /* returns true if given bucket matches key (passed as void*) */
67
68 make_func_t *make_func;
69 /* returns bucket containing new record with key (passed as void*) */
70
71 format_cols_func_t *format_cols_func;
72 /* append table columns to str */
73
74 format_row_func_t *format_row_func;
75 /* format record and append to str */
76 };
77
78 static void hashtable_reduce(struct hashtable *ht);
79 static void hashtable_free(struct hashtable *h);
80
81 #define HOST_BITS 1 /* initial size of hosts table */
82 #define PORT_BITS 1 /* initial size of ports tables */
83 #define PROTO_BITS 1 /* initial size of proto table */
84
85 /* We only use one hosts_db hashtable and this is it. */
86 static struct hashtable *hosts_db = NULL;
87
88 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
89 static const double phi_1 =
90 0.61803398874989490252573887119069695472717285156250;
91
92 /* Co-prime of u, using phi^-1 */
93 inline static uint32_t
94 coprime(const uint32_t u)
95 {
96 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
97 }
98
99 /*
100 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
101 * src/sys/netinet/tcp_hostcache.c 1.1
102 */
103 inline static uint32_t
104 ipv4_hash(const struct addr *const a)
105 {
106 uint32_t ip = a->ip.v4;
107 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
108 }
109
110 #ifndef s6_addr32
111 # ifdef sun
112 /*
113 * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/netinet/in.h#130
114 */
115 # define s6_addr32 _S6_un._S6_u32
116 # else
117 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
118 * taken care of GNU/Linux and GNU/kfreebsd. */
119 # define s6_addr32 __u6_addr.__u6_addr32
120 # endif
121 #endif
122
123 /*
124 * This is the IPv6 hash function used by FreeBSD in the same file as above,
125 * svn rev 122922.
126 */
127 inline static uint32_t
128 ipv6_hash(const struct addr *const a)
129 {
130 const struct in6_addr *const ip6 = &(a->ip.v6);
131 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
132 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
133 }
134
135 /* ---------------------------------------------------------------------------
136 * hash_func collection
137 */
138 static uint32_t
139 hash_func_host(const struct hashtable *h _unused_, const void *key)
140 {
141 const struct addr *a = key;
142 if (a->family == IPv4)
143 return (ipv4_hash(a));
144 else {
145 assert(a->family == IPv6);
146 return (ipv6_hash(a));
147 }
148 }
149
150 #define CASTKEY(type) (*((const type *)key))
151
152 static uint32_t
153 hash_func_short(const struct hashtable *h, const void *key)
154 {
155 return (CASTKEY(uint16_t) * h->coeff);
156 }
157
158 static uint32_t
159 hash_func_byte(const struct hashtable *h, const void *key)
160 {
161 return (CASTKEY(uint8_t) * h->coeff);
162 }
163
164 /* ---------------------------------------------------------------------------
165 * key_func collection
166 */
167
168 static const void *
169 key_func_host(const struct bucket *b)
170 {
171 return &(b->u.host.addr);
172 }
173
174 static const void *
175 key_func_port_tcp(const struct bucket *b)
176 {
177 return &(b->u.port_tcp.port);
178 }
179
180 static const void *
181 key_func_port_udp(const struct bucket *b)
182 {
183 return &(b->u.port_udp.port);
184 }
185
186 static const void *
187 key_func_ip_proto(const struct bucket *b)
188 {
189 return &(b->u.ip_proto.proto);
190 }
191
192 /* ---------------------------------------------------------------------------
193 * find_func collection
194 */
195
196 static int
197 find_func_host(const struct bucket *b, const void *key)
198 {
199 return (addr_equal(key, &(b->u.host.addr)));
200 }
201
202 static int
203 find_func_port_tcp(const struct bucket *b, const void *key)
204 {
205 return (b->u.port_tcp.port == CASTKEY(uint16_t));
206 }
207
208 static int
209 find_func_port_udp(const struct bucket *b, const void *key)
210 {
211 return (b->u.port_udp.port == CASTKEY(uint16_t));
212 }
213
214 static int
215 find_func_ip_proto(const struct bucket *b, const void *key)
216 {
217 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
218 }
219
220 /* ---------------------------------------------------------------------------
221 * make_func collection
222 */
223
224 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
225 struct bucket *next; \
226 uint64_t in, out, total; \
227 union { struct type t; } u; } _custom_bucket; \
228 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
229 struct type *name_content = &(name_bucket->u.type); \
230 name_bucket->next = NULL; \
231 name_bucket->in = name_bucket->out = name_bucket->total = 0;
232
233 static struct bucket *
234 make_func_host(const void *key)
235 {
236 MAKE_BUCKET(b, h, host);
237 h->addr = CASTKEY(struct addr);
238 h->dns = NULL;
239 h->lastseen = 0;
240 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
241 h->ports_tcp = NULL;
242 h->ports_udp = NULL;
243 h->ip_protos = NULL;
244 return (b);
245 }
246
247 static void
248 free_func_host(struct bucket *b)
249 {
250 struct host *h = &(b->u.host);
251 if (h->dns != NULL) free(h->dns);
252 hashtable_free(h->ports_tcp);
253 hashtable_free(h->ports_udp);
254 hashtable_free(h->ip_protos);
255 }
256
257 static struct bucket *
258 make_func_port_tcp(const void *key)
259 {
260 MAKE_BUCKET(b, p, port_tcp);
261 p->port = CASTKEY(uint16_t);
262 p->syn = 0;
263 return (b);
264 }
265
266 static struct bucket *
267 make_func_port_udp(const void *key)
268 {
269 MAKE_BUCKET(b, p, port_udp);
270 p->port = CASTKEY(uint16_t);
271 return (b);
272 }
273
274 static struct bucket *
275 make_func_ip_proto(const void *key)
276 {
277 MAKE_BUCKET(b, p, ip_proto);
278 p->proto = CASTKEY(uint8_t);
279 return (b);
280 }
281
282 static void
283 free_func_simple(struct bucket *b _unused_)
284 {
285 /* nop */
286 }
287
288 /* ---------------------------------------------------------------------------
289 * format_func collection (ordered by struct)
290 */
291
292 static void
293 format_cols_host(struct str *buf)
294 {
295 /* FIXME: don't clobber parts of the query string
296 * specifically "full" and "start"
297 * when setting sort direction
298 */
299 str_append(buf,
300 "<table>\n"
301 "<tr>\n"
302 " <th>IP</th>\n"
303 " <th>Hostname</th>\n");
304 if (hosts_db_show_macs) str_append(buf,
305 " <th>MAC Address</th>\n");
306 str_append(buf,
307 " <th><a href=\"?sort=in\">In</a></th>\n"
308 " <th><a href=\"?sort=out\">Out</a></th>\n"
309 " <th><a href=\"?sort=total\">Total</a></th>\n");
310 if (opt_want_lastseen) str_append(buf,
311 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
312 str_append(buf,
313 "</tr>\n");
314 }
315
316 static void
317 format_row_host(struct str *buf, const struct bucket *b,
318 const char *css_class)
319 {
320 const char *ip = addr_to_str(&(b->u.host.addr));
321
322 str_appendf(buf,
323 "<tr class=\"%s\">\n"
324 " <td><a href=\"./%s/\">%s</a></td>\n"
325 " <td>%s</td>\n",
326 css_class,
327 ip, ip,
328 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
329
330 if (hosts_db_show_macs)
331 str_appendf(buf,
332 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
333 b->u.host.mac_addr[0],
334 b->u.host.mac_addr[1],
335 b->u.host.mac_addr[2],
336 b->u.host.mac_addr[3],
337 b->u.host.mac_addr[4],
338 b->u.host.mac_addr[5]);
339
340 str_appendf(buf,
341 " <td class=\"num\">%'qu</td>\n"
342 " <td class=\"num\">%'qu</td>\n"
343 " <td class=\"num\">%'qu</td>\n",
344 b->in, b->out, b->total);
345
346 if (opt_want_lastseen) {
347 time_t last_t = b->u.host.lastseen;
348 struct str *last_str = NULL;
349
350 if ((now >= last_t) && (last_t > 0))
351 last_str = length_of_time(now - last_t);
352
353 str_append(buf,
354 " <td class=\"num\">");
355 if (last_str == NULL) {
356 if (last_t == 0)
357 str_append(buf, "(never)");
358 else
359 str_append(buf, "(clock error)");
360 } else {
361 str_appendstr(buf, last_str);
362 str_free(last_str);
363 }
364 str_append(buf,
365 "</td>");
366 }
367
368 str_appendf(buf,
369 "</tr>\n");
370
371 /* Only resolve hosts "on demand" */
372 if (b->u.host.dns == NULL)
373 dns_queue(&(b->u.host.addr));
374 }
375
376 static void
377 format_cols_port_tcp(struct str *buf)
378 {
379 str_append(buf,
380 "<table>\n"
381 "<tr>\n"
382 " <th>Port</td>\n"
383 " <th>Service</td>\n"
384 " <th>In</td>\n"
385 " <th>Out</td>\n"
386 " <th>Total</td>\n"
387 " <th>SYNs</td>\n"
388 "</tr>\n"
389 );
390 }
391
392 static void
393 format_row_port_tcp(struct str *buf, const struct bucket *b,
394 const char *css_class)
395 {
396 const struct port_tcp *p = &(b->u.port_tcp);
397
398 str_appendf(buf,
399 "<tr class=\"%s\">\n"
400 " <td class=\"num\">%u</td>\n"
401 " <td>%s</td>\n"
402 " <td class=\"num\">%'qu</td>\n"
403 " <td class=\"num\">%'qu</td>\n"
404 " <td class=\"num\">%'qu</td>\n"
405 " <td class=\"num\">%'qu</td>\n"
406 "</tr>\n",
407 css_class,
408 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
409 );
410 }
411
412 static void
413 format_cols_port_udp(struct str *buf)
414 {
415 str_append(buf,
416 "<table>\n"
417 "<tr>\n"
418 " <th>Port</td>\n"
419 " <th>Service</td>\n"
420 " <th>In</td>\n"
421 " <th>Out</td>\n"
422 " <th>Total</td>\n"
423 "</tr>\n"
424 );
425 }
426
427 static void
428 format_row_port_udp(struct str *buf, const struct bucket *b,
429 const char *css_class)
430 {
431 const struct port_udp *p = &(b->u.port_udp);
432
433 str_appendf(buf,
434 "<tr class=\"%s\">\n"
435 " <td class=\"num\">%u</td>\n"
436 " <td>%s</td>\n"
437 " <td class=\"num\">%'qu</td>\n"
438 " <td class=\"num\">%'qu</td>\n"
439 " <td class=\"num\">%'qu</td>\n"
440 "</tr>\n",
441 css_class,
442 p->port, getservudp(p->port), b->in, b->out, b->total
443 );
444 }
445
446 static void
447 format_cols_ip_proto(struct str *buf)
448 {
449 str_append(buf,
450 "<table>\n"
451 "<tr>\n"
452 " <th>#</td>\n"
453 " <th>Protocol</td>\n"
454 " <th>In</td>\n"
455 " <th>Out</td>\n"
456 " <th>Total</td>\n"
457 "</tr>\n"
458 );
459 }
460
461 static void
462 format_row_ip_proto(struct str *buf, const struct bucket *b,
463 const char *css_class)
464 {
465 const struct ip_proto *p = &(b->u.ip_proto);
466
467 str_appendf(buf,
468 "<tr class=\"%s\">\n"
469 " <td class=\"num\">%u</td>\n"
470 " <td>%s</td>\n"
471 " <td class=\"num\">%'qu</td>\n"
472 " <td class=\"num\">%'qu</td>\n"
473 " <td class=\"num\">%'qu</td>\n"
474 "</tr>\n",
475 css_class,
476 p->proto, getproto(p->proto),
477 b->in, b->out, b->total
478 );
479 }
480
481 /* ---------------------------------------------------------------------------
482 * Initialise a hashtable.
483 */
484 static struct hashtable *
485 hashtable_make(const uint8_t bits,
486 const unsigned int count_max,
487 const unsigned int count_keep,
488 hash_func_t *hash_func,
489 free_func_t *free_func,
490 key_func_t *key_func,
491 find_func_t *find_func,
492 make_func_t *make_func,
493 format_cols_func_t *format_cols_func,
494 format_row_func_t *format_row_func)
495 {
496 struct hashtable *hash;
497 assert(bits > 0);
498
499 hash = xmalloc(sizeof(*hash));
500 hash->bits = bits;
501 hash->count_max = count_max;
502 hash->count_keep = count_keep;
503 hash->size = 1U << bits;
504 hash->mask = hash->size - 1;
505 hash->coeff = coprime(hash->size);
506 hash->hash_func = hash_func;
507 hash->free_func = free_func;
508 hash->key_func = key_func;
509 hash->find_func = find_func;
510 hash->make_func = make_func;
511 hash->format_cols_func = format_cols_func;
512 hash->format_row_func = format_row_func;
513 hash->count = 0;
514 hash->table = xcalloc(hash->size, sizeof(*hash->table));
515 memset(&(hash->stats), 0, sizeof(hash->stats));
516 return (hash);
517 }
518
519 /* ---------------------------------------------------------------------------
520 * Initialise global hosts_db.
521 */
522 void
523 hosts_db_init(void)
524 {
525 assert(hosts_db == NULL);
526 hosts_db = hashtable_make(HOST_BITS, opt_hosts_max, opt_hosts_keep,
527 hash_func_host, free_func_host, key_func_host, find_func_host,
528 make_func_host, format_cols_host, format_row_host);
529 }
530
531 static void
532 hashtable_rehash(struct hashtable *h, const uint8_t bits)
533 {
534 struct bucket **old_table, **new_table;
535 uint32_t i, old_size;
536 assert(h != NULL);
537 assert(bits > 0);
538
539 h->stats.rehashes++;
540 old_size = h->size;
541 old_table = h->table;
542
543 h->bits = bits;
544 h->size = 1U << bits;
545 h->mask = h->size - 1;
546 h->coeff = coprime(h->size);
547 new_table = xcalloc(h->size, sizeof(*new_table));
548
549 for (i=0; i<old_size; i++) {
550 struct bucket *next, *b = old_table[i];
551 while (b != NULL) {
552 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
553 next = b->next;
554 b->next = new_table[pos];
555 new_table[pos] = b;
556 b = next;
557 }
558 }
559
560 free(h->table);
561 h->table = new_table;
562 }
563
564 static void
565 hashtable_insert(struct hashtable *h, struct bucket *b)
566 {
567 uint32_t pos;
568 assert(h != NULL);
569 assert(b != NULL);
570 assert(b->next == NULL);
571
572 /* Rehash on 80% occupancy */
573 if ((h->count > h->size) ||
574 ((h->size - h->count) < h->size / 5))
575 hashtable_rehash(h, h->bits+1);
576
577 pos = h->hash_func(h, h->key_func(b)) & h->mask;
578 if (h->table[pos] == NULL)
579 h->table[pos] = b;
580 else {
581 /* Insert at top of chain. */
582 b->next = h->table[pos];
583 h->table[pos] = b;
584 }
585 h->count++;
586 h->stats.inserts++;
587 }
588
589 /* Return bucket matching key, or NULL if no such entry. */
590 static struct bucket *
591 hashtable_search(struct hashtable *h, const void *key)
592 {
593 uint32_t pos;
594 struct bucket *b;
595
596 h->stats.searches++;
597 pos = h->hash_func(h, key) & h->mask;
598 b = h->table[pos];
599 while (b != NULL) {
600 if (h->find_func(b, key))
601 return (b);
602 else
603 b = b->next;
604 }
605 return (NULL);
606 }
607
608 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
609 /* Search for a key. If it's not there, make and insert a bucket for it. */
610 static struct bucket *
611 hashtable_find_or_insert(struct hashtable *h, const void *key,
612 const reduce_bool allow_reduce)
613 {
614 struct bucket *b = hashtable_search(h, key);
615
616 if (b == NULL) {
617 /* Not found, so insert after checking occupancy. */
618 if (allow_reduce && (h->count >= h->count_max))
619 hashtable_reduce(h);
620 b = h->make_func(key);
621 hashtable_insert(h, b);
622 }
623 return (b);
624 }
625
626 /*
627 * Frees the hashtable and the buckets. The contents are assumed to be
628 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
629 * the bucket.
630 */
631 static void
632 hashtable_free(struct hashtable *h)
633 {
634 uint32_t i;
635
636 if (h == NULL)
637 return;
638 for (i=0; i<h->size; i++) {
639 struct bucket *tmp, *b = h->table[i];
640 while (b != NULL) {
641 tmp = b;
642 b = b->next;
643 h->free_func(tmp);
644 free(tmp);
645 }
646 }
647 free(h->table);
648 free(h);
649 }
650
651 /* ---------------------------------------------------------------------------
652 * Return existing host or insert a new one.
653 */
654 struct bucket *
655 host_get(const struct addr *const a)
656 {
657 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
658 }
659
660 /* ---------------------------------------------------------------------------
661 * Find host, returns NULL if not in DB.
662 */
663 struct bucket *
664 host_find(const struct addr *const a)
665 {
666 return (hashtable_search(hosts_db, a));
667 }
668
669 /* ---------------------------------------------------------------------------
670 * Find host, returns NULL if not in DB.
671 */
672 static struct bucket *
673 host_search(const char *ipstr)
674 {
675 struct addr a;
676 struct addrinfo hints, *ai;
677
678 memset(&hints, 0, sizeof(hints));
679 hints.ai_family = AF_UNSPEC;
680 hints.ai_flags = AI_NUMERICHOST;
681
682 if (getaddrinfo(ipstr, NULL, &hints, &ai))
683 return (NULL); /* invalid addr */
684
685 if (ai->ai_family == AF_INET) {
686 a.family = IPv4;
687 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
688 }
689 else if (ai->ai_family == AF_INET6) {
690 a.family = IPv6;
691 memcpy(&(a.ip.v6),
692 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
693 sizeof(a.ip.v6));
694 } else {
695 freeaddrinfo(ai);
696 return (NULL); /* unknown family */
697 }
698 freeaddrinfo(ai);
699
700 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
701 return (hashtable_search(hosts_db, &a));
702 }
703
704 /* ---------------------------------------------------------------------------
705 * Reduce a hashtable to the top <keep> entries.
706 */
707 static void
708 hashtable_reduce(struct hashtable *ht)
709 {
710 uint32_t i, pos, rmd;
711 const struct bucket **table;
712 uint64_t cutoff;
713
714 assert(ht->count_keep < ht->count);
715
716 /* Fill table with pointers to buckets in hashtable. */
717 table = xcalloc(ht->count, sizeof(*table));
718 for (pos=0, i=0; i<ht->size; i++) {
719 struct bucket *b = ht->table[i];
720 while (b != NULL) {
721 table[pos++] = b;
722 b = b->next;
723 }
724 }
725 assert(pos == ht->count);
726 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
727 cutoff = table[ht->count_keep]->total;
728 free(table);
729
730 /* Remove all elements with total <= cutoff. */
731 rmd = 0;
732 for (i=0; i<ht->size; i++) {
733 struct bucket *last = NULL, *next, *b = ht->table[i];
734 while (b != NULL) {
735 next = b->next;
736 if (b->total <= cutoff) {
737 /* Remove this one. */
738 ht->free_func(b);
739 free(b);
740 if (last == NULL)
741 ht->table[i] = next;
742 else
743 last->next = next;
744 rmd++;
745 ht->count--;
746 } else {
747 last = b;
748 }
749 b = next;
750 }
751 }
752 verbosef("hashtable_reduce: removed %u buckets, left %u",
753 rmd, ht->count);
754 hashtable_rehash(ht, ht->bits); /* is this needed? */
755 }
756
757 /* Reduce hosts_db if needed. */
758 void hosts_db_reduce(void)
759 {
760 if (hosts_db->count >= hosts_db->count_max)
761 hashtable_reduce(hosts_db);
762 }
763
764 /* ---------------------------------------------------------------------------
765 * Reset hosts_db to empty.
766 */
767 void
768 hosts_db_reset(void)
769 {
770 unsigned int i;
771
772 for (i=0; i<hosts_db->size; i++) {
773 struct bucket *next, *b = hosts_db->table[i];
774 while (b != NULL) {
775 next = b->next;
776 hosts_db->free_func(b);
777 free(b);
778 b = next;
779 }
780 hosts_db->table[i] = NULL;
781 }
782 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
783 hosts_db->count = 0;
784 }
785
786 /* ---------------------------------------------------------------------------
787 * Deallocate hosts_db.
788 */
789 void hosts_db_free(void)
790 {
791 uint32_t i;
792
793 assert(hosts_db != NULL);
794 for (i=0; i<hosts_db->size; i++) {
795 struct bucket *tmp, *b = hosts_db->table[i];
796 while (b != NULL) {
797 tmp = b;
798 b = b->next;
799 hosts_db->free_func(tmp);
800 free(tmp);
801 }
802 }
803 free(hosts_db->table);
804 free(hosts_db);
805 hosts_db = NULL;
806 }
807
808 /* ---------------------------------------------------------------------------
809 * Find or create a port_tcp inside a host.
810 */
811 struct bucket *
812 host_get_port_tcp(struct bucket *host, const uint16_t port)
813 {
814 struct host *h = &host->u.host;
815 assert(h != NULL);
816 if (h->ports_tcp == NULL)
817 h->ports_tcp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
818 hash_func_short, free_func_simple, key_func_port_tcp,
819 find_func_port_tcp, make_func_port_tcp,
820 format_cols_port_tcp, format_row_port_tcp);
821 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
822 }
823
824 /* ---------------------------------------------------------------------------
825 * Find or create a port_udp inside a host.
826 */
827 struct bucket *
828 host_get_port_udp(struct bucket *host, const uint16_t port)
829 {
830 struct host *h = &host->u.host;
831 assert(h != NULL);
832 if (h->ports_udp == NULL)
833 h->ports_udp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
834 hash_func_short, free_func_simple, key_func_port_udp,
835 find_func_port_udp, make_func_port_udp,
836 format_cols_port_udp, format_row_port_udp);
837 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
838 }
839
840 /* ---------------------------------------------------------------------------
841 * Find or create an ip_proto inside a host.
842 */
843 struct bucket *
844 host_get_ip_proto(struct bucket *host, const uint8_t proto)
845 {
846 struct host *h = &host->u.host;
847 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
848 assert(h != NULL);
849 if (h->ip_protos == NULL)
850 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
851 hash_func_byte, free_func_simple, key_func_ip_proto,
852 find_func_ip_proto, make_func_ip_proto,
853 format_cols_ip_proto, format_row_ip_proto);
854 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
855 }
856
857 static struct str *html_hosts_main(const char *qs);
858 static struct str *html_hosts_detail(const char *ip);
859
860 /* ---------------------------------------------------------------------------
861 * Web interface: delegate the /hosts/ space.
862 */
863 struct str *
864 html_hosts(const char *uri, const char *query)
865 {
866 unsigned int i, num_elems;
867 char **elem = split('/', uri, &num_elems);
868 struct str *buf = NULL;
869
870 assert(num_elems >= 1);
871 assert(strcmp(elem[0], "hosts") == 0);
872
873 if (num_elems == 1)
874 /* /hosts/ */
875 buf = html_hosts_main(query);
876 else if (num_elems == 2)
877 /* /hosts/<IP of host>/ */
878 buf = html_hosts_detail(elem[1]);
879
880 for (i=0; i<num_elems; i++)
881 free(elem[i]);
882 free(elem);
883 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
884 other codes to be possible */
885 }
886
887 /* ---------------------------------------------------------------------------
888 * Format hashtable into HTML.
889 */
890 static void
891 format_table(struct str *buf, struct hashtable *ht, unsigned int start,
892 const enum sort_dir sort, const int full)
893 {
894 const struct bucket **table;
895 unsigned int i, pos, end;
896 int alt = 0;
897
898 if ((ht == NULL) || (ht->count == 0)) {
899 str_append(buf, "<p>The table is empty.</p>\n");
900 return;
901 }
902
903 /* Fill table with pointers to buckets in hashtable. */
904 table = xcalloc(ht->count, sizeof(*table));
905 for (pos=0, i=0; i<ht->size; i++) {
906 struct bucket *b = ht->table[i];
907 while (b != NULL) {
908 table[pos++] = b;
909 b = b->next;
910 }
911 }
912 assert(pos == ht->count);
913
914 if (full) {
915 /* full report overrides start and end */
916 start = 0;
917 end = ht->count;
918 } else
919 end = MIN(ht->count, (uint32_t)start+MAX_ENTRIES);
920
921 str_appendf(buf, "(%u-%u of %u)<br>\n", start+1, end, ht->count);
922 qsort_buckets(table, ht->count, start, end, sort);
923 ht->format_cols_func(buf);
924
925 for (i=start; i<end; i++) {
926 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
927 alt = !alt; /* alternate class for table rows */
928 }
929 free(table);
930 str_append(buf, "</table>\n");
931 }
932
933 /* ---------------------------------------------------------------------------
934 * Web interface: sorted table of hosts.
935 */
936 static struct str *
937 html_hosts_main(const char *qs)
938 {
939 struct str *buf = str_make();
940 char *qs_start, *qs_sort, *qs_full, *ep;
941 const char *sortstr;
942 int start, full = 0;
943 enum sort_dir sort;
944
945 /* parse query string */
946 qs_start = qs_get(qs, "start");
947 qs_sort = qs_get(qs, "sort");
948 qs_full = qs_get(qs, "full");
949 if (qs_full != NULL) {
950 full = 1;
951 free(qs_full);
952 }
953
954 /* validate sort */
955 if (qs_sort == NULL) sort = TOTAL;
956 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
957 else if (strcmp(qs_sort, "in") == 0) sort = IN;
958 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
959 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
960 else {
961 str_append(buf, "Error: invalid value for \"sort\".\n");
962 goto done;
963 }
964
965 /* parse start */
966 if (qs_start == NULL)
967 start = 0;
968 else {
969 start = (int)strtoul(qs_start, &ep, 10);
970 if (*ep != '\0') {
971 str_append(buf, "Error: \"start\" is not a number.\n");
972 goto done;
973 }
974 if ((errno == ERANGE) ||
975 (start < 0) || (start >= (int)hosts_db->count)) {
976 str_append(buf, "Error: \"start\" is out of bounds.\n");
977 goto done;
978 }
979 }
980
981 #define PREV "&lt;&lt;&lt; prev page"
982 #define NEXT "next page &gt;&gt;&gt;"
983 #define FULL "full table"
984
985 html_open(buf, "Hosts", /*path_depth=*/1, /*want_graph_js=*/0);
986 format_table(buf, hosts_db, start, sort, full);
987
988 /* <prev | full | stats | next> */
989 sortstr = qs_sort;
990 if (sortstr == NULL) sortstr = "total";
991 if (start > 0) {
992 int prev = start - MAX_ENTRIES;
993 if (prev < 0)
994 prev = 0;
995 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
996 prev, sortstr);
997 } else
998 str_append(buf, PREV);
999
1000 if (full)
1001 str_append(buf, " | " FULL);
1002 else
1003 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
1004 sortstr);
1005
1006 if (start+MAX_ENTRIES < (int)hosts_db->count)
1007 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
1008 start+MAX_ENTRIES, sortstr);
1009 else
1010 str_append(buf, " | " NEXT);
1011
1012 str_append(buf, "<br>\n");
1013
1014 html_close(buf);
1015 done:
1016 if (qs_start != NULL) free(qs_start);
1017 if (qs_sort != NULL) free(qs_sort);
1018 return buf;
1019 #undef PREV
1020 #undef NEXT
1021 #undef FULL
1022 }
1023
1024 /* ---------------------------------------------------------------------------
1025 * Web interface: detailed view of a single host.
1026 */
1027 static struct str *
1028 html_hosts_detail(const char *ip)
1029 {
1030 struct bucket *h;
1031 struct str *buf, *ls_len;
1032 char ls_when[100];
1033 const char *canonical;
1034 time_t ls;
1035
1036 h = host_search(ip);
1037 if (h == NULL)
1038 return (NULL); /* no such host */
1039
1040 canonical = addr_to_str(&(h->u.host.addr));
1041
1042 /* Overview. */
1043 buf = str_make();
1044 html_open(buf, ip, /*path_depth=*/2, /*want_graph_js=*/0);
1045 if (strcmp(ip, canonical) != 0)
1046 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1047 str_appendf(buf,
1048 "<p>\n"
1049 "<b>Hostname:</b> %s<br>\n",
1050 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1051
1052 /* Resolve host "on demand" */
1053 if (h->u.host.dns == NULL)
1054 dns_queue(&(h->u.host.addr));
1055
1056 if (hosts_db_show_macs)
1057 str_appendf(buf,
1058 "<b>MAC Address:</b> "
1059 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1060 h->u.host.mac_addr[0],
1061 h->u.host.mac_addr[1],
1062 h->u.host.mac_addr[2],
1063 h->u.host.mac_addr[3],
1064 h->u.host.mac_addr[4],
1065 h->u.host.mac_addr[5]);
1066
1067 str_append(buf,
1068 "</p>\n"
1069 "<p>\n"
1070 "<b>Last seen:</b> ");
1071
1072 ls = h->u.host.lastseen;
1073 if (strftime(ls_when, sizeof(ls_when),
1074 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1075 str_append(buf, ls_when);
1076
1077 if (h->u.host.lastseen <= now) {
1078 ls_len = length_of_time(now - h->u.host.lastseen);
1079 str_append(buf, " (");
1080 str_appendstr(buf, ls_len);
1081 str_free(ls_len);
1082 str_append(buf, " ago)");
1083 } else {
1084 str_append(buf, " (in the future, possible clock problem)");
1085 }
1086
1087 str_appendf(buf,
1088 "</p>\n"
1089 "<p>\n"
1090 " <b>In:</b> %'qu<br>\n"
1091 " <b>Out:</b> %'qu<br>\n"
1092 " <b>Total:</b> %'qu<br>\n"
1093 "</p>\n",
1094 h->in, h->out, h->total);
1095
1096 str_append(buf, "<h3>TCP ports</h3>\n");
1097 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1098
1099 str_append(buf, "<h3>UDP ports</h3>\n");
1100 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1101
1102 str_append(buf, "<h3>IP protocols</h3>\n");
1103 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1104
1105 html_close(buf);
1106 return (buf);
1107 }
1108
1109 /* ---------------------------------------------------------------------------
1110 * Database import and export code:
1111 * Initially written and contributed by Ben Stewart.
1112 * copyright (c) 2007-2011 Ben Stewart, Emil Mikulic.
1113 */
1114 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1115 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1116 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1117
1118 static const char
1119 export_proto_ip = 'P',
1120 export_proto_tcp = 'T',
1121 export_proto_udp = 'U';
1122
1123 static const unsigned char
1124 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1125 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1126 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1127
1128 /* ---------------------------------------------------------------------------
1129 * Load a host's ip_proto table from a file.
1130 * Returns 0 on failure, 1 on success.
1131 */
1132 static int
1133 hosts_db_import_ip(const int fd, struct bucket *host)
1134 {
1135 uint8_t count, i;
1136
1137 if (!expect8(fd, export_proto_ip)) return 0;
1138 if (!read8(fd, &count)) return 0;
1139
1140 for (i=0; i<count; i++) {
1141 struct bucket *b;
1142 uint8_t proto;
1143 uint64_t in, out;
1144
1145 if (!read8(fd, &proto)) return 0;
1146 if (!read64(fd, &in)) return 0;
1147 if (!read64(fd, &out)) return 0;
1148
1149 /* Store data */
1150 b = host_get_ip_proto(host, proto);
1151 b->in = in;
1152 b->out = out;
1153 b->total = in + out;
1154 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1155 }
1156 return 1;
1157 }
1158
1159 /* ---------------------------------------------------------------------------
1160 * Load a host's port_tcp table from a file.
1161 * Returns 0 on failure, 1 on success.
1162 */
1163 static int
1164 hosts_db_import_tcp(const int fd, struct bucket *host)
1165 {
1166 uint16_t count, i;
1167
1168 if (!expect8(fd, export_proto_tcp)) return 0;
1169 if (!read16(fd, &count)) return 0;
1170
1171 for (i=0; i<count; i++) {
1172 struct bucket *b;
1173 uint16_t port;
1174 uint64_t in, out, syn;
1175
1176 if (!read16(fd, &port)) return 0;
1177 if (!read64(fd, &syn)) return 0;
1178 if (!read64(fd, &in)) return 0;
1179 if (!read64(fd, &out)) return 0;
1180
1181 /* Store data */
1182 b = host_get_port_tcp(host, port);
1183 b->in = in;
1184 b->out = out;
1185 b->total = in + out;
1186 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1187 b->u.port_tcp.syn = syn;
1188 }
1189 return 1;
1190 }
1191
1192 /* ---------------------------------------------------------------------------
1193 * Load a host's port_tcp table from a file.
1194 * Returns 0 on failure, 1 on success.
1195 */
1196 static int
1197 hosts_db_import_udp(const int fd, struct bucket *host)
1198 {
1199 uint16_t count, i;
1200
1201 if (!expect8(fd, export_proto_udp)) return 0;
1202 if (!read16(fd, &count)) return 0;
1203
1204 for (i=0; i<count; i++) {
1205 struct bucket *b;
1206 uint16_t port;
1207 uint64_t in, out;
1208
1209 if (!read16(fd, &port)) return 0;
1210 if (!read64(fd, &in)) return 0;
1211 if (!read64(fd, &out)) return 0;
1212
1213 /* Store data */
1214 b = host_get_port_udp(host, port);
1215 b->in = in;
1216 b->out = out;
1217 b->total = in + out;
1218 assert(b->u.port_udp.port == port); /* done by make_func */
1219 }
1220 return 1;
1221 }
1222
1223 /* ---------------------------------------------------------------------------
1224 * Load all hosts from a file.
1225 * Returns 0 on failure, 1 on success.
1226 */
1227 static int
1228 hosts_db_import_host(const int fd)
1229 {
1230 struct bucket *host;
1231 struct addr a;
1232 uint8_t hostname_len;
1233 uint64_t in, out;
1234 unsigned int pos = xtell(fd);
1235 char hdr[4];
1236 int ver = 0;
1237
1238 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1239 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1240 ver = 3;
1241 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1242 ver = 2;
1243 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1244 ver = 1;
1245 else {
1246 warnx("bad host header: %02x%02x%02x%02x",
1247 hdr[0], hdr[1], hdr[2], hdr[3]);
1248 return 0;
1249 }
1250
1251 if (ver == 3) {
1252 if (!readaddr(fd, &a))
1253 return 0;
1254 } else {
1255 assert((ver == 1) || (ver == 2));
1256 if (!readaddr_ipv4(fd, &a))
1257 return 0;
1258 }
1259 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1260 host = host_get(&a);
1261 assert(addr_equal(&(host->u.host.addr), &a));
1262
1263 if (ver > 1) {
1264 uint64_t t;
1265 if (!read64(fd, &t)) return 0;
1266 host->u.host.lastseen = (time_t)t;
1267 }
1268
1269 assert(sizeof(host->u.host.mac_addr) == 6);
1270 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1271 return 0;
1272
1273 /* HOSTNAME */
1274 assert(host->u.host.dns == NULL); /* make fn? */
1275 if (!read8(fd, &hostname_len)) return 0;
1276 if (hostname_len > 0) {
1277 host->u.host.dns = xmalloc(hostname_len + 1);
1278 host->u.host.dns[0] = '\0';
1279
1280 /* At this point, the hostname is attached to a host which is in our
1281 * hosts_db, so if we bail out due to an import error, this pointer
1282 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1283 */
1284
1285 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1286 host->u.host.dns[hostname_len] = '\0';
1287 }
1288
1289 if (!read64(fd, &in)) return 0;
1290 if (!read64(fd, &out)) return 0;
1291
1292 host->in = in;
1293 host->out = out;
1294 host->total = in + out;
1295
1296 /* Host's port and proto subtables: */
1297 if (!hosts_db_import_ip(fd, host)) return 0;
1298 if (!hosts_db_import_tcp(fd, host)) return 0;
1299 if (!hosts_db_import_udp(fd, host)) return 0;
1300 return 1;
1301 }
1302
1303 /* ---------------------------------------------------------------------------
1304 * Database Import: Grab hosts_db from a file provided by the caller.
1305 *
1306 * This function will retrieve the data sans the header. We expect the caller
1307 * to have validated the header of the hosts_db segment, and left the file
1308 * sitting at the start of the data.
1309 */
1310 int hosts_db_import(const int fd)
1311 {
1312 uint32_t host_count, i;
1313
1314 if (!read32(fd, &host_count)) return 0;
1315
1316 for (i=0; i<host_count; i++)
1317 if (!hosts_db_import_host(fd)) return 0;
1318
1319 return 1;
1320 }
1321
1322 /* ---------------------------------------------------------------------------
1323 * Database Export: Dump hosts_db into a file provided by the caller.
1324 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1325 */
1326 int hosts_db_export(const int fd)
1327 {
1328 uint32_t i;
1329 struct bucket *b;
1330
1331 if (!write32(fd, hosts_db->count)) return 0;
1332
1333 for (i = 0; i<hosts_db->size; i++)
1334 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1335 /* For each host: */
1336 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1337 return 0;
1338
1339 if (!writeaddr(fd, &(b->u.host.addr))) return 0;
1340
1341 if (!write64(fd, (uint64_t)(b->u.host.lastseen))) return 0;
1342
1343 assert(sizeof(b->u.host.mac_addr) == 6);
1344 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1345 return 0;
1346
1347 /* HOSTNAME */
1348 if (b->u.host.dns == NULL) {
1349 if (!write8(fd, 0)) return 0;
1350 } else {
1351 int dnslen = strlen(b->u.host.dns);
1352
1353 if (dnslen > 255) {
1354 warnx("found a very long hostname: \"%s\"\n"
1355 "wasn't expecting one longer than 255 chars (this one is %d)",
1356 b->u.host.dns, dnslen);
1357 dnslen = 255;
1358 }
1359
1360 if (!write8(fd, (uint8_t)dnslen)) return 0;
1361 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1362 }
1363
1364 if (!write64(fd, b->in)) return 0;
1365 if (!write64(fd, b->out)) return 0;
1366
1367 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1368 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1369 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1370 }
1371 return 1;
1372 }
1373
1374 /* ---------------------------------------------------------------------------
1375 * Dump the ip_proto table of a host.
1376 */
1377 static int
1378 hosts_db_export_ip(const struct hashtable *h, const int fd)
1379 {
1380 uint32_t i, written = 0;
1381 struct bucket *b;
1382
1383 /* IP DATA */
1384 if (!write8(fd, export_proto_ip)) return 0;
1385
1386 /* If no data, write a IP Proto count of 0 and we're done. */
1387 if (h == NULL) {
1388 if (!write8(fd, 0)) return 0;
1389 return 1;
1390 }
1391
1392 assert(h->count < 256);
1393 if (!write8(fd, (uint8_t)h->count)) return 0;
1394
1395 for (i = 0; i<h->size; i++)
1396 for (b = h->table[i]; b != NULL; b = b->next) {
1397 /* For each ip_proto bucket: */
1398
1399 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1400 if (!write64(fd, b->in)) return 0;
1401 if (!write64(fd, b->out)) return 0;
1402 written++;
1403 }
1404 assert(written == h->count);
1405 return 1;
1406 }
1407
1408 /* ---------------------------------------------------------------------------
1409 * Dump the port_tcp table of a host.
1410 */
1411 static int
1412 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1413 {
1414 struct bucket *b;
1415 uint32_t i, written = 0;
1416
1417 /* TCP DATA */
1418 if (!write8(fd, export_proto_tcp)) return 0;
1419
1420 /* If no data, write a count of 0 and we're done. */
1421 if (h == NULL) {
1422 if (!write16(fd, 0)) return 0;
1423 return 1;
1424 }
1425
1426 assert(h->count < 65536);
1427 if (!write16(fd, (uint16_t)h->count)) return 0;
1428
1429 for (i = 0; i<h->size; i++)
1430 for (b = h->table[i]; b != NULL; b = b->next) {
1431 if (!write16(fd, b->u.port_tcp.port)) return 0;
1432 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1433 if (!write64(fd, b->in)) return 0;
1434 if (!write64(fd, b->out)) return 0;
1435 written++;
1436 }
1437 assert(written == h->count);
1438 return 1;
1439 }
1440
1441 /* ---------------------------------------------------------------------------
1442 * Dump the port_udp table of a host.
1443 */
1444 static int
1445 hosts_db_export_udp(const struct hashtable *h, const int fd)
1446 {
1447 struct bucket *b;
1448 uint32_t i, written = 0;
1449
1450 /* UDP DATA */
1451 if (!write8(fd, export_proto_udp)) return 0;
1452
1453 /* If no data, write a count of 0 and we're done. */
1454 if (h == NULL) {
1455 if (!write16(fd, 0)) return 0;
1456 return 1;
1457 }
1458
1459 assert(h->count < 65536);
1460 if (!write16(fd, (uint16_t)h->count)) return 0;
1461
1462 for (i = 0; i<h->size; i++)
1463 for (b = h->table[i]; b != NULL; b = b->next) {
1464 if (!write16(fd, b->u.port_udp.port)) return 0;
1465 if (!write64(fd, b->in)) return 0;
1466 if (!write64(fd, b->out)) return 0;
1467 written++;
1468 }
1469 assert(written == h->count);
1470 return 1;
1471 }
1472
1473 /* vim:set ts=3 sw=3 tw=78 expandtab: */