10f8a9a2dca5dec515e656470f63036810f79189
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2009 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "http.h" /* for base_url */
19 #include "ncache.h"
20 #include "now.h"
21 #include "str.h"
22
23 #include <arpa/inet.h> /* inet_aton() */
24 #include <netdb.h> /* struct addrinfo */
25 #include <assert.h>
26 #include <errno.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h> /* memset(), strcmp() */
30 #include <unistd.h>
31
32 extern int want_lastseen;
33 int show_mac_addrs = 0;
34 extern const char *interface;
35
36 /* FIXME: specify somewhere more sane/tunable */
37 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
38
39 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
40 typedef void (free_func_t)(struct bucket *);
41 typedef const void * (key_func_t)(const struct bucket *);
42 typedef int (find_func_t)(const struct bucket *, const void *);
43 typedef struct bucket * (make_func_t)(const void *);
44 typedef void (format_cols_func_t)(struct str *);
45 typedef void (format_row_func_t)(struct str *, const struct bucket *,
46 const char *);
47
48 struct hashtable {
49 uint8_t bits; /* size of hashtable in bits */
50 uint32_t size, mask;
51 uint32_t count, count_max, count_keep; /* items in table */
52 uint32_t coeff; /* coefficient for Fibonacci hashing */
53 struct bucket **table;
54
55 struct {
56 uint64_t inserts, searches, deletions, rehashes;
57 } stats;
58
59 hash_func_t *hash_func;
60 /* returns hash value of given key (passed as void*) */
61
62 free_func_t *free_func;
63 /* free of bucket payload */
64
65 key_func_t *key_func;
66 /* returns pointer to key of bucket (to pass to hash_func) */
67
68 find_func_t *find_func;
69 /* returns true if given bucket matches key (passed as void*) */
70
71 make_func_t *make_func;
72 /* returns bucket containing new record with key (passed as void*) */
73
74 format_cols_func_t *format_cols_func;
75 /* append table columns to str */
76
77 format_row_func_t *format_row_func;
78 /* format record and append to str */
79 };
80
81 static void hashtable_reduce(struct hashtable *ht);
82 static void hashtable_free(struct hashtable *h);
83
84 #define HOST_BITS 1 /* initial size of hosts table */
85 #define PORT_BITS 1 /* initial size of ports tables */
86 #define PROTO_BITS 1 /* initial size of proto table */
87
88 /* We only use one hosts_db hashtable and this is it. */
89 static struct hashtable *hosts_db = NULL;
90
91 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
92 static const double phi_1 =
93 0.61803398874989490252573887119069695472717285156250;
94
95 /* Co-prime of u, using phi^-1 */
96 inline static uint32_t
97 coprime(const uint32_t u)
98 {
99 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
100 }
101
102 /*
103 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
104 * src/sys/netinet/tcp_hostcache.c 1.1
105 */
106 inline static uint32_t
107 ipv4_hash(const struct addr *const a)
108 {
109 uint32_t ip = a->ip.v4;
110 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
111 }
112
113 #ifndef s6_addr32
114 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
115 * taken care of GNU/Linux and GNU/kfreebsd. */
116 # define s6_addr32 __u6_addr.__u6_addr32
117 #endif
118
119 /*
120 * This is the IPv6 hash function used by FreeBSD in the same file as above,
121 * svn rev 122922.
122 */
123 inline static uint32_t
124 ipv6_hash(const struct addr *const a)
125 {
126 const struct in6_addr *const ip6 = &(a->ip.v6);
127 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
128 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
129 }
130
131 /* ---------------------------------------------------------------------------
132 * hash_func collection
133 */
134 static uint32_t
135 hash_func_host(const struct hashtable *h _unused_, const void *key)
136 {
137 const struct addr *a = key;
138 if (a->family == IPv4)
139 return (ipv4_hash(a));
140 else {
141 assert(a->family == IPv6);
142 return (ipv6_hash(a));
143 }
144 }
145
146 #define CASTKEY(type) (*((const type *)key))
147
148 static uint32_t
149 hash_func_short(const struct hashtable *h, const void *key)
150 {
151 return (CASTKEY(uint16_t) * h->coeff);
152 }
153
154 static uint32_t
155 hash_func_byte(const struct hashtable *h, const void *key)
156 {
157 return (CASTKEY(uint8_t) * h->coeff);
158 }
159
160 /* ---------------------------------------------------------------------------
161 * key_func collection
162 */
163
164 static const void *
165 key_func_host(const struct bucket *b)
166 {
167 return &(b->u.host.addr);
168 }
169
170 static const void *
171 key_func_port_tcp(const struct bucket *b)
172 {
173 return &(b->u.port_tcp.port);
174 }
175
176 static const void *
177 key_func_port_udp(const struct bucket *b)
178 {
179 return &(b->u.port_udp.port);
180 }
181
182 static const void *
183 key_func_ip_proto(const struct bucket *b)
184 {
185 return &(b->u.ip_proto.proto);
186 }
187
188 /* ---------------------------------------------------------------------------
189 * find_func collection
190 */
191
192 static int
193 find_func_host(const struct bucket *b, const void *key)
194 {
195 return (addr_equal(key, &(b->u.host.addr)));
196 }
197
198 static int
199 find_func_port_tcp(const struct bucket *b, const void *key)
200 {
201 return (b->u.port_tcp.port == CASTKEY(uint16_t));
202 }
203
204 static int
205 find_func_port_udp(const struct bucket *b, const void *key)
206 {
207 return (b->u.port_udp.port == CASTKEY(uint16_t));
208 }
209
210 static int
211 find_func_ip_proto(const struct bucket *b, const void *key)
212 {
213 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
214 }
215
216 /* ---------------------------------------------------------------------------
217 * make_func collection
218 */
219
220 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
221 struct bucket *next; \
222 uint64_t in, out, total; \
223 union { struct type t; } u; } _custom_bucket; \
224 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
225 struct type *name_content = &(name_bucket->u.type); \
226 name_bucket->next = NULL; \
227 name_bucket->in = name_bucket->out = name_bucket->total = 0;
228
229 static struct bucket *
230 make_func_host(const void *key)
231 {
232 MAKE_BUCKET(b, h, host);
233 h->addr = CASTKEY(struct addr);
234 h->dns = NULL;
235 h->last_seen = now;
236 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
237 h->ports_tcp = NULL;
238 h->ports_udp = NULL;
239 h->ip_protos = NULL;
240 return (b);
241 }
242
243 static void
244 free_func_host(struct bucket *b)
245 {
246 struct host *h = &(b->u.host);
247 if (h->dns != NULL) free(h->dns);
248 hashtable_free(h->ports_tcp);
249 hashtable_free(h->ports_udp);
250 hashtable_free(h->ip_protos);
251 }
252
253 static struct bucket *
254 make_func_port_tcp(const void *key)
255 {
256 MAKE_BUCKET(b, p, port_tcp);
257 p->port = CASTKEY(uint16_t);
258 p->syn = 0;
259 return (b);
260 }
261
262 static struct bucket *
263 make_func_port_udp(const void *key)
264 {
265 MAKE_BUCKET(b, p, port_udp);
266 p->port = CASTKEY(uint16_t);
267 return (b);
268 }
269
270 static struct bucket *
271 make_func_ip_proto(const void *key)
272 {
273 MAKE_BUCKET(b, p, ip_proto);
274 p->proto = CASTKEY(uint8_t);
275 return (b);
276 }
277
278 static void
279 free_func_simple(struct bucket *b _unused_)
280 {
281 /* nop */
282 }
283
284 /* ---------------------------------------------------------------------------
285 * format_func collection (ordered by struct)
286 */
287
288 static void
289 format_cols_host(struct str *buf)
290 {
291 /* FIXME: don't clobber parts of the query string
292 * specifically "full" and "start"
293 * when setting sort direction
294 */
295 str_append(buf,
296 "<table>\n"
297 "<tr>\n"
298 " <th>IP</th>\n"
299 " <th>Hostname</th>\n");
300 if (show_mac_addrs) str_append(buf,
301 " <th>MAC Address</th>\n");
302 str_append(buf,
303 " <th><a href=\"?sort=in\">In</a></th>\n"
304 " <th><a href=\"?sort=out\">Out</a></th>\n"
305 " <th><a href=\"?sort=total\">Total</a></th>\n");
306 if (want_lastseen) str_append(buf,
307 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
308 str_append(buf,
309 "</tr>\n");
310 }
311
312 static void
313 format_row_host(struct str *buf, const struct bucket *b,
314 const char *css_class)
315 {
316 const char *ip = addr_to_str(&(b->u.host.addr));
317
318 str_appendf(buf,
319 "<tr class=\"%s\">\n"
320 " <td><a href=\"%shosts/%s/\">%s</a></td>\n"
321 " <td>%s</td>\n",
322 css_class,
323 base_url, ip, ip,
324 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
325
326 if (show_mac_addrs)
327 str_appendf(buf,
328 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
329 b->u.host.mac_addr[0],
330 b->u.host.mac_addr[1],
331 b->u.host.mac_addr[2],
332 b->u.host.mac_addr[3],
333 b->u.host.mac_addr[4],
334 b->u.host.mac_addr[5]);
335
336 str_appendf(buf,
337 " <td class=\"num\">%'qu</td>\n"
338 " <td class=\"num\">%'qu</td>\n"
339 " <td class=\"num\">%'qu</td>\n",
340 b->in, b->out, b->total);
341
342 if (want_lastseen) {
343 time_t last_t = b->u.host.last_seen;
344 struct str *lastseen = NULL;
345
346 if (now >= last_t)
347 lastseen = length_of_time(now - last_t);
348
349 str_append(buf,
350 " <td class=\"num\">");
351 if (lastseen == NULL)
352 str_append(buf, "(clock error)");
353 else {
354 str_appendstr(buf, lastseen);
355 str_free(lastseen);
356 }
357 str_append(buf,
358 "</td>");
359 }
360
361 str_appendf(buf,
362 "</tr>\n");
363
364 /* Only resolve hosts "on demand" */
365 if (b->u.host.dns == NULL)
366 dns_queue(&(b->u.host.addr));
367 }
368
369 static void
370 format_cols_port_tcp(struct str *buf)
371 {
372 str_append(buf,
373 "<table>\n"
374 "<tr>\n"
375 " <th>Port</td>\n"
376 " <th>Service</td>\n"
377 " <th>In</td>\n"
378 " <th>Out</td>\n"
379 " <th>Total</td>\n"
380 " <th>SYNs</td>\n"
381 "</tr>\n"
382 );
383 }
384
385 static void
386 format_row_port_tcp(struct str *buf, const struct bucket *b,
387 const char *css_class)
388 {
389 const struct port_tcp *p = &(b->u.port_tcp);
390
391 str_appendf(buf,
392 "<tr class=\"%s\">\n"
393 " <td class=\"num\">%u</td>\n"
394 " <td>%s</td>\n"
395 " <td class=\"num\">%'qu</td>\n"
396 " <td class=\"num\">%'qu</td>\n"
397 " <td class=\"num\">%'qu</td>\n"
398 " <td class=\"num\">%'qu</td>\n"
399 "</tr>\n",
400 css_class,
401 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
402 );
403 }
404
405 static void
406 format_cols_port_udp(struct str *buf)
407 {
408 str_append(buf,
409 "<table>\n"
410 "<tr>\n"
411 " <th>Port</td>\n"
412 " <th>Service</td>\n"
413 " <th>In</td>\n"
414 " <th>Out</td>\n"
415 " <th>Total</td>\n"
416 "</tr>\n"
417 );
418 }
419
420 static void
421 format_row_port_udp(struct str *buf, const struct bucket *b,
422 const char *css_class)
423 {
424 const struct port_udp *p = &(b->u.port_udp);
425
426 str_appendf(buf,
427 "<tr class=\"%s\">\n"
428 " <td class=\"num\">%u</td>\n"
429 " <td>%s</td>\n"
430 " <td class=\"num\">%'qu</td>\n"
431 " <td class=\"num\">%'qu</td>\n"
432 " <td class=\"num\">%'qu</td>\n"
433 "</tr>\n",
434 css_class,
435 p->port, getservudp(p->port), b->in, b->out, b->total
436 );
437 }
438
439 static void
440 format_cols_ip_proto(struct str *buf)
441 {
442 str_append(buf,
443 "<table>\n"
444 "<tr>\n"
445 " <th>#</td>\n"
446 " <th>Protocol</td>\n"
447 " <th>In</td>\n"
448 " <th>Out</td>\n"
449 " <th>Total</td>\n"
450 "</tr>\n"
451 );
452 }
453
454 static void
455 format_row_ip_proto(struct str *buf, const struct bucket *b,
456 const char *css_class)
457 {
458 const struct ip_proto *p = &(b->u.ip_proto);
459
460 str_appendf(buf,
461 "<tr class=\"%s\">\n"
462 " <td class=\"num\">%u</td>\n"
463 " <td>%s</td>\n"
464 " <td class=\"num\">%'qu</td>\n"
465 " <td class=\"num\">%'qu</td>\n"
466 " <td class=\"num\">%'qu</td>\n"
467 "</tr>\n",
468 css_class,
469 p->proto, getproto(p->proto),
470 b->in, b->out, b->total
471 );
472 }
473
474 /* ---------------------------------------------------------------------------
475 * Initialise a hashtable.
476 */
477 static struct hashtable *
478 hashtable_make(const uint8_t bits,
479 const unsigned int count_max,
480 const unsigned int count_keep,
481 hash_func_t *hash_func,
482 free_func_t *free_func,
483 key_func_t *key_func,
484 find_func_t *find_func,
485 make_func_t *make_func,
486 format_cols_func_t *format_cols_func,
487 format_row_func_t *format_row_func)
488 {
489 struct hashtable *hash;
490 assert(bits > 0);
491
492 hash = xmalloc(sizeof(*hash));
493 hash->bits = bits;
494 hash->count_max = count_max;
495 hash->count_keep = count_keep;
496 hash->size = 1U << bits;
497 hash->mask = hash->size - 1;
498 hash->coeff = coprime(hash->size);
499 hash->hash_func = hash_func;
500 hash->free_func = free_func;
501 hash->key_func = key_func;
502 hash->find_func = find_func;
503 hash->make_func = make_func;
504 hash->format_cols_func = format_cols_func;
505 hash->format_row_func = format_row_func;
506 hash->count = 0;
507 hash->table = xcalloc(hash->size, sizeof(*hash->table));
508 memset(&(hash->stats), 0, sizeof(hash->stats));
509 return (hash);
510 }
511
512 /* ---------------------------------------------------------------------------
513 * Initialise global hosts_db.
514 */
515 void
516 hosts_db_init(void)
517 {
518 assert(hosts_db == NULL);
519 hosts_db = hashtable_make(HOST_BITS, hosts_max, hosts_keep,
520 hash_func_host, free_func_host, key_func_host, find_func_host,
521 make_func_host, format_cols_host, format_row_host);
522 }
523
524 static void
525 hashtable_rehash(struct hashtable *h, const uint8_t bits)
526 {
527 struct bucket **old_table, **new_table;
528 uint32_t i, old_size;
529 assert(h != NULL);
530 assert(bits > 0);
531
532 h->stats.rehashes++;
533 old_size = h->size;
534 old_table = h->table;
535
536 h->bits = bits;
537 h->size = 1U << bits;
538 h->mask = h->size - 1;
539 h->coeff = coprime(h->size);
540 new_table = xcalloc(h->size, sizeof(*new_table));
541
542 for (i=0; i<old_size; i++) {
543 struct bucket *next, *b = old_table[i];
544 while (b != NULL) {
545 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
546 next = b->next;
547 b->next = new_table[pos];
548 new_table[pos] = b;
549 b = next;
550 }
551 }
552
553 free(h->table);
554 h->table = new_table;
555 }
556
557 static void
558 hashtable_insert(struct hashtable *h, struct bucket *b)
559 {
560 uint32_t pos;
561 assert(h != NULL);
562 assert(b != NULL);
563 assert(b->next == NULL);
564
565 /* Rehash on 80% occupancy */
566 if ((h->count > h->size) ||
567 ((h->size - h->count) < h->size / 5))
568 hashtable_rehash(h, h->bits+1);
569
570 pos = h->hash_func(h, h->key_func(b)) & h->mask;
571 if (h->table[pos] == NULL)
572 h->table[pos] = b;
573 else {
574 /* Insert at top of chain. */
575 b->next = h->table[pos];
576 h->table[pos] = b;
577 }
578 h->count++;
579 h->stats.inserts++;
580 }
581
582 /* Return bucket matching key, or NULL if no such entry. */
583 static struct bucket *
584 hashtable_search(struct hashtable *h, const void *key)
585 {
586 uint32_t pos;
587 struct bucket *b;
588
589 h->stats.searches++;
590 pos = h->hash_func(h, key) & h->mask;
591 b = h->table[pos];
592 while (b != NULL) {
593 if (h->find_func(b, key))
594 return (b);
595 else
596 b = b->next;
597 }
598 return (NULL);
599 }
600
601 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
602 /* Search for a key. If it's not there, make and insert a bucket for it. */
603 static struct bucket *
604 hashtable_find_or_insert(struct hashtable *h, const void *key,
605 const reduce_bool allow_reduce)
606 {
607 struct bucket *b = hashtable_search(h, key);
608
609 if (b == NULL) {
610 /* Not found, so insert after checking occupancy. */
611 if (allow_reduce && (h->count >= h->count_max))
612 hashtable_reduce(h);
613 b = h->make_func(key);
614 hashtable_insert(h, b);
615 }
616 return (b);
617 }
618
619 /*
620 * Frees the hashtable and the buckets. The contents are assumed to be
621 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
622 * the bucket.
623 */
624 static void
625 hashtable_free(struct hashtable *h)
626 {
627 uint32_t i;
628
629 if (h == NULL)
630 return;
631 for (i=0; i<h->size; i++) {
632 struct bucket *tmp, *b = h->table[i];
633 while (b != NULL) {
634 tmp = b;
635 b = b->next;
636 h->free_func(tmp);
637 free(tmp);
638 }
639 }
640 free(h->table);
641 free(h);
642 }
643
644 /* ---------------------------------------------------------------------------
645 * Return existing host or insert a new one.
646 */
647 struct bucket *
648 host_get(const struct addr *const a)
649 {
650 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
651 }
652
653 /* ---------------------------------------------------------------------------
654 * Find host, returns NULL if not in DB.
655 */
656 struct bucket *
657 host_find(const struct addr *const a)
658 {
659 return (hashtable_search(hosts_db, a));
660 }
661
662 /* ---------------------------------------------------------------------------
663 * Find host, returns NULL if not in DB.
664 */
665 static struct bucket *
666 host_search(const char *ipstr)
667 {
668 struct addr a;
669 struct addrinfo hints, *ai;
670
671 memset(&hints, 0, sizeof(hints));
672 hints.ai_family = AF_UNSPEC;
673 hints.ai_flags = AI_NUMERICHOST;
674
675 if (getaddrinfo(ipstr, NULL, &hints, &ai))
676 return (NULL); /* invalid addr */
677
678 if (ai->ai_family == AF_INET) {
679 a.family = IPv4;
680 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
681 }
682 else if (ai->ai_family == AF_INET6) {
683 a.family = IPv6;
684 memcpy(&(a.ip.v6),
685 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
686 sizeof(a.ip.v6));
687 } else {
688 freeaddrinfo(ai);
689 return (NULL); /* unknown family */
690 }
691 freeaddrinfo(ai);
692
693 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
694 return (hashtable_search(hosts_db, &a));
695 }
696
697 /* ---------------------------------------------------------------------------
698 * Reduce a hashtable to the top <keep> entries.
699 */
700 static void
701 hashtable_reduce(struct hashtable *ht)
702 {
703 uint32_t i, pos, rmd;
704 const struct bucket **table;
705 uint64_t cutoff;
706
707 assert(ht->count_keep < ht->count);
708
709 /* Fill table with pointers to buckets in hashtable. */
710 table = xcalloc(ht->count, sizeof(*table));
711 for (pos=0, i=0; i<ht->size; i++) {
712 struct bucket *b = ht->table[i];
713 while (b != NULL) {
714 table[pos++] = b;
715 b = b->next;
716 }
717 }
718 assert(pos == ht->count);
719 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
720 cutoff = table[ht->count_keep]->total;
721 free(table);
722
723 /* Remove all elements with total <= cutoff. */
724 rmd = 0;
725 for (i=0; i<ht->size; i++) {
726 struct bucket *last = NULL, *next, *b = ht->table[i];
727 while (b != NULL) {
728 next = b->next;
729 if (b->total <= cutoff) {
730 /* Remove this one. */
731 ht->free_func(b);
732 free(b);
733 if (last == NULL)
734 ht->table[i] = next;
735 else
736 last->next = next;
737 rmd++;
738 ht->count--;
739 } else {
740 last = b;
741 }
742 b = next;
743 }
744 }
745 verbosef("hashtable_reduce: removed %u buckets, left %u",
746 rmd, ht->count);
747 hashtable_rehash(ht, ht->bits); /* is this needed? */
748 }
749
750 /* Reduce hosts_db if needed. */
751 void hosts_db_reduce(void)
752 {
753 if (hosts_db->count >= hosts_db->count_max)
754 hashtable_reduce(hosts_db);
755 }
756
757 /* ---------------------------------------------------------------------------
758 * Reset hosts_db to empty.
759 */
760 void
761 hosts_db_reset(void)
762 {
763 unsigned int i;
764
765 for (i=0; i<hosts_db->size; i++) {
766 struct bucket *next, *b = hosts_db->table[i];
767 while (b != NULL) {
768 next = b->next;
769 hosts_db->free_func(b);
770 free(b);
771 b = next;
772 }
773 hosts_db->table[i] = NULL;
774 }
775 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
776 hosts_db->count = 0;
777 }
778
779 /* ---------------------------------------------------------------------------
780 * Deallocate hosts_db.
781 */
782 void hosts_db_free(void)
783 {
784 uint32_t i;
785
786 assert(hosts_db != NULL);
787 for (i=0; i<hosts_db->size; i++) {
788 struct bucket *tmp, *b = hosts_db->table[i];
789 while (b != NULL) {
790 tmp = b;
791 b = b->next;
792 hosts_db->free_func(tmp);
793 free(tmp);
794 }
795 }
796 free(hosts_db->table);
797 free(hosts_db);
798 hosts_db = NULL;
799 }
800
801 /* ---------------------------------------------------------------------------
802 * Find or create a port_tcp inside a host.
803 */
804 struct bucket *
805 host_get_port_tcp(struct bucket *host, const uint16_t port)
806 {
807 struct host *h = &host->u.host;
808 assert(h != NULL);
809 if (h->ports_tcp == NULL)
810 h->ports_tcp = hashtable_make(PORT_BITS, ports_max, ports_keep,
811 hash_func_short, free_func_simple, key_func_port_tcp,
812 find_func_port_tcp, make_func_port_tcp,
813 format_cols_port_tcp, format_row_port_tcp);
814 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
815 }
816
817 /* ---------------------------------------------------------------------------
818 * Find or create a port_udp inside a host.
819 */
820 struct bucket *
821 host_get_port_udp(struct bucket *host, const uint16_t port)
822 {
823 struct host *h = &host->u.host;
824 assert(h != NULL);
825 if (h->ports_udp == NULL)
826 h->ports_udp = hashtable_make(PORT_BITS, ports_max, ports_keep,
827 hash_func_short, free_func_simple, key_func_port_udp,
828 find_func_port_udp, make_func_port_udp,
829 format_cols_port_udp, format_row_port_udp);
830 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
831 }
832
833 /* ---------------------------------------------------------------------------
834 * Find or create an ip_proto inside a host.
835 */
836 struct bucket *
837 host_get_ip_proto(struct bucket *host, const uint8_t proto)
838 {
839 struct host *h = &host->u.host;
840 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
841 assert(h != NULL);
842 if (h->ip_protos == NULL)
843 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
844 hash_func_byte, free_func_simple, key_func_ip_proto,
845 find_func_ip_proto, make_func_ip_proto,
846 format_cols_ip_proto, format_row_ip_proto);
847 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
848 }
849
850 static struct str *html_hosts_main(const char *qs);
851 static struct str *html_hosts_detail(const char *ip);
852
853 /* ---------------------------------------------------------------------------
854 * Web interface: delegate the /hosts/ space.
855 */
856 struct str *
857 html_hosts(const char *uri, const char *query)
858 {
859 unsigned int i, num_elems;
860 char **elem = split('/', uri, &num_elems);
861 struct str *buf = NULL;
862
863 assert(num_elems >= 1);
864 assert(strcmp(elem[0], "hosts") == 0);
865
866 if (num_elems == 1)
867 /* /hosts/ */
868 buf = html_hosts_main(query);
869 else if (num_elems == 2)
870 /* /hosts/<IP of host>/ */
871 buf = html_hosts_detail(elem[1]);
872
873 for (i=0; i<num_elems; i++)
874 free(elem[i]);
875 free(elem);
876 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
877 other codes to be possible */
878 }
879
880 /* ---------------------------------------------------------------------------
881 * Format hashtable into HTML.
882 */
883 static void
884 format_table(struct str *buf, struct hashtable *ht, unsigned int start,
885 const enum sort_dir sort, const int full)
886 {
887 const struct bucket **table;
888 unsigned int i, pos, end;
889 int alt = 0;
890
891 if ((ht == NULL) || (ht->count == 0)) {
892 str_append(buf, "<p>The table is empty.</p>\n");
893 return;
894 }
895
896 /* Fill table with pointers to buckets in hashtable. */
897 table = xcalloc(ht->count, sizeof(*table));
898 for (pos=0, i=0; i<ht->size; i++) {
899 struct bucket *b = ht->table[i];
900 while (b != NULL) {
901 table[pos++] = b;
902 b = b->next;
903 }
904 }
905 assert(pos == ht->count);
906
907 if (full) {
908 /* full report overrides start and end */
909 start = 0;
910 end = ht->count;
911 } else
912 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
913
914 str_appendf(buf, "(%u-%u of %u)<br>\n", start+1, end, ht->count);
915 qsort_buckets(table, ht->count, start, end, sort);
916 ht->format_cols_func(buf);
917
918 for (i=start; i<end; i++) {
919 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
920 alt = !alt; /* alternate class for table rows */
921 }
922 free(table);
923 str_append(buf, "</table>\n");
924 }
925
926 /* ---------------------------------------------------------------------------
927 * Web interface: sorted table of hosts.
928 */
929 static struct str *
930 html_hosts_main(const char *qs)
931 {
932 struct str *buf = str_make();
933 char *qs_start, *qs_sort, *qs_full, *ep;
934 const char *sortstr;
935 int start, full = 0;
936 enum sort_dir sort;
937
938 /* parse query string */
939 qs_start = qs_get(qs, "start");
940 qs_sort = qs_get(qs, "sort");
941 qs_full = qs_get(qs, "full");
942 if (qs_full != NULL) {
943 full = 1;
944 free(qs_full);
945 }
946
947 /* validate sort */
948 if (qs_sort == NULL) sort = TOTAL;
949 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
950 else if (strcmp(qs_sort, "in") == 0) sort = IN;
951 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
952 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
953 else {
954 str_append(buf, "Error: invalid value for \"sort\".\n");
955 goto done;
956 }
957
958 /* parse start */
959 if (qs_start == NULL)
960 start = 0;
961 else {
962 start = (int)strtoul(qs_start, &ep, 10);
963 if (*ep != '\0') {
964 str_append(buf, "Error: \"start\" is not a number.\n");
965 goto done;
966 }
967 if ((errno == ERANGE) ||
968 (start < 0) || (start >= (int)hosts_db->count)) {
969 str_append(buf, "Error: \"start\" is out of bounds.\n");
970 goto done;
971 }
972 }
973
974 #define PREV "&lt;&lt;&lt; prev page"
975 #define NEXT "next page &gt;&gt;&gt;"
976 #define FULL "full table"
977
978 html_open(buf, "Hosts", interface, /*want_graph_js=*/0);
979 format_table(buf, hosts_db, start, sort, full);
980
981 /* <prev | full | stats | next> */
982 sortstr = qs_sort;
983 if (sortstr == NULL) sortstr = "total";
984 if (start > 0) {
985 int prev = start - MAX_ENTRIES;
986 if (prev < 0)
987 prev = 0;
988 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
989 prev, sortstr);
990 } else
991 str_append(buf, PREV);
992
993 if (full)
994 str_append(buf, " | " FULL);
995 else
996 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
997 sortstr);
998
999 if (start+MAX_ENTRIES < (int)hosts_db->count)
1000 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
1001 start+MAX_ENTRIES, sortstr);
1002 else
1003 str_append(buf, " | " NEXT);
1004
1005 str_append(buf, "<br>\n");
1006
1007 html_close(buf);
1008 done:
1009 if (qs_start != NULL) free(qs_start);
1010 if (qs_sort != NULL) free(qs_sort);
1011 return buf;
1012 #undef PREV
1013 #undef NEXT
1014 #undef FULL
1015 }
1016
1017 /* ---------------------------------------------------------------------------
1018 * Web interface: detailed view of a single host.
1019 */
1020 static struct str *
1021 html_hosts_detail(const char *ip)
1022 {
1023 struct bucket *h;
1024 struct str *buf, *ls_len;
1025 char ls_when[100];
1026 const char *canonical;
1027 time_t ls;
1028
1029 h = host_search(ip);
1030 if (h == NULL)
1031 return (NULL); /* no such host */
1032
1033 canonical = addr_to_str(&(h->u.host.addr));
1034
1035 /* Overview. */
1036 buf = str_make();
1037 html_open(buf, ip, interface, /*want_graph_js=*/0);
1038 if (strcmp(ip, canonical) != 0)
1039 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1040 str_appendf(buf,
1041 "<p>\n"
1042 "<b>Hostname:</b> %s<br>\n",
1043 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1044
1045 /* Resolve host "on demand" */
1046 if (h->u.host.dns == NULL)
1047 dns_queue(&(h->u.host.addr));
1048
1049 if (show_mac_addrs)
1050 str_appendf(buf,
1051 "<b>MAC Address:</b> "
1052 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1053 h->u.host.mac_addr[0],
1054 h->u.host.mac_addr[1],
1055 h->u.host.mac_addr[2],
1056 h->u.host.mac_addr[3],
1057 h->u.host.mac_addr[4],
1058 h->u.host.mac_addr[5]);
1059
1060 str_append(buf,
1061 "</p>\n"
1062 "<p>\n"
1063 "<b>Last seen:</b> ");
1064
1065 ls = h->u.host.last_seen;
1066 if (strftime(ls_when, sizeof(ls_when),
1067 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1068 str_append(buf, ls_when);
1069
1070 if (h->u.host.last_seen <= now) {
1071 ls_len = length_of_time(now - h->u.host.last_seen);
1072 str_append(buf, " (");
1073 str_appendstr(buf, ls_len);
1074 str_free(ls_len);
1075 str_append(buf, " ago)");
1076 } else {
1077 str_append(buf, " (in the future, possible clock problem)");
1078 }
1079
1080 str_appendf(buf,
1081 "</p>\n"
1082 "<p>\n"
1083 " <b>In:</b> %'qu<br>\n"
1084 " <b>Out:</b> %'qu<br>\n"
1085 " <b>Total:</b> %'qu<br>\n"
1086 "</p>\n",
1087 h->in, h->out, h->total);
1088
1089 str_append(buf, "<h3>TCP ports</h3>\n");
1090 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1091
1092 str_append(buf, "<h3>UDP ports</h3>\n");
1093 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1094
1095 str_append(buf, "<h3>IP protocols</h3>\n");
1096 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1097
1098 html_close(buf);
1099 return (buf);
1100 }
1101
1102 /* ---------------------------------------------------------------------------
1103 * Database import and export code:
1104 * Initially written and contributed by Ben Stewart.
1105 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1106 */
1107 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1108 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1109 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1110
1111 static const char
1112 export_proto_ip = 'P',
1113 export_proto_tcp = 'T',
1114 export_proto_udp = 'U';
1115
1116 static const unsigned char
1117 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1118 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1119 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1120
1121 /* ---------------------------------------------------------------------------
1122 * Load a host's ip_proto table from a file.
1123 * Returns 0 on failure, 1 on success.
1124 */
1125 static int
1126 hosts_db_import_ip(const int fd, struct bucket *host)
1127 {
1128 uint8_t count, i;
1129
1130 if (!expect8(fd, export_proto_ip)) return 0;
1131 if (!read8(fd, &count)) return 0;
1132
1133 for (i=0; i<count; i++) {
1134 struct bucket *b;
1135 uint8_t proto;
1136 uint64_t in, out;
1137
1138 if (!read8(fd, &proto)) return 0;
1139 if (!read64(fd, &in)) return 0;
1140 if (!read64(fd, &out)) return 0;
1141
1142 /* Store data */
1143 b = host_get_ip_proto(host, proto);
1144 b->in = in;
1145 b->out = out;
1146 b->total = in + out;
1147 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1148 }
1149 return 1;
1150 }
1151
1152 /* ---------------------------------------------------------------------------
1153 * Load a host's port_tcp table from a file.
1154 * Returns 0 on failure, 1 on success.
1155 */
1156 static int
1157 hosts_db_import_tcp(const int fd, struct bucket *host)
1158 {
1159 uint16_t count, i;
1160
1161 if (!expect8(fd, export_proto_tcp)) return 0;
1162 if (!read16(fd, &count)) return 0;
1163
1164 for (i=0; i<count; i++) {
1165 struct bucket *b;
1166 uint16_t port;
1167 uint64_t in, out, syn;
1168
1169 if (!read16(fd, &port)) return 0;
1170 if (!read64(fd, &syn)) return 0;
1171 if (!read64(fd, &in)) return 0;
1172 if (!read64(fd, &out)) return 0;
1173
1174 /* Store data */
1175 b = host_get_port_tcp(host, port);
1176 b->in = in;
1177 b->out = out;
1178 b->total = in + out;
1179 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1180 b->u.port_tcp.syn = syn;
1181 }
1182 return 1;
1183 }
1184
1185 /* ---------------------------------------------------------------------------
1186 * Load a host's port_tcp table from a file.
1187 * Returns 0 on failure, 1 on success.
1188 */
1189 static int
1190 hosts_db_import_udp(const int fd, struct bucket *host)
1191 {
1192 uint16_t count, i;
1193
1194 if (!expect8(fd, export_proto_udp)) return 0;
1195 if (!read16(fd, &count)) return 0;
1196
1197 for (i=0; i<count; i++) {
1198 struct bucket *b;
1199 uint16_t port;
1200 uint64_t in, out;
1201
1202 if (!read16(fd, &port)) return 0;
1203 if (!read64(fd, &in)) return 0;
1204 if (!read64(fd, &out)) return 0;
1205
1206 /* Store data */
1207 b = host_get_port_udp(host, port);
1208 b->in = in;
1209 b->out = out;
1210 b->total = in + out;
1211 assert(b->u.port_udp.port == port); /* done by make_func */
1212 }
1213 return 1;
1214 }
1215
1216 /* ---------------------------------------------------------------------------
1217 * Load all hosts from a file.
1218 * Returns 0 on failure, 1 on success.
1219 */
1220 static int
1221 hosts_db_import_host(const int fd)
1222 {
1223 struct bucket *host;
1224 struct addr a;
1225 uint8_t hostname_len;
1226 uint64_t in, out;
1227 unsigned int pos = xtell(fd);
1228 char hdr[4];
1229 int ver = 0;
1230
1231 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1232 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1233 ver = 3;
1234 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1235 ver = 2;
1236 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1237 ver = 1;
1238 else {
1239 warnx("bad host header: %02x%02x%02x%02x",
1240 hdr[0], hdr[1], hdr[2], hdr[3]);
1241 return 0;
1242 }
1243
1244 if (ver == 3) {
1245 if (!readaddr(fd, &a))
1246 return 0;
1247 } else {
1248 assert((ver == 1) || (ver == 2));
1249 if (!readaddr_ipv4(fd, &a))
1250 return 0;
1251 }
1252 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1253 host = host_get(&a);
1254 assert(addr_equal(&(host->u.host.addr), &a));
1255
1256 if (ver > 1) {
1257 uint64_t t;
1258 if (!read64(fd, &t)) return 0;
1259 host->u.host.last_seen = (time_t)t;
1260 }
1261
1262 assert(sizeof(host->u.host.mac_addr) == 6);
1263 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1264 return 0;
1265
1266 /* HOSTNAME */
1267 assert(host->u.host.dns == NULL); /* make fn? */
1268 if (!read8(fd, &hostname_len)) return 0;
1269 if (hostname_len > 0) {
1270 host->u.host.dns = xmalloc(hostname_len + 1);
1271 host->u.host.dns[0] = '\0';
1272
1273 /* At this point, the hostname is attached to a host which is in our
1274 * hosts_db, so if we bail out due to an import error, this pointer
1275 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1276 */
1277
1278 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1279 host->u.host.dns[hostname_len] = '\0';
1280 }
1281
1282 if (!read64(fd, &in)) return 0;
1283 if (!read64(fd, &out)) return 0;
1284
1285 host->in = in;
1286 host->out = out;
1287 host->total = in + out;
1288
1289 /* Host's port and proto subtables: */
1290 if (!hosts_db_import_ip(fd, host)) return 0;
1291 if (!hosts_db_import_tcp(fd, host)) return 0;
1292 if (!hosts_db_import_udp(fd, host)) return 0;
1293 return 1;
1294 }
1295
1296 /* ---------------------------------------------------------------------------
1297 * Database Import: Grab hosts_db from a file provided by the caller.
1298 *
1299 * This function will retrieve the data sans the header. We expect the caller
1300 * to have validated the header of the hosts_db segment, and left the file
1301 * sitting at the start of the data.
1302 */
1303 int hosts_db_import(const int fd)
1304 {
1305 uint32_t host_count, i;
1306
1307 if (!read32(fd, &host_count)) return 0;
1308
1309 for (i=0; i<host_count; i++)
1310 if (!hosts_db_import_host(fd)) return 0;
1311
1312 return 1;
1313 }
1314
1315 /* ---------------------------------------------------------------------------
1316 * Database Export: Dump hosts_db into a file provided by the caller.
1317 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1318 */
1319 int hosts_db_export(const int fd)
1320 {
1321 uint32_t i;
1322 struct bucket *b;
1323
1324 if (!write32(fd, hosts_db->count)) return 0;
1325
1326 for (i = 0; i<hosts_db->size; i++)
1327 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1328 /* For each host: */
1329 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1330 return 0;
1331
1332 if (!writeaddr(fd, &(b->u.host.addr))) return 0;
1333
1334 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1335
1336 assert(sizeof(b->u.host.mac_addr) == 6);
1337 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1338 return 0;
1339
1340 /* HOSTNAME */
1341 if (b->u.host.dns == NULL) {
1342 if (!write8(fd, 0)) return 0;
1343 } else {
1344 int dnslen = strlen(b->u.host.dns);
1345
1346 if (dnslen > 255) {
1347 warnx("found a very long hostname: \"%s\"\n"
1348 "wasn't expecting one longer than 255 chars (this one is %d)",
1349 b->u.host.dns, dnslen);
1350 dnslen = 255;
1351 }
1352
1353 if (!write8(fd, (uint8_t)dnslen)) return 0;
1354 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1355 }
1356
1357 if (!write64(fd, b->in)) return 0;
1358 if (!write64(fd, b->out)) return 0;
1359
1360 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1361 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1362 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1363 }
1364 return 1;
1365 }
1366
1367 /* ---------------------------------------------------------------------------
1368 * Dump the ip_proto table of a host.
1369 */
1370 static int
1371 hosts_db_export_ip(const struct hashtable *h, const int fd)
1372 {
1373 uint32_t i, written = 0;
1374 struct bucket *b;
1375
1376 /* IP DATA */
1377 if (!write8(fd, export_proto_ip)) return 0;
1378
1379 /* If no data, write a IP Proto count of 0 and we're done. */
1380 if (h == NULL) {
1381 if (!write8(fd, 0)) return 0;
1382 return 1;
1383 }
1384
1385 assert(h->count < 256);
1386 if (!write8(fd, (uint8_t)h->count)) return 0;
1387
1388 for (i = 0; i<h->size; i++)
1389 for (b = h->table[i]; b != NULL; b = b->next) {
1390 /* For each ip_proto bucket: */
1391
1392 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1393 if (!write64(fd, b->in)) return 0;
1394 if (!write64(fd, b->out)) return 0;
1395 written++;
1396 }
1397 assert(written == h->count);
1398 return 1;
1399 }
1400
1401 /* ---------------------------------------------------------------------------
1402 * Dump the port_tcp table of a host.
1403 */
1404 static int
1405 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1406 {
1407 struct bucket *b;
1408 uint32_t i, written = 0;
1409
1410 /* TCP DATA */
1411 if (!write8(fd, export_proto_tcp)) return 0;
1412
1413 /* If no data, write a count of 0 and we're done. */
1414 if (h == NULL) {
1415 if (!write16(fd, 0)) return 0;
1416 return 1;
1417 }
1418
1419 assert(h->count < 65536);
1420 if (!write16(fd, (uint16_t)h->count)) return 0;
1421
1422 for (i = 0; i<h->size; i++)
1423 for (b = h->table[i]; b != NULL; b = b->next) {
1424 if (!write16(fd, b->u.port_tcp.port)) return 0;
1425 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1426 if (!write64(fd, b->in)) return 0;
1427 if (!write64(fd, b->out)) return 0;
1428 written++;
1429 }
1430 assert(written == h->count);
1431 return 1;
1432 }
1433
1434 /* ---------------------------------------------------------------------------
1435 * Dump the port_udp table of a host.
1436 */
1437 static int
1438 hosts_db_export_udp(const struct hashtable *h, const int fd)
1439 {
1440 struct bucket *b;
1441 uint32_t i, written = 0;
1442
1443 /* UDP DATA */
1444 if (!write8(fd, export_proto_udp)) return 0;
1445
1446 /* If no data, write a count of 0 and we're done. */
1447 if (h == NULL) {
1448 if (!write16(fd, 0)) return 0;
1449 return 1;
1450 }
1451
1452 assert(h->count < 65536);
1453 if (!write16(fd, (uint16_t)h->count)) return 0;
1454
1455 for (i = 0; i<h->size; i++)
1456 for (b = h->table[i]; b != NULL; b = b->next) {
1457 if (!write16(fd, b->u.port_udp.port)) return 0;
1458 if (!write64(fd, b->in)) return 0;
1459 if (!write64(fd, b->out)) return 0;
1460 written++;
1461 }
1462 assert(written == h->count);
1463 return 1;
1464 }
1465
1466 /* vim:set ts=3 sw=3 tw=78 expandtab: */