allow sort on "last seen"
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2009 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20
21 #include <arpa/inet.h> /* inet_aton() */
22 #include <netdb.h> /* struct addrinfo */
23 #include <assert.h>
24 #include <errno.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h> /* memset(), strcmp() */
28 #include <unistd.h>
29
30 extern int want_lastseen;
31 int show_mac_addrs = 0;
32 extern const char *interface;
33
34 /* FIXME: specify somewhere more sane/tunable */
35 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
36
37 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
38 typedef void (free_func_t)(struct bucket *);
39 typedef const void * (key_func_t)(const struct bucket *);
40 typedef int (find_func_t)(const struct bucket *, const void *);
41 typedef struct bucket * (make_func_t)(const void *);
42 typedef void (format_cols_func_t)(struct str *);
43 typedef void (format_row_func_t)(struct str *, const struct bucket *,
44 const char *);
45
46 struct hashtable {
47 uint8_t bits; /* size of hashtable in bits */
48 uint32_t size, mask;
49 uint32_t count, count_max, count_keep; /* items in table */
50 uint32_t coeff; /* coefficient for Fibonacci hashing */
51 struct bucket **table;
52
53 struct {
54 uint64_t inserts, searches, deletions, rehashes;
55 } stats;
56
57 hash_func_t *hash_func;
58 /* returns hash value of given key (passed as void*) */
59
60 free_func_t *free_func;
61 /* free of bucket payload */
62
63 key_func_t *key_func;
64 /* returns pointer to key of bucket (to pass to hash_func) */
65
66 find_func_t *find_func;
67 /* returns true if given bucket matches key (passed as void*) */
68
69 make_func_t *make_func;
70 /* returns bucket containing new record with key (passed as void*) */
71
72 format_cols_func_t *format_cols_func;
73 /* append table columns to str */
74
75 format_row_func_t *format_row_func;
76 /* format record and append to str */
77 };
78
79 static void hashtable_reduce(struct hashtable *ht);
80 static void hashtable_free(struct hashtable *h);
81
82 #define HOST_BITS 1 /* initial size of hosts table */
83 #define PORT_BITS 1 /* initial size of ports tables */
84 #define PROTO_BITS 1 /* initial size of proto table */
85
86 /* We only use one hosts_db hashtable and this is it. */
87 static struct hashtable *hosts_db = NULL;
88
89 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
90 static const double phi_1 =
91 0.61803398874989490252573887119069695472717285156250;
92
93 /* Co-prime of u, using phi^-1 */
94 inline static uint32_t
95 coprime(const uint32_t u)
96 {
97 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
98 }
99
100 /*
101 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
102 * src/sys/netinet/tcp_hostcache.c 1.1
103 */
104 inline static uint32_t
105 ipv4_hash(const struct in_addr *const ip)
106 {
107 return ( (ip->s_addr) ^ ((ip->s_addr) >> 7) ^ ((ip->s_addr) >> 17) );
108 }
109
110 #ifndef s6_addr32
111 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
112 * taken care of GNU/Linux and GNU/kfreebsd. */
113 # define s6_addr32 __u6_addr.__u6_addr32
114 #endif
115
116 /*
117 * This is the IPv6 hash function used by FreeBSD in the same file as above,
118 * svn rev 122922.
119 */
120 inline static uint32_t
121 ipv6_hash(const struct in6_addr *const ip6)
122 {
123 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^ ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
124 }
125
126 /* ---------------------------------------------------------------------------
127 * hash_func collection
128 */
129 #define CASTKEY(type) (*((const type *)key))
130
131 static uint32_t
132 hash_func_host(const struct hashtable *h _unused_, const void *key)
133 {
134 struct addr46 *ip = (struct addr46 *) key;
135
136 switch (ip->af) {
137 case AF_INET:
138 return (ipv4_hash(&ip->addr.ip));
139 case AF_INET6:
140 return (ipv6_hash(&ip->addr.ip6));
141 default:
142 return 0;
143 }
144 }
145
146 static uint32_t
147 hash_func_short(const struct hashtable *h, const void *key)
148 {
149 return (CASTKEY(uint16_t) * h->coeff);
150 }
151
152 static uint32_t
153 hash_func_byte(const struct hashtable *h, const void *key)
154 {
155 return (CASTKEY(uint8_t) * h->coeff);
156 }
157
158 /* ---------------------------------------------------------------------------
159 * key_func collection
160 */
161
162 static const void *
163 key_func_host(const struct bucket *b)
164 {
165 return &(b->u.host.ipaddr);
166 }
167
168 static const void *
169 key_func_port_tcp(const struct bucket *b)
170 {
171 return &(b->u.port_tcp.port);
172 }
173
174 static const void *
175 key_func_port_udp(const struct bucket *b)
176 {
177 return &(b->u.port_udp.port);
178 }
179
180 static const void *
181 key_func_ip_proto(const struct bucket *b)
182 {
183 return &(b->u.ip_proto.proto);
184 }
185
186 /* ---------------------------------------------------------------------------
187 * find_func collection
188 */
189
190 static int
191 find_func_host(const struct bucket *b, const void *key)
192 {
193 struct addr46 *ipaddr = (struct addr46 *) key;
194
195 if (b->u.host.ipaddr.af != ipaddr->af)
196 return 0; /* not comparable */
197
198 switch (ipaddr->af) {
199 case AF_INET:
200 return (memcmp(&b->u.host.ipaddr.addr.ip,
201 &ipaddr->addr.ip, sizeof(ipaddr->addr.ip))
202 == 0);
203 case AF_INET6:
204 return (memcmp(&b->u.host.ipaddr.addr.ip6,
205 &ipaddr->addr.ip6, sizeof(ipaddr->addr.ip6))
206 == 0);
207 default:
208 return 0;
209 }
210 }
211
212 static int
213 find_func_port_tcp(const struct bucket *b, const void *key)
214 {
215 return (b->u.port_tcp.port == CASTKEY(uint16_t));
216 }
217
218 static int
219 find_func_port_udp(const struct bucket *b, const void *key)
220 {
221 return (b->u.port_udp.port == CASTKEY(uint16_t));
222 }
223
224 static int
225 find_func_ip_proto(const struct bucket *b, const void *key)
226 {
227 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
228 }
229
230 /* ---------------------------------------------------------------------------
231 * make_func collection
232 */
233
234 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
235 struct bucket *next; \
236 uint64_t in, out, total; \
237 union { struct type t; } u; } _custom_bucket; \
238 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
239 struct type *name_content = &(name_bucket->u.type); \
240 name_bucket->next = NULL; \
241 name_bucket->in = name_bucket->out = name_bucket->total = 0;
242
243 static struct bucket *
244 make_func_host(const void *key)
245 {
246 MAKE_BUCKET(b, h, host);
247 memcpy(&h->ipaddr, key, sizeof(h->ipaddr));
248 h->dns = NULL;
249 h->last_seen = now;
250 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
251 h->ports_tcp = NULL;
252 h->ports_udp = NULL;
253 h->ip_protos = NULL;
254 return (b);
255 }
256
257 static void
258 free_func_host(struct bucket *b)
259 {
260 struct host *h = &(b->u.host);
261 if (h->dns != NULL) free(h->dns);
262 hashtable_free(h->ports_tcp);
263 hashtable_free(h->ports_udp);
264 hashtable_free(h->ip_protos);
265 }
266
267 static struct bucket *
268 make_func_port_tcp(const void *key)
269 {
270 MAKE_BUCKET(b, p, port_tcp);
271 p->port = CASTKEY(uint16_t);
272 p->syn = 0;
273 return (b);
274 }
275
276 static struct bucket *
277 make_func_port_udp(const void *key)
278 {
279 MAKE_BUCKET(b, p, port_udp);
280 p->port = CASTKEY(uint16_t);
281 return (b);
282 }
283
284 static struct bucket *
285 make_func_ip_proto(const void *key)
286 {
287 MAKE_BUCKET(b, p, ip_proto);
288 p->proto = CASTKEY(uint8_t);
289 return (b);
290 }
291
292 static void
293 free_func_simple(struct bucket *b _unused_)
294 {
295 /* nop */
296 }
297
298 /* ---------------------------------------------------------------------------
299 * format_func collection (ordered by struct)
300 */
301
302 static void
303 format_cols_host(struct str *buf)
304 {
305 /* FIXME: don't clobber parts of the query string
306 * specifically "full" and "start"
307 * when setting sort direction
308 */
309 str_append(buf,
310 "<table>\n"
311 "<tr>\n"
312 " <th>IP</th>\n"
313 " <th>Hostname</th>\n");
314 if (show_mac_addrs) str_append(buf,
315 " <th>MAC Address</th>\n");
316 str_append(buf,
317 " <th><a href=\"?sort=in\">In</a></th>\n"
318 " <th><a href=\"?sort=out\">Out</a></th>\n"
319 " <th><a href=\"?sort=total\">Total</a></th>\n");
320 if (want_lastseen) str_append(buf,
321 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
322 str_append(buf,
323 "</tr>\n");
324 }
325
326 static void
327 format_row_host(struct str *buf, const struct bucket *b,
328 const char *css_class)
329 {
330 const char *ip = ip_to_str( &b->u.host.ipaddr );
331
332 str_appendf(buf,
333 "<tr class=\"%s\">\n"
334 " <td><a href=\"/hosts/%s/\">%s</a></td>\n"
335 " <td>%s</td>\n",
336 css_class,
337 ip, ip,
338 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
339
340 if (show_mac_addrs)
341 str_appendf(buf,
342 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
343 b->u.host.mac_addr[0],
344 b->u.host.mac_addr[1],
345 b->u.host.mac_addr[2],
346 b->u.host.mac_addr[3],
347 b->u.host.mac_addr[4],
348 b->u.host.mac_addr[5]);
349
350 str_appendf(buf,
351 " <td class=\"num\">%'qu</td>\n"
352 " <td class=\"num\">%'qu</td>\n"
353 " <td class=\"num\">%'qu</td>\n",
354 b->in, b->out, b->total);
355
356 if (want_lastseen) {
357 time_t last_t = b->u.host.last_seen;
358 struct str *lastseen = NULL;
359
360 if (now >= last_t)
361 lastseen = length_of_time(now - last_t);
362
363 str_append(buf,
364 " <td class=\"num\">");
365 if (lastseen == NULL)
366 str_append(buf, "(clock error)");
367 else {
368 str_appendstr(buf, lastseen);
369 str_free(lastseen);
370 }
371 str_append(buf,
372 "</td>");
373 }
374
375 str_appendf(buf,
376 "</tr>\n");
377
378 /* Only resolve hosts "on demand" */
379 if (b->u.host.dns == NULL)
380 dns_queue(&b->u.host.ipaddr);
381 }
382
383 static void
384 format_cols_port_tcp(struct str *buf)
385 {
386 str_append(buf,
387 "<table>\n"
388 "<tr>\n"
389 " <th>Port</td>\n"
390 " <th>Service</td>\n"
391 " <th>In</td>\n"
392 " <th>Out</td>\n"
393 " <th>Total</td>\n"
394 " <th>SYNs</td>\n"
395 "</tr>\n"
396 );
397 }
398
399 static void
400 format_row_port_tcp(struct str *buf, const struct bucket *b,
401 const char *css_class)
402 {
403 const struct port_tcp *p = &(b->u.port_tcp);
404
405 str_appendf(buf,
406 "<tr class=\"%s\">\n"
407 " <td class=\"num\">%u</td>\n"
408 " <td>%s</td>\n"
409 " <td class=\"num\">%'qu</td>\n"
410 " <td class=\"num\">%'qu</td>\n"
411 " <td class=\"num\">%'qu</td>\n"
412 " <td class=\"num\">%'qu</td>\n"
413 "</tr>\n",
414 css_class,
415 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
416 );
417 }
418
419 static void
420 format_cols_port_udp(struct str *buf)
421 {
422 str_append(buf,
423 "<table>\n"
424 "<tr>\n"
425 " <th>Port</td>\n"
426 " <th>Service</td>\n"
427 " <th>In</td>\n"
428 " <th>Out</td>\n"
429 " <th>Total</td>\n"
430 "</tr>\n"
431 );
432 }
433
434 static void
435 format_row_port_udp(struct str *buf, const struct bucket *b,
436 const char *css_class)
437 {
438 const struct port_udp *p = &(b->u.port_udp);
439
440 str_appendf(buf,
441 "<tr class=\"%s\">\n"
442 " <td class=\"num\">%u</td>\n"
443 " <td>%s</td>\n"
444 " <td class=\"num\">%'qu</td>\n"
445 " <td class=\"num\">%'qu</td>\n"
446 " <td class=\"num\">%'qu</td>\n"
447 "</tr>\n",
448 css_class,
449 p->port, getservudp(p->port), b->in, b->out, b->total
450 );
451 }
452
453 static void
454 format_cols_ip_proto(struct str *buf)
455 {
456 str_append(buf,
457 "<table>\n"
458 "<tr>\n"
459 " <th>#</td>\n"
460 " <th>Protocol</td>\n"
461 " <th>In</td>\n"
462 " <th>Out</td>\n"
463 " <th>Total</td>\n"
464 "</tr>\n"
465 );
466 }
467
468 static void
469 format_row_ip_proto(struct str *buf, const struct bucket *b,
470 const char *css_class)
471 {
472 const struct ip_proto *p = &(b->u.ip_proto);
473
474 str_appendf(buf,
475 "<tr class=\"%s\">\n"
476 " <td class=\"num\">%u</td>\n"
477 " <td>%s</td>\n"
478 " <td class=\"num\">%'qu</td>\n"
479 " <td class=\"num\">%'qu</td>\n"
480 " <td class=\"num\">%'qu</td>\n"
481 "</tr>\n",
482 css_class,
483 p->proto, getproto(p->proto),
484 b->in, b->out, b->total
485 );
486 }
487
488 /* ---------------------------------------------------------------------------
489 * Initialise a hashtable.
490 */
491 static struct hashtable *
492 hashtable_make(const uint8_t bits,
493 const unsigned int count_max,
494 const unsigned int count_keep,
495 hash_func_t *hash_func,
496 free_func_t *free_func,
497 key_func_t *key_func,
498 find_func_t *find_func,
499 make_func_t *make_func,
500 format_cols_func_t *format_cols_func,
501 format_row_func_t *format_row_func)
502 {
503 struct hashtable *hash;
504 assert(bits > 0);
505
506 hash = xmalloc(sizeof(*hash));
507 hash->bits = bits;
508 hash->count_max = count_max;
509 hash->count_keep = count_keep;
510 hash->size = 1U << bits;
511 hash->mask = hash->size - 1;
512 hash->coeff = coprime(hash->size);
513 hash->hash_func = hash_func;
514 hash->free_func = free_func;
515 hash->key_func = key_func;
516 hash->find_func = find_func;
517 hash->make_func = make_func;
518 hash->format_cols_func = format_cols_func;
519 hash->format_row_func = format_row_func;
520 hash->count = 0;
521 hash->table = xcalloc(hash->size, sizeof(*hash->table));
522 memset(&(hash->stats), 0, sizeof(hash->stats));
523 return (hash);
524 }
525
526 /* ---------------------------------------------------------------------------
527 * Initialise global hosts_db.
528 */
529 void
530 hosts_db_init(void)
531 {
532 assert(hosts_db == NULL);
533 hosts_db = hashtable_make(HOST_BITS, hosts_max, hosts_keep,
534 hash_func_host, free_func_host, key_func_host, find_func_host,
535 make_func_host, format_cols_host, format_row_host);
536 }
537
538 static void
539 hashtable_rehash(struct hashtable *h, const uint8_t bits)
540 {
541 struct bucket **old_table, **new_table;
542 uint32_t i, old_size;
543 assert(h != NULL);
544 assert(bits > 0);
545
546 h->stats.rehashes++;
547 old_size = h->size;
548 old_table = h->table;
549
550 h->bits = bits;
551 h->size = 1U << bits;
552 h->mask = h->size - 1;
553 h->coeff = coprime(h->size);
554 new_table = xcalloc(h->size, sizeof(*new_table));
555
556 for (i=0; i<old_size; i++) {
557 struct bucket *next, *b = old_table[i];
558 while (b != NULL) {
559 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
560 next = b->next;
561 b->next = new_table[pos];
562 new_table[pos] = b;
563 b = next;
564 }
565 }
566
567 free(h->table);
568 h->table = new_table;
569 }
570
571 static void
572 hashtable_insert(struct hashtable *h, struct bucket *b)
573 {
574 uint32_t pos;
575 assert(h != NULL);
576 assert(b != NULL);
577 assert(b->next == NULL);
578
579 /* Rehash on 80% occupancy */
580 if ((h->count > h->size) ||
581 ((h->size - h->count) < h->size / 5))
582 hashtable_rehash(h, h->bits+1);
583
584 pos = h->hash_func(h, h->key_func(b)) & h->mask;
585 if (h->table[pos] == NULL)
586 h->table[pos] = b;
587 else {
588 /* Insert at top of chain. */
589 b->next = h->table[pos];
590 h->table[pos] = b;
591 }
592 h->count++;
593 h->stats.inserts++;
594 }
595
596 /* Return bucket matching key, or NULL if no such entry. */
597 static struct bucket *
598 hashtable_search(struct hashtable *h, const void *key)
599 {
600 uint32_t pos;
601 struct bucket *b;
602
603 h->stats.searches++;
604 pos = h->hash_func(h, key) & h->mask;
605 b = h->table[pos];
606 while (b != NULL) {
607 if (h->find_func(b, key))
608 return (b);
609 else
610 b = b->next;
611 }
612 return (NULL);
613 }
614
615 /* Search for a key. If it's not there, make and insert a bucket for it. */
616 static struct bucket *
617 hashtable_find_or_insert(struct hashtable *h, const void *key)
618 {
619 struct bucket *b = hashtable_search(h, key);
620
621 if (b == NULL) {
622 /* Not found, so insert after checking occupancy. */
623 /*assert(h->count <= h->count_max);*/
624 if (h->count >= h->count_max) hashtable_reduce(h);
625 b = h->make_func(key);
626 hashtable_insert(h, b);
627 }
628 return (b);
629 }
630
631 /*
632 * Frees the hashtable and the buckets. The contents are assumed to be
633 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
634 * the bucket.
635 */
636 static void
637 hashtable_free(struct hashtable *h)
638 {
639 uint32_t i;
640
641 if (h == NULL)
642 return;
643 for (i=0; i<h->size; i++) {
644 struct bucket *tmp, *b = h->table[i];
645 while (b != NULL) {
646 tmp = b;
647 b = b->next;
648 h->free_func(tmp);
649 free(tmp);
650 }
651 }
652 free(h->table);
653 free(h);
654 }
655
656 /* ---------------------------------------------------------------------------
657 * Return existing host or insert a new one.
658 */
659 struct bucket *
660 host_get(const struct addr46 *const ip)
661 {
662 return (hashtable_find_or_insert(hosts_db, ip));
663 }
664
665 /* ---------------------------------------------------------------------------
666 * Find host, returns NULL if not in DB.
667 */
668 struct bucket *
669 host_find(const struct addr46 *const ip)
670 {
671 return (hashtable_search(hosts_db, ip));
672 }
673
674 /* ---------------------------------------------------------------------------
675 * Find host, returns NULL if not in DB.
676 */
677 static struct bucket *
678 host_search(const char *ipstr)
679 {
680 struct addr46 ipaddr;
681 struct sockaddr_storage ss;
682 struct addrinfo hints, *ai;
683
684 memset(&hints, 0, sizeof(hints));
685 hints.ai_family = AF_UNSPEC;
686 hints.ai_flags = AI_NUMERICHOST;
687
688 if (getaddrinfo(ipstr, NULL, &hints, &ai))
689 return (NULL); /* invalid addr */
690
691 memcpy(&ss, ai->ai_addr, ai->ai_addrlen);
692 freeaddrinfo(ai);
693
694 ipaddr.af = ss.ss_family;
695 switch (ss.ss_family) {
696 case AF_INET:
697 memcpy(&ipaddr.addr.ip, &((struct sockaddr_in *) &ss)->sin_addr,
698 sizeof(ipaddr.addr.ip));
699 break;
700 case AF_INET6:
701 memcpy(&ipaddr.addr.ip6, &((struct sockaddr_in6 *) &ss)->sin6_addr,
702 sizeof(ipaddr.addr.ip6));
703 break;
704 default:
705 return (NULL);
706 }
707
708 return (hashtable_search(hosts_db, &ipaddr));
709 }
710
711 /* ---------------------------------------------------------------------------
712 * Reduce a hashtable to the top <keep> entries.
713 */
714 static void
715 hashtable_reduce(struct hashtable *ht)
716 {
717 uint32_t i, pos, rmd;
718 const struct bucket **table;
719 uint64_t cutoff;
720
721 assert(ht->count_keep < ht->count);
722
723 /* Fill table with pointers to buckets in hashtable. */
724 table = xcalloc(ht->count, sizeof(*table));
725 for (pos=0, i=0; i<ht->size; i++) {
726 struct bucket *b = ht->table[i];
727 while (b != NULL) {
728 table[pos++] = b;
729 b = b->next;
730 }
731 }
732 assert(pos == ht->count);
733 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
734 cutoff = table[ht->count_keep]->total;
735 free(table);
736
737 /* Remove all elements with total <= cutoff. */
738 rmd = 0;
739 for (i=0; i<ht->size; i++) {
740 struct bucket *last = NULL, *next, *b = ht->table[i];
741 while (b != NULL) {
742 next = b->next;
743 if (b->total <= cutoff) {
744 /* Remove this one. */
745 ht->free_func(b);
746 free(b);
747 if (last == NULL)
748 ht->table[i] = next;
749 else
750 last->next = next;
751 rmd++;
752 ht->count--;
753 } else {
754 last = b;
755 }
756 b = next;
757 }
758 }
759 verbosef("hashtable_reduce: removed %u buckets, left %u",
760 rmd, ht->count);
761 hashtable_rehash(ht, ht->bits); /* is this needed? */
762 }
763
764 /* ---------------------------------------------------------------------------
765 * Reset hosts_db to empty.
766 */
767 void
768 hosts_db_reset(void)
769 {
770 unsigned int i;
771
772 for (i=0; i<hosts_db->size; i++) {
773 struct bucket *next, *b = hosts_db->table[i];
774 while (b != NULL) {
775 next = b->next;
776 hosts_db->free_func(b);
777 free(b);
778 b = next;
779 }
780 hosts_db->table[i] = NULL;
781 }
782 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
783 hosts_db->count = 0;
784 }
785
786 /* ---------------------------------------------------------------------------
787 * Deallocate hosts_db.
788 */
789 void hosts_db_free(void)
790 {
791 uint32_t i;
792
793 assert(hosts_db != NULL);
794 for (i=0; i<hosts_db->size; i++) {
795 struct bucket *tmp, *b = hosts_db->table[i];
796 while (b != NULL) {
797 tmp = b;
798 b = b->next;
799 hosts_db->free_func(tmp);
800 free(tmp);
801 }
802 }
803 free(hosts_db->table);
804 free(hosts_db);
805 hosts_db = NULL;
806 }
807
808 /* ---------------------------------------------------------------------------
809 * Find or create a port_tcp inside a host.
810 */
811 struct bucket *
812 host_get_port_tcp(struct bucket *host, const uint16_t port)
813 {
814 struct host *h = &host->u.host;
815 assert(h != NULL);
816 if (h->ports_tcp == NULL)
817 h->ports_tcp = hashtable_make(PORT_BITS, ports_max, ports_keep,
818 hash_func_short, free_func_simple, key_func_port_tcp,
819 find_func_port_tcp, make_func_port_tcp,
820 format_cols_port_tcp, format_row_port_tcp);
821 return (hashtable_find_or_insert(h->ports_tcp, &port));
822 }
823
824 /* ---------------------------------------------------------------------------
825 * Find or create a port_udp inside a host.
826 */
827 struct bucket *
828 host_get_port_udp(struct bucket *host, const uint16_t port)
829 {
830 struct host *h = &host->u.host;
831 assert(h != NULL);
832 if (h->ports_udp == NULL)
833 h->ports_udp = hashtable_make(PORT_BITS, ports_max, ports_keep,
834 hash_func_short, free_func_simple, key_func_port_udp,
835 find_func_port_udp, make_func_port_udp,
836 format_cols_port_udp, format_row_port_udp);
837 return (hashtable_find_or_insert(h->ports_udp, &port));
838 }
839
840 /* ---------------------------------------------------------------------------
841 * Find or create an ip_proto inside a host.
842 */
843 struct bucket *
844 host_get_ip_proto(struct bucket *host, const uint8_t proto)
845 {
846 struct host *h = &host->u.host;
847 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
848 assert(h != NULL);
849 if (h->ip_protos == NULL)
850 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
851 hash_func_byte, free_func_simple, key_func_ip_proto,
852 find_func_ip_proto, make_func_ip_proto,
853 format_cols_ip_proto, format_row_ip_proto);
854 return (hashtable_find_or_insert(h->ip_protos, &proto));
855 }
856
857 static struct str *html_hosts_main(const char *qs);
858 static struct str *html_hosts_detail(const char *ip);
859
860 /* ---------------------------------------------------------------------------
861 * Web interface: delegate the /hosts/ space.
862 */
863 struct str *
864 html_hosts(const char *uri, const char *query)
865 {
866 int i, num_elems;
867 char **elem = split('/', uri, &num_elems);
868 struct str *buf = NULL;
869
870 assert(num_elems >= 1);
871 assert(strcmp(elem[0], "hosts") == 0);
872
873 if (num_elems == 1)
874 /* /hosts/ */
875 buf = html_hosts_main(query);
876 else if (num_elems == 2)
877 /* /hosts/<IP of host>/ */
878 buf = html_hosts_detail(elem[1]);
879
880 for (i=0; i<num_elems; i++)
881 free(elem[i]);
882 free(elem);
883 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
884 other codes to be possible */
885 }
886
887 /* ---------------------------------------------------------------------------
888 * Format hashtable into HTML.
889 */
890 static void
891 format_table(struct str *buf, struct hashtable *ht, int start,
892 const enum sort_dir sort, const int full)
893 {
894 const struct bucket **table;
895 uint32_t i, pos, end;
896 int alt = 0;
897
898 if ((ht == NULL) || (ht->count == 0)) {
899 str_append(buf, "<p>The table is empty.</p>\n");
900 return;
901 }
902
903 /* Fill table with pointers to buckets in hashtable. */
904 table = xcalloc(ht->count, sizeof(*table));
905 for (pos=0, i=0; i<ht->size; i++) {
906 struct bucket *b = ht->table[i];
907 while (b != NULL) {
908 table[pos++] = b;
909 b = b->next;
910 }
911 }
912 assert(pos == ht->count);
913
914 if (full) {
915 /* full report overrides start and end */
916 start = 0;
917 end = ht->count;
918 } else
919 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
920
921 str_appendf(buf, "(%u-%u of %u)<br/>\n", start+1, end, ht->count);
922 qsort_buckets(table, ht->count, start, end, sort);
923 ht->format_cols_func(buf);
924
925 for (i=start; i<end; i++) {
926 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
927 alt = !alt; /* alternate class for table rows */
928 }
929 free(table);
930 str_append(buf, "</table>\n");
931 }
932
933 /* ---------------------------------------------------------------------------
934 * Web interface: sorted table of hosts.
935 */
936 static struct str *
937 html_hosts_main(const char *qs)
938 {
939 struct str *buf = str_make();
940 char *qs_start, *qs_sort, *qs_full, *ep;
941 const char *sortstr;
942 int start, full = 0;
943 enum sort_dir sort;
944
945 /* parse query string */
946 qs_start = qs_get(qs, "start");
947 qs_sort = qs_get(qs, "sort");
948 qs_full = qs_get(qs, "full");
949 if (qs_full != NULL) {
950 full = 1;
951 free(qs_full);
952 }
953
954 /* validate sort */
955 if (qs_sort == NULL) sort = TOTAL;
956 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
957 else if (strcmp(qs_sort, "in") == 0) sort = IN;
958 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
959 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
960 else {
961 str_append(buf, "Error: invalid value for \"sort\".\n");
962 goto done;
963 }
964
965 /* parse start */
966 if (qs_start == NULL)
967 start = 0;
968 else {
969 start = (int)strtoul(qs_start, &ep, 10);
970 if (*ep != '\0') {
971 str_append(buf, "Error: \"start\" is not a number.\n");
972 goto done;
973 }
974 if ((errno == ERANGE) ||
975 (start < 0) || (start >= (int)hosts_db->count)) {
976 str_append(buf, "Error: \"start\" is out of bounds.\n");
977 goto done;
978 }
979 }
980
981 #define PREV "&lt;&lt;&lt; prev page"
982 #define NEXT "next page &gt;&gt;&gt;"
983 #define FULL "full table"
984
985 str_append(buf, html_header_1);
986 str_appendf(buf, " <title>darkstat3: Hosts (%s)</title>\n", interface);
987 str_append(buf, html_header_2);
988 str_appendf(buf, "<h2 class=\"pageheader\">Hosts (%s)</h2>\n", interface);
989 format_table(buf, hosts_db, start, sort, full);
990
991 /* <prev | full | stats | next> */
992 sortstr = qs_sort;
993 if (sortstr == NULL) sortstr = "total";
994 if (start > 0) {
995 int prev = max(start - MAX_ENTRIES, 0);
996 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
997 prev, sortstr);
998 } else
999 str_append(buf, PREV);
1000
1001 if (full)
1002 str_append(buf, " | " FULL);
1003 else
1004 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
1005 sortstr);
1006
1007 if (start+MAX_ENTRIES < (int)hosts_db->count)
1008 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
1009 start+MAX_ENTRIES, sortstr);
1010 else
1011 str_append(buf, " | " NEXT);
1012
1013 str_append(buf, "<br/>\n");
1014 str_append(buf, html_footer);
1015 done:
1016 if (qs_start != NULL) free(qs_start);
1017 if (qs_sort != NULL) free(qs_sort);
1018 return buf;
1019 #undef PREV
1020 #undef NEXT
1021 #undef FULL
1022 }
1023
1024 /* ---------------------------------------------------------------------------
1025 * Web interface: detailed view of a single host.
1026 */
1027 static struct str *
1028 html_hosts_detail(const char *ip)
1029 {
1030 struct bucket *h;
1031 struct str *buf, *ls_len;
1032 char ls_when[100];
1033 time_t ls;
1034
1035 h = host_search(ip);
1036 if (h == NULL)
1037 return (NULL); /* no such host */
1038
1039 /* Overview. */
1040 buf = str_make();
1041 str_append(buf, html_header_1);
1042 str_appendf(buf, " <title>%s</title>\n", ip);
1043 str_append(buf, html_header_2);
1044 str_appendf(buf,
1045 "<h2>%s</h2>\n"
1046 "<p>\n"
1047 "<b>Hostname:</b> %s<br/>\n",
1048 ip,
1049 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1050
1051 /* Resolve host "on demand" */
1052 if (h->u.host.dns == NULL)
1053 dns_queue(&h->u.host.ipaddr);
1054
1055 if (show_mac_addrs)
1056 str_appendf(buf,
1057 "<b>MAC Address:</b> "
1058 "<tt>%x:%x:%x:%x:%x:%x</tt><br/>\n",
1059 h->u.host.mac_addr[0],
1060 h->u.host.mac_addr[1],
1061 h->u.host.mac_addr[2],
1062 h->u.host.mac_addr[3],
1063 h->u.host.mac_addr[4],
1064 h->u.host.mac_addr[5]);
1065
1066 str_append(buf,
1067 "</p>\n"
1068 "<p>\n"
1069 "<b>Last seen:</b> ");
1070
1071 ls = h->u.host.last_seen;
1072 if (strftime(ls_when, sizeof(ls_when),
1073 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1074 str_append(buf, ls_when);
1075
1076 if (h->u.host.last_seen <= now) {
1077 ls_len = length_of_time(now - h->u.host.last_seen);
1078 str_append(buf, " (");
1079 str_appendstr(buf, ls_len);
1080 str_free(ls_len);
1081 str_append(buf, " ago)");
1082 } else {
1083 str_append(buf, " (in the future, possible clock problem)");
1084 }
1085
1086 str_appendf(buf,
1087 "</p>\n"
1088 "<p>\n"
1089 " <b>In:</b> %'qu<br/>\n"
1090 " <b>Out:</b> %'qu<br/>\n"
1091 " <b>Total:</b> %'qu<br/>\n"
1092 "</p>\n",
1093 h->in, h->out, h->total);
1094
1095 str_append(buf, "<h3>TCP ports</h3>\n");
1096 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1097
1098 str_append(buf, "<h3>UDP ports</h3>\n");
1099 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1100
1101 str_append(buf, "<h3>IP protocols</h3>\n");
1102 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1103
1104 str_append(buf, html_footer);
1105 return (buf);
1106 }
1107
1108 /* ---------------------------------------------------------------------------
1109 * Database import and export code:
1110 * Initially written and contributed by Ben Stewart.
1111 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1112 */
1113 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1114 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1115 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1116
1117 static const char
1118 export_proto_ip = 'P',
1119 export_proto_tcp = 'T',
1120 export_proto_udp = 'U';
1121
1122 static const unsigned char
1123 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1124 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02};
1125
1126 /* ---------------------------------------------------------------------------
1127 * Load a host's ip_proto table from a file.
1128 * Returns 0 on failure, 1 on success.
1129 */
1130 static int
1131 hosts_db_import_ip(const int fd, struct bucket *host)
1132 {
1133 uint8_t count, i;
1134
1135 if (!expect8(fd, export_proto_ip)) return 0;
1136 if (!read8(fd, &count)) return 0;
1137
1138 for (i=0; i<count; i++) {
1139 struct bucket *b;
1140 uint8_t proto;
1141 uint64_t in, out;
1142
1143 if (!read8(fd, &proto)) return 0;
1144 if (!read64(fd, &in)) return 0;
1145 if (!read64(fd, &out)) return 0;
1146
1147 /* Store data */
1148 b = host_get_ip_proto(host, proto);
1149 b->in = in;
1150 b->out = out;
1151 b->total = in + out;
1152 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1153 }
1154 return 1;
1155 }
1156
1157 /* ---------------------------------------------------------------------------
1158 * Load a host's port_tcp table from a file.
1159 * Returns 0 on failure, 1 on success.
1160 */
1161 static int
1162 hosts_db_import_tcp(const int fd, struct bucket *host)
1163 {
1164 uint16_t count, i;
1165
1166 if (!expect8(fd, export_proto_tcp)) return 0;
1167 if (!read16(fd, &count)) return 0;
1168
1169 for (i=0; i<count; i++) {
1170 struct bucket *b;
1171 uint16_t port;
1172 uint64_t in, out, syn;
1173
1174 if (!read16(fd, &port)) return 0;
1175 if (!read64(fd, &syn)) return 0;
1176 if (!read64(fd, &in)) return 0;
1177 if (!read64(fd, &out)) return 0;
1178
1179 /* Store data */
1180 b = host_get_port_tcp(host, port);
1181 b->in = in;
1182 b->out = out;
1183 b->total = in + out;
1184 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1185 b->u.port_tcp.syn = syn;
1186 }
1187 return 1;
1188 }
1189
1190 /* ---------------------------------------------------------------------------
1191 * Load a host's port_tcp table from a file.
1192 * Returns 0 on failure, 1 on success.
1193 */
1194 static int
1195 hosts_db_import_udp(const int fd, struct bucket *host)
1196 {
1197 uint16_t count, i;
1198
1199 if (!expect8(fd, export_proto_udp)) return 0;
1200 if (!read16(fd, &count)) return 0;
1201
1202 for (i=0; i<count; i++) {
1203 struct bucket *b;
1204 uint16_t port;
1205 uint64_t in, out;
1206
1207 if (!read16(fd, &port)) return 0;
1208 if (!read64(fd, &in)) return 0;
1209 if (!read64(fd, &out)) return 0;
1210
1211 /* Store data */
1212 b = host_get_port_udp(host, port);
1213 b->in = in;
1214 b->out = out;
1215 b->total = in + out;
1216 assert(b->u.port_udp.port == port); /* done by make_func */
1217 }
1218 return 1;
1219 }
1220
1221 /* ---------------------------------------------------------------------------
1222 * Load all hosts from a file.
1223 * Returns 0 on failure, 1 on success.
1224 */
1225 static int
1226 hosts_db_import_host(const int fd)
1227 {
1228 struct bucket *host;
1229 struct addr46 ipaddr;
1230 uint8_t hostname_len;
1231 uint64_t in, out;
1232 unsigned int pos = xtell(fd);
1233 char hdr[4];
1234 int ver = 0;
1235
1236 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1237 if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1238 ver = 2;
1239 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1240 ver = 1;
1241 else {
1242 warnx("bad host header: %02x%02x%02x%02x",
1243 hdr[0], hdr[1], hdr[2], hdr[3]);
1244 return 0;
1245 }
1246
1247 ipaddr.af = AF_INET;
1248 if (!readaddr(fd, &ipaddr)) return 0;
1249 verbosef("at file pos %u, importing host %s", pos, ip_to_str(&ipaddr));
1250 host = host_get(&ipaddr);
1251 assert(host->u.host.ipaddr.addr.ip.s_addr == ipaddr.addr.ip.s_addr); /* make fn? */
1252
1253 if (ver > 1) {
1254 uint64_t t;
1255 if (!read64(fd, &t)) return 0;
1256 host->u.host.last_seen = (time_t)t;
1257 }
1258
1259 assert(sizeof(host->u.host.mac_addr) == 6);
1260 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1261 return 0;
1262
1263 /* HOSTNAME */
1264 assert(host->u.host.dns == NULL); /* make fn? */
1265 if (!read8(fd, &hostname_len)) return 0;
1266 if (hostname_len > 0) {
1267 host->u.host.dns = xmalloc(hostname_len + 1);
1268 host->u.host.dns[0] = '\0';
1269
1270 /* At this point, the hostname is attached to a host which is in our
1271 * hosts_db, so if we bail out due to an import error, this pointer
1272 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1273 */
1274
1275 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1276 host->u.host.dns[hostname_len] = '\0';
1277 }
1278
1279 if (!read64(fd, &in)) return 0;
1280 if (!read64(fd, &out)) return 0;
1281
1282 host->in = in;
1283 host->out = out;
1284 host->total = in + out;
1285
1286 /* Host's port and proto subtables: */
1287 if (!hosts_db_import_ip(fd, host)) return 0;
1288 if (!hosts_db_import_tcp(fd, host)) return 0;
1289 if (!hosts_db_import_udp(fd, host)) return 0;
1290 return 1;
1291 }
1292
1293 /* ---------------------------------------------------------------------------
1294 * Database Import: Grab hosts_db from a file provided by the caller.
1295 *
1296 * This function will retrieve the data sans the header. We expect the caller
1297 * to have validated the header of the hosts_db segment, and left the file
1298 * sitting at the start of the data.
1299 */
1300 int hosts_db_import(const int fd)
1301 {
1302 uint32_t host_count, i;
1303
1304 if (!read32(fd, &host_count)) return 0;
1305
1306 for (i=0; i<host_count; i++)
1307 if (!hosts_db_import_host(fd)) return 0;
1308
1309 return 1;
1310 }
1311
1312 /* ---------------------------------------------------------------------------
1313 * Database Export: Dump hosts_db into a file provided by the caller.
1314 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1315 */
1316 int hosts_db_export(const int fd)
1317 {
1318 uint32_t i;
1319 struct bucket *b;
1320
1321 if (!write32(fd, hosts_db->count)) return 0;
1322
1323 for (i = 0; i<hosts_db->size; i++)
1324 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1325 /* For each host: */
1326 if (!writen(fd, export_tag_host_ver2, sizeof(export_tag_host_ver2)))
1327 return 0;
1328
1329 if (!writeaddr(fd, &b->u.host.ipaddr)) return 0;
1330
1331 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1332
1333 assert(sizeof(b->u.host.mac_addr) == 6);
1334 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1335 return 0;
1336
1337 /* HOSTNAME */
1338 if (b->u.host.dns == NULL) {
1339 if (!write8(fd, 0)) return 0;
1340 } else {
1341 int dnslen = strlen(b->u.host.dns);
1342
1343 if (dnslen > 255) {
1344 warnx("found a very long hostname: \"%s\"\n"
1345 "wasn't expecting one longer than 255 chars (this one is %d)",
1346 b->u.host.dns, dnslen);
1347 dnslen = 255;
1348 }
1349
1350 if (!write8(fd, (uint8_t)dnslen)) return 0;
1351 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1352 }
1353
1354 if (!write64(fd, b->in)) return 0;
1355 if (!write64(fd, b->out)) return 0;
1356
1357 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1358 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1359 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1360 }
1361 return 1;
1362 }
1363
1364 /* ---------------------------------------------------------------------------
1365 * Dump the ip_proto table of a host.
1366 */
1367 static int
1368 hosts_db_export_ip(const struct hashtable *h, const int fd)
1369 {
1370 uint32_t i, written = 0;
1371 struct bucket *b;
1372
1373 /* IP DATA */
1374 if (!write8(fd, export_proto_ip)) return 0;
1375
1376 /* If no data, write a IP Proto count of 0 and we're done. */
1377 if (h == NULL) {
1378 if (!write8(fd, 0)) return 0;
1379 return 1;
1380 }
1381
1382 assert(h->count < 256);
1383 if (!write8(fd, (uint8_t)h->count)) return 0;
1384
1385 for (i = 0; i<h->size; i++)
1386 for (b = h->table[i]; b != NULL; b = b->next) {
1387 /* For each ip_proto bucket: */
1388
1389 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1390 if (!write64(fd, b->in)) return 0;
1391 if (!write64(fd, b->out)) return 0;
1392 written++;
1393 }
1394 assert(written == h->count);
1395 return 1;
1396 }
1397
1398 /* ---------------------------------------------------------------------------
1399 * Dump the port_tcp table of a host.
1400 */
1401 static int
1402 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1403 {
1404 struct bucket *b;
1405 uint32_t i, written = 0;
1406
1407 /* TCP DATA */
1408 if (!write8(fd, export_proto_tcp)) return 0;
1409
1410 /* If no data, write a count of 0 and we're done. */
1411 if (h == NULL) {
1412 if (!write16(fd, 0)) return 0;
1413 return 1;
1414 }
1415
1416 assert(h->count < 65536);
1417 if (!write16(fd, (uint16_t)h->count)) return 0;
1418
1419 for (i = 0; i<h->size; i++)
1420 for (b = h->table[i]; b != NULL; b = b->next) {
1421 if (!write16(fd, b->u.port_tcp.port)) return 0;
1422 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1423 if (!write64(fd, b->in)) return 0;
1424 if (!write64(fd, b->out)) return 0;
1425 written++;
1426 }
1427 assert(written == h->count);
1428 return 1;
1429 }
1430
1431 /* ---------------------------------------------------------------------------
1432 * Dump the port_udp table of a host.
1433 */
1434 static int
1435 hosts_db_export_udp(const struct hashtable *h, const int fd)
1436 {
1437 struct bucket *b;
1438 uint32_t i, written = 0;
1439
1440 /* UDP DATA */
1441 if (!write8(fd, export_proto_udp)) return 0;
1442
1443 /* If no data, write a count of 0 and we're done. */
1444 if (h == NULL) {
1445 if (!write16(fd, 0)) return 0;
1446 return 1;
1447 }
1448
1449 assert(h->count < 65536);
1450 if (!write16(fd, (uint16_t)h->count)) return 0;
1451
1452 for (i = 0; i<h->size; i++)
1453 for (b = h->table[i]; b != NULL; b = b->next) {
1454 if (!write16(fd, b->u.port_udp.port)) return 0;
1455 if (!write64(fd, b->in)) return 0;
1456 if (!write64(fd, b->out)) return 0;
1457 written++;
1458 }
1459 assert(written == h->count);
1460 return 1;
1461 }
1462
1463 /* vim:set ts=3 sw=3 tw=78 expandtab: */