Make HTML template more flexible.
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2009 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20 #include "str.h"
21
22 #include <arpa/inet.h> /* inet_aton() */
23 #include <netdb.h> /* struct addrinfo */
24 #include <assert.h>
25 #include <errno.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h> /* memset(), strcmp() */
29 #include <unistd.h>
30
31 extern int want_lastseen;
32 int show_mac_addrs = 0;
33 extern const char *interface;
34
35 /* FIXME: specify somewhere more sane/tunable */
36 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
37
38 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
39 typedef void (free_func_t)(struct bucket *);
40 typedef const void * (key_func_t)(const struct bucket *);
41 typedef int (find_func_t)(const struct bucket *, const void *);
42 typedef struct bucket * (make_func_t)(const void *);
43 typedef void (format_cols_func_t)(struct str *);
44 typedef void (format_row_func_t)(struct str *, const struct bucket *,
45 const char *);
46
47 struct hashtable {
48 uint8_t bits; /* size of hashtable in bits */
49 uint32_t size, mask;
50 uint32_t count, count_max, count_keep; /* items in table */
51 uint32_t coeff; /* coefficient for Fibonacci hashing */
52 struct bucket **table;
53
54 struct {
55 uint64_t inserts, searches, deletions, rehashes;
56 } stats;
57
58 hash_func_t *hash_func;
59 /* returns hash value of given key (passed as void*) */
60
61 free_func_t *free_func;
62 /* free of bucket payload */
63
64 key_func_t *key_func;
65 /* returns pointer to key of bucket (to pass to hash_func) */
66
67 find_func_t *find_func;
68 /* returns true if given bucket matches key (passed as void*) */
69
70 make_func_t *make_func;
71 /* returns bucket containing new record with key (passed as void*) */
72
73 format_cols_func_t *format_cols_func;
74 /* append table columns to str */
75
76 format_row_func_t *format_row_func;
77 /* format record and append to str */
78 };
79
80 static void hashtable_reduce(struct hashtable *ht);
81 static void hashtable_free(struct hashtable *h);
82
83 #define HOST_BITS 1 /* initial size of hosts table */
84 #define PORT_BITS 1 /* initial size of ports tables */
85 #define PROTO_BITS 1 /* initial size of proto table */
86
87 /* We only use one hosts_db hashtable and this is it. */
88 static struct hashtable *hosts_db = NULL;
89
90 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
91 static const double phi_1 =
92 0.61803398874989490252573887119069695472717285156250;
93
94 /* Co-prime of u, using phi^-1 */
95 inline static uint32_t
96 coprime(const uint32_t u)
97 {
98 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
99 }
100
101 /*
102 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
103 * src/sys/netinet/tcp_hostcache.c 1.1
104 */
105 inline static uint32_t
106 ipv4_hash(const struct addr *const a)
107 {
108 uint32_t ip = a->ip.v4;
109 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
110 }
111
112 #ifndef s6_addr32
113 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
114 * taken care of GNU/Linux and GNU/kfreebsd. */
115 # define s6_addr32 __u6_addr.__u6_addr32
116 #endif
117
118 /*
119 * This is the IPv6 hash function used by FreeBSD in the same file as above,
120 * svn rev 122922.
121 */
122 inline static uint32_t
123 ipv6_hash(const struct addr *const a)
124 {
125 const struct in6_addr *const ip6 = &(a->ip.v6);
126 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
127 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
128 }
129
130 /* ---------------------------------------------------------------------------
131 * hash_func collection
132 */
133 static uint32_t
134 hash_func_host(const struct hashtable *h _unused_, const void *key)
135 {
136 const struct addr *a = key;
137 if (a->family == IPv4)
138 return (ipv4_hash(a));
139 else {
140 assert(a->family == IPv6);
141 return (ipv6_hash(a));
142 }
143 }
144
145 #define CASTKEY(type) (*((const type *)key))
146
147 static uint32_t
148 hash_func_short(const struct hashtable *h, const void *key)
149 {
150 return (CASTKEY(uint16_t) * h->coeff);
151 }
152
153 static uint32_t
154 hash_func_byte(const struct hashtable *h, const void *key)
155 {
156 return (CASTKEY(uint8_t) * h->coeff);
157 }
158
159 /* ---------------------------------------------------------------------------
160 * key_func collection
161 */
162
163 static const void *
164 key_func_host(const struct bucket *b)
165 {
166 return &(b->u.host.addr);
167 }
168
169 static const void *
170 key_func_port_tcp(const struct bucket *b)
171 {
172 return &(b->u.port_tcp.port);
173 }
174
175 static const void *
176 key_func_port_udp(const struct bucket *b)
177 {
178 return &(b->u.port_udp.port);
179 }
180
181 static const void *
182 key_func_ip_proto(const struct bucket *b)
183 {
184 return &(b->u.ip_proto.proto);
185 }
186
187 /* ---------------------------------------------------------------------------
188 * find_func collection
189 */
190
191 static int
192 find_func_host(const struct bucket *b, const void *key)
193 {
194 return (addr_equal(key, &(b->u.host.addr)));
195 }
196
197 static int
198 find_func_port_tcp(const struct bucket *b, const void *key)
199 {
200 return (b->u.port_tcp.port == CASTKEY(uint16_t));
201 }
202
203 static int
204 find_func_port_udp(const struct bucket *b, const void *key)
205 {
206 return (b->u.port_udp.port == CASTKEY(uint16_t));
207 }
208
209 static int
210 find_func_ip_proto(const struct bucket *b, const void *key)
211 {
212 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
213 }
214
215 /* ---------------------------------------------------------------------------
216 * make_func collection
217 */
218
219 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
220 struct bucket *next; \
221 uint64_t in, out, total; \
222 union { struct type t; } u; } _custom_bucket; \
223 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
224 struct type *name_content = &(name_bucket->u.type); \
225 name_bucket->next = NULL; \
226 name_bucket->in = name_bucket->out = name_bucket->total = 0;
227
228 static struct bucket *
229 make_func_host(const void *key)
230 {
231 MAKE_BUCKET(b, h, host);
232 h->addr = CASTKEY(struct addr);
233 h->dns = NULL;
234 h->last_seen = now;
235 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
236 h->ports_tcp = NULL;
237 h->ports_udp = NULL;
238 h->ip_protos = NULL;
239 return (b);
240 }
241
242 static void
243 free_func_host(struct bucket *b)
244 {
245 struct host *h = &(b->u.host);
246 if (h->dns != NULL) free(h->dns);
247 hashtable_free(h->ports_tcp);
248 hashtable_free(h->ports_udp);
249 hashtable_free(h->ip_protos);
250 }
251
252 static struct bucket *
253 make_func_port_tcp(const void *key)
254 {
255 MAKE_BUCKET(b, p, port_tcp);
256 p->port = CASTKEY(uint16_t);
257 p->syn = 0;
258 return (b);
259 }
260
261 static struct bucket *
262 make_func_port_udp(const void *key)
263 {
264 MAKE_BUCKET(b, p, port_udp);
265 p->port = CASTKEY(uint16_t);
266 return (b);
267 }
268
269 static struct bucket *
270 make_func_ip_proto(const void *key)
271 {
272 MAKE_BUCKET(b, p, ip_proto);
273 p->proto = CASTKEY(uint8_t);
274 return (b);
275 }
276
277 static void
278 free_func_simple(struct bucket *b _unused_)
279 {
280 /* nop */
281 }
282
283 /* ---------------------------------------------------------------------------
284 * format_func collection (ordered by struct)
285 */
286
287 static void
288 format_cols_host(struct str *buf)
289 {
290 /* FIXME: don't clobber parts of the query string
291 * specifically "full" and "start"
292 * when setting sort direction
293 */
294 str_append(buf,
295 "<table>\n"
296 "<tr>\n"
297 " <th>IP</th>\n"
298 " <th>Hostname</th>\n");
299 if (show_mac_addrs) str_append(buf,
300 " <th>MAC Address</th>\n");
301 str_append(buf,
302 " <th><a href=\"?sort=in\">In</a></th>\n"
303 " <th><a href=\"?sort=out\">Out</a></th>\n"
304 " <th><a href=\"?sort=total\">Total</a></th>\n");
305 if (want_lastseen) str_append(buf,
306 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
307 str_append(buf,
308 "</tr>\n");
309 }
310
311 static void
312 format_row_host(struct str *buf, const struct bucket *b,
313 const char *css_class)
314 {
315 const char *ip = addr_to_str(&(b->u.host.addr));
316
317 str_appendf(buf,
318 "<tr class=\"%s\">\n"
319 " <td><a href=\"/hosts/%s/\">%s</a></td>\n"
320 " <td>%s</td>\n",
321 css_class,
322 ip, ip,
323 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
324
325 if (show_mac_addrs)
326 str_appendf(buf,
327 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
328 b->u.host.mac_addr[0],
329 b->u.host.mac_addr[1],
330 b->u.host.mac_addr[2],
331 b->u.host.mac_addr[3],
332 b->u.host.mac_addr[4],
333 b->u.host.mac_addr[5]);
334
335 str_appendf(buf,
336 " <td class=\"num\">%'qu</td>\n"
337 " <td class=\"num\">%'qu</td>\n"
338 " <td class=\"num\">%'qu</td>\n",
339 b->in, b->out, b->total);
340
341 if (want_lastseen) {
342 time_t last_t = b->u.host.last_seen;
343 struct str *lastseen = NULL;
344
345 if (now >= last_t)
346 lastseen = length_of_time(now - last_t);
347
348 str_append(buf,
349 " <td class=\"num\">");
350 if (lastseen == NULL)
351 str_append(buf, "(clock error)");
352 else {
353 str_appendstr(buf, lastseen);
354 str_free(lastseen);
355 }
356 str_append(buf,
357 "</td>");
358 }
359
360 str_appendf(buf,
361 "</tr>\n");
362
363 /* Only resolve hosts "on demand" */
364 if (b->u.host.dns == NULL)
365 dns_queue(&(b->u.host.addr));
366 }
367
368 static void
369 format_cols_port_tcp(struct str *buf)
370 {
371 str_append(buf,
372 "<table>\n"
373 "<tr>\n"
374 " <th>Port</td>\n"
375 " <th>Service</td>\n"
376 " <th>In</td>\n"
377 " <th>Out</td>\n"
378 " <th>Total</td>\n"
379 " <th>SYNs</td>\n"
380 "</tr>\n"
381 );
382 }
383
384 static void
385 format_row_port_tcp(struct str *buf, const struct bucket *b,
386 const char *css_class)
387 {
388 const struct port_tcp *p = &(b->u.port_tcp);
389
390 str_appendf(buf,
391 "<tr class=\"%s\">\n"
392 " <td class=\"num\">%u</td>\n"
393 " <td>%s</td>\n"
394 " <td class=\"num\">%'qu</td>\n"
395 " <td class=\"num\">%'qu</td>\n"
396 " <td class=\"num\">%'qu</td>\n"
397 " <td class=\"num\">%'qu</td>\n"
398 "</tr>\n",
399 css_class,
400 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
401 );
402 }
403
404 static void
405 format_cols_port_udp(struct str *buf)
406 {
407 str_append(buf,
408 "<table>\n"
409 "<tr>\n"
410 " <th>Port</td>\n"
411 " <th>Service</td>\n"
412 " <th>In</td>\n"
413 " <th>Out</td>\n"
414 " <th>Total</td>\n"
415 "</tr>\n"
416 );
417 }
418
419 static void
420 format_row_port_udp(struct str *buf, const struct bucket *b,
421 const char *css_class)
422 {
423 const struct port_udp *p = &(b->u.port_udp);
424
425 str_appendf(buf,
426 "<tr class=\"%s\">\n"
427 " <td class=\"num\">%u</td>\n"
428 " <td>%s</td>\n"
429 " <td class=\"num\">%'qu</td>\n"
430 " <td class=\"num\">%'qu</td>\n"
431 " <td class=\"num\">%'qu</td>\n"
432 "</tr>\n",
433 css_class,
434 p->port, getservudp(p->port), b->in, b->out, b->total
435 );
436 }
437
438 static void
439 format_cols_ip_proto(struct str *buf)
440 {
441 str_append(buf,
442 "<table>\n"
443 "<tr>\n"
444 " <th>#</td>\n"
445 " <th>Protocol</td>\n"
446 " <th>In</td>\n"
447 " <th>Out</td>\n"
448 " <th>Total</td>\n"
449 "</tr>\n"
450 );
451 }
452
453 static void
454 format_row_ip_proto(struct str *buf, const struct bucket *b,
455 const char *css_class)
456 {
457 const struct ip_proto *p = &(b->u.ip_proto);
458
459 str_appendf(buf,
460 "<tr class=\"%s\">\n"
461 " <td class=\"num\">%u</td>\n"
462 " <td>%s</td>\n"
463 " <td class=\"num\">%'qu</td>\n"
464 " <td class=\"num\">%'qu</td>\n"
465 " <td class=\"num\">%'qu</td>\n"
466 "</tr>\n",
467 css_class,
468 p->proto, getproto(p->proto),
469 b->in, b->out, b->total
470 );
471 }
472
473 /* ---------------------------------------------------------------------------
474 * Initialise a hashtable.
475 */
476 static struct hashtable *
477 hashtable_make(const uint8_t bits,
478 const unsigned int count_max,
479 const unsigned int count_keep,
480 hash_func_t *hash_func,
481 free_func_t *free_func,
482 key_func_t *key_func,
483 find_func_t *find_func,
484 make_func_t *make_func,
485 format_cols_func_t *format_cols_func,
486 format_row_func_t *format_row_func)
487 {
488 struct hashtable *hash;
489 assert(bits > 0);
490
491 hash = xmalloc(sizeof(*hash));
492 hash->bits = bits;
493 hash->count_max = count_max;
494 hash->count_keep = count_keep;
495 hash->size = 1U << bits;
496 hash->mask = hash->size - 1;
497 hash->coeff = coprime(hash->size);
498 hash->hash_func = hash_func;
499 hash->free_func = free_func;
500 hash->key_func = key_func;
501 hash->find_func = find_func;
502 hash->make_func = make_func;
503 hash->format_cols_func = format_cols_func;
504 hash->format_row_func = format_row_func;
505 hash->count = 0;
506 hash->table = xcalloc(hash->size, sizeof(*hash->table));
507 memset(&(hash->stats), 0, sizeof(hash->stats));
508 return (hash);
509 }
510
511 /* ---------------------------------------------------------------------------
512 * Initialise global hosts_db.
513 */
514 void
515 hosts_db_init(void)
516 {
517 assert(hosts_db == NULL);
518 hosts_db = hashtable_make(HOST_BITS, hosts_max, hosts_keep,
519 hash_func_host, free_func_host, key_func_host, find_func_host,
520 make_func_host, format_cols_host, format_row_host);
521 }
522
523 static void
524 hashtable_rehash(struct hashtable *h, const uint8_t bits)
525 {
526 struct bucket **old_table, **new_table;
527 uint32_t i, old_size;
528 assert(h != NULL);
529 assert(bits > 0);
530
531 h->stats.rehashes++;
532 old_size = h->size;
533 old_table = h->table;
534
535 h->bits = bits;
536 h->size = 1U << bits;
537 h->mask = h->size - 1;
538 h->coeff = coprime(h->size);
539 new_table = xcalloc(h->size, sizeof(*new_table));
540
541 for (i=0; i<old_size; i++) {
542 struct bucket *next, *b = old_table[i];
543 while (b != NULL) {
544 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
545 next = b->next;
546 b->next = new_table[pos];
547 new_table[pos] = b;
548 b = next;
549 }
550 }
551
552 free(h->table);
553 h->table = new_table;
554 }
555
556 static void
557 hashtable_insert(struct hashtable *h, struct bucket *b)
558 {
559 uint32_t pos;
560 assert(h != NULL);
561 assert(b != NULL);
562 assert(b->next == NULL);
563
564 /* Rehash on 80% occupancy */
565 if ((h->count > h->size) ||
566 ((h->size - h->count) < h->size / 5))
567 hashtable_rehash(h, h->bits+1);
568
569 pos = h->hash_func(h, h->key_func(b)) & h->mask;
570 if (h->table[pos] == NULL)
571 h->table[pos] = b;
572 else {
573 /* Insert at top of chain. */
574 b->next = h->table[pos];
575 h->table[pos] = b;
576 }
577 h->count++;
578 h->stats.inserts++;
579 }
580
581 /* Return bucket matching key, or NULL if no such entry. */
582 static struct bucket *
583 hashtable_search(struct hashtable *h, const void *key)
584 {
585 uint32_t pos;
586 struct bucket *b;
587
588 h->stats.searches++;
589 pos = h->hash_func(h, key) & h->mask;
590 b = h->table[pos];
591 while (b != NULL) {
592 if (h->find_func(b, key))
593 return (b);
594 else
595 b = b->next;
596 }
597 return (NULL);
598 }
599
600 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
601 /* Search for a key. If it's not there, make and insert a bucket for it. */
602 static struct bucket *
603 hashtable_find_or_insert(struct hashtable *h, const void *key,
604 const reduce_bool allow_reduce)
605 {
606 struct bucket *b = hashtable_search(h, key);
607
608 if (b == NULL) {
609 /* Not found, so insert after checking occupancy. */
610 if (allow_reduce && (h->count >= h->count_max))
611 hashtable_reduce(h);
612 b = h->make_func(key);
613 hashtable_insert(h, b);
614 }
615 return (b);
616 }
617
618 /*
619 * Frees the hashtable and the buckets. The contents are assumed to be
620 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
621 * the bucket.
622 */
623 static void
624 hashtable_free(struct hashtable *h)
625 {
626 uint32_t i;
627
628 if (h == NULL)
629 return;
630 for (i=0; i<h->size; i++) {
631 struct bucket *tmp, *b = h->table[i];
632 while (b != NULL) {
633 tmp = b;
634 b = b->next;
635 h->free_func(tmp);
636 free(tmp);
637 }
638 }
639 free(h->table);
640 free(h);
641 }
642
643 /* ---------------------------------------------------------------------------
644 * Return existing host or insert a new one.
645 */
646 struct bucket *
647 host_get(const struct addr *const a)
648 {
649 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
650 }
651
652 /* ---------------------------------------------------------------------------
653 * Find host, returns NULL if not in DB.
654 */
655 struct bucket *
656 host_find(const struct addr *const a)
657 {
658 return (hashtable_search(hosts_db, a));
659 }
660
661 /* ---------------------------------------------------------------------------
662 * Find host, returns NULL if not in DB.
663 */
664 static struct bucket *
665 host_search(const char *ipstr)
666 {
667 struct addr a;
668 struct addrinfo hints, *ai;
669
670 memset(&hints, 0, sizeof(hints));
671 hints.ai_family = AF_UNSPEC;
672 hints.ai_flags = AI_NUMERICHOST;
673
674 if (getaddrinfo(ipstr, NULL, &hints, &ai))
675 return (NULL); /* invalid addr */
676
677 if (ai->ai_family == AF_INET) {
678 a.family = IPv4;
679 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
680 }
681 else if (ai->ai_family == AF_INET6) {
682 a.family = IPv6;
683 memcpy(&(a.ip.v6),
684 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
685 sizeof(a.ip.v6));
686 } else {
687 freeaddrinfo(ai);
688 return (NULL); /* unknown family */
689 }
690 freeaddrinfo(ai);
691
692 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
693 return (hashtable_search(hosts_db, &a));
694 }
695
696 /* ---------------------------------------------------------------------------
697 * Reduce a hashtable to the top <keep> entries.
698 */
699 static void
700 hashtable_reduce(struct hashtable *ht)
701 {
702 uint32_t i, pos, rmd;
703 const struct bucket **table;
704 uint64_t cutoff;
705
706 assert(ht->count_keep < ht->count);
707
708 /* Fill table with pointers to buckets in hashtable. */
709 table = xcalloc(ht->count, sizeof(*table));
710 for (pos=0, i=0; i<ht->size; i++) {
711 struct bucket *b = ht->table[i];
712 while (b != NULL) {
713 table[pos++] = b;
714 b = b->next;
715 }
716 }
717 assert(pos == ht->count);
718 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
719 cutoff = table[ht->count_keep]->total;
720 free(table);
721
722 /* Remove all elements with total <= cutoff. */
723 rmd = 0;
724 for (i=0; i<ht->size; i++) {
725 struct bucket *last = NULL, *next, *b = ht->table[i];
726 while (b != NULL) {
727 next = b->next;
728 if (b->total <= cutoff) {
729 /* Remove this one. */
730 ht->free_func(b);
731 free(b);
732 if (last == NULL)
733 ht->table[i] = next;
734 else
735 last->next = next;
736 rmd++;
737 ht->count--;
738 } else {
739 last = b;
740 }
741 b = next;
742 }
743 }
744 verbosef("hashtable_reduce: removed %u buckets, left %u",
745 rmd, ht->count);
746 hashtable_rehash(ht, ht->bits); /* is this needed? */
747 }
748
749 /* Reduce hosts_db if needed. */
750 void hosts_db_reduce(void)
751 {
752 if (hosts_db->count >= hosts_db->count_max)
753 hashtable_reduce(hosts_db);
754 }
755
756 /* ---------------------------------------------------------------------------
757 * Reset hosts_db to empty.
758 */
759 void
760 hosts_db_reset(void)
761 {
762 unsigned int i;
763
764 for (i=0; i<hosts_db->size; i++) {
765 struct bucket *next, *b = hosts_db->table[i];
766 while (b != NULL) {
767 next = b->next;
768 hosts_db->free_func(b);
769 free(b);
770 b = next;
771 }
772 hosts_db->table[i] = NULL;
773 }
774 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
775 hosts_db->count = 0;
776 }
777
778 /* ---------------------------------------------------------------------------
779 * Deallocate hosts_db.
780 */
781 void hosts_db_free(void)
782 {
783 uint32_t i;
784
785 assert(hosts_db != NULL);
786 for (i=0; i<hosts_db->size; i++) {
787 struct bucket *tmp, *b = hosts_db->table[i];
788 while (b != NULL) {
789 tmp = b;
790 b = b->next;
791 hosts_db->free_func(tmp);
792 free(tmp);
793 }
794 }
795 free(hosts_db->table);
796 free(hosts_db);
797 hosts_db = NULL;
798 }
799
800 /* ---------------------------------------------------------------------------
801 * Find or create a port_tcp inside a host.
802 */
803 struct bucket *
804 host_get_port_tcp(struct bucket *host, const uint16_t port)
805 {
806 struct host *h = &host->u.host;
807 assert(h != NULL);
808 if (h->ports_tcp == NULL)
809 h->ports_tcp = hashtable_make(PORT_BITS, ports_max, ports_keep,
810 hash_func_short, free_func_simple, key_func_port_tcp,
811 find_func_port_tcp, make_func_port_tcp,
812 format_cols_port_tcp, format_row_port_tcp);
813 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
814 }
815
816 /* ---------------------------------------------------------------------------
817 * Find or create a port_udp inside a host.
818 */
819 struct bucket *
820 host_get_port_udp(struct bucket *host, const uint16_t port)
821 {
822 struct host *h = &host->u.host;
823 assert(h != NULL);
824 if (h->ports_udp == NULL)
825 h->ports_udp = hashtable_make(PORT_BITS, ports_max, ports_keep,
826 hash_func_short, free_func_simple, key_func_port_udp,
827 find_func_port_udp, make_func_port_udp,
828 format_cols_port_udp, format_row_port_udp);
829 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
830 }
831
832 /* ---------------------------------------------------------------------------
833 * Find or create an ip_proto inside a host.
834 */
835 struct bucket *
836 host_get_ip_proto(struct bucket *host, const uint8_t proto)
837 {
838 struct host *h = &host->u.host;
839 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
840 assert(h != NULL);
841 if (h->ip_protos == NULL)
842 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
843 hash_func_byte, free_func_simple, key_func_ip_proto,
844 find_func_ip_proto, make_func_ip_proto,
845 format_cols_ip_proto, format_row_ip_proto);
846 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
847 }
848
849 static struct str *html_hosts_main(const char *qs);
850 static struct str *html_hosts_detail(const char *ip);
851
852 /* ---------------------------------------------------------------------------
853 * Web interface: delegate the /hosts/ space.
854 */
855 struct str *
856 html_hosts(const char *uri, const char *query)
857 {
858 int i, num_elems;
859 char **elem = split('/', uri, &num_elems);
860 struct str *buf = NULL;
861
862 assert(num_elems >= 1);
863 assert(strcmp(elem[0], "hosts") == 0);
864
865 if (num_elems == 1)
866 /* /hosts/ */
867 buf = html_hosts_main(query);
868 else if (num_elems == 2)
869 /* /hosts/<IP of host>/ */
870 buf = html_hosts_detail(elem[1]);
871
872 for (i=0; i<num_elems; i++)
873 free(elem[i]);
874 free(elem);
875 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
876 other codes to be possible */
877 }
878
879 /* ---------------------------------------------------------------------------
880 * Format hashtable into HTML.
881 */
882 static void
883 format_table(struct str *buf, struct hashtable *ht, int start,
884 const enum sort_dir sort, const int full)
885 {
886 const struct bucket **table;
887 uint32_t i, pos, end;
888 int alt = 0;
889
890 if ((ht == NULL) || (ht->count == 0)) {
891 str_append(buf, "<p>The table is empty.</p>\n");
892 return;
893 }
894
895 /* Fill table with pointers to buckets in hashtable. */
896 table = xcalloc(ht->count, sizeof(*table));
897 for (pos=0, i=0; i<ht->size; i++) {
898 struct bucket *b = ht->table[i];
899 while (b != NULL) {
900 table[pos++] = b;
901 b = b->next;
902 }
903 }
904 assert(pos == ht->count);
905
906 if (full) {
907 /* full report overrides start and end */
908 start = 0;
909 end = ht->count;
910 } else
911 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
912
913 str_appendf(buf, "(%u-%u of %u)<br/>\n", start+1, end, ht->count);
914 qsort_buckets(table, ht->count, start, end, sort);
915 ht->format_cols_func(buf);
916
917 for (i=start; i<end; i++) {
918 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
919 alt = !alt; /* alternate class for table rows */
920 }
921 free(table);
922 str_append(buf, "</table>\n");
923 }
924
925 /* ---------------------------------------------------------------------------
926 * Web interface: sorted table of hosts.
927 */
928 static struct str *
929 html_hosts_main(const char *qs)
930 {
931 struct str *buf = str_make();
932 char *qs_start, *qs_sort, *qs_full, *ep;
933 const char *sortstr;
934 int start, full = 0;
935 enum sort_dir sort;
936
937 /* parse query string */
938 qs_start = qs_get(qs, "start");
939 qs_sort = qs_get(qs, "sort");
940 qs_full = qs_get(qs, "full");
941 if (qs_full != NULL) {
942 full = 1;
943 free(qs_full);
944 }
945
946 /* validate sort */
947 if (qs_sort == NULL) sort = TOTAL;
948 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
949 else if (strcmp(qs_sort, "in") == 0) sort = IN;
950 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
951 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
952 else {
953 str_append(buf, "Error: invalid value for \"sort\".\n");
954 goto done;
955 }
956
957 /* parse start */
958 if (qs_start == NULL)
959 start = 0;
960 else {
961 start = (int)strtoul(qs_start, &ep, 10);
962 if (*ep != '\0') {
963 str_append(buf, "Error: \"start\" is not a number.\n");
964 goto done;
965 }
966 if ((errno == ERANGE) ||
967 (start < 0) || (start >= (int)hosts_db->count)) {
968 str_append(buf, "Error: \"start\" is out of bounds.\n");
969 goto done;
970 }
971 }
972
973 #define PREV "&lt;&lt;&lt; prev page"
974 #define NEXT "next page &gt;&gt;&gt;"
975 #define FULL "full table"
976
977 html_open(buf, "Hosts", interface, NULL);
978 format_table(buf, hosts_db, start, sort, full);
979
980 /* <prev | full | stats | next> */
981 sortstr = qs_sort;
982 if (sortstr == NULL) sortstr = "total";
983 if (start > 0) {
984 int prev = max(start - MAX_ENTRIES, 0);
985 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
986 prev, sortstr);
987 } else
988 str_append(buf, PREV);
989
990 if (full)
991 str_append(buf, " | " FULL);
992 else
993 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
994 sortstr);
995
996 if (start+MAX_ENTRIES < (int)hosts_db->count)
997 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
998 start+MAX_ENTRIES, sortstr);
999 else
1000 str_append(buf, " | " NEXT);
1001
1002 str_append(buf, "<br/>\n");
1003
1004 html_close(buf);
1005 done:
1006 if (qs_start != NULL) free(qs_start);
1007 if (qs_sort != NULL) free(qs_sort);
1008 return buf;
1009 #undef PREV
1010 #undef NEXT
1011 #undef FULL
1012 }
1013
1014 /* ---------------------------------------------------------------------------
1015 * Web interface: detailed view of a single host.
1016 */
1017 static struct str *
1018 html_hosts_detail(const char *ip)
1019 {
1020 struct bucket *h;
1021 struct str *buf, *ls_len;
1022 char ls_when[100];
1023 const char *canonical;
1024 time_t ls;
1025
1026 h = host_search(ip);
1027 if (h == NULL)
1028 return (NULL); /* no such host */
1029
1030 canonical = addr_to_str(&(h->u.host.addr));
1031
1032 /* Overview. */
1033 buf = str_make();
1034 html_open(buf, ip, interface, NULL);
1035 if (strcmp(ip, canonical) != 0)
1036 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1037 str_appendf(buf,
1038 "<p>\n"
1039 "<b>Hostname:</b> %s<br/>\n",
1040 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1041
1042 /* Resolve host "on demand" */
1043 if (h->u.host.dns == NULL)
1044 dns_queue(&(h->u.host.addr));
1045
1046 if (show_mac_addrs)
1047 str_appendf(buf,
1048 "<b>MAC Address:</b> "
1049 "<tt>%x:%x:%x:%x:%x:%x</tt><br/>\n",
1050 h->u.host.mac_addr[0],
1051 h->u.host.mac_addr[1],
1052 h->u.host.mac_addr[2],
1053 h->u.host.mac_addr[3],
1054 h->u.host.mac_addr[4],
1055 h->u.host.mac_addr[5]);
1056
1057 str_append(buf,
1058 "</p>\n"
1059 "<p>\n"
1060 "<b>Last seen:</b> ");
1061
1062 ls = h->u.host.last_seen;
1063 if (strftime(ls_when, sizeof(ls_when),
1064 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1065 str_append(buf, ls_when);
1066
1067 if (h->u.host.last_seen <= now) {
1068 ls_len = length_of_time(now - h->u.host.last_seen);
1069 str_append(buf, " (");
1070 str_appendstr(buf, ls_len);
1071 str_free(ls_len);
1072 str_append(buf, " ago)");
1073 } else {
1074 str_append(buf, " (in the future, possible clock problem)");
1075 }
1076
1077 str_appendf(buf,
1078 "</p>\n"
1079 "<p>\n"
1080 " <b>In:</b> %'qu<br/>\n"
1081 " <b>Out:</b> %'qu<br/>\n"
1082 " <b>Total:</b> %'qu<br/>\n"
1083 "</p>\n",
1084 h->in, h->out, h->total);
1085
1086 str_append(buf, "<h3>TCP ports</h3>\n");
1087 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1088
1089 str_append(buf, "<h3>UDP ports</h3>\n");
1090 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1091
1092 str_append(buf, "<h3>IP protocols</h3>\n");
1093 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1094
1095 html_close(buf);
1096 return (buf);
1097 }
1098
1099 /* ---------------------------------------------------------------------------
1100 * Database import and export code:
1101 * Initially written and contributed by Ben Stewart.
1102 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1103 */
1104 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1105 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1106 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1107
1108 static const char
1109 export_proto_ip = 'P',
1110 export_proto_tcp = 'T',
1111 export_proto_udp = 'U';
1112
1113 static const unsigned char
1114 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1115 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1116 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1117
1118 /* ---------------------------------------------------------------------------
1119 * Load a host's ip_proto table from a file.
1120 * Returns 0 on failure, 1 on success.
1121 */
1122 static int
1123 hosts_db_import_ip(const int fd, struct bucket *host)
1124 {
1125 uint8_t count, i;
1126
1127 if (!expect8(fd, export_proto_ip)) return 0;
1128 if (!read8(fd, &count)) return 0;
1129
1130 for (i=0; i<count; i++) {
1131 struct bucket *b;
1132 uint8_t proto;
1133 uint64_t in, out;
1134
1135 if (!read8(fd, &proto)) return 0;
1136 if (!read64(fd, &in)) return 0;
1137 if (!read64(fd, &out)) return 0;
1138
1139 /* Store data */
1140 b = host_get_ip_proto(host, proto);
1141 b->in = in;
1142 b->out = out;
1143 b->total = in + out;
1144 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1145 }
1146 return 1;
1147 }
1148
1149 /* ---------------------------------------------------------------------------
1150 * Load a host's port_tcp table from a file.
1151 * Returns 0 on failure, 1 on success.
1152 */
1153 static int
1154 hosts_db_import_tcp(const int fd, struct bucket *host)
1155 {
1156 uint16_t count, i;
1157
1158 if (!expect8(fd, export_proto_tcp)) return 0;
1159 if (!read16(fd, &count)) return 0;
1160
1161 for (i=0; i<count; i++) {
1162 struct bucket *b;
1163 uint16_t port;
1164 uint64_t in, out, syn;
1165
1166 if (!read16(fd, &port)) return 0;
1167 if (!read64(fd, &syn)) return 0;
1168 if (!read64(fd, &in)) return 0;
1169 if (!read64(fd, &out)) return 0;
1170
1171 /* Store data */
1172 b = host_get_port_tcp(host, port);
1173 b->in = in;
1174 b->out = out;
1175 b->total = in + out;
1176 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1177 b->u.port_tcp.syn = syn;
1178 }
1179 return 1;
1180 }
1181
1182 /* ---------------------------------------------------------------------------
1183 * Load a host's port_tcp table from a file.
1184 * Returns 0 on failure, 1 on success.
1185 */
1186 static int
1187 hosts_db_import_udp(const int fd, struct bucket *host)
1188 {
1189 uint16_t count, i;
1190
1191 if (!expect8(fd, export_proto_udp)) return 0;
1192 if (!read16(fd, &count)) return 0;
1193
1194 for (i=0; i<count; i++) {
1195 struct bucket *b;
1196 uint16_t port;
1197 uint64_t in, out;
1198
1199 if (!read16(fd, &port)) return 0;
1200 if (!read64(fd, &in)) return 0;
1201 if (!read64(fd, &out)) return 0;
1202
1203 /* Store data */
1204 b = host_get_port_udp(host, port);
1205 b->in = in;
1206 b->out = out;
1207 b->total = in + out;
1208 assert(b->u.port_udp.port == port); /* done by make_func */
1209 }
1210 return 1;
1211 }
1212
1213 /* ---------------------------------------------------------------------------
1214 * Load all hosts from a file.
1215 * Returns 0 on failure, 1 on success.
1216 */
1217 static int
1218 hosts_db_import_host(const int fd)
1219 {
1220 struct bucket *host;
1221 struct addr a;
1222 uint8_t hostname_len;
1223 uint64_t in, out;
1224 unsigned int pos = xtell(fd);
1225 char hdr[4];
1226 int ver = 0;
1227
1228 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1229 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1230 ver = 3;
1231 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1232 ver = 2;
1233 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1234 ver = 1;
1235 else {
1236 warnx("bad host header: %02x%02x%02x%02x",
1237 hdr[0], hdr[1], hdr[2], hdr[3]);
1238 return 0;
1239 }
1240
1241 if (ver == 3) {
1242 if (!readaddr(fd, &a))
1243 return 0;
1244 } else {
1245 assert((ver == 1) || (ver == 2));
1246 if (!readaddr_ipv4(fd, &a))
1247 return 0;
1248 }
1249 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1250 host = host_get(&a);
1251 assert(addr_equal(&(host->u.host.addr), &a));
1252
1253 if (ver > 1) {
1254 uint64_t t;
1255 if (!read64(fd, &t)) return 0;
1256 host->u.host.last_seen = (time_t)t;
1257 }
1258
1259 assert(sizeof(host->u.host.mac_addr) == 6);
1260 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1261 return 0;
1262
1263 /* HOSTNAME */
1264 assert(host->u.host.dns == NULL); /* make fn? */
1265 if (!read8(fd, &hostname_len)) return 0;
1266 if (hostname_len > 0) {
1267 host->u.host.dns = xmalloc(hostname_len + 1);
1268 host->u.host.dns[0] = '\0';
1269
1270 /* At this point, the hostname is attached to a host which is in our
1271 * hosts_db, so if we bail out due to an import error, this pointer
1272 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1273 */
1274
1275 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1276 host->u.host.dns[hostname_len] = '\0';
1277 }
1278
1279 if (!read64(fd, &in)) return 0;
1280 if (!read64(fd, &out)) return 0;
1281
1282 host->in = in;
1283 host->out = out;
1284 host->total = in + out;
1285
1286 /* Host's port and proto subtables: */
1287 if (!hosts_db_import_ip(fd, host)) return 0;
1288 if (!hosts_db_import_tcp(fd, host)) return 0;
1289 if (!hosts_db_import_udp(fd, host)) return 0;
1290 return 1;
1291 }
1292
1293 /* ---------------------------------------------------------------------------
1294 * Database Import: Grab hosts_db from a file provided by the caller.
1295 *
1296 * This function will retrieve the data sans the header. We expect the caller
1297 * to have validated the header of the hosts_db segment, and left the file
1298 * sitting at the start of the data.
1299 */
1300 int hosts_db_import(const int fd)
1301 {
1302 uint32_t host_count, i;
1303
1304 if (!read32(fd, &host_count)) return 0;
1305
1306 for (i=0; i<host_count; i++)
1307 if (!hosts_db_import_host(fd)) return 0;
1308
1309 return 1;
1310 }
1311
1312 /* ---------------------------------------------------------------------------
1313 * Database Export: Dump hosts_db into a file provided by the caller.
1314 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1315 */
1316 int hosts_db_export(const int fd)
1317 {
1318 uint32_t i;
1319 struct bucket *b;
1320
1321 if (!write32(fd, hosts_db->count)) return 0;
1322
1323 for (i = 0; i<hosts_db->size; i++)
1324 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1325 /* For each host: */
1326 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1327 return 0;
1328
1329 if (!writeaddr(fd, &(b->u.host.addr))) return 0;
1330
1331 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1332
1333 assert(sizeof(b->u.host.mac_addr) == 6);
1334 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1335 return 0;
1336
1337 /* HOSTNAME */
1338 if (b->u.host.dns == NULL) {
1339 if (!write8(fd, 0)) return 0;
1340 } else {
1341 int dnslen = strlen(b->u.host.dns);
1342
1343 if (dnslen > 255) {
1344 warnx("found a very long hostname: \"%s\"\n"
1345 "wasn't expecting one longer than 255 chars (this one is %d)",
1346 b->u.host.dns, dnslen);
1347 dnslen = 255;
1348 }
1349
1350 if (!write8(fd, (uint8_t)dnslen)) return 0;
1351 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1352 }
1353
1354 if (!write64(fd, b->in)) return 0;
1355 if (!write64(fd, b->out)) return 0;
1356
1357 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1358 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1359 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1360 }
1361 return 1;
1362 }
1363
1364 /* ---------------------------------------------------------------------------
1365 * Dump the ip_proto table of a host.
1366 */
1367 static int
1368 hosts_db_export_ip(const struct hashtable *h, const int fd)
1369 {
1370 uint32_t i, written = 0;
1371 struct bucket *b;
1372
1373 /* IP DATA */
1374 if (!write8(fd, export_proto_ip)) return 0;
1375
1376 /* If no data, write a IP Proto count of 0 and we're done. */
1377 if (h == NULL) {
1378 if (!write8(fd, 0)) return 0;
1379 return 1;
1380 }
1381
1382 assert(h->count < 256);
1383 if (!write8(fd, (uint8_t)h->count)) return 0;
1384
1385 for (i = 0; i<h->size; i++)
1386 for (b = h->table[i]; b != NULL; b = b->next) {
1387 /* For each ip_proto bucket: */
1388
1389 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1390 if (!write64(fd, b->in)) return 0;
1391 if (!write64(fd, b->out)) return 0;
1392 written++;
1393 }
1394 assert(written == h->count);
1395 return 1;
1396 }
1397
1398 /* ---------------------------------------------------------------------------
1399 * Dump the port_tcp table of a host.
1400 */
1401 static int
1402 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1403 {
1404 struct bucket *b;
1405 uint32_t i, written = 0;
1406
1407 /* TCP DATA */
1408 if (!write8(fd, export_proto_tcp)) return 0;
1409
1410 /* If no data, write a count of 0 and we're done. */
1411 if (h == NULL) {
1412 if (!write16(fd, 0)) return 0;
1413 return 1;
1414 }
1415
1416 assert(h->count < 65536);
1417 if (!write16(fd, (uint16_t)h->count)) return 0;
1418
1419 for (i = 0; i<h->size; i++)
1420 for (b = h->table[i]; b != NULL; b = b->next) {
1421 if (!write16(fd, b->u.port_tcp.port)) return 0;
1422 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1423 if (!write64(fd, b->in)) return 0;
1424 if (!write64(fd, b->out)) return 0;
1425 written++;
1426 }
1427 assert(written == h->count);
1428 return 1;
1429 }
1430
1431 /* ---------------------------------------------------------------------------
1432 * Dump the port_udp table of a host.
1433 */
1434 static int
1435 hosts_db_export_udp(const struct hashtable *h, const int fd)
1436 {
1437 struct bucket *b;
1438 uint32_t i, written = 0;
1439
1440 /* UDP DATA */
1441 if (!write8(fd, export_proto_udp)) return 0;
1442
1443 /* If no data, write a count of 0 and we're done. */
1444 if (h == NULL) {
1445 if (!write16(fd, 0)) return 0;
1446 return 1;
1447 }
1448
1449 assert(h->count < 65536);
1450 if (!write16(fd, (uint16_t)h->count)) return 0;
1451
1452 for (i = 0; i<h->size; i++)
1453 for (b = h->table[i]; b != NULL; b = b->next) {
1454 if (!write16(fd, b->u.port_udp.port)) return 0;
1455 if (!write64(fd, b->in)) return 0;
1456 if (!write64(fd, b->out)) return 0;
1457 written++;
1458 }
1459 assert(written == h->count);
1460 return 1;
1461 }
1462
1463 /* vim:set ts=3 sw=3 tw=78 expandtab: */