Prefix all global variables to aid debugging.
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2009 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "http.h" /* for http_base_url */
19 #include "ncache.h"
20 #include "now.h"
21 #include "str.h"
22
23 #include <arpa/inet.h> /* inet_aton() */
24 #include <netdb.h> /* struct addrinfo */
25 #include <assert.h>
26 #include <errno.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h> /* memset(), strcmp() */
30 #include <unistd.h>
31
32 int hosts_db_show_macs = 0;
33
34 /* FIXME: specify somewhere more sane/tunable */
35 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
36
37 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
38 typedef void (free_func_t)(struct bucket *);
39 typedef const void * (key_func_t)(const struct bucket *);
40 typedef int (find_func_t)(const struct bucket *, const void *);
41 typedef struct bucket * (make_func_t)(const void *);
42 typedef void (format_cols_func_t)(struct str *);
43 typedef void (format_row_func_t)(struct str *, const struct bucket *,
44 const char *);
45
46 struct hashtable {
47 uint8_t bits; /* size of hashtable in bits */
48 uint32_t size, mask;
49 uint32_t count, count_max, count_keep; /* items in table */
50 uint32_t coeff; /* coefficient for Fibonacci hashing */
51 struct bucket **table;
52
53 struct {
54 uint64_t inserts, searches, deletions, rehashes;
55 } stats;
56
57 hash_func_t *hash_func;
58 /* returns hash value of given key (passed as void*) */
59
60 free_func_t *free_func;
61 /* free of bucket payload */
62
63 key_func_t *key_func;
64 /* returns pointer to key of bucket (to pass to hash_func) */
65
66 find_func_t *find_func;
67 /* returns true if given bucket matches key (passed as void*) */
68
69 make_func_t *make_func;
70 /* returns bucket containing new record with key (passed as void*) */
71
72 format_cols_func_t *format_cols_func;
73 /* append table columns to str */
74
75 format_row_func_t *format_row_func;
76 /* format record and append to str */
77 };
78
79 static void hashtable_reduce(struct hashtable *ht);
80 static void hashtable_free(struct hashtable *h);
81
82 #define HOST_BITS 1 /* initial size of hosts table */
83 #define PORT_BITS 1 /* initial size of ports tables */
84 #define PROTO_BITS 1 /* initial size of proto table */
85
86 /* We only use one hosts_db hashtable and this is it. */
87 static struct hashtable *hosts_db = NULL;
88
89 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
90 static const double phi_1 =
91 0.61803398874989490252573887119069695472717285156250;
92
93 /* Co-prime of u, using phi^-1 */
94 inline static uint32_t
95 coprime(const uint32_t u)
96 {
97 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
98 }
99
100 /*
101 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
102 * src/sys/netinet/tcp_hostcache.c 1.1
103 */
104 inline static uint32_t
105 ipv4_hash(const struct addr *const a)
106 {
107 uint32_t ip = a->ip.v4;
108 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
109 }
110
111 #ifndef s6_addr32
112 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
113 * taken care of GNU/Linux and GNU/kfreebsd. */
114 # define s6_addr32 __u6_addr.__u6_addr32
115 #endif
116
117 /*
118 * This is the IPv6 hash function used by FreeBSD in the same file as above,
119 * svn rev 122922.
120 */
121 inline static uint32_t
122 ipv6_hash(const struct addr *const a)
123 {
124 const struct in6_addr *const ip6 = &(a->ip.v6);
125 return ( ip6->s6_addr32[0] ^ ip6->s6_addr32[1] ^
126 ip6->s6_addr32[2] ^ ip6->s6_addr32[3] );
127 }
128
129 /* ---------------------------------------------------------------------------
130 * hash_func collection
131 */
132 static uint32_t
133 hash_func_host(const struct hashtable *h _unused_, const void *key)
134 {
135 const struct addr *a = key;
136 if (a->family == IPv4)
137 return (ipv4_hash(a));
138 else {
139 assert(a->family == IPv6);
140 return (ipv6_hash(a));
141 }
142 }
143
144 #define CASTKEY(type) (*((const type *)key))
145
146 static uint32_t
147 hash_func_short(const struct hashtable *h, const void *key)
148 {
149 return (CASTKEY(uint16_t) * h->coeff);
150 }
151
152 static uint32_t
153 hash_func_byte(const struct hashtable *h, const void *key)
154 {
155 return (CASTKEY(uint8_t) * h->coeff);
156 }
157
158 /* ---------------------------------------------------------------------------
159 * key_func collection
160 */
161
162 static const void *
163 key_func_host(const struct bucket *b)
164 {
165 return &(b->u.host.addr);
166 }
167
168 static const void *
169 key_func_port_tcp(const struct bucket *b)
170 {
171 return &(b->u.port_tcp.port);
172 }
173
174 static const void *
175 key_func_port_udp(const struct bucket *b)
176 {
177 return &(b->u.port_udp.port);
178 }
179
180 static const void *
181 key_func_ip_proto(const struct bucket *b)
182 {
183 return &(b->u.ip_proto.proto);
184 }
185
186 /* ---------------------------------------------------------------------------
187 * find_func collection
188 */
189
190 static int
191 find_func_host(const struct bucket *b, const void *key)
192 {
193 return (addr_equal(key, &(b->u.host.addr)));
194 }
195
196 static int
197 find_func_port_tcp(const struct bucket *b, const void *key)
198 {
199 return (b->u.port_tcp.port == CASTKEY(uint16_t));
200 }
201
202 static int
203 find_func_port_udp(const struct bucket *b, const void *key)
204 {
205 return (b->u.port_udp.port == CASTKEY(uint16_t));
206 }
207
208 static int
209 find_func_ip_proto(const struct bucket *b, const void *key)
210 {
211 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
212 }
213
214 /* ---------------------------------------------------------------------------
215 * make_func collection
216 */
217
218 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
219 struct bucket *next; \
220 uint64_t in, out, total; \
221 union { struct type t; } u; } _custom_bucket; \
222 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
223 struct type *name_content = &(name_bucket->u.type); \
224 name_bucket->next = NULL; \
225 name_bucket->in = name_bucket->out = name_bucket->total = 0;
226
227 static struct bucket *
228 make_func_host(const void *key)
229 {
230 MAKE_BUCKET(b, h, host);
231 h->addr = CASTKEY(struct addr);
232 h->dns = NULL;
233 h->last_seen = now;
234 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
235 h->ports_tcp = NULL;
236 h->ports_udp = NULL;
237 h->ip_protos = NULL;
238 return (b);
239 }
240
241 static void
242 free_func_host(struct bucket *b)
243 {
244 struct host *h = &(b->u.host);
245 if (h->dns != NULL) free(h->dns);
246 hashtable_free(h->ports_tcp);
247 hashtable_free(h->ports_udp);
248 hashtable_free(h->ip_protos);
249 }
250
251 static struct bucket *
252 make_func_port_tcp(const void *key)
253 {
254 MAKE_BUCKET(b, p, port_tcp);
255 p->port = CASTKEY(uint16_t);
256 p->syn = 0;
257 return (b);
258 }
259
260 static struct bucket *
261 make_func_port_udp(const void *key)
262 {
263 MAKE_BUCKET(b, p, port_udp);
264 p->port = CASTKEY(uint16_t);
265 return (b);
266 }
267
268 static struct bucket *
269 make_func_ip_proto(const void *key)
270 {
271 MAKE_BUCKET(b, p, ip_proto);
272 p->proto = CASTKEY(uint8_t);
273 return (b);
274 }
275
276 static void
277 free_func_simple(struct bucket *b _unused_)
278 {
279 /* nop */
280 }
281
282 /* ---------------------------------------------------------------------------
283 * format_func collection (ordered by struct)
284 */
285
286 static void
287 format_cols_host(struct str *buf)
288 {
289 /* FIXME: don't clobber parts of the query string
290 * specifically "full" and "start"
291 * when setting sort direction
292 */
293 str_append(buf,
294 "<table>\n"
295 "<tr>\n"
296 " <th>IP</th>\n"
297 " <th>Hostname</th>\n");
298 if (hosts_db_show_macs) str_append(buf,
299 " <th>MAC Address</th>\n");
300 str_append(buf,
301 " <th><a href=\"?sort=in\">In</a></th>\n"
302 " <th><a href=\"?sort=out\">Out</a></th>\n"
303 " <th><a href=\"?sort=total\">Total</a></th>\n");
304 if (opt_want_lastseen) str_append(buf,
305 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
306 str_append(buf,
307 "</tr>\n");
308 }
309
310 static void
311 format_row_host(struct str *buf, const struct bucket *b,
312 const char *css_class)
313 {
314 const char *ip = addr_to_str(&(b->u.host.addr));
315
316 str_appendf(buf,
317 "<tr class=\"%s\">\n"
318 " <td><a href=\"%shosts/%s/\">%s</a></td>\n"
319 " <td>%s</td>\n",
320 css_class,
321 http_base_url, ip, ip,
322 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
323
324 if (hosts_db_show_macs)
325 str_appendf(buf,
326 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
327 b->u.host.mac_addr[0],
328 b->u.host.mac_addr[1],
329 b->u.host.mac_addr[2],
330 b->u.host.mac_addr[3],
331 b->u.host.mac_addr[4],
332 b->u.host.mac_addr[5]);
333
334 str_appendf(buf,
335 " <td class=\"num\">%'qu</td>\n"
336 " <td class=\"num\">%'qu</td>\n"
337 " <td class=\"num\">%'qu</td>\n",
338 b->in, b->out, b->total);
339
340 if (opt_want_lastseen) {
341 time_t last_t = b->u.host.last_seen;
342 struct str *lastseen = NULL;
343
344 if (now >= last_t)
345 lastseen = length_of_time(now - last_t);
346
347 str_append(buf,
348 " <td class=\"num\">");
349 if (lastseen == NULL)
350 str_append(buf, "(clock error)");
351 else {
352 str_appendstr(buf, lastseen);
353 str_free(lastseen);
354 }
355 str_append(buf,
356 "</td>");
357 }
358
359 str_appendf(buf,
360 "</tr>\n");
361
362 /* Only resolve hosts "on demand" */
363 if (b->u.host.dns == NULL)
364 dns_queue(&(b->u.host.addr));
365 }
366
367 static void
368 format_cols_port_tcp(struct str *buf)
369 {
370 str_append(buf,
371 "<table>\n"
372 "<tr>\n"
373 " <th>Port</td>\n"
374 " <th>Service</td>\n"
375 " <th>In</td>\n"
376 " <th>Out</td>\n"
377 " <th>Total</td>\n"
378 " <th>SYNs</td>\n"
379 "</tr>\n"
380 );
381 }
382
383 static void
384 format_row_port_tcp(struct str *buf, const struct bucket *b,
385 const char *css_class)
386 {
387 const struct port_tcp *p = &(b->u.port_tcp);
388
389 str_appendf(buf,
390 "<tr class=\"%s\">\n"
391 " <td class=\"num\">%u</td>\n"
392 " <td>%s</td>\n"
393 " <td class=\"num\">%'qu</td>\n"
394 " <td class=\"num\">%'qu</td>\n"
395 " <td class=\"num\">%'qu</td>\n"
396 " <td class=\"num\">%'qu</td>\n"
397 "</tr>\n",
398 css_class,
399 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
400 );
401 }
402
403 static void
404 format_cols_port_udp(struct str *buf)
405 {
406 str_append(buf,
407 "<table>\n"
408 "<tr>\n"
409 " <th>Port</td>\n"
410 " <th>Service</td>\n"
411 " <th>In</td>\n"
412 " <th>Out</td>\n"
413 " <th>Total</td>\n"
414 "</tr>\n"
415 );
416 }
417
418 static void
419 format_row_port_udp(struct str *buf, const struct bucket *b,
420 const char *css_class)
421 {
422 const struct port_udp *p = &(b->u.port_udp);
423
424 str_appendf(buf,
425 "<tr class=\"%s\">\n"
426 " <td class=\"num\">%u</td>\n"
427 " <td>%s</td>\n"
428 " <td class=\"num\">%'qu</td>\n"
429 " <td class=\"num\">%'qu</td>\n"
430 " <td class=\"num\">%'qu</td>\n"
431 "</tr>\n",
432 css_class,
433 p->port, getservudp(p->port), b->in, b->out, b->total
434 );
435 }
436
437 static void
438 format_cols_ip_proto(struct str *buf)
439 {
440 str_append(buf,
441 "<table>\n"
442 "<tr>\n"
443 " <th>#</td>\n"
444 " <th>Protocol</td>\n"
445 " <th>In</td>\n"
446 " <th>Out</td>\n"
447 " <th>Total</td>\n"
448 "</tr>\n"
449 );
450 }
451
452 static void
453 format_row_ip_proto(struct str *buf, const struct bucket *b,
454 const char *css_class)
455 {
456 const struct ip_proto *p = &(b->u.ip_proto);
457
458 str_appendf(buf,
459 "<tr class=\"%s\">\n"
460 " <td class=\"num\">%u</td>\n"
461 " <td>%s</td>\n"
462 " <td class=\"num\">%'qu</td>\n"
463 " <td class=\"num\">%'qu</td>\n"
464 " <td class=\"num\">%'qu</td>\n"
465 "</tr>\n",
466 css_class,
467 p->proto, getproto(p->proto),
468 b->in, b->out, b->total
469 );
470 }
471
472 /* ---------------------------------------------------------------------------
473 * Initialise a hashtable.
474 */
475 static struct hashtable *
476 hashtable_make(const uint8_t bits,
477 const unsigned int count_max,
478 const unsigned int count_keep,
479 hash_func_t *hash_func,
480 free_func_t *free_func,
481 key_func_t *key_func,
482 find_func_t *find_func,
483 make_func_t *make_func,
484 format_cols_func_t *format_cols_func,
485 format_row_func_t *format_row_func)
486 {
487 struct hashtable *hash;
488 assert(bits > 0);
489
490 hash = xmalloc(sizeof(*hash));
491 hash->bits = bits;
492 hash->count_max = count_max;
493 hash->count_keep = count_keep;
494 hash->size = 1U << bits;
495 hash->mask = hash->size - 1;
496 hash->coeff = coprime(hash->size);
497 hash->hash_func = hash_func;
498 hash->free_func = free_func;
499 hash->key_func = key_func;
500 hash->find_func = find_func;
501 hash->make_func = make_func;
502 hash->format_cols_func = format_cols_func;
503 hash->format_row_func = format_row_func;
504 hash->count = 0;
505 hash->table = xcalloc(hash->size, sizeof(*hash->table));
506 memset(&(hash->stats), 0, sizeof(hash->stats));
507 return (hash);
508 }
509
510 /* ---------------------------------------------------------------------------
511 * Initialise global hosts_db.
512 */
513 void
514 hosts_db_init(void)
515 {
516 assert(hosts_db == NULL);
517 hosts_db = hashtable_make(HOST_BITS, opt_hosts_max, opt_hosts_keep,
518 hash_func_host, free_func_host, key_func_host, find_func_host,
519 make_func_host, format_cols_host, format_row_host);
520 }
521
522 static void
523 hashtable_rehash(struct hashtable *h, const uint8_t bits)
524 {
525 struct bucket **old_table, **new_table;
526 uint32_t i, old_size;
527 assert(h != NULL);
528 assert(bits > 0);
529
530 h->stats.rehashes++;
531 old_size = h->size;
532 old_table = h->table;
533
534 h->bits = bits;
535 h->size = 1U << bits;
536 h->mask = h->size - 1;
537 h->coeff = coprime(h->size);
538 new_table = xcalloc(h->size, sizeof(*new_table));
539
540 for (i=0; i<old_size; i++) {
541 struct bucket *next, *b = old_table[i];
542 while (b != NULL) {
543 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
544 next = b->next;
545 b->next = new_table[pos];
546 new_table[pos] = b;
547 b = next;
548 }
549 }
550
551 free(h->table);
552 h->table = new_table;
553 }
554
555 static void
556 hashtable_insert(struct hashtable *h, struct bucket *b)
557 {
558 uint32_t pos;
559 assert(h != NULL);
560 assert(b != NULL);
561 assert(b->next == NULL);
562
563 /* Rehash on 80% occupancy */
564 if ((h->count > h->size) ||
565 ((h->size - h->count) < h->size / 5))
566 hashtable_rehash(h, h->bits+1);
567
568 pos = h->hash_func(h, h->key_func(b)) & h->mask;
569 if (h->table[pos] == NULL)
570 h->table[pos] = b;
571 else {
572 /* Insert at top of chain. */
573 b->next = h->table[pos];
574 h->table[pos] = b;
575 }
576 h->count++;
577 h->stats.inserts++;
578 }
579
580 /* Return bucket matching key, or NULL if no such entry. */
581 static struct bucket *
582 hashtable_search(struct hashtable *h, const void *key)
583 {
584 uint32_t pos;
585 struct bucket *b;
586
587 h->stats.searches++;
588 pos = h->hash_func(h, key) & h->mask;
589 b = h->table[pos];
590 while (b != NULL) {
591 if (h->find_func(b, key))
592 return (b);
593 else
594 b = b->next;
595 }
596 return (NULL);
597 }
598
599 typedef enum { NO_REDUCE = 0, ALLOW_REDUCE = 1 } reduce_bool;
600 /* Search for a key. If it's not there, make and insert a bucket for it. */
601 static struct bucket *
602 hashtable_find_or_insert(struct hashtable *h, const void *key,
603 const reduce_bool allow_reduce)
604 {
605 struct bucket *b = hashtable_search(h, key);
606
607 if (b == NULL) {
608 /* Not found, so insert after checking occupancy. */
609 if (allow_reduce && (h->count >= h->count_max))
610 hashtable_reduce(h);
611 b = h->make_func(key);
612 hashtable_insert(h, b);
613 }
614 return (b);
615 }
616
617 /*
618 * Frees the hashtable and the buckets. The contents are assumed to be
619 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
620 * the bucket.
621 */
622 static void
623 hashtable_free(struct hashtable *h)
624 {
625 uint32_t i;
626
627 if (h == NULL)
628 return;
629 for (i=0; i<h->size; i++) {
630 struct bucket *tmp, *b = h->table[i];
631 while (b != NULL) {
632 tmp = b;
633 b = b->next;
634 h->free_func(tmp);
635 free(tmp);
636 }
637 }
638 free(h->table);
639 free(h);
640 }
641
642 /* ---------------------------------------------------------------------------
643 * Return existing host or insert a new one.
644 */
645 struct bucket *
646 host_get(const struct addr *const a)
647 {
648 return (hashtable_find_or_insert(hosts_db, a, NO_REDUCE));
649 }
650
651 /* ---------------------------------------------------------------------------
652 * Find host, returns NULL if not in DB.
653 */
654 struct bucket *
655 host_find(const struct addr *const a)
656 {
657 return (hashtable_search(hosts_db, a));
658 }
659
660 /* ---------------------------------------------------------------------------
661 * Find host, returns NULL if not in DB.
662 */
663 static struct bucket *
664 host_search(const char *ipstr)
665 {
666 struct addr a;
667 struct addrinfo hints, *ai;
668
669 memset(&hints, 0, sizeof(hints));
670 hints.ai_family = AF_UNSPEC;
671 hints.ai_flags = AI_NUMERICHOST;
672
673 if (getaddrinfo(ipstr, NULL, &hints, &ai))
674 return (NULL); /* invalid addr */
675
676 if (ai->ai_family == AF_INET) {
677 a.family = IPv4;
678 a.ip.v4 = ((const struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr;
679 }
680 else if (ai->ai_family == AF_INET6) {
681 a.family = IPv6;
682 memcpy(&(a.ip.v6),
683 ((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr.s6_addr,
684 sizeof(a.ip.v6));
685 } else {
686 freeaddrinfo(ai);
687 return (NULL); /* unknown family */
688 }
689 freeaddrinfo(ai);
690
691 verbosef("search(%s) turned into %s", ipstr, addr_to_str(&a));
692 return (hashtable_search(hosts_db, &a));
693 }
694
695 /* ---------------------------------------------------------------------------
696 * Reduce a hashtable to the top <keep> entries.
697 */
698 static void
699 hashtable_reduce(struct hashtable *ht)
700 {
701 uint32_t i, pos, rmd;
702 const struct bucket **table;
703 uint64_t cutoff;
704
705 assert(ht->count_keep < ht->count);
706
707 /* Fill table with pointers to buckets in hashtable. */
708 table = xcalloc(ht->count, sizeof(*table));
709 for (pos=0, i=0; i<ht->size; i++) {
710 struct bucket *b = ht->table[i];
711 while (b != NULL) {
712 table[pos++] = b;
713 b = b->next;
714 }
715 }
716 assert(pos == ht->count);
717 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
718 cutoff = table[ht->count_keep]->total;
719 free(table);
720
721 /* Remove all elements with total <= cutoff. */
722 rmd = 0;
723 for (i=0; i<ht->size; i++) {
724 struct bucket *last = NULL, *next, *b = ht->table[i];
725 while (b != NULL) {
726 next = b->next;
727 if (b->total <= cutoff) {
728 /* Remove this one. */
729 ht->free_func(b);
730 free(b);
731 if (last == NULL)
732 ht->table[i] = next;
733 else
734 last->next = next;
735 rmd++;
736 ht->count--;
737 } else {
738 last = b;
739 }
740 b = next;
741 }
742 }
743 verbosef("hashtable_reduce: removed %u buckets, left %u",
744 rmd, ht->count);
745 hashtable_rehash(ht, ht->bits); /* is this needed? */
746 }
747
748 /* Reduce hosts_db if needed. */
749 void hosts_db_reduce(void)
750 {
751 if (hosts_db->count >= hosts_db->count_max)
752 hashtable_reduce(hosts_db);
753 }
754
755 /* ---------------------------------------------------------------------------
756 * Reset hosts_db to empty.
757 */
758 void
759 hosts_db_reset(void)
760 {
761 unsigned int i;
762
763 for (i=0; i<hosts_db->size; i++) {
764 struct bucket *next, *b = hosts_db->table[i];
765 while (b != NULL) {
766 next = b->next;
767 hosts_db->free_func(b);
768 free(b);
769 b = next;
770 }
771 hosts_db->table[i] = NULL;
772 }
773 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
774 hosts_db->count = 0;
775 }
776
777 /* ---------------------------------------------------------------------------
778 * Deallocate hosts_db.
779 */
780 void hosts_db_free(void)
781 {
782 uint32_t i;
783
784 assert(hosts_db != NULL);
785 for (i=0; i<hosts_db->size; i++) {
786 struct bucket *tmp, *b = hosts_db->table[i];
787 while (b != NULL) {
788 tmp = b;
789 b = b->next;
790 hosts_db->free_func(tmp);
791 free(tmp);
792 }
793 }
794 free(hosts_db->table);
795 free(hosts_db);
796 hosts_db = NULL;
797 }
798
799 /* ---------------------------------------------------------------------------
800 * Find or create a port_tcp inside a host.
801 */
802 struct bucket *
803 host_get_port_tcp(struct bucket *host, const uint16_t port)
804 {
805 struct host *h = &host->u.host;
806 assert(h != NULL);
807 if (h->ports_tcp == NULL)
808 h->ports_tcp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
809 hash_func_short, free_func_simple, key_func_port_tcp,
810 find_func_port_tcp, make_func_port_tcp,
811 format_cols_port_tcp, format_row_port_tcp);
812 return (hashtable_find_or_insert(h->ports_tcp, &port, ALLOW_REDUCE));
813 }
814
815 /* ---------------------------------------------------------------------------
816 * Find or create a port_udp inside a host.
817 */
818 struct bucket *
819 host_get_port_udp(struct bucket *host, const uint16_t port)
820 {
821 struct host *h = &host->u.host;
822 assert(h != NULL);
823 if (h->ports_udp == NULL)
824 h->ports_udp = hashtable_make(PORT_BITS, opt_ports_max, opt_ports_keep,
825 hash_func_short, free_func_simple, key_func_port_udp,
826 find_func_port_udp, make_func_port_udp,
827 format_cols_port_udp, format_row_port_udp);
828 return (hashtable_find_or_insert(h->ports_udp, &port, ALLOW_REDUCE));
829 }
830
831 /* ---------------------------------------------------------------------------
832 * Find or create an ip_proto inside a host.
833 */
834 struct bucket *
835 host_get_ip_proto(struct bucket *host, const uint8_t proto)
836 {
837 struct host *h = &host->u.host;
838 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
839 assert(h != NULL);
840 if (h->ip_protos == NULL)
841 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
842 hash_func_byte, free_func_simple, key_func_ip_proto,
843 find_func_ip_proto, make_func_ip_proto,
844 format_cols_ip_proto, format_row_ip_proto);
845 return (hashtable_find_or_insert(h->ip_protos, &proto, ALLOW_REDUCE));
846 }
847
848 static struct str *html_hosts_main(const char *qs);
849 static struct str *html_hosts_detail(const char *ip);
850
851 /* ---------------------------------------------------------------------------
852 * Web interface: delegate the /hosts/ space.
853 */
854 struct str *
855 html_hosts(const char *uri, const char *query)
856 {
857 unsigned int i, num_elems;
858 char **elem = split('/', uri, &num_elems);
859 struct str *buf = NULL;
860
861 assert(num_elems >= 1);
862 assert(strcmp(elem[0], "hosts") == 0);
863
864 if (num_elems == 1)
865 /* /hosts/ */
866 buf = html_hosts_main(query);
867 else if (num_elems == 2)
868 /* /hosts/<IP of host>/ */
869 buf = html_hosts_detail(elem[1]);
870
871 for (i=0; i<num_elems; i++)
872 free(elem[i]);
873 free(elem);
874 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
875 other codes to be possible */
876 }
877
878 /* ---------------------------------------------------------------------------
879 * Format hashtable into HTML.
880 */
881 static void
882 format_table(struct str *buf, struct hashtable *ht, unsigned int start,
883 const enum sort_dir sort, const int full)
884 {
885 const struct bucket **table;
886 unsigned int i, pos, end;
887 int alt = 0;
888
889 if ((ht == NULL) || (ht->count == 0)) {
890 str_append(buf, "<p>The table is empty.</p>\n");
891 return;
892 }
893
894 /* Fill table with pointers to buckets in hashtable. */
895 table = xcalloc(ht->count, sizeof(*table));
896 for (pos=0, i=0; i<ht->size; i++) {
897 struct bucket *b = ht->table[i];
898 while (b != NULL) {
899 table[pos++] = b;
900 b = b->next;
901 }
902 }
903 assert(pos == ht->count);
904
905 if (full) {
906 /* full report overrides start and end */
907 start = 0;
908 end = ht->count;
909 } else
910 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
911
912 str_appendf(buf, "(%u-%u of %u)<br>\n", start+1, end, ht->count);
913 qsort_buckets(table, ht->count, start, end, sort);
914 ht->format_cols_func(buf);
915
916 for (i=start; i<end; i++) {
917 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
918 alt = !alt; /* alternate class for table rows */
919 }
920 free(table);
921 str_append(buf, "</table>\n");
922 }
923
924 /* ---------------------------------------------------------------------------
925 * Web interface: sorted table of hosts.
926 */
927 static struct str *
928 html_hosts_main(const char *qs)
929 {
930 struct str *buf = str_make();
931 char *qs_start, *qs_sort, *qs_full, *ep;
932 const char *sortstr;
933 int start, full = 0;
934 enum sort_dir sort;
935
936 /* parse query string */
937 qs_start = qs_get(qs, "start");
938 qs_sort = qs_get(qs, "sort");
939 qs_full = qs_get(qs, "full");
940 if (qs_full != NULL) {
941 full = 1;
942 free(qs_full);
943 }
944
945 /* validate sort */
946 if (qs_sort == NULL) sort = TOTAL;
947 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
948 else if (strcmp(qs_sort, "in") == 0) sort = IN;
949 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
950 else if (strcmp(qs_sort, "lastseen") == 0) sort = LASTSEEN;
951 else {
952 str_append(buf, "Error: invalid value for \"sort\".\n");
953 goto done;
954 }
955
956 /* parse start */
957 if (qs_start == NULL)
958 start = 0;
959 else {
960 start = (int)strtoul(qs_start, &ep, 10);
961 if (*ep != '\0') {
962 str_append(buf, "Error: \"start\" is not a number.\n");
963 goto done;
964 }
965 if ((errno == ERANGE) ||
966 (start < 0) || (start >= (int)hosts_db->count)) {
967 str_append(buf, "Error: \"start\" is out of bounds.\n");
968 goto done;
969 }
970 }
971
972 #define PREV "&lt;&lt;&lt; prev page"
973 #define NEXT "next page &gt;&gt;&gt;"
974 #define FULL "full table"
975
976 html_open(buf, "Hosts", opt_interface, /*want_graph_js=*/0);
977 format_table(buf, hosts_db, start, sort, full);
978
979 /* <prev | full | stats | next> */
980 sortstr = qs_sort;
981 if (sortstr == NULL) sortstr = "total";
982 if (start > 0) {
983 int prev = start - MAX_ENTRIES;
984 if (prev < 0)
985 prev = 0;
986 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
987 prev, sortstr);
988 } else
989 str_append(buf, PREV);
990
991 if (full)
992 str_append(buf, " | " FULL);
993 else
994 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
995 sortstr);
996
997 if (start+MAX_ENTRIES < (int)hosts_db->count)
998 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
999 start+MAX_ENTRIES, sortstr);
1000 else
1001 str_append(buf, " | " NEXT);
1002
1003 str_append(buf, "<br>\n");
1004
1005 html_close(buf);
1006 done:
1007 if (qs_start != NULL) free(qs_start);
1008 if (qs_sort != NULL) free(qs_sort);
1009 return buf;
1010 #undef PREV
1011 #undef NEXT
1012 #undef FULL
1013 }
1014
1015 /* ---------------------------------------------------------------------------
1016 * Web interface: detailed view of a single host.
1017 */
1018 static struct str *
1019 html_hosts_detail(const char *ip)
1020 {
1021 struct bucket *h;
1022 struct str *buf, *ls_len;
1023 char ls_when[100];
1024 const char *canonical;
1025 time_t ls;
1026
1027 h = host_search(ip);
1028 if (h == NULL)
1029 return (NULL); /* no such host */
1030
1031 canonical = addr_to_str(&(h->u.host.addr));
1032
1033 /* Overview. */
1034 buf = str_make();
1035 html_open(buf, ip, opt_interface, /*want_graph_js=*/0);
1036 if (strcmp(ip, canonical) != 0)
1037 str_appendf(buf, "(canonically <b>%s</b>)\n", canonical);
1038 str_appendf(buf,
1039 "<p>\n"
1040 "<b>Hostname:</b> %s<br>\n",
1041 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
1042
1043 /* Resolve host "on demand" */
1044 if (h->u.host.dns == NULL)
1045 dns_queue(&(h->u.host.addr));
1046
1047 if (hosts_db_show_macs)
1048 str_appendf(buf,
1049 "<b>MAC Address:</b> "
1050 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1051 h->u.host.mac_addr[0],
1052 h->u.host.mac_addr[1],
1053 h->u.host.mac_addr[2],
1054 h->u.host.mac_addr[3],
1055 h->u.host.mac_addr[4],
1056 h->u.host.mac_addr[5]);
1057
1058 str_append(buf,
1059 "</p>\n"
1060 "<p>\n"
1061 "<b>Last seen:</b> ");
1062
1063 ls = h->u.host.last_seen;
1064 if (strftime(ls_when, sizeof(ls_when),
1065 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1066 str_append(buf, ls_when);
1067
1068 if (h->u.host.last_seen <= now) {
1069 ls_len = length_of_time(now - h->u.host.last_seen);
1070 str_append(buf, " (");
1071 str_appendstr(buf, ls_len);
1072 str_free(ls_len);
1073 str_append(buf, " ago)");
1074 } else {
1075 str_append(buf, " (in the future, possible clock problem)");
1076 }
1077
1078 str_appendf(buf,
1079 "</p>\n"
1080 "<p>\n"
1081 " <b>In:</b> %'qu<br>\n"
1082 " <b>Out:</b> %'qu<br>\n"
1083 " <b>Total:</b> %'qu<br>\n"
1084 "</p>\n",
1085 h->in, h->out, h->total);
1086
1087 str_append(buf, "<h3>TCP ports</h3>\n");
1088 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1089
1090 str_append(buf, "<h3>UDP ports</h3>\n");
1091 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1092
1093 str_append(buf, "<h3>IP protocols</h3>\n");
1094 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1095
1096 html_close(buf);
1097 return (buf);
1098 }
1099
1100 /* ---------------------------------------------------------------------------
1101 * Database import and export code:
1102 * Initially written and contributed by Ben Stewart.
1103 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1104 */
1105 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1106 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1107 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1108
1109 static const char
1110 export_proto_ip = 'P',
1111 export_proto_tcp = 'T',
1112 export_proto_udp = 'U';
1113
1114 static const unsigned char
1115 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1116 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02},
1117 export_tag_host_ver3[] = {'H', 'S', 'T', 0x03};
1118
1119 /* ---------------------------------------------------------------------------
1120 * Load a host's ip_proto table from a file.
1121 * Returns 0 on failure, 1 on success.
1122 */
1123 static int
1124 hosts_db_import_ip(const int fd, struct bucket *host)
1125 {
1126 uint8_t count, i;
1127
1128 if (!expect8(fd, export_proto_ip)) return 0;
1129 if (!read8(fd, &count)) return 0;
1130
1131 for (i=0; i<count; i++) {
1132 struct bucket *b;
1133 uint8_t proto;
1134 uint64_t in, out;
1135
1136 if (!read8(fd, &proto)) return 0;
1137 if (!read64(fd, &in)) return 0;
1138 if (!read64(fd, &out)) return 0;
1139
1140 /* Store data */
1141 b = host_get_ip_proto(host, proto);
1142 b->in = in;
1143 b->out = out;
1144 b->total = in + out;
1145 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1146 }
1147 return 1;
1148 }
1149
1150 /* ---------------------------------------------------------------------------
1151 * Load a host's port_tcp table from a file.
1152 * Returns 0 on failure, 1 on success.
1153 */
1154 static int
1155 hosts_db_import_tcp(const int fd, struct bucket *host)
1156 {
1157 uint16_t count, i;
1158
1159 if (!expect8(fd, export_proto_tcp)) return 0;
1160 if (!read16(fd, &count)) return 0;
1161
1162 for (i=0; i<count; i++) {
1163 struct bucket *b;
1164 uint16_t port;
1165 uint64_t in, out, syn;
1166
1167 if (!read16(fd, &port)) return 0;
1168 if (!read64(fd, &syn)) return 0;
1169 if (!read64(fd, &in)) return 0;
1170 if (!read64(fd, &out)) return 0;
1171
1172 /* Store data */
1173 b = host_get_port_tcp(host, port);
1174 b->in = in;
1175 b->out = out;
1176 b->total = in + out;
1177 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1178 b->u.port_tcp.syn = syn;
1179 }
1180 return 1;
1181 }
1182
1183 /* ---------------------------------------------------------------------------
1184 * Load a host's port_tcp table from a file.
1185 * Returns 0 on failure, 1 on success.
1186 */
1187 static int
1188 hosts_db_import_udp(const int fd, struct bucket *host)
1189 {
1190 uint16_t count, i;
1191
1192 if (!expect8(fd, export_proto_udp)) return 0;
1193 if (!read16(fd, &count)) return 0;
1194
1195 for (i=0; i<count; i++) {
1196 struct bucket *b;
1197 uint16_t port;
1198 uint64_t in, out;
1199
1200 if (!read16(fd, &port)) return 0;
1201 if (!read64(fd, &in)) return 0;
1202 if (!read64(fd, &out)) return 0;
1203
1204 /* Store data */
1205 b = host_get_port_udp(host, port);
1206 b->in = in;
1207 b->out = out;
1208 b->total = in + out;
1209 assert(b->u.port_udp.port == port); /* done by make_func */
1210 }
1211 return 1;
1212 }
1213
1214 /* ---------------------------------------------------------------------------
1215 * Load all hosts from a file.
1216 * Returns 0 on failure, 1 on success.
1217 */
1218 static int
1219 hosts_db_import_host(const int fd)
1220 {
1221 struct bucket *host;
1222 struct addr a;
1223 uint8_t hostname_len;
1224 uint64_t in, out;
1225 unsigned int pos = xtell(fd);
1226 char hdr[4];
1227 int ver = 0;
1228
1229 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1230 if (memcmp(hdr, export_tag_host_ver3, sizeof(hdr)) == 0)
1231 ver = 3;
1232 else if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1233 ver = 2;
1234 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1235 ver = 1;
1236 else {
1237 warnx("bad host header: %02x%02x%02x%02x",
1238 hdr[0], hdr[1], hdr[2], hdr[3]);
1239 return 0;
1240 }
1241
1242 if (ver == 3) {
1243 if (!readaddr(fd, &a))
1244 return 0;
1245 } else {
1246 assert((ver == 1) || (ver == 2));
1247 if (!readaddr_ipv4(fd, &a))
1248 return 0;
1249 }
1250 verbosef("at file pos %u, importing host %s", pos, addr_to_str(&a));
1251 host = host_get(&a);
1252 assert(addr_equal(&(host->u.host.addr), &a));
1253
1254 if (ver > 1) {
1255 uint64_t t;
1256 if (!read64(fd, &t)) return 0;
1257 host->u.host.last_seen = (time_t)t;
1258 }
1259
1260 assert(sizeof(host->u.host.mac_addr) == 6);
1261 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1262 return 0;
1263
1264 /* HOSTNAME */
1265 assert(host->u.host.dns == NULL); /* make fn? */
1266 if (!read8(fd, &hostname_len)) return 0;
1267 if (hostname_len > 0) {
1268 host->u.host.dns = xmalloc(hostname_len + 1);
1269 host->u.host.dns[0] = '\0';
1270
1271 /* At this point, the hostname is attached to a host which is in our
1272 * hosts_db, so if we bail out due to an import error, this pointer
1273 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1274 */
1275
1276 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1277 host->u.host.dns[hostname_len] = '\0';
1278 }
1279
1280 if (!read64(fd, &in)) return 0;
1281 if (!read64(fd, &out)) return 0;
1282
1283 host->in = in;
1284 host->out = out;
1285 host->total = in + out;
1286
1287 /* Host's port and proto subtables: */
1288 if (!hosts_db_import_ip(fd, host)) return 0;
1289 if (!hosts_db_import_tcp(fd, host)) return 0;
1290 if (!hosts_db_import_udp(fd, host)) return 0;
1291 return 1;
1292 }
1293
1294 /* ---------------------------------------------------------------------------
1295 * Database Import: Grab hosts_db from a file provided by the caller.
1296 *
1297 * This function will retrieve the data sans the header. We expect the caller
1298 * to have validated the header of the hosts_db segment, and left the file
1299 * sitting at the start of the data.
1300 */
1301 int hosts_db_import(const int fd)
1302 {
1303 uint32_t host_count, i;
1304
1305 if (!read32(fd, &host_count)) return 0;
1306
1307 for (i=0; i<host_count; i++)
1308 if (!hosts_db_import_host(fd)) return 0;
1309
1310 return 1;
1311 }
1312
1313 /* ---------------------------------------------------------------------------
1314 * Database Export: Dump hosts_db into a file provided by the caller.
1315 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1316 */
1317 int hosts_db_export(const int fd)
1318 {
1319 uint32_t i;
1320 struct bucket *b;
1321
1322 if (!write32(fd, hosts_db->count)) return 0;
1323
1324 for (i = 0; i<hosts_db->size; i++)
1325 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1326 /* For each host: */
1327 if (!writen(fd, export_tag_host_ver3, sizeof(export_tag_host_ver3)))
1328 return 0;
1329
1330 if (!writeaddr(fd, &(b->u.host.addr))) return 0;
1331
1332 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1333
1334 assert(sizeof(b->u.host.mac_addr) == 6);
1335 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1336 return 0;
1337
1338 /* HOSTNAME */
1339 if (b->u.host.dns == NULL) {
1340 if (!write8(fd, 0)) return 0;
1341 } else {
1342 int dnslen = strlen(b->u.host.dns);
1343
1344 if (dnslen > 255) {
1345 warnx("found a very long hostname: \"%s\"\n"
1346 "wasn't expecting one longer than 255 chars (this one is %d)",
1347 b->u.host.dns, dnslen);
1348 dnslen = 255;
1349 }
1350
1351 if (!write8(fd, (uint8_t)dnslen)) return 0;
1352 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1353 }
1354
1355 if (!write64(fd, b->in)) return 0;
1356 if (!write64(fd, b->out)) return 0;
1357
1358 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1359 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1360 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1361 }
1362 return 1;
1363 }
1364
1365 /* ---------------------------------------------------------------------------
1366 * Dump the ip_proto table of a host.
1367 */
1368 static int
1369 hosts_db_export_ip(const struct hashtable *h, const int fd)
1370 {
1371 uint32_t i, written = 0;
1372 struct bucket *b;
1373
1374 /* IP DATA */
1375 if (!write8(fd, export_proto_ip)) return 0;
1376
1377 /* If no data, write a IP Proto count of 0 and we're done. */
1378 if (h == NULL) {
1379 if (!write8(fd, 0)) return 0;
1380 return 1;
1381 }
1382
1383 assert(h->count < 256);
1384 if (!write8(fd, (uint8_t)h->count)) return 0;
1385
1386 for (i = 0; i<h->size; i++)
1387 for (b = h->table[i]; b != NULL; b = b->next) {
1388 /* For each ip_proto bucket: */
1389
1390 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1391 if (!write64(fd, b->in)) return 0;
1392 if (!write64(fd, b->out)) return 0;
1393 written++;
1394 }
1395 assert(written == h->count);
1396 return 1;
1397 }
1398
1399 /* ---------------------------------------------------------------------------
1400 * Dump the port_tcp table of a host.
1401 */
1402 static int
1403 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1404 {
1405 struct bucket *b;
1406 uint32_t i, written = 0;
1407
1408 /* TCP DATA */
1409 if (!write8(fd, export_proto_tcp)) return 0;
1410
1411 /* If no data, write a count of 0 and we're done. */
1412 if (h == NULL) {
1413 if (!write16(fd, 0)) return 0;
1414 return 1;
1415 }
1416
1417 assert(h->count < 65536);
1418 if (!write16(fd, (uint16_t)h->count)) return 0;
1419
1420 for (i = 0; i<h->size; i++)
1421 for (b = h->table[i]; b != NULL; b = b->next) {
1422 if (!write16(fd, b->u.port_tcp.port)) return 0;
1423 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1424 if (!write64(fd, b->in)) return 0;
1425 if (!write64(fd, b->out)) return 0;
1426 written++;
1427 }
1428 assert(written == h->count);
1429 return 1;
1430 }
1431
1432 /* ---------------------------------------------------------------------------
1433 * Dump the port_udp table of a host.
1434 */
1435 static int
1436 hosts_db_export_udp(const struct hashtable *h, const int fd)
1437 {
1438 struct bucket *b;
1439 uint32_t i, written = 0;
1440
1441 /* UDP DATA */
1442 if (!write8(fd, export_proto_udp)) return 0;
1443
1444 /* If no data, write a count of 0 and we're done. */
1445 if (h == NULL) {
1446 if (!write16(fd, 0)) return 0;
1447 return 1;
1448 }
1449
1450 assert(h->count < 65536);
1451 if (!write16(fd, (uint16_t)h->count)) return 0;
1452
1453 for (i = 0; i<h->size; i++)
1454 for (b = h->table[i]; b != NULL; b = b->next) {
1455 if (!write16(fd, b->u.port_udp.port)) return 0;
1456 if (!write64(fd, b->in)) return 0;
1457 if (!write64(fd, b->out)) return 0;
1458 written++;
1459 }
1460 assert(written == h->count);
1461 return 1;
1462 }
1463
1464 /* vim:set ts=3 sw=3 tw=78 expandtab: */