Implement --no-lastseen
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2009 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20
21 #include <arpa/inet.h> /* inet_aton() */
22 #include <assert.h>
23 #include <errno.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h> /* memset(), strcmp() */
27 #include <unistd.h>
28
29 extern int want_lastseen;
30 int show_mac_addrs = 0;
31 extern const char *interface;
32
33 /* FIXME: specify somewhere more sane/tunable */
34 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
35
36 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
37 typedef void (free_func_t)(struct bucket *);
38 typedef const void * (key_func_t)(const struct bucket *);
39 typedef int (find_func_t)(const struct bucket *, const void *);
40 typedef struct bucket * (make_func_t)(const void *);
41 typedef void (format_cols_func_t)(struct str *);
42 typedef void (format_row_func_t)(struct str *, const struct bucket *,
43 const char *);
44
45 struct hashtable {
46 uint8_t bits; /* size of hashtable in bits */
47 uint32_t size, mask;
48 uint32_t count, count_max, count_keep; /* items in table */
49 uint32_t coeff; /* coefficient for Fibonacci hashing */
50 struct bucket **table;
51
52 struct {
53 uint64_t inserts, searches, deletions, rehashes;
54 } stats;
55
56 hash_func_t *hash_func;
57 /* returns hash value of given key (passed as void*) */
58
59 free_func_t *free_func;
60 /* free of bucket payload */
61
62 key_func_t *key_func;
63 /* returns pointer to key of bucket (to pass to hash_func) */
64
65 find_func_t *find_func;
66 /* returns true if given bucket matches key (passed as void*) */
67
68 make_func_t *make_func;
69 /* returns bucket containing new record with key (passed as void*) */
70
71 format_cols_func_t *format_cols_func;
72 /* append table columns to str */
73
74 format_row_func_t *format_row_func;
75 /* format record and append to str */
76 };
77
78 static void hashtable_reduce(struct hashtable *ht);
79 static void hashtable_free(struct hashtable *h);
80
81 #define HOST_BITS 1 /* initial size of hosts table */
82 #define PORT_BITS 1 /* initial size of ports tables */
83 #define PROTO_BITS 1 /* initial size of proto table */
84
85 /* We only use one hosts_db hashtable and this is it. */
86 static struct hashtable *hosts_db = NULL;
87
88 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
89 static const double phi_1 =
90 0.61803398874989490252573887119069695472717285156250;
91
92 /* Co-prime of u, using phi^-1 */
93 inline static uint32_t
94 coprime(const uint32_t u)
95 {
96 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
97 }
98
99 /*
100 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
101 * src/sys/netinet/tcp_hostcache.c 1.1
102 */
103 inline static uint32_t
104 ipv4_hash(const uint32_t ip)
105 {
106 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
107 }
108
109 /* ---------------------------------------------------------------------------
110 * hash_func collection
111 */
112 #define CASTKEY(type) (*((const type *)key))
113
114 static uint32_t
115 hash_func_host(const struct hashtable *h _unused_, const void *key)
116 {
117 return (ipv4_hash(CASTKEY(in_addr_t)));
118 }
119
120 static uint32_t
121 hash_func_short(const struct hashtable *h, const void *key)
122 {
123 return (CASTKEY(uint16_t) * h->coeff);
124 }
125
126 static uint32_t
127 hash_func_byte(const struct hashtable *h, const void *key)
128 {
129 return (CASTKEY(uint8_t) * h->coeff);
130 }
131
132 /* ---------------------------------------------------------------------------
133 * key_func collection
134 */
135
136 static const void *
137 key_func_host(const struct bucket *b)
138 {
139 return &(b->u.host.ip);
140 }
141
142 static const void *
143 key_func_port_tcp(const struct bucket *b)
144 {
145 return &(b->u.port_tcp.port);
146 }
147
148 static const void *
149 key_func_port_udp(const struct bucket *b)
150 {
151 return &(b->u.port_udp.port);
152 }
153
154 static const void *
155 key_func_ip_proto(const struct bucket *b)
156 {
157 return &(b->u.ip_proto.proto);
158 }
159
160 /* ---------------------------------------------------------------------------
161 * find_func collection
162 */
163
164 static int
165 find_func_host(const struct bucket *b, const void *key)
166 {
167 return (b->u.host.ip == CASTKEY(in_addr_t));
168 }
169
170 static int
171 find_func_port_tcp(const struct bucket *b, const void *key)
172 {
173 return (b->u.port_tcp.port == CASTKEY(uint16_t));
174 }
175
176 static int
177 find_func_port_udp(const struct bucket *b, const void *key)
178 {
179 return (b->u.port_udp.port == CASTKEY(uint16_t));
180 }
181
182 static int
183 find_func_ip_proto(const struct bucket *b, const void *key)
184 {
185 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
186 }
187
188 /* ---------------------------------------------------------------------------
189 * make_func collection
190 */
191
192 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
193 struct bucket *next; \
194 uint64_t in, out, total; \
195 union { struct type t; } u; } _custom_bucket; \
196 struct bucket *name_bucket = xmalloc(sizeof(_custom_bucket)); \
197 struct type *name_content = &(name_bucket->u.type); \
198 name_bucket->next = NULL; \
199 name_bucket->in = name_bucket->out = name_bucket->total = 0;
200
201 static struct bucket *
202 make_func_host(const void *key)
203 {
204 MAKE_BUCKET(b, h, host);
205 h->ip = CASTKEY(in_addr_t);
206 h->dns = NULL;
207 h->last_seen = now;
208 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
209 h->ports_tcp = NULL;
210 h->ports_udp = NULL;
211 h->ip_protos = NULL;
212 return (b);
213 }
214
215 static void
216 free_func_host(struct bucket *b)
217 {
218 struct host *h = &(b->u.host);
219 if (h->dns != NULL) free(h->dns);
220 hashtable_free(h->ports_tcp);
221 hashtable_free(h->ports_udp);
222 hashtable_free(h->ip_protos);
223 }
224
225 static struct bucket *
226 make_func_port_tcp(const void *key)
227 {
228 MAKE_BUCKET(b, p, port_tcp);
229 p->port = CASTKEY(uint16_t);
230 p->syn = 0;
231 return (b);
232 }
233
234 static struct bucket *
235 make_func_port_udp(const void *key)
236 {
237 MAKE_BUCKET(b, p, port_udp);
238 p->port = CASTKEY(uint16_t);
239 return (b);
240 }
241
242 static struct bucket *
243 make_func_ip_proto(const void *key)
244 {
245 MAKE_BUCKET(b, p, ip_proto);
246 p->proto = CASTKEY(uint8_t);
247 return (b);
248 }
249
250 static void
251 free_func_simple(struct bucket *b _unused_)
252 {
253 /* nop */
254 }
255
256 /* ---------------------------------------------------------------------------
257 * format_func collection (ordered by struct)
258 */
259
260 static void
261 format_cols_host(struct str *buf)
262 {
263 /* FIXME: don't clobber parts of the query string
264 * specifically "full" and "start"
265 * when setting sort direction
266 */
267 str_append(buf,
268 "<table>\n"
269 "<tr>\n"
270 " <th>IP</th>\n"
271 " <th>Hostname</th>\n");
272 if (show_mac_addrs) str_append(buf,
273 " <th>MAC Address</th>\n");
274 str_append(buf,
275 " <th><a href=\"?sort=in\">In</a></th>\n"
276 " <th><a href=\"?sort=out\">Out</a></th>\n"
277 " <th><a href=\"?sort=total\">Total</a></th>\n");
278 if (want_lastseen) str_append(buf,
279 " <th>Last seen</th>\n");
280 str_append(buf,
281 "</tr>\n");
282 }
283
284 static void
285 format_row_host(struct str *buf, const struct bucket *b,
286 const char *css_class)
287 {
288 const char *ip = ip_to_str( b->u.host.ip );
289
290 str_appendf(buf,
291 "<tr class=\"%s\">\n"
292 " <td><a href=\"%s/\">%s</a></td>\n"
293 " <td>%s</td>\n",
294 css_class,
295 ip, ip,
296 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
297
298 if (show_mac_addrs)
299 str_appendf(buf,
300 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
301 b->u.host.mac_addr[0],
302 b->u.host.mac_addr[1],
303 b->u.host.mac_addr[2],
304 b->u.host.mac_addr[3],
305 b->u.host.mac_addr[4],
306 b->u.host.mac_addr[5]);
307
308 str_appendf(buf,
309 " <td class=\"num\">%'qu</td>\n"
310 " <td class=\"num\">%'qu</td>\n"
311 " <td class=\"num\">%'qu</td>\n",
312 b->in, b->out, b->total);
313
314 if (want_lastseen) {
315 time_t last_t = b->u.host.last_seen;
316 struct str *lastseen = NULL;
317
318 last_t = b->u.host.last_seen;
319 if (now >= last_t)
320 lastseen = length_of_time(now - last_t);
321
322 str_append(buf,
323 " <td class=\"num\">");
324 if (lastseen == NULL)
325 str_append(buf, "(clock error)");
326 else {
327 str_appendstr(buf, lastseen);
328 str_free(lastseen);
329 }
330 str_append(buf,
331 "</td>");
332 }
333
334 str_appendf(buf,
335 "</tr>\n");
336
337 /* Only resolve hosts "on demand" */
338 if (b->u.host.dns == NULL)
339 dns_queue(b->u.host.ip);
340 }
341
342 static void
343 format_cols_port_tcp(struct str *buf)
344 {
345 str_append(buf,
346 "<table>\n"
347 "<tr>\n"
348 " <th>Port</td>\n"
349 " <th>Service</td>\n"
350 " <th>In</td>\n"
351 " <th>Out</td>\n"
352 " <th>Total</td>\n"
353 " <th>SYNs</td>\n"
354 "</tr>\n"
355 );
356 }
357
358 static void
359 format_row_port_tcp(struct str *buf, const struct bucket *b,
360 const char *css_class)
361 {
362 const struct port_tcp *p = &(b->u.port_tcp);
363
364 str_appendf(buf,
365 "<tr class=\"%s\">\n"
366 " <td class=\"num\">%u</td>\n"
367 " <td>%s</td>\n"
368 " <td class=\"num\">%'qu</td>\n"
369 " <td class=\"num\">%'qu</td>\n"
370 " <td class=\"num\">%'qu</td>\n"
371 " <td class=\"num\">%'qu</td>\n"
372 "</tr>\n",
373 css_class,
374 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
375 );
376 }
377
378 static void
379 format_cols_port_udp(struct str *buf)
380 {
381 str_append(buf,
382 "<table>\n"
383 "<tr>\n"
384 " <th>Port</td>\n"
385 " <th>Service</td>\n"
386 " <th>In</td>\n"
387 " <th>Out</td>\n"
388 " <th>Total</td>\n"
389 "</tr>\n"
390 );
391 }
392
393 static void
394 format_row_port_udp(struct str *buf, const struct bucket *b,
395 const char *css_class)
396 {
397 const struct port_udp *p = &(b->u.port_udp);
398
399 str_appendf(buf,
400 "<tr class=\"%s\">\n"
401 " <td class=\"num\">%u</td>\n"
402 " <td>%s</td>\n"
403 " <td class=\"num\">%'qu</td>\n"
404 " <td class=\"num\">%'qu</td>\n"
405 " <td class=\"num\">%'qu</td>\n"
406 "</tr>\n",
407 css_class,
408 p->port, getservudp(p->port), b->in, b->out, b->total
409 );
410 }
411
412 static void
413 format_cols_ip_proto(struct str *buf)
414 {
415 str_append(buf,
416 "<table>\n"
417 "<tr>\n"
418 " <th>#</td>\n"
419 " <th>Protocol</td>\n"
420 " <th>In</td>\n"
421 " <th>Out</td>\n"
422 " <th>Total</td>\n"
423 "</tr>\n"
424 );
425 }
426
427 static void
428 format_row_ip_proto(struct str *buf, const struct bucket *b,
429 const char *css_class)
430 {
431 const struct ip_proto *p = &(b->u.ip_proto);
432
433 str_appendf(buf,
434 "<tr class=\"%s\">\n"
435 " <td class=\"num\">%u</td>\n"
436 " <td>%s</td>\n"
437 " <td class=\"num\">%'qu</td>\n"
438 " <td class=\"num\">%'qu</td>\n"
439 " <td class=\"num\">%'qu</td>\n"
440 "</tr>\n",
441 css_class,
442 p->proto, getproto(p->proto),
443 b->in, b->out, b->total
444 );
445 }
446
447 /* ---------------------------------------------------------------------------
448 * Initialise a hashtable.
449 */
450 static struct hashtable *
451 hashtable_make(const uint8_t bits,
452 const unsigned int count_max,
453 const unsigned int count_keep,
454 hash_func_t *hash_func,
455 free_func_t *free_func,
456 key_func_t *key_func,
457 find_func_t *find_func,
458 make_func_t *make_func,
459 format_cols_func_t *format_cols_func,
460 format_row_func_t *format_row_func)
461 {
462 struct hashtable *hash;
463 assert(bits > 0);
464
465 hash = xmalloc(sizeof(*hash));
466 hash->bits = bits;
467 hash->count_max = count_max;
468 hash->count_keep = count_keep;
469 hash->size = 1U << bits;
470 hash->mask = hash->size - 1;
471 hash->coeff = coprime(hash->size);
472 hash->hash_func = hash_func;
473 hash->free_func = free_func;
474 hash->key_func = key_func;
475 hash->find_func = find_func;
476 hash->make_func = make_func;
477 hash->format_cols_func = format_cols_func;
478 hash->format_row_func = format_row_func;
479 hash->count = 0;
480 hash->table = xcalloc(hash->size, sizeof(*hash->table));
481 memset(&(hash->stats), 0, sizeof(hash->stats));
482 return (hash);
483 }
484
485 /* ---------------------------------------------------------------------------
486 * Initialise global hosts_db.
487 */
488 void
489 hosts_db_init(void)
490 {
491 assert(hosts_db == NULL);
492 hosts_db = hashtable_make(HOST_BITS, hosts_max, hosts_keep,
493 hash_func_host, free_func_host, key_func_host, find_func_host,
494 make_func_host, format_cols_host, format_row_host);
495 }
496
497 static void
498 hashtable_rehash(struct hashtable *h, const uint8_t bits)
499 {
500 struct bucket **old_table, **new_table;
501 uint32_t i, old_size;
502 assert(h != NULL);
503 assert(bits > 0);
504
505 h->stats.rehashes++;
506 old_size = h->size;
507 old_table = h->table;
508
509 h->bits = bits;
510 h->size = 1U << bits;
511 h->mask = h->size - 1;
512 h->coeff = coprime(h->size);
513 new_table = xcalloc(h->size, sizeof(*new_table));
514
515 for (i=0; i<old_size; i++) {
516 struct bucket *next, *b = old_table[i];
517 while (b != NULL) {
518 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
519 next = b->next;
520 b->next = new_table[pos];
521 new_table[pos] = b;
522 b = next;
523 }
524 }
525
526 free(h->table);
527 h->table = new_table;
528 }
529
530 static void
531 hashtable_insert(struct hashtable *h, struct bucket *b)
532 {
533 uint32_t pos;
534 assert(h != NULL);
535 assert(b != NULL);
536 assert(b->next == NULL);
537
538 /* Rehash on 80% occupancy */
539 if ((h->count > h->size) ||
540 ((h->size - h->count) < h->size / 5))
541 hashtable_rehash(h, h->bits+1);
542
543 pos = h->hash_func(h, h->key_func(b)) & h->mask;
544 if (h->table[pos] == NULL)
545 h->table[pos] = b;
546 else {
547 /* Insert at top of chain. */
548 b->next = h->table[pos];
549 h->table[pos] = b;
550 }
551 h->count++;
552 h->stats.inserts++;
553 }
554
555 /* Return bucket matching key, or NULL if no such entry. */
556 static struct bucket *
557 hashtable_search(struct hashtable *h, const void *key)
558 {
559 uint32_t pos;
560 struct bucket *b;
561
562 h->stats.searches++;
563 pos = h->hash_func(h, key) & h->mask;
564 b = h->table[pos];
565 while (b != NULL) {
566 if (h->find_func(b, key))
567 return (b);
568 else
569 b = b->next;
570 }
571 return (NULL);
572 }
573
574 /* Search for a key. If it's not there, make and insert a bucket for it. */
575 static struct bucket *
576 hashtable_find_or_insert(struct hashtable *h, const void *key)
577 {
578 struct bucket *b = hashtable_search(h, key);
579
580 if (b == NULL) {
581 /* Not found, so insert after checking occupancy. */
582 /*assert(h->count <= h->count_max);*/
583 if (h->count >= h->count_max) hashtable_reduce(h);
584 b = h->make_func(key);
585 hashtable_insert(h, b);
586 }
587 return (b);
588 }
589
590 /*
591 * Frees the hashtable and the buckets. The contents are assumed to be
592 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
593 * the bucket.
594 */
595 static void
596 hashtable_free(struct hashtable *h)
597 {
598 uint32_t i;
599
600 if (h == NULL)
601 return;
602 for (i=0; i<h->size; i++) {
603 struct bucket *tmp, *b = h->table[i];
604 while (b != NULL) {
605 tmp = b;
606 b = b->next;
607 h->free_func(tmp);
608 free(tmp);
609 }
610 }
611 free(h->table);
612 free(h);
613 }
614
615 /* ---------------------------------------------------------------------------
616 * Return existing host or insert a new one.
617 */
618 struct bucket *
619 host_get(const in_addr_t ip)
620 {
621 return (hashtable_find_or_insert(hosts_db, &ip));
622 }
623
624 /* ---------------------------------------------------------------------------
625 * Find host, returns NULL if not in DB.
626 */
627 struct bucket *
628 host_find(const in_addr_t ip)
629 {
630 return (hashtable_search(hosts_db, &ip));
631 }
632
633 /* ---------------------------------------------------------------------------
634 * Find host, returns NULL if not in DB.
635 */
636 static struct bucket *
637 host_search(const char *ipstr)
638 {
639 struct in_addr addr;
640
641 if (inet_aton(ipstr, &addr) != 1)
642 return (NULL); /* invalid addr */
643 return (hashtable_search(hosts_db, &(addr.s_addr)));
644 }
645
646 /* ---------------------------------------------------------------------------
647 * Reduce a hashtable to the top <keep> entries.
648 */
649 static void
650 hashtable_reduce(struct hashtable *ht)
651 {
652 uint32_t i, pos, rmd;
653 const struct bucket **table;
654 uint64_t cutoff;
655
656 assert(ht->count_keep < ht->count);
657
658 /* Fill table with pointers to buckets in hashtable. */
659 table = xmalloc(sizeof(*table) * ht->count);
660 for (pos=0, i=0; i<ht->size; i++) {
661 struct bucket *b = ht->table[i];
662 while (b != NULL) {
663 table[pos++] = b;
664 b = b->next;
665 }
666 }
667 assert(pos == ht->count);
668 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
669 cutoff = table[ht->count_keep]->total;
670 free(table);
671
672 /* Remove all elements with total <= cutoff. */
673 rmd = 0;
674 for (i=0; i<ht->size; i++) {
675 struct bucket *last = NULL, *next, *b = ht->table[i];
676 while (b != NULL) {
677 next = b->next;
678 if (b->total <= cutoff) {
679 /* Remove this one. */
680 ht->free_func(b);
681 free(b);
682 if (last == NULL)
683 ht->table[i] = next;
684 else
685 last->next = next;
686 rmd++;
687 ht->count--;
688 } else {
689 last = b;
690 }
691 b = next;
692 }
693 }
694 verbosef("hashtable_reduce: removed %u buckets, left %u",
695 rmd, ht->count);
696 hashtable_rehash(ht, ht->bits); /* is this needed? */
697 }
698
699 /* ---------------------------------------------------------------------------
700 * Reset hosts_db to empty.
701 */
702 void
703 hosts_db_reset(void)
704 {
705 unsigned int i;
706
707 for (i=0; i<hosts_db->size; i++) {
708 struct bucket *next, *b = hosts_db->table[i];
709 while (b != NULL) {
710 next = b->next;
711 hosts_db->free_func(b);
712 free(b);
713 b = next;
714 }
715 hosts_db->table[i] = NULL;
716 }
717 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
718 hosts_db->count = 0;
719 }
720
721 /* ---------------------------------------------------------------------------
722 * Deallocate hosts_db.
723 */
724 void hosts_db_free(void)
725 {
726 uint32_t i;
727
728 assert(hosts_db != NULL);
729 for (i=0; i<hosts_db->size; i++) {
730 struct bucket *tmp, *b = hosts_db->table[i];
731 while (b != NULL) {
732 tmp = b;
733 b = b->next;
734 hosts_db->free_func(tmp);
735 free(tmp);
736 }
737 }
738 free(hosts_db->table);
739 free(hosts_db);
740 hosts_db = NULL;
741 }
742
743 /* ---------------------------------------------------------------------------
744 * Find or create a port_tcp inside a host.
745 */
746 struct bucket *
747 host_get_port_tcp(struct bucket *host, const uint16_t port)
748 {
749 struct host *h = &host->u.host;
750 assert(h != NULL);
751 if (h->ports_tcp == NULL)
752 h->ports_tcp = hashtable_make(PORT_BITS, ports_max, ports_keep,
753 hash_func_short, free_func_simple, key_func_port_tcp,
754 find_func_port_tcp, make_func_port_tcp,
755 format_cols_port_tcp, format_row_port_tcp);
756 return (hashtable_find_or_insert(h->ports_tcp, &port));
757 }
758
759 /* ---------------------------------------------------------------------------
760 * Find or create a port_udp inside a host.
761 */
762 struct bucket *
763 host_get_port_udp(struct bucket *host, const uint16_t port)
764 {
765 struct host *h = &host->u.host;
766 assert(h != NULL);
767 if (h->ports_udp == NULL)
768 h->ports_udp = hashtable_make(PORT_BITS, ports_max, ports_keep,
769 hash_func_short, free_func_simple, key_func_port_udp,
770 find_func_port_udp, make_func_port_udp,
771 format_cols_port_udp, format_row_port_udp);
772 return (hashtable_find_or_insert(h->ports_udp, &port));
773 }
774
775 /* ---------------------------------------------------------------------------
776 * Find or create an ip_proto inside a host.
777 */
778 struct bucket *
779 host_get_ip_proto(struct bucket *host, const uint8_t proto)
780 {
781 struct host *h = &host->u.host;
782 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
783 assert(h != NULL);
784 if (h->ip_protos == NULL)
785 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
786 hash_func_byte, free_func_simple, key_func_ip_proto,
787 find_func_ip_proto, make_func_ip_proto,
788 format_cols_ip_proto, format_row_ip_proto);
789 return (hashtable_find_or_insert(h->ip_protos, &proto));
790 }
791
792 static struct str *html_hosts_main(const char *qs);
793 static struct str *html_hosts_detail(const char *ip);
794
795 /* ---------------------------------------------------------------------------
796 * Web interface: delegate the /hosts/ space.
797 */
798 struct str *
799 html_hosts(const char *uri, const char *query)
800 {
801 int i, num_elems;
802 char **elem = split('/', uri, &num_elems);
803 struct str *buf = NULL;
804
805 assert(num_elems >= 1);
806 assert(strcmp(elem[0], "hosts") == 0);
807
808 if (num_elems == 1)
809 /* /hosts/ */
810 buf = html_hosts_main(query);
811 else if (num_elems == 2)
812 /* /hosts/<IP of host>/ */
813 buf = html_hosts_detail(elem[1]);
814
815 for (i=0; i<num_elems; i++)
816 free(elem[i]);
817 free(elem);
818 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
819 other codes to be possible */
820 }
821
822 /* ---------------------------------------------------------------------------
823 * Format hashtable into HTML.
824 */
825 static void
826 format_table(struct str *buf, struct hashtable *ht, int start,
827 const enum sort_dir sort, const int full)
828 {
829 const struct bucket **table;
830 uint32_t i, pos, end;
831 int alt = 0;
832
833 if ((ht == NULL) || (ht->count == 0)) {
834 str_append(buf, "<p>The table is empty.</p>\n");
835 return;
836 }
837
838 /* Fill table with pointers to buckets in hashtable. */
839 table = xmalloc(sizeof(*table) * ht->count);
840 for (pos=0, i=0; i<ht->size; i++) {
841 struct bucket *b = ht->table[i];
842 while (b != NULL) {
843 table[pos++] = b;
844 b = b->next;
845 }
846 }
847 assert(pos == ht->count);
848
849 if (full) {
850 /* full report overrides start and end */
851 start = 0;
852 end = ht->count;
853 } else
854 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
855
856 str_appendf(buf, "(%u-%u of %u)<br/>\n", start+1, end, ht->count);
857 qsort_buckets(table, ht->count, start, end, sort);
858 ht->format_cols_func(buf);
859
860 for (i=start; i<end; i++) {
861 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
862 alt = !alt; /* alternate class for table rows */
863 }
864 free(table);
865 str_append(buf, "</table>\n");
866 }
867
868 /* ---------------------------------------------------------------------------
869 * Web interface: sorted table of hosts.
870 */
871 static struct str *
872 html_hosts_main(const char *qs)
873 {
874 struct str *buf = str_make();
875 char *qs_start, *qs_sort, *qs_full, *ep;
876 const char *sortstr;
877 int start, full = 0;
878 enum sort_dir sort;
879
880 /* parse query string */
881 qs_start = qs_get(qs, "start");
882 qs_sort = qs_get(qs, "sort");
883 qs_full = qs_get(qs, "full");
884 if (qs_full != NULL) {
885 full = 1;
886 free(qs_full);
887 }
888
889 /* validate sort */
890 if (qs_sort == NULL) sort = TOTAL;
891 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
892 else if (strcmp(qs_sort, "in") == 0) sort = IN;
893 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
894 else {
895 str_append(buf, "Error: invalid value for \"sort\".\n");
896 goto done;
897 }
898
899 /* parse start */
900 if (qs_start == NULL)
901 start = 0;
902 else {
903 start = (int)strtoul(qs_start, &ep, 10);
904 if (*ep != '\0') {
905 str_append(buf, "Error: \"start\" is not a number.\n");
906 goto done;
907 }
908 if ((errno == ERANGE) ||
909 (start < 0) || (start >= (int)hosts_db->count)) {
910 str_append(buf, "Error: \"start\" is out of bounds.\n");
911 goto done;
912 }
913 }
914
915 #define PREV "&lt;&lt;&lt; prev page"
916 #define NEXT "next page &gt;&gt;&gt;"
917 #define FULL "full table"
918
919 str_append(buf, html_header_1);
920 str_appendf(buf, " <title>darkstat3: Hosts (%s)</title>\n", interface);
921 str_append(buf, html_header_2);
922 str_appendf(buf, "<h2 class=\"pageheader\">Hosts (%s)</h2>\n", interface);
923 format_table(buf, hosts_db, start, sort, full);
924
925 /* <prev | full | stats | next> */
926 sortstr = qs_sort;
927 if (sortstr == NULL) sortstr = "total";
928 if (start > 0) {
929 int prev = max(start - MAX_ENTRIES, 0);
930 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
931 prev, sortstr);
932 } else
933 str_append(buf, PREV);
934
935 if (full)
936 str_append(buf, " | " FULL);
937 else
938 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
939 sortstr);
940
941 if (start+MAX_ENTRIES < (int)hosts_db->count)
942 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
943 start+MAX_ENTRIES, sortstr);
944 else
945 str_append(buf, " | " NEXT);
946
947 str_append(buf, "<br/>\n");
948 str_append(buf, html_footer);
949 done:
950 if (qs_start != NULL) free(qs_start);
951 if (qs_sort != NULL) free(qs_sort);
952 return buf;
953 #undef PREV
954 #undef NEXT
955 #undef FULL
956 }
957
958 /* ---------------------------------------------------------------------------
959 * Web interface: detailed view of a single host.
960 */
961 static struct str *
962 html_hosts_detail(const char *ip)
963 {
964 struct bucket *h;
965 struct str *buf, *ls_len;
966 char ls_when[100];
967 time_t ls;
968
969 h = host_search(ip);
970 if (h == NULL)
971 return (NULL); /* no such host */
972
973 /* Overview. */
974 buf = str_make();
975 str_append(buf, html_header_1);
976 str_appendf(buf, " <title>%s</title>\n", ip);
977 str_append(buf, html_header_2);
978 str_appendf(buf,
979 "<h2>%s</h2>\n"
980 "<p>\n"
981 "<b>Hostname:</b> %s<br/>\n",
982 ip,
983 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
984
985 /* Resolve host "on demand" */
986 if (h->u.host.dns == NULL)
987 dns_queue(h->u.host.ip);
988
989 if (show_mac_addrs)
990 str_appendf(buf,
991 "<b>MAC Address:</b> "
992 "<tt>%x:%x:%x:%x:%x:%x</tt><br/>\n",
993 h->u.host.mac_addr[0],
994 h->u.host.mac_addr[1],
995 h->u.host.mac_addr[2],
996 h->u.host.mac_addr[3],
997 h->u.host.mac_addr[4],
998 h->u.host.mac_addr[5]);
999
1000 str_append(buf,
1001 "</p>\n"
1002 "<p>\n"
1003 "<b>Last seen:</b> ");
1004
1005 ls = h->u.host.last_seen;
1006 if (strftime(ls_when, sizeof(ls_when),
1007 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1008 str_append(buf, ls_when);
1009
1010 if (h->u.host.last_seen <= now) {
1011 ls_len = length_of_time(now - h->u.host.last_seen);
1012 str_append(buf, " (");
1013 str_appendstr(buf, ls_len);
1014 str_free(ls_len);
1015 str_append(buf, " ago)");
1016 } else {
1017 str_append(buf, " (in the future, possible clock problem)");
1018 }
1019
1020 str_appendf(buf,
1021 "</p>\n"
1022 "<p>\n"
1023 " <b>In:</b> %'qu<br/>\n"
1024 " <b>Out:</b> %'qu<br/>\n"
1025 " <b>Total:</b> %'qu<br/>\n"
1026 "</p>\n",
1027 h->in, h->out, h->total);
1028
1029 str_append(buf, "<h3>TCP ports</h3>\n");
1030 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1031
1032 str_append(buf, "<h3>UDP ports</h3>\n");
1033 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1034
1035 str_append(buf, "<h3>IP protocols</h3>\n");
1036 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1037
1038 str_append(buf, html_footer);
1039 return (buf);
1040 }
1041
1042 /* ---------------------------------------------------------------------------
1043 * Database import and export code:
1044 * Initially written and contributed by Ben Stewart.
1045 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1046 */
1047 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1048 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1049 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1050
1051 static const char
1052 export_proto_ip = 'P',
1053 export_proto_tcp = 'T',
1054 export_proto_udp = 'U';
1055
1056 static const unsigned char
1057 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1058 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02};
1059
1060 /* ---------------------------------------------------------------------------
1061 * Load a host's ip_proto table from a file.
1062 * Returns 0 on failure, 1 on success.
1063 */
1064 static int
1065 hosts_db_import_ip(const int fd, struct bucket *host)
1066 {
1067 uint8_t count, i;
1068
1069 if (!expect8(fd, export_proto_ip)) return 0;
1070 if (!read8(fd, &count)) return 0;
1071
1072 for (i=0; i<count; i++) {
1073 struct bucket *b;
1074 uint8_t proto;
1075 uint64_t in, out;
1076
1077 if (!read8(fd, &proto)) return 0;
1078 if (!read64(fd, &in)) return 0;
1079 if (!read64(fd, &out)) return 0;
1080
1081 /* Store data */
1082 b = host_get_ip_proto(host, proto);
1083 b->in = in;
1084 b->out = out;
1085 b->total = in + out;
1086 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1087 }
1088 return 1;
1089 }
1090
1091 /* ---------------------------------------------------------------------------
1092 * Load a host's port_tcp table from a file.
1093 * Returns 0 on failure, 1 on success.
1094 */
1095 static int
1096 hosts_db_import_tcp(const int fd, struct bucket *host)
1097 {
1098 uint16_t count, i;
1099
1100 if (!expect8(fd, export_proto_tcp)) return 0;
1101 if (!read16(fd, &count)) return 0;
1102
1103 for (i=0; i<count; i++) {
1104 struct bucket *b;
1105 uint16_t port;
1106 uint64_t in, out, syn;
1107
1108 if (!read16(fd, &port)) return 0;
1109 if (!read64(fd, &syn)) return 0;
1110 if (!read64(fd, &in)) return 0;
1111 if (!read64(fd, &out)) return 0;
1112
1113 /* Store data */
1114 b = host_get_port_tcp(host, port);
1115 b->in = in;
1116 b->out = out;
1117 b->total = in + out;
1118 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1119 b->u.port_tcp.syn = syn;
1120 }
1121 return 1;
1122 }
1123
1124 /* ---------------------------------------------------------------------------
1125 * Load a host's port_tcp table from a file.
1126 * Returns 0 on failure, 1 on success.
1127 */
1128 static int
1129 hosts_db_import_udp(const int fd, struct bucket *host)
1130 {
1131 uint16_t count, i;
1132
1133 if (!expect8(fd, export_proto_udp)) return 0;
1134 if (!read16(fd, &count)) return 0;
1135
1136 for (i=0; i<count; i++) {
1137 struct bucket *b;
1138 uint16_t port;
1139 uint64_t in, out;
1140
1141 if (!read16(fd, &port)) return 0;
1142 if (!read64(fd, &in)) return 0;
1143 if (!read64(fd, &out)) return 0;
1144
1145 /* Store data */
1146 b = host_get_port_udp(host, port);
1147 b->in = in;
1148 b->out = out;
1149 b->total = in + out;
1150 assert(b->u.port_udp.port == port); /* done by make_func */
1151 }
1152 return 1;
1153 }
1154
1155 /* ---------------------------------------------------------------------------
1156 * Load all hosts from a file.
1157 * Returns 0 on failure, 1 on success.
1158 */
1159 static int
1160 hosts_db_import_host(const int fd)
1161 {
1162 struct bucket *host;
1163 in_addr_t addr;
1164 uint8_t hostname_len;
1165 uint64_t in, out;
1166 unsigned int pos = xtell(fd);
1167 char hdr[4];
1168 int ver = 0;
1169
1170 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1171 if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1172 ver = 2;
1173 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1174 ver = 1;
1175 else {
1176 warnx("bad host header: %02x%02x%02x%02x",
1177 hdr[0], hdr[1], hdr[2], hdr[3]);
1178 return 0;
1179 }
1180
1181 if (!readaddr(fd, &addr)) return 0;
1182 verbosef("at file pos %u, importing host %s", pos, ip_to_str(addr));
1183 host = host_get(addr);
1184 assert(host->u.host.ip == addr); /* make fn? */
1185
1186 if (ver > 1) {
1187 uint64_t t;
1188 if (!read64(fd, &t)) return 0;
1189 host->u.host.last_seen = (time_t)t;
1190 }
1191
1192 assert(sizeof(host->u.host.mac_addr) == 6);
1193 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1194 return 0;
1195
1196 /* HOSTNAME */
1197 assert(host->u.host.dns == NULL); /* make fn? */
1198 if (!read8(fd, &hostname_len)) return 0;
1199 if (hostname_len > 0) {
1200 host->u.host.dns = xmalloc(hostname_len + 1);
1201 host->u.host.dns[0] = '\0';
1202
1203 /* At this point, the hostname is attached to a host which is in our
1204 * hosts_db, so if we bail out due to an import error, this pointer
1205 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1206 */
1207
1208 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1209 host->u.host.dns[hostname_len] = '\0';
1210 }
1211
1212 if (!read64(fd, &in)) return 0;
1213 if (!read64(fd, &out)) return 0;
1214
1215 host->in = in;
1216 host->out = out;
1217 host->total = in + out;
1218
1219 /* Host's port and proto subtables: */
1220 if (!hosts_db_import_ip(fd, host)) return 0;
1221 if (!hosts_db_import_tcp(fd, host)) return 0;
1222 if (!hosts_db_import_udp(fd, host)) return 0;
1223 return 1;
1224 }
1225
1226 /* ---------------------------------------------------------------------------
1227 * Database Import: Grab hosts_db from a file provided by the caller.
1228 *
1229 * This function will retrieve the data sans the header. We expect the caller
1230 * to have validated the header of the hosts_db segment, and left the file
1231 * sitting at the start of the data.
1232 */
1233 int hosts_db_import(const int fd)
1234 {
1235 uint32_t host_count, i;
1236
1237 if (!read32(fd, &host_count)) return 0;
1238
1239 for (i=0; i<host_count; i++)
1240 if (!hosts_db_import_host(fd)) return 0;
1241
1242 return 1;
1243 }
1244
1245 /* ---------------------------------------------------------------------------
1246 * Database Export: Dump hosts_db into a file provided by the caller.
1247 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1248 */
1249 int hosts_db_export(const int fd)
1250 {
1251 uint32_t i;
1252 struct bucket *b;
1253
1254 if (!write32(fd, hosts_db->count)) return 0;
1255
1256 for (i = 0; i<hosts_db->size; i++)
1257 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1258 /* For each host: */
1259 if (!writen(fd, export_tag_host_ver2, sizeof(export_tag_host_ver2)))
1260 return 0;
1261
1262 if (!writeaddr(fd, b->u.host.ip)) return 0;
1263
1264 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1265
1266 assert(sizeof(b->u.host.mac_addr) == 6);
1267 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1268 return 0;
1269
1270 /* HOSTNAME */
1271 if (b->u.host.dns == NULL) {
1272 if (!write8(fd, 0)) return 0;
1273 } else {
1274 int dnslen = strlen(b->u.host.dns);
1275
1276 if (dnslen > 255) {
1277 warnx("found a very long hostname: \"%s\"\n"
1278 "wasn't expecting one longer than 255 chars (this one is %d)",
1279 b->u.host.dns, dnslen);
1280 dnslen = 255;
1281 }
1282
1283 if (!write8(fd, (uint8_t)dnslen)) return 0;
1284 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1285 }
1286
1287 if (!write64(fd, b->in)) return 0;
1288 if (!write64(fd, b->out)) return 0;
1289
1290 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1291 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1292 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1293 }
1294 return 1;
1295 }
1296
1297 /* ---------------------------------------------------------------------------
1298 * Dump the ip_proto table of a host.
1299 */
1300 static int
1301 hosts_db_export_ip(const struct hashtable *h, const int fd)
1302 {
1303 uint32_t i, written = 0;
1304 struct bucket *b;
1305
1306 /* IP DATA */
1307 if (!write8(fd, export_proto_ip)) return 0;
1308
1309 /* If no data, write a IP Proto count of 0 and we're done. */
1310 if (h == NULL) {
1311 if (!write8(fd, 0)) return 0;
1312 return 1;
1313 }
1314
1315 assert(h->count < 256);
1316 if (!write8(fd, (uint8_t)h->count)) return 0;
1317
1318 for (i = 0; i<h->size; i++)
1319 for (b = h->table[i]; b != NULL; b = b->next) {
1320 /* For each ip_proto bucket: */
1321
1322 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1323 if (!write64(fd, b->in)) return 0;
1324 if (!write64(fd, b->out)) return 0;
1325 written++;
1326 }
1327 assert(written == h->count);
1328 return 1;
1329 }
1330
1331 /* ---------------------------------------------------------------------------
1332 * Dump the port_tcp table of a host.
1333 */
1334 static int
1335 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1336 {
1337 struct bucket *b;
1338 uint32_t i, written = 0;
1339
1340 /* TCP DATA */
1341 if (!write8(fd, export_proto_tcp)) return 0;
1342
1343 /* If no data, write a count of 0 and we're done. */
1344 if (h == NULL) {
1345 if (!write16(fd, 0)) return 0;
1346 return 1;
1347 }
1348
1349 assert(h->count < 65536);
1350 if (!write16(fd, (uint16_t)h->count)) return 0;
1351
1352 for (i = 0; i<h->size; i++)
1353 for (b = h->table[i]; b != NULL; b = b->next) {
1354 if (!write16(fd, b->u.port_tcp.port)) return 0;
1355 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1356 if (!write64(fd, b->in)) return 0;
1357 if (!write64(fd, b->out)) return 0;
1358 written++;
1359 }
1360 assert(written == h->count);
1361 return 1;
1362 }
1363
1364 /* ---------------------------------------------------------------------------
1365 * Dump the port_udp table of a host.
1366 */
1367 static int
1368 hosts_db_export_udp(const struct hashtable *h, const int fd)
1369 {
1370 struct bucket *b;
1371 uint32_t i, written = 0;
1372
1373 /* UDP DATA */
1374 if (!write8(fd, export_proto_udp)) return 0;
1375
1376 /* If no data, write a count of 0 and we're done. */
1377 if (h == NULL) {
1378 if (!write16(fd, 0)) return 0;
1379 return 1;
1380 }
1381
1382 assert(h->count < 65536);
1383 if (!write16(fd, (uint16_t)h->count)) return 0;
1384
1385 for (i = 0; i<h->size; i++)
1386 for (b = h->table[i]; b != NULL; b = b->next) {
1387 if (!write16(fd, b->u.port_udp.port)) return 0;
1388 if (!write64(fd, b->in)) return 0;
1389 if (!write64(fd, b->out)) return 0;
1390 written++;
1391 }
1392 assert(written == h->count);
1393 return 1;
1394 }
1395
1396 /* vim:set ts=3 sw=3 tw=78 expandtab: */