Don't apply ports-{max,keep} to protocols.
[darkstat] / hosts_db.c
1 /* darkstat 3
2 * copyright (c) 2001-2008 Emil Mikulic.
3 *
4 * hosts_db.c: database of hosts, ports, protocols.
5 *
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
8 */
9
10 #include "darkstat.h"
11 #include "conv.h"
12 #include "decode.h"
13 #include "dns.h"
14 #include "err.h"
15 #include "hosts_db.h"
16 #include "db.h"
17 #include "html.h"
18 #include "ncache.h"
19 #include "now.h"
20
21 #include <arpa/inet.h> /* inet_aton() */
22 #include <assert.h>
23 #include <errno.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h> /* memset(), strcmp() */
27 #include <unistd.h>
28
29 int show_mac_addrs = 0;
30 extern const char *interface;
31
32 /* FIXME: specify somewhere more sane/tunable */
33 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
34
35 typedef uint32_t (hash_func_t)(const struct hashtable *, const void *);
36 typedef void (free_func_t)(struct bucket *);
37 typedef const void * (key_func_t)(const struct bucket *);
38 typedef int (find_func_t)(const struct bucket *, const void *);
39 typedef struct bucket * (make_func_t)(const void *);
40 typedef void (format_cols_func_t)(struct str *);
41 typedef void (format_row_func_t)(struct str *, const struct bucket *,
42 const char *);
43
44 struct hashtable {
45 uint8_t bits; /* size of hashtable in bits */
46 uint32_t size, mask;
47 uint32_t count, count_max, count_keep; /* items in table */
48 uint32_t coeff; /* coefficient for Fibonacci hashing */
49 struct bucket **table;
50
51 struct {
52 uint64_t inserts, searches, deletions, rehashes;
53 } stats;
54
55 hash_func_t *hash_func;
56 /* returns hash value of given key (passed as void*) */
57
58 free_func_t *free_func;
59 /* free of bucket payload */
60
61 key_func_t *key_func;
62 /* returns pointer to key of bucket (to pass to hash_func) */
63
64 find_func_t *find_func;
65 /* returns true if given bucket matches key (passed as void*) */
66
67 make_func_t *make_func;
68 /* returns bucket containing new record with key (passed as void*) */
69
70 format_cols_func_t *format_cols_func;
71 /* append table columns to str */
72
73 format_row_func_t *format_row_func;
74 /* format record and append to str */
75 };
76
77 static void hashtable_reduce(struct hashtable *ht);
78 static void hashtable_free(struct hashtable *h);
79
80 #define HOST_BITS 1 /* initial size of hosts table */
81 #define PORT_BITS 1 /* initial size of ports tables */
82 #define PROTO_BITS 1 /* initial size of proto table */
83
84 /* We only use one hosts_db hashtable and this is it. */
85 static struct hashtable *hosts_db = NULL;
86
87 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
88 static const double phi_1 =
89 0.61803398874989490252573887119069695472717285156250;
90
91 /* Co-prime of u, using phi^-1 */
92 inline static uint32_t
93 coprime(const uint32_t u)
94 {
95 return ( (uint32_t)( (double)(u) * phi_1 ) | 1U );
96 }
97
98 /*
99 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
100 * src/sys/netinet/tcp_hostcache.c 1.1
101 */
102 inline static uint32_t
103 ipv4_hash(const uint32_t ip)
104 {
105 return ( (ip) ^ ((ip) >> 7) ^ ((ip) >> 17) );
106 }
107
108 /* ---------------------------------------------------------------------------
109 * hash_func collection
110 */
111 #define CASTKEY(type) (*((const type *)key))
112
113 static uint32_t
114 hash_func_host(const struct hashtable *h _unused_, const void *key)
115 {
116 return (ipv4_hash(CASTKEY(in_addr_t)));
117 }
118
119 static uint32_t
120 hash_func_short(const struct hashtable *h, const void *key)
121 {
122 return (CASTKEY(uint16_t) * h->coeff);
123 }
124
125 static uint32_t
126 hash_func_byte(const struct hashtable *h, const void *key)
127 {
128 return (CASTKEY(uint8_t) * h->coeff);
129 }
130
131 /* ---------------------------------------------------------------------------
132 * key_func collection
133 */
134
135 static const void *
136 key_func_host(const struct bucket *b)
137 {
138 return &(b->u.host.ip);
139 }
140
141 static const void *
142 key_func_port_tcp(const struct bucket *b)
143 {
144 return &(b->u.port_tcp.port);
145 }
146
147 static const void *
148 key_func_port_udp(const struct bucket *b)
149 {
150 return &(b->u.port_udp.port);
151 }
152
153 static const void *
154 key_func_ip_proto(const struct bucket *b)
155 {
156 return &(b->u.ip_proto.proto);
157 }
158
159 /* ---------------------------------------------------------------------------
160 * find_func collection
161 */
162
163 static int
164 find_func_host(const struct bucket *b, const void *key)
165 {
166 return (b->u.host.ip == CASTKEY(in_addr_t));
167 }
168
169 static int
170 find_func_port_tcp(const struct bucket *b, const void *key)
171 {
172 return (b->u.port_tcp.port == CASTKEY(uint16_t));
173 }
174
175 static int
176 find_func_port_udp(const struct bucket *b, const void *key)
177 {
178 return (b->u.port_udp.port == CASTKEY(uint16_t));
179 }
180
181 static int
182 find_func_ip_proto(const struct bucket *b, const void *key)
183 {
184 return (b->u.ip_proto.proto == CASTKEY(uint8_t));
185 }
186
187 /* ---------------------------------------------------------------------------
188 * make_func collection
189 */
190
191 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
192 struct bucket *next; \
193 uint64_t in, out, total; \
194 union { struct type t; } u; } _custom_bucket; \
195 struct bucket *name_bucket = xmalloc(sizeof(_custom_bucket)); \
196 struct type *name_content = &(name_bucket->u.type); \
197 name_bucket->next = NULL; \
198 name_bucket->in = name_bucket->out = name_bucket->total = 0;
199
200 static struct bucket *
201 make_func_host(const void *key)
202 {
203 MAKE_BUCKET(b, h, host);
204 h->ip = CASTKEY(in_addr_t);
205 h->dns = NULL;
206 h->last_seen = now;
207 memset(&h->mac_addr, 0, sizeof(h->mac_addr));
208 h->ports_tcp = NULL;
209 h->ports_udp = NULL;
210 h->ip_protos = NULL;
211 return (b);
212 }
213
214 static void
215 free_func_host(struct bucket *b)
216 {
217 struct host *h = &(b->u.host);
218 if (h->dns != NULL) free(h->dns);
219 hashtable_free(h->ports_tcp);
220 hashtable_free(h->ports_udp);
221 hashtable_free(h->ip_protos);
222 }
223
224 static struct bucket *
225 make_func_port_tcp(const void *key)
226 {
227 MAKE_BUCKET(b, p, port_tcp);
228 p->port = CASTKEY(uint16_t);
229 p->syn = 0;
230 return (b);
231 }
232
233 static struct bucket *
234 make_func_port_udp(const void *key)
235 {
236 MAKE_BUCKET(b, p, port_udp);
237 p->port = CASTKEY(uint16_t);
238 return (b);
239 }
240
241 static struct bucket *
242 make_func_ip_proto(const void *key)
243 {
244 MAKE_BUCKET(b, p, ip_proto);
245 p->proto = CASTKEY(uint8_t);
246 return (b);
247 }
248
249 static void
250 free_func_simple(struct bucket *b _unused_)
251 {
252 /* nop */
253 }
254
255 /* ---------------------------------------------------------------------------
256 * format_func collection (ordered by struct)
257 */
258
259 static void
260 format_cols_host(struct str *buf)
261 {
262 /* FIXME: don't clobber parts of the query string
263 * specifically "full" and "start"
264 * when setting sort direction
265 */
266 str_append(buf,
267 "<table>\n"
268 "<tr>\n"
269 " <th>IP</th>\n"
270 " <th>Hostname</th>\n");
271 if (show_mac_addrs) str_append(buf,
272 " <th>MAC Address</th>\n");
273 str_append(buf,
274 " <th><a href=\"?sort=in\">In</a></th>\n"
275 " <th><a href=\"?sort=out\">Out</a></th>\n"
276 " <th><a href=\"?sort=total\">Total</a></th>\n"
277 " <th>Last seen</th>\n"
278 "</tr>\n");
279 }
280
281 static void
282 format_row_host(struct str *buf, const struct bucket *b,
283 const char *css_class)
284 {
285 const char *ip = ip_to_str( b->u.host.ip );
286 struct str *lastseen = NULL;
287 time_t last_t;
288
289 str_appendf(buf,
290 "<tr class=\"%s\">\n"
291 " <td><a href=\"%s/\">%s</a></td>\n"
292 " <td>%s</td>\n",
293 css_class,
294 ip, ip,
295 (b->u.host.dns == NULL) ? "" : b->u.host.dns);
296
297 if (show_mac_addrs)
298 str_appendf(buf,
299 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
300 b->u.host.mac_addr[0],
301 b->u.host.mac_addr[1],
302 b->u.host.mac_addr[2],
303 b->u.host.mac_addr[3],
304 b->u.host.mac_addr[4],
305 b->u.host.mac_addr[5]);
306
307 last_t = b->u.host.last_seen;
308 if (now >= last_t)
309 lastseen = length_of_time(now - last_t);
310
311 str_appendf(buf,
312 " <td class=\"num\">%'qu</td>\n"
313 " <td class=\"num\">%'qu</td>\n"
314 " <td class=\"num\">%'qu</td>\n"
315 " <td class=\"num\">",
316 b->in, b->out, b->total);
317
318 if (lastseen == NULL)
319 str_append(buf, "(clock error)");
320 else {
321 str_appendstr(buf, lastseen);
322 str_free(lastseen);
323 }
324
325 str_appendf(buf,
326 "</td>\n"
327 "</tr>\n");
328
329 /* Only resolve hosts "on demand" */
330 if (b->u.host.dns == NULL)
331 dns_queue(b->u.host.ip);
332 }
333
334 static void
335 format_cols_port_tcp(struct str *buf)
336 {
337 str_append(buf,
338 "<table>\n"
339 "<tr>\n"
340 " <th>Port</td>\n"
341 " <th>Service</td>\n"
342 " <th>In</td>\n"
343 " <th>Out</td>\n"
344 " <th>Total</td>\n"
345 " <th>SYNs</td>\n"
346 "</tr>\n"
347 );
348 }
349
350 static void
351 format_row_port_tcp(struct str *buf, const struct bucket *b,
352 const char *css_class)
353 {
354 const struct port_tcp *p = &(b->u.port_tcp);
355
356 str_appendf(buf,
357 "<tr class=\"%s\">\n"
358 " <td class=\"num\">%u</td>\n"
359 " <td>%s</td>\n"
360 " <td class=\"num\">%'qu</td>\n"
361 " <td class=\"num\">%'qu</td>\n"
362 " <td class=\"num\">%'qu</td>\n"
363 " <td class=\"num\">%'qu</td>\n"
364 "</tr>\n",
365 css_class,
366 p->port, getservtcp(p->port), b->in, b->out, b->total, p->syn
367 );
368 }
369
370 static void
371 format_cols_port_udp(struct str *buf)
372 {
373 str_append(buf,
374 "<table>\n"
375 "<tr>\n"
376 " <th>Port</td>\n"
377 " <th>Service</td>\n"
378 " <th>In</td>\n"
379 " <th>Out</td>\n"
380 " <th>Total</td>\n"
381 "</tr>\n"
382 );
383 }
384
385 static void
386 format_row_port_udp(struct str *buf, const struct bucket *b,
387 const char *css_class)
388 {
389 const struct port_udp *p = &(b->u.port_udp);
390
391 str_appendf(buf,
392 "<tr class=\"%s\">\n"
393 " <td class=\"num\">%u</td>\n"
394 " <td>%s</td>\n"
395 " <td class=\"num\">%'qu</td>\n"
396 " <td class=\"num\">%'qu</td>\n"
397 " <td class=\"num\">%'qu</td>\n"
398 "</tr>\n",
399 css_class,
400 p->port, getservudp(p->port), b->in, b->out, b->total
401 );
402 }
403
404 static void
405 format_cols_ip_proto(struct str *buf)
406 {
407 str_append(buf,
408 "<table>\n"
409 "<tr>\n"
410 " <th>#</td>\n"
411 " <th>Protocol</td>\n"
412 " <th>In</td>\n"
413 " <th>Out</td>\n"
414 " <th>Total</td>\n"
415 "</tr>\n"
416 );
417 }
418
419 static void
420 format_row_ip_proto(struct str *buf, const struct bucket *b,
421 const char *css_class)
422 {
423 const struct ip_proto *p = &(b->u.ip_proto);
424
425 str_appendf(buf,
426 "<tr class=\"%s\">\n"
427 " <td class=\"num\">%u</td>\n"
428 " <td>%s</td>\n"
429 " <td class=\"num\">%'qu</td>\n"
430 " <td class=\"num\">%'qu</td>\n"
431 " <td class=\"num\">%'qu</td>\n"
432 "</tr>\n",
433 css_class,
434 p->proto, getproto(p->proto),
435 b->in, b->out, b->total
436 );
437 }
438
439 /* ---------------------------------------------------------------------------
440 * Initialise a hashtable.
441 */
442 static struct hashtable *
443 hashtable_make(const uint8_t bits,
444 const unsigned int count_max,
445 const unsigned int count_keep,
446 hash_func_t *hash_func,
447 free_func_t *free_func,
448 key_func_t *key_func,
449 find_func_t *find_func,
450 make_func_t *make_func,
451 format_cols_func_t *format_cols_func,
452 format_row_func_t *format_row_func)
453 {
454 struct hashtable *hash;
455 assert(bits > 0);
456
457 hash = xmalloc(sizeof(*hash));
458 hash->bits = bits;
459 hash->count_max = count_max;
460 hash->count_keep = count_keep;
461 hash->size = 1U << bits;
462 hash->mask = hash->size - 1;
463 hash->coeff = coprime(hash->size);
464 hash->hash_func = hash_func;
465 hash->free_func = free_func;
466 hash->key_func = key_func;
467 hash->find_func = find_func;
468 hash->make_func = make_func;
469 hash->format_cols_func = format_cols_func;
470 hash->format_row_func = format_row_func;
471 hash->count = 0;
472 hash->table = xcalloc(hash->size, sizeof(*hash->table));
473 memset(&(hash->stats), 0, sizeof(hash->stats));
474 return (hash);
475 }
476
477 /* ---------------------------------------------------------------------------
478 * Initialise global hosts_db.
479 */
480 void
481 hosts_db_init(void)
482 {
483 assert(hosts_db == NULL);
484 hosts_db = hashtable_make(HOST_BITS, hosts_max, hosts_keep,
485 hash_func_host, free_func_host, key_func_host, find_func_host,
486 make_func_host, format_cols_host, format_row_host);
487 }
488
489 static void
490 hashtable_rehash(struct hashtable *h, const uint8_t bits)
491 {
492 struct bucket **old_table, **new_table;
493 uint32_t i, old_size;
494 assert(h != NULL);
495 assert(bits > 0);
496
497 h->stats.rehashes++;
498 old_size = h->size;
499 old_table = h->table;
500
501 h->bits = bits;
502 h->size = 1U << bits;
503 h->mask = h->size - 1;
504 h->coeff = coprime(h->size);
505 new_table = xcalloc(h->size, sizeof(*new_table));
506
507 for (i=0; i<old_size; i++) {
508 struct bucket *next, *b = old_table[i];
509 while (b != NULL) {
510 uint32_t pos = h->hash_func(h, h->key_func(b)) & h->mask;
511 next = b->next;
512 b->next = new_table[pos];
513 new_table[pos] = b;
514 b = next;
515 }
516 }
517
518 free(h->table);
519 h->table = new_table;
520 }
521
522 static void
523 hashtable_insert(struct hashtable *h, struct bucket *b)
524 {
525 uint32_t pos;
526 assert(h != NULL);
527 assert(b != NULL);
528 assert(b->next == NULL);
529
530 /* Rehash on 80% occupancy */
531 if ((h->count > h->size) ||
532 ((h->size - h->count) < h->size / 5))
533 hashtable_rehash(h, h->bits+1);
534
535 pos = h->hash_func(h, h->key_func(b)) & h->mask;
536 if (h->table[pos] == NULL)
537 h->table[pos] = b;
538 else {
539 /* Insert at top of chain. */
540 b->next = h->table[pos];
541 h->table[pos] = b;
542 }
543 h->count++;
544 h->stats.inserts++;
545 }
546
547 /* Return bucket matching key, or NULL if no such entry. */
548 static struct bucket *
549 hashtable_search(struct hashtable *h, const void *key)
550 {
551 uint32_t pos;
552 struct bucket *b;
553
554 h->stats.searches++;
555 pos = h->hash_func(h, key) & h->mask;
556 b = h->table[pos];
557 while (b != NULL) {
558 if (h->find_func(b, key))
559 return (b);
560 else
561 b = b->next;
562 }
563 return (NULL);
564 }
565
566 /* Search for a key. If it's not there, make and insert a bucket for it. */
567 static struct bucket *
568 hashtable_find_or_insert(struct hashtable *h, const void *key)
569 {
570 struct bucket *b = hashtable_search(h, key);
571
572 if (b == NULL) {
573 /* Not found, so insert after checking occupancy. */
574 /*assert(h->count <= h->count_max);*/
575 if (h->count >= h->count_max) hashtable_reduce(h);
576 b = h->make_func(key);
577 hashtable_insert(h, b);
578 }
579 return (b);
580 }
581
582 /*
583 * Frees the hashtable and the buckets. The contents are assumed to be
584 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
585 * the bucket.
586 */
587 static void
588 hashtable_free(struct hashtable *h)
589 {
590 uint32_t i;
591
592 if (h == NULL)
593 return;
594 for (i=0; i<h->size; i++) {
595 struct bucket *tmp, *b = h->table[i];
596 while (b != NULL) {
597 tmp = b;
598 b = b->next;
599 h->free_func(tmp);
600 free(tmp);
601 }
602 }
603 free(h->table);
604 free(h);
605 }
606
607 /* ---------------------------------------------------------------------------
608 * Return existing host or insert a new one.
609 */
610 struct bucket *
611 host_get(const in_addr_t ip)
612 {
613 return (hashtable_find_or_insert(hosts_db, &ip));
614 }
615
616 /* ---------------------------------------------------------------------------
617 * Find host, returns NULL if not in DB.
618 */
619 struct bucket *
620 host_find(const in_addr_t ip)
621 {
622 return (hashtable_search(hosts_db, &ip));
623 }
624
625 /* ---------------------------------------------------------------------------
626 * Find host, returns NULL if not in DB.
627 */
628 static struct bucket *
629 host_search(const char *ipstr)
630 {
631 struct in_addr addr;
632
633 if (inet_aton(ipstr, &addr) != 1)
634 return (NULL); /* invalid addr */
635 return (hashtable_search(hosts_db, &(addr.s_addr)));
636 }
637
638 /* ---------------------------------------------------------------------------
639 * Reduce a hashtable to the top <keep> entries.
640 */
641 static void
642 hashtable_reduce(struct hashtable *ht)
643 {
644 uint32_t i, pos, rmd;
645 const struct bucket **table;
646 uint64_t cutoff;
647
648 assert(ht->count_keep < ht->count);
649
650 /* Fill table with pointers to buckets in hashtable. */
651 table = xmalloc(sizeof(*table) * ht->count);
652 for (pos=0, i=0; i<ht->size; i++) {
653 struct bucket *b = ht->table[i];
654 while (b != NULL) {
655 table[pos++] = b;
656 b = b->next;
657 }
658 }
659 assert(pos == ht->count);
660 qsort_buckets(table, ht->count, 0, ht->count_keep, TOTAL);
661 cutoff = table[ht->count_keep]->total;
662 free(table);
663
664 /* Remove all elements with total <= cutoff. */
665 rmd = 0;
666 for (i=0; i<ht->size; i++) {
667 struct bucket *last = NULL, *next, *b = ht->table[i];
668 while (b != NULL) {
669 next = b->next;
670 if (b->total <= cutoff) {
671 /* Remove this one. */
672 ht->free_func(b);
673 free(b);
674 if (last == NULL)
675 ht->table[i] = next;
676 else
677 last->next = next;
678 rmd++;
679 ht->count--;
680 } else {
681 last = b;
682 }
683 b = next;
684 }
685 }
686 verbosef("hashtable_reduce: removed %u buckets, left %u",
687 rmd, ht->count);
688 hashtable_rehash(ht, ht->bits); /* is this needed? */
689 }
690
691 /* ---------------------------------------------------------------------------
692 * Reset hosts_db to empty.
693 */
694 void
695 hosts_db_reset(void)
696 {
697 unsigned int i;
698
699 for (i=0; i<hosts_db->size; i++) {
700 struct bucket *next, *b = hosts_db->table[i];
701 while (b != NULL) {
702 next = b->next;
703 hosts_db->free_func(b);
704 free(b);
705 b = next;
706 }
707 hosts_db->table[i] = NULL;
708 }
709 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db->count);
710 hosts_db->count = 0;
711 }
712
713 /* ---------------------------------------------------------------------------
714 * Deallocate hosts_db.
715 */
716 void hosts_db_free(void)
717 {
718 uint32_t i;
719
720 assert(hosts_db != NULL);
721 for (i=0; i<hosts_db->size; i++) {
722 struct bucket *tmp, *b = hosts_db->table[i];
723 while (b != NULL) {
724 tmp = b;
725 b = b->next;
726 hosts_db->free_func(tmp);
727 free(tmp);
728 }
729 }
730 free(hosts_db->table);
731 free(hosts_db);
732 hosts_db = NULL;
733 }
734
735 /* ---------------------------------------------------------------------------
736 * Find or create a port_tcp inside a host.
737 */
738 struct bucket *
739 host_get_port_tcp(struct bucket *host, const uint16_t port)
740 {
741 struct host *h = &host->u.host;
742 assert(h != NULL);
743 if (h->ports_tcp == NULL)
744 h->ports_tcp = hashtable_make(PORT_BITS, ports_max, ports_keep,
745 hash_func_short, free_func_simple, key_func_port_tcp,
746 find_func_port_tcp, make_func_port_tcp,
747 format_cols_port_tcp, format_row_port_tcp);
748 return (hashtable_find_or_insert(h->ports_tcp, &port));
749 }
750
751 /* ---------------------------------------------------------------------------
752 * Find or create a port_udp inside a host.
753 */
754 struct bucket *
755 host_get_port_udp(struct bucket *host, const uint16_t port)
756 {
757 struct host *h = &host->u.host;
758 assert(h != NULL);
759 if (h->ports_udp == NULL)
760 h->ports_udp = hashtable_make(PORT_BITS, ports_max, ports_keep,
761 hash_func_short, free_func_simple, key_func_port_udp,
762 find_func_port_udp, make_func_port_udp,
763 format_cols_port_udp, format_row_port_udp);
764 return (hashtable_find_or_insert(h->ports_udp, &port));
765 }
766
767 /* ---------------------------------------------------------------------------
768 * Find or create an ip_proto inside a host.
769 */
770 struct bucket *
771 host_get_ip_proto(struct bucket *host, const uint8_t proto)
772 {
773 struct host *h = &host->u.host;
774 static const unsigned int PROTOS_MAX = 512, PROTOS_KEEP = 256;
775 assert(h != NULL);
776 if (h->ip_protos == NULL)
777 h->ip_protos = hashtable_make(PROTO_BITS, PROTOS_MAX, PROTOS_KEEP,
778 hash_func_byte, free_func_simple, key_func_ip_proto,
779 find_func_ip_proto, make_func_ip_proto,
780 format_cols_ip_proto, format_row_ip_proto);
781 return (hashtable_find_or_insert(h->ip_protos, &proto));
782 }
783
784 static struct str *html_hosts_main(const char *qs);
785 static struct str *html_hosts_detail(const char *ip);
786
787 /* ---------------------------------------------------------------------------
788 * Web interface: delegate the /hosts/ space.
789 */
790 struct str *
791 html_hosts(const char *uri, const char *query)
792 {
793 int i, num_elems;
794 char **elem = split('/', uri, &num_elems);
795 struct str *buf = NULL;
796
797 assert(num_elems >= 1);
798 assert(strcmp(elem[0], "hosts") == 0);
799
800 if (num_elems == 1)
801 /* /hosts/ */
802 buf = html_hosts_main(query);
803 else if (num_elems == 2)
804 /* /hosts/<IP of host>/ */
805 buf = html_hosts_detail(elem[1]);
806
807 for (i=0; i<num_elems; i++)
808 free(elem[i]);
809 free(elem);
810 return (buf); /* FIXME: a NULL here becomes 404 Not Found, we might want
811 other codes to be possible */
812 }
813
814 /* ---------------------------------------------------------------------------
815 * Format hashtable into HTML.
816 */
817 static void
818 format_table(struct str *buf, struct hashtable *ht, int start,
819 const enum sort_dir sort, const int full)
820 {
821 const struct bucket **table;
822 uint32_t i, pos, end;
823 int alt = 0;
824
825 if ((ht == NULL) || (ht->count == 0)) {
826 str_append(buf, "<p>The table is empty.</p>\n");
827 return;
828 }
829
830 /* Fill table with pointers to buckets in hashtable. */
831 table = xmalloc(sizeof(*table) * ht->count);
832 for (pos=0, i=0; i<ht->size; i++) {
833 struct bucket *b = ht->table[i];
834 while (b != NULL) {
835 table[pos++] = b;
836 b = b->next;
837 }
838 }
839 assert(pos == ht->count);
840
841 if (full) {
842 /* full report overrides start and end */
843 start = 0;
844 end = ht->count;
845 } else
846 end = min(ht->count, (uint32_t)start+MAX_ENTRIES);
847
848 str_appendf(buf, "(%u-%u of %u)<br/>\n", start+1, end, ht->count);
849 qsort_buckets(table, ht->count, start, end, sort);
850 ht->format_cols_func(buf);
851
852 for (i=start; i<end; i++) {
853 ht->format_row_func(buf, table[i], alt ? "alt1" : "alt2");
854 alt = !alt; /* alternate class for table rows */
855 }
856 free(table);
857 str_append(buf, "</table>\n");
858 }
859
860 /* ---------------------------------------------------------------------------
861 * Web interface: sorted table of hosts.
862 */
863 static struct str *
864 html_hosts_main(const char *qs)
865 {
866 struct str *buf = str_make();
867 char *qs_start, *qs_sort, *qs_full, *ep;
868 const char *sortstr;
869 int start, full = 0;
870 enum sort_dir sort;
871
872 /* parse query string */
873 qs_start = qs_get(qs, "start");
874 qs_sort = qs_get(qs, "sort");
875 qs_full = qs_get(qs, "full");
876 if (qs_full != NULL) {
877 full = 1;
878 free(qs_full);
879 }
880
881 /* validate sort */
882 if (qs_sort == NULL) sort = TOTAL;
883 else if (strcmp(qs_sort, "total") == 0) sort = TOTAL;
884 else if (strcmp(qs_sort, "in") == 0) sort = IN;
885 else if (strcmp(qs_sort, "out") == 0) sort = OUT;
886 else {
887 str_append(buf, "Error: invalid value for \"sort\".\n");
888 goto done;
889 }
890
891 /* parse start */
892 if (qs_start == NULL)
893 start = 0;
894 else {
895 start = (int)strtoul(qs_start, &ep, 10);
896 if (*ep != '\0') {
897 str_append(buf, "Error: \"start\" is not a number.\n");
898 goto done;
899 }
900 if ((errno == ERANGE) ||
901 (start < 0) || (start >= (int)hosts_db->count)) {
902 str_append(buf, "Error: \"start\" is out of bounds.\n");
903 goto done;
904 }
905 }
906
907 #define PREV "&lt;&lt;&lt; prev page"
908 #define NEXT "next page &gt;&gt;&gt;"
909 #define FULL "full table"
910
911 str_append(buf, html_header_1);
912 str_appendf(buf, " <title>darkstat3: Hosts (%s)</title>\n", interface);
913 str_append(buf, html_header_2);
914 str_appendf(buf, "<h2 class=\"pageheader\">Hosts (%s)</h2>\n", interface);
915 format_table(buf, hosts_db, start, sort, full);
916
917 /* <prev | full | stats | next> */
918 sortstr = qs_sort;
919 if (sortstr == NULL) sortstr = "total";
920 if (start > 0) {
921 int prev = max(start - MAX_ENTRIES, 0);
922 str_appendf(buf, "<a href=\"?start=%d&sort=%s\">" PREV "</a>",
923 prev, sortstr);
924 } else
925 str_append(buf, PREV);
926
927 if (full)
928 str_append(buf, " | " FULL);
929 else
930 str_appendf(buf, " | <a href=\"?full=yes&sort=%s\">" FULL "</a>",
931 sortstr);
932
933 if (start+MAX_ENTRIES < (int)hosts_db->count)
934 str_appendf(buf, " | <a href=\"?start=%d&sort=%s\">" NEXT "</a>",
935 start+MAX_ENTRIES, sortstr);
936 else
937 str_append(buf, " | " NEXT);
938
939 str_append(buf, "<br/>\n");
940 str_append(buf, html_footer);
941 done:
942 if (qs_start != NULL) free(qs_start);
943 if (qs_sort != NULL) free(qs_sort);
944 return buf;
945 #undef PREV
946 #undef NEXT
947 #undef FULL
948 }
949
950 /* ---------------------------------------------------------------------------
951 * Web interface: detailed view of a single host.
952 */
953 static struct str *
954 html_hosts_detail(const char *ip)
955 {
956 struct bucket *h;
957 struct str *buf, *ls_len;
958 char ls_when[100];
959 time_t ls;
960
961 h = host_search(ip);
962 if (h == NULL)
963 return (NULL); /* no such host */
964
965 /* Overview. */
966 buf = str_make();
967 str_append(buf, html_header_1);
968 str_appendf(buf, " <title>%s</title>\n", ip);
969 str_append(buf, html_header_2);
970 str_appendf(buf,
971 "<h2>%s</h2>\n"
972 "<p>\n"
973 "<b>Hostname:</b> %s<br/>\n",
974 ip,
975 (h->u.host.dns == NULL)?"(resolving...)":h->u.host.dns);
976
977 /* Resolve host "on demand" */
978 if (h->u.host.dns == NULL)
979 dns_queue(h->u.host.ip);
980
981 if (show_mac_addrs)
982 str_appendf(buf,
983 "<b>MAC Address:</b> "
984 "<tt>%x:%x:%x:%x:%x:%x</tt><br/>\n",
985 h->u.host.mac_addr[0],
986 h->u.host.mac_addr[1],
987 h->u.host.mac_addr[2],
988 h->u.host.mac_addr[3],
989 h->u.host.mac_addr[4],
990 h->u.host.mac_addr[5]);
991
992 str_append(buf,
993 "</p>\n"
994 "<p>\n"
995 "<b>Last seen:</b> ");
996
997 ls = h->u.host.last_seen;
998 if (strftime(ls_when, sizeof(ls_when),
999 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&ls)) != 0)
1000 str_append(buf, ls_when);
1001
1002 if (h->u.host.last_seen <= now) {
1003 ls_len = length_of_time(now - h->u.host.last_seen);
1004 str_append(buf, " (");
1005 str_appendstr(buf, ls_len);
1006 str_free(ls_len);
1007 str_append(buf, " ago)");
1008 } else {
1009 str_append(buf, " (in the future, possible clock problem)");
1010 }
1011
1012 str_appendf(buf,
1013 "</p>\n"
1014 "<p>\n"
1015 " <b>In:</b> %'qu<br/>\n"
1016 " <b>Out:</b> %'qu<br/>\n"
1017 " <b>Total:</b> %'qu<br/>\n"
1018 "</p>\n",
1019 h->in, h->out, h->total);
1020
1021 str_append(buf, "<h3>TCP ports</h3>\n");
1022 format_table(buf, h->u.host.ports_tcp, 0,TOTAL,0);
1023
1024 str_append(buf, "<h3>UDP ports</h3>\n");
1025 format_table(buf, h->u.host.ports_udp, 0,TOTAL,0);
1026
1027 str_append(buf, "<h3>IP protocols</h3>\n");
1028 format_table(buf, h->u.host.ip_protos, 0,TOTAL,0);
1029
1030 str_append(buf, html_footer);
1031 return (buf);
1032 }
1033
1034 /* ---------------------------------------------------------------------------
1035 * Database import and export code:
1036 * Initially written and contributed by Ben Stewart.
1037 * copyright (c) 2007 Ben Stewart, Emil Mikulic.
1038 */
1039 static int hosts_db_export_ip(const struct hashtable *h, const int fd);
1040 static int hosts_db_export_tcp(const struct hashtable *h, const int fd);
1041 static int hosts_db_export_udp(const struct hashtable *h, const int fd);
1042
1043 static const char
1044 export_proto_ip = 'P',
1045 export_proto_tcp = 'T',
1046 export_proto_udp = 'U';
1047
1048 static const unsigned char
1049 export_tag_host_ver1[] = {'H', 'S', 'T', 0x01},
1050 export_tag_host_ver2[] = {'H', 'S', 'T', 0x02};
1051
1052 /* ---------------------------------------------------------------------------
1053 * Load a host's ip_proto table from a file.
1054 * Returns 0 on failure, 1 on success.
1055 */
1056 static int
1057 hosts_db_import_ip(const int fd, struct bucket *host)
1058 {
1059 uint8_t count, i;
1060
1061 if (!expect8(fd, export_proto_ip)) return 0;
1062 if (!read8(fd, &count)) return 0;
1063
1064 for (i=0; i<count; i++) {
1065 struct bucket *b;
1066 uint8_t proto;
1067 uint64_t in, out;
1068
1069 if (!read8(fd, &proto)) return 0;
1070 if (!read64(fd, &in)) return 0;
1071 if (!read64(fd, &out)) return 0;
1072
1073 /* Store data */
1074 b = host_get_ip_proto(host, proto);
1075 b->in = in;
1076 b->out = out;
1077 b->total = in + out;
1078 assert(b->u.ip_proto.proto == proto); /* should be done by make fn */
1079 }
1080 return 1;
1081 }
1082
1083 /* ---------------------------------------------------------------------------
1084 * Load a host's port_tcp table from a file.
1085 * Returns 0 on failure, 1 on success.
1086 */
1087 static int
1088 hosts_db_import_tcp(const int fd, struct bucket *host)
1089 {
1090 uint16_t count, i;
1091
1092 if (!expect8(fd, export_proto_tcp)) return 0;
1093 if (!read16(fd, &count)) return 0;
1094
1095 for (i=0; i<count; i++) {
1096 struct bucket *b;
1097 uint16_t port;
1098 uint64_t in, out, syn;
1099
1100 if (!read16(fd, &port)) return 0;
1101 if (!read64(fd, &syn)) return 0;
1102 if (!read64(fd, &in)) return 0;
1103 if (!read64(fd, &out)) return 0;
1104
1105 /* Store data */
1106 b = host_get_port_tcp(host, port);
1107 b->in = in;
1108 b->out = out;
1109 b->total = in + out;
1110 assert(b->u.port_tcp.port == port); /* done by make_func_port_tcp */
1111 b->u.port_tcp.syn = syn;
1112 }
1113 return 1;
1114 }
1115
1116 /* ---------------------------------------------------------------------------
1117 * Load a host's port_tcp table from a file.
1118 * Returns 0 on failure, 1 on success.
1119 */
1120 static int
1121 hosts_db_import_udp(const int fd, struct bucket *host)
1122 {
1123 uint16_t count, i;
1124
1125 if (!expect8(fd, export_proto_udp)) return 0;
1126 if (!read16(fd, &count)) return 0;
1127
1128 for (i=0; i<count; i++) {
1129 struct bucket *b;
1130 uint16_t port;
1131 uint64_t in, out;
1132
1133 if (!read16(fd, &port)) return 0;
1134 if (!read64(fd, &in)) return 0;
1135 if (!read64(fd, &out)) return 0;
1136
1137 /* Store data */
1138 b = host_get_port_udp(host, port);
1139 b->in = in;
1140 b->out = out;
1141 b->total = in + out;
1142 assert(b->u.port_udp.port == port); /* done by make_func */
1143 }
1144 return 1;
1145 }
1146
1147 /* ---------------------------------------------------------------------------
1148 * Load all hosts from a file.
1149 * Returns 0 on failure, 1 on success.
1150 */
1151 static int
1152 hosts_db_import_host(const int fd)
1153 {
1154 struct bucket *host;
1155 in_addr_t addr;
1156 uint8_t hostname_len;
1157 uint64_t in, out;
1158 unsigned int pos = xtell(fd);
1159 char hdr[4];
1160 int ver = 0;
1161
1162 if (!readn(fd, hdr, sizeof(hdr))) return 0;
1163 if (memcmp(hdr, export_tag_host_ver2, sizeof(hdr)) == 0)
1164 ver = 2;
1165 else if (memcmp(hdr, export_tag_host_ver1, sizeof(hdr)) == 0)
1166 ver = 1;
1167 else {
1168 warnx("bad host header: %02x%02x%02x%02x",
1169 hdr[0], hdr[1], hdr[2], hdr[3]);
1170 return 0;
1171 }
1172
1173 if (!readaddr(fd, &addr)) return 0;
1174 verbosef("at file pos %u, importing host %s", pos, ip_to_str(addr));
1175 host = host_get(addr);
1176 assert(host->u.host.ip == addr); /* make fn? */
1177
1178 if (ver > 1) {
1179 uint64_t t;
1180 if (!read64(fd, &t)) return 0;
1181 host->u.host.last_seen = (time_t)t;
1182 }
1183
1184 assert(sizeof(host->u.host.mac_addr) == 6);
1185 if (!readn(fd, host->u.host.mac_addr, sizeof(host->u.host.mac_addr)))
1186 return 0;
1187
1188 /* HOSTNAME */
1189 assert(host->u.host.dns == NULL); /* make fn? */
1190 if (!read8(fd, &hostname_len)) return 0;
1191 if (hostname_len > 0) {
1192 host->u.host.dns = xmalloc(hostname_len + 1);
1193 host->u.host.dns[0] = '\0';
1194
1195 /* At this point, the hostname is attached to a host which is in our
1196 * hosts_db, so if we bail out due to an import error, this pointer
1197 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1198 */
1199
1200 if (!readn(fd, host->u.host.dns, hostname_len)) return 0;
1201 host->u.host.dns[hostname_len] = '\0';
1202 }
1203
1204 if (!read64(fd, &in)) return 0;
1205 if (!read64(fd, &out)) return 0;
1206
1207 host->in = in;
1208 host->out = out;
1209 host->total = in + out;
1210
1211 /* Host's port and proto subtables: */
1212 if (!hosts_db_import_ip(fd, host)) return 0;
1213 if (!hosts_db_import_tcp(fd, host)) return 0;
1214 if (!hosts_db_import_udp(fd, host)) return 0;
1215 return 1;
1216 }
1217
1218 /* ---------------------------------------------------------------------------
1219 * Database Import: Grab hosts_db from a file provided by the caller.
1220 *
1221 * This function will retrieve the data sans the header. We expect the caller
1222 * to have validated the header of the hosts_db segment, and left the file
1223 * sitting at the start of the data.
1224 */
1225 int hosts_db_import(const int fd)
1226 {
1227 uint32_t host_count, i;
1228
1229 if (!read32(fd, &host_count)) return 0;
1230
1231 for (i=0; i<host_count; i++)
1232 if (!hosts_db_import_host(fd)) return 0;
1233
1234 return 1;
1235 }
1236
1237 /* ---------------------------------------------------------------------------
1238 * Database Export: Dump hosts_db into a file provided by the caller.
1239 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1240 */
1241 int hosts_db_export(const int fd)
1242 {
1243 uint32_t i;
1244 struct bucket *b;
1245
1246 if (!write32(fd, hosts_db->count)) return 0;
1247
1248 for (i = 0; i<hosts_db->size; i++)
1249 for (b = hosts_db->table[i]; b != NULL; b = b->next) {
1250 /* For each host: */
1251 if (!writen(fd, export_tag_host_ver2, sizeof(export_tag_host_ver2)))
1252 return 0;
1253
1254 if (!writeaddr(fd, b->u.host.ip)) return 0;
1255
1256 if (!write64(fd, (uint64_t)(b->u.host.last_seen))) return 0;
1257
1258 assert(sizeof(b->u.host.mac_addr) == 6);
1259 if (!writen(fd, b->u.host.mac_addr, sizeof(b->u.host.mac_addr)))
1260 return 0;
1261
1262 /* HOSTNAME */
1263 if (b->u.host.dns == NULL) {
1264 if (!write8(fd, 0)) return 0;
1265 } else {
1266 int dnslen = strlen(b->u.host.dns);
1267
1268 if (dnslen > 255) {
1269 warnx("found a very long hostname: \"%s\"\n"
1270 "wasn't expecting one longer than 255 chars (this one is %d)",
1271 b->u.host.dns, dnslen);
1272 dnslen = 255;
1273 }
1274
1275 if (!write8(fd, (uint8_t)dnslen)) return 0;
1276 if (!writen(fd, b->u.host.dns, dnslen)) return 0;
1277 }
1278
1279 if (!write64(fd, b->in)) return 0;
1280 if (!write64(fd, b->out)) return 0;
1281
1282 if (!hosts_db_export_ip(b->u.host.ip_protos, fd)) return 0;
1283 if (!hosts_db_export_tcp(b->u.host.ports_tcp, fd)) return 0;
1284 if (!hosts_db_export_udp(b->u.host.ports_udp, fd)) return 0;
1285 }
1286 return 1;
1287 }
1288
1289 /* ---------------------------------------------------------------------------
1290 * Dump the ip_proto table of a host.
1291 */
1292 static int
1293 hosts_db_export_ip(const struct hashtable *h, const int fd)
1294 {
1295 uint32_t i, written = 0;
1296 struct bucket *b;
1297
1298 /* IP DATA */
1299 if (!write8(fd, export_proto_ip)) return 0;
1300
1301 /* If no data, write a IP Proto count of 0 and we're done. */
1302 if (h == NULL) {
1303 if (!write8(fd, 0)) return 0;
1304 return 1;
1305 }
1306
1307 assert(h->count < 256);
1308 if (!write8(fd, (uint8_t)h->count)) return 0;
1309
1310 for (i = 0; i<h->size; i++)
1311 for (b = h->table[i]; b != NULL; b = b->next) {
1312 /* For each ip_proto bucket: */
1313
1314 if (!write8(fd, b->u.ip_proto.proto)) return 0;
1315 if (!write64(fd, b->in)) return 0;
1316 if (!write64(fd, b->out)) return 0;
1317 written++;
1318 }
1319 assert(written == h->count);
1320 return 1;
1321 }
1322
1323 /* ---------------------------------------------------------------------------
1324 * Dump the port_tcp table of a host.
1325 */
1326 static int
1327 hosts_db_export_tcp(const struct hashtable *h, const int fd)
1328 {
1329 struct bucket *b;
1330 uint32_t i, written = 0;
1331
1332 /* TCP DATA */
1333 if (!write8(fd, export_proto_tcp)) return 0;
1334
1335 /* If no data, write a count of 0 and we're done. */
1336 if (h == NULL) {
1337 if (!write16(fd, 0)) return 0;
1338 return 1;
1339 }
1340
1341 assert(h->count < 65536);
1342 if (!write16(fd, (uint16_t)h->count)) return 0;
1343
1344 for (i = 0; i<h->size; i++)
1345 for (b = h->table[i]; b != NULL; b = b->next) {
1346 if (!write16(fd, b->u.port_tcp.port)) return 0;
1347 if (!write64(fd, b->u.port_tcp.syn)) return 0;
1348 if (!write64(fd, b->in)) return 0;
1349 if (!write64(fd, b->out)) return 0;
1350 written++;
1351 }
1352 assert(written == h->count);
1353 return 1;
1354 }
1355
1356 /* ---------------------------------------------------------------------------
1357 * Dump the port_udp table of a host.
1358 */
1359 static int
1360 hosts_db_export_udp(const struct hashtable *h, const int fd)
1361 {
1362 struct bucket *b;
1363 uint32_t i, written = 0;
1364
1365 /* UDP DATA */
1366 if (!write8(fd, export_proto_udp)) return 0;
1367
1368 /* If no data, write a count of 0 and we're done. */
1369 if (h == NULL) {
1370 if (!write16(fd, 0)) return 0;
1371 return 1;
1372 }
1373
1374 assert(h->count < 65536);
1375 if (!write16(fd, (uint16_t)h->count)) return 0;
1376
1377 for (i = 0; i<h->size; i++)
1378 for (b = h->table[i]; b != NULL; b = b->next) {
1379 if (!write16(fd, b->u.port_udp.port)) return 0;
1380 if (!write64(fd, b->in)) return 0;
1381 if (!write64(fd, b->out)) return 0;
1382 written++;
1383 }
1384 assert(written == h->count);
1385 return 1;
1386 }
1387
1388 /* vim:set ts=3 sw=3 tw=78 expandtab: */