2 * copyright (c) 2001-2014 Emil Mikulic.
4 * hosts_db.c: database of hosts, ports, protocols.
6 * You may use, modify and redistribute this file under the terms of the
7 * GNU General Public License version 2. (see COPYING.GPL)
23 #include <netdb.h> /* struct addrinfo */
28 #include <string.h> /* memset(), strcmp() */
32 int hosts_db_show_macs
= 0;
34 /* FIXME: specify somewhere more sane/tunable */
35 #define MAX_ENTRIES 30 /* in an HTML table rendered from a hashtable */
37 typedef uint32_t (hash_func_t
)(const struct hashtable
*, const void *);
38 typedef void (free_func_t
)(struct bucket
*);
39 typedef const void * (key_func_t
)(const struct bucket
*);
40 typedef int (find_func_t
)(const struct bucket
*, const void *);
41 typedef struct bucket
* (make_func_t
)(const void *);
42 typedef void (format_cols_func_t
)(struct str
*);
43 typedef void (format_row_func_t
)(struct str
*, const struct bucket
*);
46 uint8_t bits
; /* size of hashtable in bits */
48 uint32_t count
, count_max
, count_keep
; /* items in table */
49 uint32_t coeff
; /* coefficient for Fibonacci hashing */
50 struct bucket
**table
;
53 uint64_t inserts
, searches
, deletions
, rehashes
;
56 hash_func_t
*hash_func
;
57 /* returns hash value of given key (passed as void*) */
59 free_func_t
*free_func
;
60 /* free of bucket payload */
63 /* returns pointer to key of bucket (to pass to hash_func) */
65 find_func_t
*find_func
;
66 /* returns true if given bucket matches key (passed as void*) */
68 make_func_t
*make_func
;
69 /* returns bucket containing new record with key (passed as void*) */
71 format_cols_func_t
*format_cols_func
;
72 /* append table columns to str */
74 format_row_func_t
*format_row_func
;
75 /* format record and append to str */
78 static void hashtable_reduce(struct hashtable
*ht
);
79 static void hashtable_free(struct hashtable
*h
);
81 #define HOST_BITS 1 /* initial size of hosts table */
82 #define PORT_BITS 1 /* initial size of ports tables */
83 #define PROTO_BITS 1 /* initial size of proto table */
85 /* We only use one hosts_db hashtable and this is it. */
86 static struct hashtable
*hosts_db
= NULL
;
88 /* phi^-1 (reciprocal of golden ratio) = (sqrt(5) - 1) / 2 */
89 static const double phi_1
=
90 0.61803398874989490252573887119069695472717285156250;
92 /* Co-prime of u, using phi^-1 */
93 static uint32_t coprime(const uint32_t u
) {
94 return ( (uint32_t)( (double)(u
) * phi_1
) | 1U );
98 * This is the "recommended" IPv4 hash function, as seen in FreeBSD's
99 * src/sys/netinet/tcp_hostcache.c 1.1
101 static uint32_t ipv4_hash(const struct addr
*const a
) {
102 uint32_t ip
= a
->ip
.v4
;
103 return ( (ip
) ^ ((ip
) >> 7) ^ ((ip
) >> 17) );
109 * http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/netinet/in.h#130
111 # define s6_addr32 _S6_un._S6_u32
113 /* Covers OpenBSD and FreeBSD. The macro __USE_GNU has
114 * taken care of GNU/Linux and GNU/kfreebsd. */
115 # define s6_addr32 __u6_addr.__u6_addr32
120 * This is the IPv6 hash function used by FreeBSD in the same file as above,
123 static uint32_t ipv6_hash(const struct addr
*const a
) {
124 const struct in6_addr
*const ip6
= &(a
->ip
.v6
);
125 return ( ip6
->s6_addr32
[0] ^ ip6
->s6_addr32
[1] ^
126 ip6
->s6_addr32
[2] ^ ip6
->s6_addr32
[3] );
129 /* ---------------------------------------------------------------------------
130 * hash_func collection
133 hash_func_host(const struct hashtable
*h _unused_
, const void *key
)
135 const struct addr
*a
= key
;
136 if (a
->family
== IPv4
)
137 return (ipv4_hash(a
));
139 assert(a
->family
== IPv6
);
140 return (ipv6_hash(a
));
144 #define CASTKEY(type) (*((const type *)key))
147 hash_func_short(const struct hashtable
*h
, const void *key
)
149 return (CASTKEY(uint16_t) * h
->coeff
);
153 hash_func_byte(const struct hashtable
*h
, const void *key
)
155 return (CASTKEY(uint8_t) * h
->coeff
);
158 /* ---------------------------------------------------------------------------
159 * key_func collection
163 key_func_host(const struct bucket
*b
)
165 return &(b
->u
.host
.addr
);
169 key_func_port_tcp(const struct bucket
*b
)
171 return &(b
->u
.port_tcp
.port
);
175 key_func_port_udp(const struct bucket
*b
)
177 return &(b
->u
.port_udp
.port
);
181 key_func_ip_proto(const struct bucket
*b
)
183 return &(b
->u
.ip_proto
.proto
);
186 /* ---------------------------------------------------------------------------
187 * find_func collection
191 find_func_host(const struct bucket
*b
, const void *key
)
193 return (addr_equal(key
, &(b
->u
.host
.addr
)));
197 find_func_port_tcp(const struct bucket
*b
, const void *key
)
199 return (b
->u
.port_tcp
.port
== CASTKEY(uint16_t));
203 find_func_port_udp(const struct bucket
*b
, const void *key
)
205 return (b
->u
.port_udp
.port
== CASTKEY(uint16_t));
209 find_func_ip_proto(const struct bucket
*b
, const void *key
)
211 return (b
->u
.ip_proto
.proto
== CASTKEY(uint8_t));
214 /* ---------------------------------------------------------------------------
215 * make_func collection
218 #define MAKE_BUCKET(name_bucket, name_content, type) struct { \
219 struct bucket *next; \
220 uint64_t in, out, total; \
221 union { struct type t; } u; } _custom_bucket; \
222 struct bucket *name_bucket = xcalloc(1, sizeof(_custom_bucket)); \
223 struct type *name_content = &(name_bucket->u.type); \
224 name_bucket->next = NULL; \
225 name_bucket->in = name_bucket->out = name_bucket->total = 0;
227 static struct bucket
*
228 make_func_host(const void *key
)
230 MAKE_BUCKET(b
, h
, host
);
231 h
->addr
= CASTKEY(struct addr
);
233 h
->last_seen_mono
= 0;
234 memset(&h
->mac_addr
, 0, sizeof(h
->mac_addr
));
236 h
->ports_tcp_remote
= NULL
;
238 h
->ports_udp_remote
= NULL
;
244 free_func_host(struct bucket
*b
)
246 struct host
*h
= &(b
->u
.host
);
247 if (h
->dns
!= NULL
) free(h
->dns
);
248 hashtable_free(h
->ports_tcp
);
249 hashtable_free(h
->ports_tcp_remote
);
250 hashtable_free(h
->ports_udp
);
251 hashtable_free(h
->ports_udp_remote
);
252 hashtable_free(h
->ip_protos
);
255 static struct bucket
*
256 make_func_port_tcp(const void *key
)
258 MAKE_BUCKET(b
, p
, port_tcp
);
259 p
->port
= CASTKEY(uint16_t);
264 static struct bucket
*
265 make_func_port_udp(const void *key
)
267 MAKE_BUCKET(b
, p
, port_udp
);
268 p
->port
= CASTKEY(uint16_t);
272 static struct bucket
*
273 make_func_ip_proto(const void *key
)
275 MAKE_BUCKET(b
, p
, ip_proto
);
276 p
->proto
= CASTKEY(uint8_t);
281 free_func_simple(struct bucket
*b _unused_
)
286 /* ---------------------------------------------------------------------------
287 * format_func collection (ordered by struct)
291 format_cols_host(struct str
*buf
)
293 /* FIXME: don't clobber parts of the query string
294 * specifically "full" and "start"
295 * when setting sort direction
301 " <th>Hostname</th>\n");
302 if (hosts_db_show_macs
) str_append(buf
,
303 " <th>MAC Address</th>\n");
305 " <th><a href=\"?sort=in\">In</a></th>\n"
306 " <th><a href=\"?sort=out\">Out</a></th>\n"
307 " <th><a href=\"?sort=total\">Total</a></th>\n");
308 if (opt_want_lastseen
) str_append(buf
,
309 " <th><a href=\"?sort=lastseen\">Last seen</a></th>\n");
315 format_row_host(struct str
*buf
, const struct bucket
*b
)
317 const char *ip
= addr_to_str(&(b
->u
.host
.addr
));
321 " <td><a href=\"./%s/\">%s</a></td>\n"
324 (b
->u
.host
.dns
== NULL
) ? "" : b
->u
.host
.dns
);
326 if (hosts_db_show_macs
)
328 " <td><tt>%x:%x:%x:%x:%x:%x</tt></td>\n",
329 b
->u
.host
.mac_addr
[0],
330 b
->u
.host
.mac_addr
[1],
331 b
->u
.host
.mac_addr
[2],
332 b
->u
.host
.mac_addr
[3],
333 b
->u
.host
.mac_addr
[4],
334 b
->u
.host
.mac_addr
[5]);
337 " <td class=\"num\">%'qu</td>\n"
338 " <td class=\"num\">%'qu</td>\n"
339 " <td class=\"num\">%'qu</td>\n",
344 if (opt_want_lastseen
) {
345 int64_t last
= b
->u
.host
.last_seen_mono
;
346 int64_t now
= (int64_t)now_mono();
347 struct str
*last_str
= NULL
;
349 if ((now
>= last
) && (last
!= 0))
350 last_str
= length_of_time(now
- last
);
352 str_append(buf
, " <td class=\"num\">");
353 if (last_str
== NULL
) {
355 str_append(buf
, "(never)");
357 str_appendf(buf
, "(clock error: last = %qd, now = %qu)",
361 str_appendstr(buf
, last_str
);
364 str_append(buf
, "</td>");
367 str_appendf(buf
, "</tr>\n");
369 /* Only resolve hosts "on demand" */
370 if (b
->u
.host
.dns
== NULL
)
371 dns_queue(&(b
->u
.host
.addr
));
375 format_cols_port_tcp(struct str
*buf
)
381 " <th>Service</td>\n"
391 format_row_port_tcp(struct str
*buf
, const struct bucket
*b
)
393 const struct port_tcp
*p
= &(b
->u
.port_tcp
);
397 " <td class=\"num\">%u</td>\n"
399 " <td class=\"num\">%'qu</td>\n"
400 " <td class=\"num\">%'qu</td>\n"
401 " <td class=\"num\">%'qu</td>\n"
402 " <td class=\"num\">%'qu</td>\n"
414 format_cols_port_udp(struct str
*buf
)
420 " <th>Service</td>\n"
429 format_row_port_udp(struct str
*buf
, const struct bucket
*b
)
431 const struct port_udp
*p
= &(b
->u
.port_udp
);
435 " <td class=\"num\">%u</td>\n"
437 " <td class=\"num\">%'qu</td>\n"
438 " <td class=\"num\">%'qu</td>\n"
439 " <td class=\"num\">%'qu</td>\n"
450 format_cols_ip_proto(struct str
*buf
)
456 " <th>Protocol</td>\n"
465 format_row_ip_proto(struct str
*buf
, const struct bucket
*b
)
467 const struct ip_proto
*p
= &(b
->u
.ip_proto
);
471 " <td class=\"num\">%u</td>\n"
473 " <td class=\"num\">%'qu</td>\n"
474 " <td class=\"num\">%'qu</td>\n"
475 " <td class=\"num\">%'qu</td>\n"
485 /* ---------------------------------------------------------------------------
486 * Initialise a hashtable.
488 static struct hashtable
*
489 hashtable_make(const uint8_t bits
,
490 const unsigned int count_max
,
491 const unsigned int count_keep
,
492 hash_func_t
*hash_func
,
493 free_func_t
*free_func
,
494 key_func_t
*key_func
,
495 find_func_t
*find_func
,
496 make_func_t
*make_func
,
497 format_cols_func_t
*format_cols_func
,
498 format_row_func_t
*format_row_func
)
500 struct hashtable
*hash
;
503 hash
= xmalloc(sizeof(*hash
));
505 hash
->count_max
= count_max
;
506 hash
->count_keep
= count_keep
;
507 hash
->size
= 1U << bits
;
508 hash
->mask
= hash
->size
- 1;
509 hash
->coeff
= coprime(hash
->size
);
510 hash
->hash_func
= hash_func
;
511 hash
->free_func
= free_func
;
512 hash
->key_func
= key_func
;
513 hash
->find_func
= find_func
;
514 hash
->make_func
= make_func
;
515 hash
->format_cols_func
= format_cols_func
;
516 hash
->format_row_func
= format_row_func
;
518 hash
->table
= xcalloc(hash
->size
, sizeof(*hash
->table
));
519 memset(&(hash
->stats
), 0, sizeof(hash
->stats
));
523 /* ---------------------------------------------------------------------------
524 * Initialise global hosts_db.
529 assert(hosts_db
== NULL
);
530 hosts_db
= hashtable_make(HOST_BITS
, opt_hosts_max
, opt_hosts_keep
,
531 hash_func_host
, free_func_host
, key_func_host
, find_func_host
,
532 make_func_host
, format_cols_host
, format_row_host
);
536 hashtable_rehash(struct hashtable
*h
, const uint8_t bits
)
538 struct bucket
**old_table
, **new_table
;
539 uint32_t i
, old_size
;
545 old_table
= h
->table
;
548 h
->size
= 1U << bits
;
549 h
->mask
= h
->size
- 1;
550 h
->coeff
= coprime(h
->size
);
551 new_table
= xcalloc(h
->size
, sizeof(*new_table
));
553 for (i
=0; i
<old_size
; i
++) {
554 struct bucket
*next
, *b
= old_table
[i
];
556 uint32_t pos
= h
->hash_func(h
, h
->key_func(b
)) & h
->mask
;
558 b
->next
= new_table
[pos
];
565 h
->table
= new_table
;
569 hashtable_insert(struct hashtable
*h
, struct bucket
*b
)
574 assert(b
->next
== NULL
);
576 /* Rehash on 80% occupancy */
577 if ((h
->count
> h
->size
) ||
578 ((h
->size
- h
->count
) < h
->size
/ 5))
579 hashtable_rehash(h
, h
->bits
+1);
581 pos
= h
->hash_func(h
, h
->key_func(b
)) & h
->mask
;
582 if (h
->table
[pos
] == NULL
)
585 /* Insert at top of chain. */
586 b
->next
= h
->table
[pos
];
593 /* Return bucket matching key, or NULL if no such entry. */
594 static struct bucket
*
595 hashtable_search(struct hashtable
*h
, const void *key
)
601 pos
= h
->hash_func(h
, key
) & h
->mask
;
604 if (h
->find_func(b
, key
))
612 typedef enum { NO_REDUCE
= 0, ALLOW_REDUCE
= 1 } reduce_bool
;
613 /* Search for a key. If it's not there, make and insert a bucket for it. */
614 static struct bucket
*
615 hashtable_find_or_insert(struct hashtable
*h
, const void *key
,
616 const reduce_bool allow_reduce
)
618 struct bucket
*b
= hashtable_search(h
, key
);
621 /* Not found, so insert after checking occupancy. */
622 if (allow_reduce
&& (h
->count
>= h
->count_max
))
624 b
= h
->make_func(key
);
625 hashtable_insert(h
, b
);
631 * Frees the hashtable and the buckets. The contents are assumed to be
632 * "simple" -- i.e. no "destructor" action is required beyond simply freeing
636 hashtable_free(struct hashtable
*h
)
642 for (i
=0; i
<h
->size
; i
++) {
643 struct bucket
*tmp
, *b
= h
->table
[i
];
655 /* ---------------------------------------------------------------------------
656 * Return existing host or insert a new one.
659 host_get(const struct addr
*const a
)
661 return (hashtable_find_or_insert(hosts_db
, a
, NO_REDUCE
));
664 /* ---------------------------------------------------------------------------
665 * Find host, returns NULL if not in DB.
668 host_find(const struct addr
*const a
)
670 return (hashtable_search(hosts_db
, a
));
673 /* ---------------------------------------------------------------------------
674 * Find host, returns NULL if not in DB.
676 static struct bucket
*
677 host_search(const char *ipstr
)
680 struct addrinfo hints
, *ai
;
682 memset(&hints
, 0, sizeof(hints
));
683 hints
.ai_family
= AF_UNSPEC
;
684 hints
.ai_flags
= AI_NUMERICHOST
;
686 if (getaddrinfo(ipstr
, NULL
, &hints
, &ai
))
687 return (NULL
); /* invalid addr */
689 if (ai
->ai_family
== AF_INET
) {
691 a
.ip
.v4
= ((const struct sockaddr_in
*)ai
->ai_addr
)->sin_addr
.s_addr
;
693 else if (ai
->ai_family
== AF_INET6
) {
696 ((struct sockaddr_in6
*)ai
->ai_addr
)->sin6_addr
.s6_addr
,
700 return (NULL
); /* unknown family */
704 verbosef("search(%s) turned into %s", ipstr
, addr_to_str(&a
));
705 return (hashtable_search(hosts_db
, &a
));
708 /* ---------------------------------------------------------------------------
709 * Reduce a hashtable to the top <keep> entries.
712 hashtable_reduce(struct hashtable
*ht
)
714 uint32_t i
, pos
, rmd
;
715 const struct bucket
**table
;
718 assert(ht
->count_keep
< ht
->count
);
720 /* Fill table with pointers to buckets in hashtable. */
721 table
= xcalloc(ht
->count
, sizeof(*table
));
722 for (pos
=0, i
=0; i
<ht
->size
; i
++) {
723 struct bucket
*b
= ht
->table
[i
];
729 assert(pos
== ht
->count
);
730 qsort_buckets(table
, ht
->count
, 0, ht
->count_keep
, TOTAL
);
731 cutoff
= table
[ht
->count_keep
]->total
;
734 /* Remove all elements with total <= cutoff. */
736 for (i
=0; i
<ht
->size
; i
++) {
737 struct bucket
*last
= NULL
, *next
, *b
= ht
->table
[i
];
740 if (b
->total
<= cutoff
) {
741 /* Remove this one. */
756 verbosef("hashtable_reduce: removed %u buckets, left %u",
758 hashtable_rehash(ht
, ht
->bits
); /* is this needed? */
761 /* Reduce hosts_db if needed. */
762 void hosts_db_reduce(void)
764 if (hosts_db
->count
>= hosts_db
->count_max
)
765 hashtable_reduce(hosts_db
);
768 /* ---------------------------------------------------------------------------
769 * Reset hosts_db to empty.
776 for (i
=0; i
<hosts_db
->size
; i
++) {
777 struct bucket
*next
, *b
= hosts_db
->table
[i
];
780 hosts_db
->free_func(b
);
784 hosts_db
->table
[i
] = NULL
;
786 verbosef("hosts_db reset to empty, freed %u hosts", hosts_db
->count
);
790 /* ---------------------------------------------------------------------------
791 * Deallocate hosts_db.
793 void hosts_db_free(void)
797 assert(hosts_db
!= NULL
);
798 for (i
=0; i
<hosts_db
->size
; i
++) {
799 struct bucket
*tmp
, *b
= hosts_db
->table
[i
];
803 hosts_db
->free_func(tmp
);
807 free(hosts_db
->table
);
812 /* ---------------------------------------------------------------------------
813 * Find or create a port_tcp inside a host.
816 host_get_port_tcp(struct bucket
*host
, const uint16_t port
)
818 struct host
*h
= &host
->u
.host
;
819 if (h
->ports_tcp
== NULL
)
820 h
->ports_tcp
= hashtable_make(PORT_BITS
, opt_ports_max
, opt_ports_keep
,
821 hash_func_short
, free_func_simple
, key_func_port_tcp
,
822 find_func_port_tcp
, make_func_port_tcp
,
823 format_cols_port_tcp
, format_row_port_tcp
);
824 return (hashtable_find_or_insert(h
->ports_tcp
, &port
, ALLOW_REDUCE
));
828 host_get_port_tcp_remote(struct bucket
*host
, const uint16_t port
)
830 struct host
*h
= &host
->u
.host
;
831 if (h
->ports_tcp_remote
== NULL
)
832 h
->ports_tcp_remote
= hashtable_make(
833 PORT_BITS
, opt_ports_max
, opt_ports_keep
, hash_func_short
,
834 free_func_simple
, key_func_port_tcp
, find_func_port_tcp
,
835 make_func_port_tcp
, format_cols_port_tcp
, format_row_port_tcp
);
836 return (hashtable_find_or_insert(h
->ports_tcp_remote
, &port
, ALLOW_REDUCE
));
839 /* ---------------------------------------------------------------------------
840 * Find or create a port_udp inside a host.
843 host_get_port_udp(struct bucket
*host
, const uint16_t port
)
845 struct host
*h
= &host
->u
.host
;
846 if (h
->ports_udp
== NULL
)
847 h
->ports_udp
= hashtable_make(PORT_BITS
, opt_ports_max
, opt_ports_keep
,
848 hash_func_short
, free_func_simple
, key_func_port_udp
,
849 find_func_port_udp
, make_func_port_udp
,
850 format_cols_port_udp
, format_row_port_udp
);
851 return (hashtable_find_or_insert(h
->ports_udp
, &port
, ALLOW_REDUCE
));
855 host_get_port_udp_remote(struct bucket
*host
, const uint16_t port
)
857 struct host
*h
= &host
->u
.host
;
858 if (h
->ports_udp_remote
== NULL
)
859 h
->ports_udp_remote
= hashtable_make(
860 PORT_BITS
, opt_ports_max
, opt_ports_keep
, hash_func_short
,
861 free_func_simple
, key_func_port_udp
, find_func_port_udp
,
862 make_func_port_udp
, format_cols_port_udp
, format_row_port_udp
);
863 return (hashtable_find_or_insert(h
->ports_udp_remote
, &port
, ALLOW_REDUCE
));
866 /* ---------------------------------------------------------------------------
867 * Find or create an ip_proto inside a host.
870 host_get_ip_proto(struct bucket
*host
, const uint8_t proto
)
872 struct host
*h
= &host
->u
.host
;
873 static const unsigned int PROTOS_MAX
= 512, PROTOS_KEEP
= 256;
875 if (h
->ip_protos
== NULL
)
876 h
->ip_protos
= hashtable_make(PROTO_BITS
, PROTOS_MAX
, PROTOS_KEEP
,
877 hash_func_byte
, free_func_simple
, key_func_ip_proto
,
878 find_func_ip_proto
, make_func_ip_proto
,
879 format_cols_ip_proto
, format_row_ip_proto
);
880 return (hashtable_find_or_insert(h
->ip_protos
, &proto
, ALLOW_REDUCE
));
883 static struct str
*html_hosts_main(const char *qs
);
884 static struct str
*html_hosts_detail(const char *ip
);
886 /* ---------------------------------------------------------------------------
887 * Web interface: delegate the /hosts/ space.
890 html_hosts(const char *uri
, const char *query
)
892 unsigned int i
, num_elems
;
893 char **elem
= split('/', uri
, &num_elems
);
894 struct str
*buf
= NULL
;
896 assert(num_elems
>= 1);
897 assert(strcmp(elem
[0], "hosts") == 0);
901 buf
= html_hosts_main(query
);
902 else if (num_elems
== 2)
903 /* /hosts/<IP of host>/ */
904 buf
= html_hosts_detail(elem
[1]);
906 for (i
=0; i
<num_elems
; i
++)
909 return (buf
); /* FIXME: a NULL here becomes 404 Not Found, we might want
910 other codes to be possible */
913 /* ---------------------------------------------------------------------------
914 * Get an array of pointers to all the buckets in the hashtable,
915 * or NULL if the hashtable is NULL or empty.
916 * The returned pointer should be free'd by the caller.
918 const struct bucket
**
919 hashtable_list_buckets(struct hashtable
*ht
)
921 const struct bucket
**table
;
924 if ((ht
== NULL
) || (ht
->count
== 0)) {
928 /* Fill table with pointers to buckets in hashtable. */
929 table
= xcalloc(ht
->count
, sizeof(*table
));
930 for (pos
=0, i
=0; i
<ht
->size
; i
++) {
931 struct bucket
*b
= ht
->table
[i
];
937 assert(pos
== ht
->count
);
941 typedef void (hashtable_foreach_func_t
)(const struct bucket
*, const void *);
943 /* ---------------------------------------------------------------------------
944 * Loop over all buckets in the given hashtable, calling the supplied function
945 * with each bucket and the supplied user_data.
948 hashtable_foreach(struct hashtable
*ht
,
949 hashtable_foreach_func_t
*hashtable_foreach_func
,
950 const void *user_data
)
952 const struct bucket
**table
;
955 table
= hashtable_list_buckets(ht
);
959 for (i
= 0; i
<ht
->count
; i
++) {
960 const struct bucket
*b
= table
[i
];
961 (*hashtable_foreach_func
)(b
, user_data
);
966 /* ---------------------------------------------------------------------------
967 * Format hashtable into HTML.
970 format_table(struct str
*buf
, struct hashtable
*ht
, unsigned int start
,
971 const enum sort_dir sort
, const int full
)
973 const struct bucket
**table
;
977 table
= hashtable_list_buckets(ht
);
980 str_append(buf
, "<p>The table is empty.</p>\n");
985 /* full report overrides start and end */
989 end
= MIN(ht
->count
, (uint32_t)start
+MAX_ENTRIES
);
991 str_appendf(buf
, "(%u-%u of %u)<br>\n", start
+1, end
, ht
->count
);
992 qsort_buckets(table
, ht
->count
, start
, end
, sort
);
993 ht
->format_cols_func(buf
);
995 for (i
=start
; i
<end
; i
++) {
996 ht
->format_row_func(buf
, table
[i
]);
997 alt
= !alt
; /* alternate class for table rows */
1000 str_append(buf
, "</table>\n");
1003 /* ---------------------------------------------------------------------------
1004 * Web interface: sorted table of hosts.
1007 html_hosts_main(const char *qs
)
1009 struct str
*buf
= str_make();
1010 char *qs_start
, *qs_sort
, *qs_full
, *ep
;
1011 const char *sortstr
;
1012 int start
, full
= 0;
1015 /* parse query string */
1016 qs_start
= qs_get(qs
, "start");
1017 qs_sort
= qs_get(qs
, "sort");
1018 qs_full
= qs_get(qs
, "full");
1019 if (qs_full
!= NULL
) {
1025 if (qs_sort
== NULL
) sort
= TOTAL
;
1026 else if (strcmp(qs_sort
, "total") == 0) sort
= TOTAL
;
1027 else if (strcmp(qs_sort
, "in") == 0) sort
= IN
;
1028 else if (strcmp(qs_sort
, "out") == 0) sort
= OUT
;
1029 else if (strcmp(qs_sort
, "lastseen") == 0) sort
= LASTSEEN
;
1031 str_append(buf
, "Error: invalid value for \"sort\".\n");
1036 if (qs_start
== NULL
)
1039 start
= (int)strtoul(qs_start
, &ep
, 10);
1041 str_append(buf
, "Error: \"start\" is not a number.\n");
1044 if ((errno
== ERANGE
) ||
1045 (start
< 0) || (start
>= (int)hosts_db
->count
)) {
1046 str_append(buf
, "Error: \"start\" is out of bounds.\n");
1051 #define PREV "<<< prev page"
1052 #define NEXT "next page >>>"
1053 #define FULL "full table"
1055 html_open(buf
, "Hosts", /*path_depth=*/1, /*want_graph_js=*/0);
1056 format_table(buf
, hosts_db
, start
, sort
, full
);
1058 /* <prev | full | stats | next> */
1060 if (sortstr
== NULL
) sortstr
= "total";
1062 int prev
= start
- MAX_ENTRIES
;
1065 str_appendf(buf
, "<a href=\"?start=%d&sort=%s\">" PREV
"</a>",
1068 str_append(buf
, PREV
);
1071 str_append(buf
, " | " FULL
);
1073 str_appendf(buf
, " | <a href=\"?full=yes&sort=%s\">" FULL
"</a>",
1076 if (start
+MAX_ENTRIES
< (int)hosts_db
->count
)
1077 str_appendf(buf
, " | <a href=\"?start=%d&sort=%s\">" NEXT
"</a>",
1078 start
+MAX_ENTRIES
, sortstr
);
1080 str_append(buf
, " | " NEXT
);
1082 str_append(buf
, "<br>\n");
1086 if (qs_start
!= NULL
) free(qs_start
);
1087 if (qs_sort
!= NULL
) free(qs_sort
);
1094 /* ---------------------------------------------------------------------------
1095 * Web interface: detailed view of a single host.
1097 static struct str
*html_hosts_detail(const char *ip
) {
1099 struct str
*buf
, *ls_len
;
1101 const char *canonical
;
1102 time_t last_seen_real
;
1104 h
= host_search(ip
);
1106 return (NULL
); /* no such host */
1108 canonical
= addr_to_str(&(h
->u
.host
.addr
));
1112 html_open(buf
, ip
, /*path_depth=*/2, /*want_graph_js=*/0);
1113 if (strcmp(ip
, canonical
) != 0)
1114 str_appendf(buf
, "(canonically <b>%s</b>)\n", canonical
);
1117 "<b>Hostname:</b> %s<br>\n",
1118 (h
->u
.host
.dns
== NULL
)?"(resolving...)":h
->u
.host
.dns
);
1120 /* Resolve host "on demand" */
1121 if (h
->u
.host
.dns
== NULL
)
1122 dns_queue(&(h
->u
.host
.addr
));
1124 if (hosts_db_show_macs
)
1126 "<b>MAC Address:</b> "
1127 "<tt>%x:%x:%x:%x:%x:%x</tt><br>\n",
1128 h
->u
.host
.mac_addr
[0],
1129 h
->u
.host
.mac_addr
[1],
1130 h
->u
.host
.mac_addr
[2],
1131 h
->u
.host
.mac_addr
[3],
1132 h
->u
.host
.mac_addr
[4],
1133 h
->u
.host
.mac_addr
[5]);
1138 "<b>Last seen:</b> ");
1140 if (h
->u
.host
.last_seen_mono
== 0) {
1141 str_append(buf
, "(never)");
1143 last_seen_real
= mono_to_real(h
->u
.host
.last_seen_mono
);
1144 if (strftime(ls_when
, sizeof(ls_when
),
1145 "%Y-%m-%d %H:%M:%S %Z%z", localtime(&last_seen_real
)) != 0)
1146 str_append(buf
, ls_when
);
1148 if (h
->u
.host
.last_seen_mono
<= now_mono()) {
1150 length_of_time((int64_t)now_mono() - h
->u
.host
.last_seen_mono
);
1151 str_append(buf
, " (");
1152 str_appendstr(buf
, ls_len
);
1154 str_append(buf
, " ago)");
1156 str_appendf(buf
, " (in the future, possible clock problem, "
1157 "last = %qd, now = %qu)",
1158 (qd
)h
->u
.host
.last_seen_mono
,
1166 " <b>In:</b> %'qu<br>\n"
1167 " <b>Out:</b> %'qu<br>\n"
1168 " <b>Total:</b> %'qu<br>\n"
1174 str_append(buf
, "<h3>TCP ports on this host</h3>\n");
1175 format_table(buf
, h
->u
.host
.ports_tcp
, 0,TOTAL
,0);
1177 str_append(buf
, "<h3>TCP ports on remote hosts</h3>\n");
1178 format_table(buf
, h
->u
.host
.ports_tcp_remote
, 0,TOTAL
,0);
1180 str_append(buf
, "<h3>UDP ports on this host</h3>\n");
1181 format_table(buf
, h
->u
.host
.ports_udp
, 0,TOTAL
,0);
1183 str_append(buf
, "<h3>UDP ports on remote hosts</h3>\n");
1184 format_table(buf
, h
->u
.host
.ports_udp_remote
, 0,TOTAL
,0);
1186 str_append(buf
, "<h3>IP protocols</h3>\n");
1187 format_table(buf
, h
->u
.host
.ip_protos
, 0,TOTAL
,0);
1189 str_append(buf
, "<br>\n");
1194 /* ---------------------------------------------------------------------------
1195 * Database import and export code:
1196 * Initially written and contributed by Ben Stewart.
1197 * copyright (c) 2007-2014 Ben Stewart, Emil Mikulic.
1199 static int hosts_db_export_ip(const struct hashtable
*h
, const int fd
);
1200 static int hosts_db_export_tcp(const char magic
, const struct hashtable
*h
,
1202 static int hosts_db_export_udp(const char magic
, const struct hashtable
*h
,
1206 export_proto_ip
= 'P',
1207 export_proto_tcp
= 'T',
1208 export_proto_tcp_remote
= 't',
1209 export_proto_udp
= 'U',
1210 export_proto_udp_remote
= 'u';
1212 static const unsigned char
1213 export_tag_host_ver1
[] = {'H', 'S', 'T', 0x01},
1214 export_tag_host_ver2
[] = {'H', 'S', 'T', 0x02},
1215 export_tag_host_ver3
[] = {'H', 'S', 'T', 0x03},
1216 export_tag_host_ver4
[] = {'H', 'S', 'T', 0x04};
1218 static void text_metrics_counter(struct str
*buf
, const char *metric
, const char *type
, const char *help
);
1219 static void text_metrics_format_host(const struct bucket
*b
, const void *user_data
);
1221 /* ---------------------------------------------------------------------------
1222 * Web interface: export stats in Prometheus text format on /metrics
1227 struct str
*buf
= str_make();
1229 text_metrics_counter(buf
,
1232 "Total number of network bytes by host and direction.");
1233 hashtable_foreach(hosts_db
, &text_metrics_format_host
, (void *)buf
);
1239 text_metrics_counter(struct str
*buf
,
1244 str_appendf(buf
, "# HELP %s %s\n", metric
, help
);
1245 str_appendf(buf
, "# TYPE %s %s\n", metric
, type
);
1249 text_metrics_format_host_key(struct str
*buf
, const struct bucket
*b
) {
1250 const char *ip
= addr_to_str(&(b
->u
.host
.addr
));
1253 "host_bytes_total{interface=\"%s\",ip=\"%s\"",
1254 title_interfaces
, ip
);
1256 if (hosts_db_show_macs
)
1257 str_appendf(buf
, ",mac=\"%x:%x:%x:%x:%x:%x\"",
1258 b
->u
.host
.mac_addr
[0],
1259 b
->u
.host
.mac_addr
[1],
1260 b
->u
.host
.mac_addr
[2],
1261 b
->u
.host
.mac_addr
[3],
1262 b
->u
.host
.mac_addr
[4],
1263 b
->u
.host
.mac_addr
[5]);
1267 text_metrics_format_host(const struct bucket
*b
,
1268 const void *user_data
)
1270 struct str
*buf
= (struct str
*)user_data
;
1272 text_metrics_format_host_key(buf
, b
);
1273 str_appendf(buf
, ",dir=\"in\"} %qu\n", (qu
)b
->in
);
1275 text_metrics_format_host_key(buf
, b
);
1276 str_appendf(buf
, ",dir=\"out\"} %qu\n", (qu
)b
->out
);
1279 /* ---------------------------------------------------------------------------
1280 * Load a host's ip_proto table from a file.
1281 * Returns 0 on failure, 1 on success.
1284 hosts_db_import_ip(const int fd
, struct bucket
*host
)
1288 if (!expect8(fd
, export_proto_ip
)) return 0;
1289 if (!read8(fd
, &count
)) return 0;
1291 for (i
=0; i
<count
; i
++) {
1296 if (!read8(fd
, &proto
)) return 0;
1297 if (!read64(fd
, &in
)) return 0;
1298 if (!read64(fd
, &out
)) return 0;
1301 b
= host_get_ip_proto(host
, proto
);
1304 b
->total
= in
+ out
;
1305 assert(b
->u
.ip_proto
.proto
== proto
); /* should be done by make fn */
1310 /* ---------------------------------------------------------------------------
1311 * Load a host's port_tcp{,_remote} table from a file.
1312 * Returns 0 on failure, 1 on success.
1314 static int hosts_db_import_tcp(const int fd
, const char magic
,
1315 struct bucket
*host
,
1316 struct bucket
*(get_port_fn
)(struct bucket
*host
,
1320 if (!expect8(fd
, magic
)) return 0;
1321 if (!read16(fd
, &count
)) return 0;
1323 for (i
=0; i
<count
; i
++) {
1326 uint64_t in
, out
, syn
;
1328 if (!read16(fd
, &port
)) return 0;
1329 if (!read64(fd
, &syn
)) return 0;
1330 if (!read64(fd
, &in
)) return 0;
1331 if (!read64(fd
, &out
)) return 0;
1334 b
= get_port_fn(host
, port
);
1337 b
->total
= in
+ out
;
1338 assert(b
->u
.port_tcp
.port
== port
); /* done by make_func_port_tcp */
1339 b
->u
.port_tcp
.syn
= syn
;
1344 /* ---------------------------------------------------------------------------
1345 * Load a host's port_tcp table from a file.
1346 * Returns 0 on failure, 1 on success.
1348 static int hosts_db_import_udp(const int fd
, const char magic
,
1349 struct bucket
*host
,
1350 struct bucket
*(get_port_fn
)(struct bucket
*host
,
1354 if (!expect8(fd
, magic
)) return 0;
1355 if (!read16(fd
, &count
)) return 0;
1357 for (i
=0; i
<count
; i
++) {
1362 if (!read16(fd
, &port
)) return 0;
1363 if (!read64(fd
, &in
)) return 0;
1364 if (!read64(fd
, &out
)) return 0;
1367 b
= get_port_fn(host
, port
);
1370 b
->total
= in
+ out
;
1371 assert(b
->u
.port_udp
.port
== port
); /* done by make_func */
1376 /* ---------------------------------------------------------------------------
1377 * Load all hosts from a file.
1378 * Returns 0 on failure, 1 on success.
1381 hosts_db_import_host(const int fd
)
1383 struct bucket
*host
;
1385 uint8_t hostname_len
;
1387 unsigned int pos
= xtell(fd
);
1391 if (!readn(fd
, hdr
, sizeof(hdr
))) return 0;
1392 if (memcmp(hdr
, export_tag_host_ver4
, sizeof(hdr
)) == 0)
1394 else if (memcmp(hdr
, export_tag_host_ver3
, sizeof(hdr
)) == 0)
1396 else if (memcmp(hdr
, export_tag_host_ver2
, sizeof(hdr
)) == 0)
1398 else if (memcmp(hdr
, export_tag_host_ver1
, sizeof(hdr
)) == 0)
1401 warnx("bad host header: %02x%02x%02x%02x",
1402 hdr
[0], hdr
[1], hdr
[2], hdr
[3]);
1407 if (!readaddr(fd
, &a
))
1410 assert((ver
== 1) || (ver
== 2));
1411 if (!readaddr_ipv4(fd
, &a
))
1414 verbosef("at file pos %u, importing host %s", pos
, addr_to_str(&a
));
1415 host
= host_get(&a
);
1416 assert(addr_equal(&(host
->u
.host
.addr
), &a
));
1420 if (!read64(fd
, &t
)) return 0;
1421 host
->u
.host
.last_seen_mono
= real_to_mono(t
);
1424 assert(sizeof(host
->u
.host
.mac_addr
) == 6);
1425 if (!readn(fd
, host
->u
.host
.mac_addr
, sizeof(host
->u
.host
.mac_addr
)))
1429 assert(host
->u
.host
.dns
== NULL
); /* make fn? */
1430 if (!read8(fd
, &hostname_len
)) return 0;
1431 if (hostname_len
> 0) {
1432 host
->u
.host
.dns
= xmalloc(hostname_len
+ 1);
1433 host
->u
.host
.dns
[0] = '\0';
1435 /* At this point, the hostname is attached to a host which is in our
1436 * hosts_db, so if we bail out due to an import error, this pointer
1437 * isn't lost and leaked, it can be cleaned up in hosts_db_{free,reset}
1440 if (!readn(fd
, host
->u
.host
.dns
, hostname_len
)) return 0;
1441 host
->u
.host
.dns
[hostname_len
] = '\0';
1444 if (!read64(fd
, &in
)) return 0;
1445 if (!read64(fd
, &out
)) return 0;
1449 host
->total
= in
+ out
;
1451 /* Host's port and proto subtables: */
1452 if (!hosts_db_import_ip(fd
, host
)) return 0;
1453 if (!hosts_db_import_tcp(fd
, export_proto_tcp
, host
, host_get_port_tcp
))
1455 if (!hosts_db_import_udp(fd
, export_proto_udp
, host
, host_get_port_udp
))
1459 if (!hosts_db_import_tcp(fd
, export_proto_tcp_remote
, host
,
1460 host_get_port_tcp_remote
))
1462 if (!hosts_db_import_udp(fd
, export_proto_udp_remote
, host
,
1463 host_get_port_udp_remote
))
1469 /* ---------------------------------------------------------------------------
1470 * Database Import: Grab hosts_db from a file provided by the caller.
1472 * This function will retrieve the data sans the header. We expect the caller
1473 * to have validated the header of the hosts_db segment, and left the file
1474 * sitting at the start of the data.
1476 int hosts_db_import(const int fd
)
1478 uint32_t host_count
, i
;
1480 if (!read32(fd
, &host_count
)) return 0;
1482 for (i
=0; i
<host_count
; i
++)
1483 if (!hosts_db_import_host(fd
)) return 0;
1488 /* ---------------------------------------------------------------------------
1489 * Database Export: Dump hosts_db into a file provided by the caller.
1490 * The caller is responsible for writing out export_tag_hosts_ver1 first.
1492 int hosts_db_export(const int fd
)
1497 if (!write32(fd
, hosts_db
->count
)) return 0;
1499 for (i
= 0; i
<hosts_db
->size
; i
++)
1500 for (b
= hosts_db
->table
[i
]; b
!= NULL
; b
= b
->next
) {
1501 /* For each host: */
1502 if (!writen(fd
, export_tag_host_ver4
, sizeof(export_tag_host_ver4
)))
1505 if (!writeaddr(fd
, &(b
->u
.host
.addr
)))
1508 if (!write64(fd
, (uint64_t)mono_to_real(b
->u
.host
.last_seen_mono
)))
1511 assert(sizeof(b
->u
.host
.mac_addr
) == 6);
1512 if (!writen(fd
, b
->u
.host
.mac_addr
, sizeof(b
->u
.host
.mac_addr
)))
1516 if (b
->u
.host
.dns
== NULL
) {
1517 if (!write8(fd
, 0)) return 0;
1519 int dnslen
= strlen(b
->u
.host
.dns
);
1522 warnx("found a very long hostname: \"%s\"\n"
1523 "wasn't expecting one longer than 255 chars (this one is %d)",
1524 b
->u
.host
.dns
, dnslen
);
1528 if (!write8(fd
, (uint8_t)dnslen
)) return 0;
1529 if (!writen(fd
, b
->u
.host
.dns
, dnslen
)) return 0;
1532 if (!write64(fd
, b
->in
)) return 0;
1533 if (!write64(fd
, b
->out
)) return 0;
1535 if (!hosts_db_export_ip(b
->u
.host
.ip_protos
, fd
)) return 0;
1536 if (!hosts_db_export_tcp(export_proto_tcp
, b
->u
.host
.ports_tcp
, fd
))
1538 if (!hosts_db_export_udp(export_proto_udp
, b
->u
.host
.ports_udp
, fd
))
1540 if (!hosts_db_export_tcp(export_proto_tcp_remote
,
1541 b
->u
.host
.ports_tcp_remote
, fd
))
1543 if (!hosts_db_export_udp(export_proto_udp_remote
,
1544 b
->u
.host
.ports_udp_remote
, fd
))
1550 /* ---------------------------------------------------------------------------
1551 * Dump the ip_proto table of a host.
1554 hosts_db_export_ip(const struct hashtable
*h
, const int fd
)
1556 uint32_t i
, written
= 0;
1560 if (!write8(fd
, export_proto_ip
)) return 0;
1562 /* If no data, write a IP Proto count of 0 and we're done. */
1564 if (!write8(fd
, 0)) return 0;
1568 assert(h
->count
< 256);
1569 if (!write8(fd
, (uint8_t)h
->count
)) return 0;
1571 for (i
= 0; i
<h
->size
; i
++)
1572 for (b
= h
->table
[i
]; b
!= NULL
; b
= b
->next
) {
1573 /* For each ip_proto bucket: */
1575 if (!write8(fd
, b
->u
.ip_proto
.proto
)) return 0;
1576 if (!write64(fd
, b
->in
)) return 0;
1577 if (!write64(fd
, b
->out
)) return 0;
1580 assert(written
== h
->count
);
1584 /* ---------------------------------------------------------------------------
1585 * Dump the port_tcp table of a host.
1588 hosts_db_export_tcp(const char magic
, const struct hashtable
*h
, const int fd
)
1591 uint32_t i
, written
= 0;
1594 if (!write8(fd
, magic
)) return 0;
1596 /* If no data, write a count of 0 and we're done. */
1598 if (!write16(fd
, 0)) return 0;
1602 assert(h
->count
< 65536);
1603 if (!write16(fd
, (uint16_t)h
->count
)) return 0;
1605 for (i
= 0; i
<h
->size
; i
++)
1606 for (b
= h
->table
[i
]; b
!= NULL
; b
= b
->next
) {
1607 if (!write16(fd
, b
->u
.port_tcp
.port
)) return 0;
1608 if (!write64(fd
, b
->u
.port_tcp
.syn
)) return 0;
1609 if (!write64(fd
, b
->in
)) return 0;
1610 if (!write64(fd
, b
->out
)) return 0;
1613 assert(written
== h
->count
);
1617 /* ---------------------------------------------------------------------------
1618 * Dump the port_udp table of a host.
1621 hosts_db_export_udp(const char magic
, const struct hashtable
*h
, const int fd
)
1624 uint32_t i
, written
= 0;
1627 if (!write8(fd
, magic
)) return 0;
1629 /* If no data, write a count of 0 and we're done. */
1631 if (!write16(fd
, 0)) return 0;
1635 assert(h
->count
< 65536);
1636 if (!write16(fd
, (uint16_t)h
->count
)) return 0;
1638 for (i
= 0; i
<h
->size
; i
++)
1639 for (b
= h
->table
[i
]; b
!= NULL
; b
= b
->next
) {
1640 if (!write16(fd
, b
->u
.port_udp
.port
)) return 0;
1641 if (!write64(fd
, b
->in
)) return 0;
1642 if (!write64(fd
, b
->out
)) return 0;
1645 assert(written
== h
->count
);
1649 /* vim:set ts=3 sw=3 tw=80 expandtab: */