+#!/bin/sh
+
+# Copyright 2013 MediaMobil Communication GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# This script converts a binary .db file into a .csv file.
+# The .db file was generated by darkstat with the --export option.
+# The .csv file shall be read by any spreadsheet application.
+SCRIPTNAME=$( basename $0)
+if test -z "$( type -P awk )" ; then
+ echo "${SCRIPTNAME}: missing AWK interpreter, at least not found in PATH"
+ echo "${SCRIPTNAME}: every POSIX compliant OS has one; add the location to PATH"
+ exit 1
+fi
+if test -z "$( type -P od )" ; then
+ echo "${SCRIPTNAME}: missing od file dump tool, at least not found in PATH"
+ echo "${SCRIPTNAME}: every POSIX compliant OS has one; add the location to PATH"
+ exit 1
+fi
+if test $# -ne 1; then
+ echo "${SCRIPTNAME}: missing parameter; need file name of .db file"
+ exit 1
+fi
+DBFILENAME=$1
+if test -r ${DBFILENAME}; then
+ echo ${SCRIPTNAME}: Found file ${DBFILENAME}
+else
+ echo ${SCRIPTNAME}: file ${DBFILENAME} does not exist
+ exit 1
+fi
+CSVFILENAME=${DBFILENAME%%.*}.csv
+echo ${SCRIPTNAME}: Writing output into ${CSVFILENAME}
+
+# The spec of the .db export format exists for different versions:
+# http://unix4lyfe.org/gitweb/darkstat/blob/0a152e51f5d9c1771308caa7135d363a722aee18:/export-format.txt
+# http://git.msquadrat.de/darkstat.git/blob_plain/master:/export-format.txt
+# http://phil.lavin.me.uk/downloads/parse.phps
+# Only file format version 1 is supported by us.
+# Obviously, darkstat itself distinguishes 3 different host format versions.
+# Only host format version 2 is supported by us.
+# The darkstat database file is converted from binary format
+# to ASCII by the standard Unix command od.
+
+# Some things don't work correctly yet.
+# Probably because there is no DNS server configured in our embedded device
+# that produces .db files within OpenWRT.
+# - host name contains nonsense at constant length 5
+# - "last seen" timing information contains always 0
+# - we read the graphics section of the file but ignore it
+
+# Let the od tool convert each binary byte into several textual formats.
+# The AWK script reads all variants and later picks the format it needs.
+od -Ad -v -tx1 -tu1 -ta -w1 < ${DBFILENAME} |
+awk '
+ NF==2 { addr = 0 + $1; hex[addr] = $2; next }
+ NF==1 && addr in dec { ascii[addr]=$1; next }
+ NF==1 && ! (addr in dec) { dec[addr]=$1; next }
+ # Now all variants of the bytes are available in certain arrays.
+ # The array indices cover the range 0 .. addr.
+
+ function read_bytes(array, address, count, retval, c) {
+ retval=""
+ for (c=0; c<count; c++)
+ retval = retval array[address+c]
+ return retval
+ }
+ function read_number(address, count, retval, c) {
+ retval=0
+ for (c=0; c<count; c++)
+ retval = retval*256 + dec[address+c]
+ return retval
+ }
+ function read_text(address, count, retval, c) {
+ retval=""
+ for (c=0; c<count; c++)
+ retval = retval ascii[address+c]
+ return retval
+ }
+ function quit(reason, terminate, retval) {
+ if (length(reason) > 0)
+ print reason
+ if (terminate != 0) {
+ # Any remaining bytes in the file shall be dumped.
+ for (i=ai; i<=addr; i++)
+ print i, hex[i], ascii[i]
+ exit(retval)
+ }
+ }
+ function readIPsection() {
+ ip_protos_data=read_bytes(ascii, ai, 1)
+ if (ip_protos_data != "P")
+ quit("expected ip_protos_data P, found " ip_protos_data, 1, 1)
+ ai += 1
+ ip_proto_count=read_number(ai, 1)
+ ai += 1
+ for (pi=0; pi<ip_proto_count; pi++) {
+ ip_proto_type=read_number(ai, 1)
+ ai += 1
+ IPprotos = IPprotos " " ip_proto_type
+ ip_proto_in += read_number(ai, 8)
+ ai += 8
+ ip_proto_out += read_number(ai, 8)
+ ai += 8
+ }
+ }
+ function readTCPsection() {
+ tcp_protos_data=read_bytes(ascii, ai, 1)
+ if (tcp_protos_data != "T")
+ quit("expected tcp_protos_data T, found " tcp_protos_data, 1, 1)
+ ai += 1
+ tcp_proto_count=read_number(ai, 2)
+ ai += 2
+ for (ti=0; ti<tcp_proto_count; ti++) {
+ tcp_proto_port=read_number(ai, 2)
+ ai += 2
+ TCPports = TCPports " " tcp_proto_port
+ tcp_proto_syn=read_number(ai, 8)
+ ai += 8
+ tcp_proto_in += read_number(ai, 8)
+ ai += 8
+ tcp_proto_out += read_number(ai, 8)
+ ai += 8
+ if (tcp_proto_port == 22) {
+ ssh_in += tcp_proto_in
+ ssh_out += tcp_proto_out
+ }
+ if (tcp_proto_port == 3389) {
+ rdp_in += tcp_proto_in
+ rdp_out += tcp_proto_out
+ }
+ }
+ }
+ function readUDPsection() {
+ udp_protos_data=read_bytes(ascii, ai, 1)
+ if (udp_protos_data != "U")
+ quit("expected udp_protos_data U, found " udp_protos_data, 1, 1)
+ ai += 1
+ udp_proto_count=read_number(ai, 2)
+ ai += 2
+ for (ui=0; ui<udp_proto_count; ui++) {
+ udp_proto_port=read_number(ai, 2)
+ ai += 2
+ UDPports = UDPports " " udp_proto_port
+ udp_proto_in += read_number(ai, 8)
+ ai += 8
+ udp_proto_out += read_number(ai, 8)
+ ai += 8
+ if (udp_proto_port == 22) {
+ ssh_in += udp_proto_in
+ ssh_out += udp_proto_out
+ }
+ if (udp_proto_port == 3389) {
+ rdp_in += udp_proto_in
+ rdp_out += udp_proto_out
+ }
+ }
+ }
+ function readGraphsection(interval) {
+ n_bars=read_number(ai++, 1)
+ i_bars=read_number(ai++, 1)
+ for (bi=0; bi<n_bars; bi++) {
+ graph_bytes_in=read_number(ai, 8)
+ ai += 8
+ graph_bytes_out=read_number(ai, 8)
+ ai += 8
+ }
+ }
+
+ END {
+ file_header=read_bytes(hex, 0, 4)
+ if (file_header != "da314159")
+ quit("input data is not an exported darkstat .db file, wrong header: " file_header, 1, 1)
+ section_header=read_bytes(hex, 4, 3)
+ if (section_header != "da4853")
+ quit("section header da4853 expected: " section_header, 1, 1)
+ db_version=read_bytes(hex, 7, 1)
+ if (db_version != "01")
+ quit("file format supported only in version 01", 1, 1)
+ host_count=read_number(8, 4)
+ ai=12
+ # Print a header into the .csv file.
+ printf("IP address;MAC address;host in bytes;host out bytes;IP protos;IP in bytes;IP out bytes;TCP port count;TCP in bytes;TCP out bytes;UDP port count;UDP in bytes;UDP out bytes;ssh in bytes;ssh out bytes;rdp in bytes;rdp out bytes;TCP ports;UDP ports\n")
+ for (hi=1; hi<=host_count; hi++) {
+ # Make sure all variables to be printed are initially empty.
+ ip_address=mac_address=""
+ host_bytes_in=host_bytes_out=ip_proto_in=ip_proto_out=tcp_proto_in=tcp_proto_out=udp_proto_in=udp_proto_out=ssh_in=ssh_out=rdp_in=rdp_out=0
+ IPprotos=TCPports=UDPports=""
+ tcp_proto_count=udp_proto_count=0
+ host_header=read_bytes(hex, ai, 3)
+ host_version=read_bytes(hex, ai+3, 1)
+ ai += 4
+ if (host_version == "02") {
+ ip_address=read_number(ai+0,1) "." read_number(ai+1,1) "." read_number(ai+2,1) "." read_number(ai+3,1)
+ ai += 4
+ if ((host_version+0) > 1) {
+ last_seen=read_number(ai, 4)
+ # This value is always 0 in our files.
+ ai += 4
+ }
+ mac_address=hex[ai+0] ":" hex[ai+1] ":" hex[ai+2] ":" hex[ai+3] ":" hex[ai+4] ":" hex[ai+5]
+ ai += 6
+ # Weird stuff: the host name should be read.
+ # But there are only 5 bytes of nonsense.
+ # The first byte should be the length counter, but it isnt.
+ # The last byte is in fact a 0 byte.
+ # Probably caused by the missing DNS server.
+ # ignore 5 bytes with nonsense
+ nonsense=read_text(ai, 5)
+ ai += 5
+ host_bytes_in=read_number(ai, 8)
+ ai += 8
+ host_bytes_out=read_number(ai, 8)
+ ai += 8
+ readIPsection()
+ readTCPsection()
+ readUDPsection()
+ } else {
+ quit("host format supported only in version 02: " host_version, 1, 1)
+ #address_familiy=read_bytes(hex, ai, 1)
+ #print "address familiy = " address_familiy
+ }
+ printf("\"%s\";\"%s\";%d;%d;%s;%d;%d;%d;%d;%d;%d;%d;%d;%d;%d;%d;%d;%s;%s\n",
+ ip_address, mac_address, host_bytes_in, host_bytes_out,
+ IPprotos, ip_proto_in, ip_proto_out,
+ tcp_proto_count, tcp_proto_in, tcp_proto_out,
+ udp_proto_count, udp_proto_in, udp_proto_out,
+ ssh_in, ssh_out, rdp_in, rdp_out,
+ TCPports, UDPports)
+ }
+ section_header=read_bytes(hex, ai, 3)
+ if (section_header != "da4752")
+ quit("section header da4752 expected: " section_header, 1, 1)
+ ai += 3
+ db_version=read_bytes(hex, ai, 1)
+ if (db_version != "01")
+ quit("file format supported only in version 01", 1, 1)
+ ai += 1
+ last_time=read_number(ai, 8)
+ ai += 8
+ readGraphsection("60 seconds")
+ readGraphsection("60 minutes")
+ readGraphsection("24 hours")
+ readGraphsection("31 days")
+ # The complete file has been parsed, no bytes should be left over.
+ # Terminate with return value 0 if the byte numbers match.
+ quit("", (addr != ai+1) ?0:1, addr != ai+1)
+ }
+ ' > ${CSVFILENAME}
projects / darkstat-debian / commitdiff