#!/usr/bin/perl -w
#2345678911234567892123456789312345678941234567895123456789612345678971234567898

# License: GPL v3
# Author: Marc MERLIN <marc_soft at merlins.org> 2010/08/11
# $Id: cacti_xpl 526 2012-04-15 19:24:09Z svnuser $

use strict;
use Date::Manip;
use Getopt::Long;

my $VERBOSE = 0;

my $LOGFILE = "/var/log/xpl.log";
$LOGFILE=$ENV{'XPLLOGFILE'} if ($ENV{'XPLLOGFILE'});
if ($LOGFILE eq "-")
{
    warn("reading from STDIN\n");
    $LOGFILE="<&STDIN";
}
# how many minutes between each data sample
my $SAMPLE_TIME = 2;

# global shared vars
my %data;
my @timeslot;

my $RRDTOOL = 0;
my $CACTI = 0;
my $CACTI_DUMP = 0;
my $CACTI_DUMP_HEADER;
my $VAR;

# whe adding fields here, go to Data Input Methods
# https://server/cacti/data_input.php?action=edit&id=14
# you can add fields more quickly by clicking add, submit, then back,
# change, and submit again.
# After adding fields, you have to add them to a Data Source
# https://server/cacti/data_sources.php?action=ds_edit&id=17

# This script will also convert the format of UV and Moisture readings
# as they are logged raw in the original file and the conversion table 
# may change over time, so it's better to log raw data and convert here.
# this is the painful and slow step. Cacti takes a while to add fields
# one per one (add field creates 'ds', which you then edit).

# This is the output I get from rfx-xpl after it's been parsed by
# http://marc.merlins.org/linux/scripts/parse_rfx-xpl
#
# BTHR918N,BTHR968 addr: 230, chan: 0 temp: 23.70 °C / 74.66 °F hum: 53 % comfort baro: 1014 hPa / 29.94 inHg, forecast: cloudy 
# RGR126,RGR682,RGR918 addr: 147, chan: 0 rain: total 1163 mm, 0 mm/hr, count 1 
# STR918,WGR918 addr: 232, chan: 0 wind: 13 ° NNE, speed 1.20 m/s / 2 kts average 1.40 m/s / 2 kts battery level 100% 
# STR918,WGR918 addr: 232, chan: 0 wind: 1 ° N, speed 2.00 m/s / 3 kts average 1.80 m/s / 3 kts battery level 100% 
# STR918,WGR918 addr: 232, chan: 0 wind: 23 ° NNE, speed 2.40 m/s / 4 kts average 1.40 m/s / 2 kts battery level 100% 
# STR918,WGR918 addr: 232, chan: 0 wind: 357 ° N, speed 2.80 m/s / 5 kts average 1.40 m/s / 2 kts battery level 100% 
# STR918,WGR918 addr: 232, chan: 0 wind: 37 ° NE, speed 1.40 m/s / 2 kts average 1.80 m/s / 3 kts battery level 100% 
# STR918,WGR918 addr: 232, chan: 0 wind: 5 ° N, speed 1.80 m/s / 3 kts average 1.40 m/s / 2 kts battery level 100%
# THGR918 addr: 225, chan: 1 temp: 17.40 °C / 63.32 °F hum: 70 % normal battery level 80% 


# when all done adding the fields, Turn On Data Source Debug Mode.
# and go to the end of this file

my @probes = ( 
	"fmr_temp", 
	"fmr_hum", 
	"fmr_pres", 
	"in_temp2", 
	"in_hum2", 
	"in_pres2", 
	"out_temp",
	"out_hum",
	"rain", 
	"rain2", 
	"winddir", 
	"windavgspd", 
	"windpeakspd", 
	"mbr_temp", 
	"mbr_hum", 
	"mbr_pres", 
	"spare4", 
	"spare5", 
	"spare6", 
	"spare7", 
	"spare8", 
	"spare9", 
	"spareabs1", 
	"spareabs2", 
	"spareabs3", 
);

# Used to detect when 
my %unused_probes = ( 
	"in_temp2" => 1, 
	"in_hum2" => 1, 
	"in_pres2" => 1, 
	"rain2" => 1, 
	"winddir" => 1, # shows undefined when speed is 0
	"spare4" => 1, 
	"spare5" => 1, 
	"spare6" => 1, 
	"spare7" => 1, 
	"spare8" => 1, 
	"spare9" => 1, 
	"spareabs1" => 1, 
	"spareabs2" => 1, 
	"spareabs3" => 1, 
);

sub reset_idx_hash
{
    my $hash;

    foreach my $probe (@probes)
    {
	$hash->{$probe} = "U";
    }
    return $hash;
}

my $label_to_idx = reset_idx_hash();
my $HASH_ELTS = keys(%{$label_to_idx});
#my $REC_LINES = $HASH_ELTS * 2;
my $REC_LINES = 100;

# Compat with other owfs scripts that use a different storage format.
my @template_labels = @probes;
my %label_to_idx = %{$label_to_idx};

my $template_str = join(":", @probes);


sub verbose
{
    my ($mesg, $level) = @_;
    $level = 1 if (not $level);

    warn("$mesg\n") if ($VERBOSE >= $level);
}

sub delta_mins
{
    $_ = DateCalc($_[0], $_[1]);
    #print "Got delta $_ from ".join("|", @_)."\n";

    return sprintf("%5.2f", Delta_Format($_, 1, "%mh"));
}

sub printable_date
{
    return UnixDate($_[0], "%Y/%m/%d %T");
}

sub printable_time
{
    return UnixDate($_[0], "%T");
}

sub date_min
{
    $_[0] =~ /\d\d:(\d\d):\d\d$/ or die "Couldn't get minute from $_\n";
    return $1;
}

sub epoch
{
    return UnixDate($_[0], "%s");
}


# load all the data samples in %data and the timeslots keys in @timeslot
# %date time indexes are converted from UTC to local time at read time
# and are stored in Date::Manip format (YYYMMDDHH:MM:SS)
sub load_file_data
{
    # fromdate and todate are given in localtime as per locale
    my ($FH, $fromdate, $todate) = @_;
    # convert them once to parsed format so that comparisons can be done as strings
    my ($parsed_fromdate, $parsed_todate) = (ParseDate($fromdate), ParseDate($todate));

    my $time = 0;
    my $process_min;
    my $new_sample = 0;
    # previous data record is used to see if we had a hole in data parsing
    my $datatime;
    my $prev_datatime;
    my $values;

    # reset to null since we can be called multiple times
    %data= ();
    @timeslot = ();

    verbose("load_file_data: between $parsed_fromdate and $parsed_todate");

    for (my $line=1; $_ = ${$FH}[$line-1] and $line <= $#{$FH} + 1; $line++)
    {
	my $date;
	# this will get inserted into $values hash ref once we know which one below
	my %new_data = ();

	next if (/^\s*$/);
	# 2010/04/18 07:44:46: X10 Sec: addr 234 (F8) DS10/90/SD90/Visonic Alert
	next if (/ X10 Sec: /);
	next if (/ X10: [A-Z]/);
	# 2010/04/18 07:44:46: process[1] cnt=4 buffer="$57$58$04$fb"
	next if (/ process\[/);
	# ignore some neighbour's readings
	# 2010/05/18 01:23:28: THR128,THx138 addr: 10, chan: 1 temp: 14.10 °C / 57.38 °F 
	next if (/ THR128,THx138 /);
	# 2010/07/26 05:21:41: BWR101,BWR102 addr: 0, chan: 0 weight: 15.20 kg, 33.44 lb 2.39:0.00 st-lb
	next if (/ BWR101,BWR102 /);

	# 2010/04/17 23:32:03: BTHR918N,BTHR968 addr: 230, chan: 0 temp: 23.70 °C / 74.66 °F hum: 53 % comfort baro: 1014 hPa / 29.94 inHg, forecast: cloudy 
	if (/^(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d): BTHR918N,BTHR968 addr: (\d+), chan: (\d+).* (\d+.\d+) ?°F hum: (\d+) % \w+ baro: \d+ hPa \/ (\d+.\d+) inHg/)
	{
	    my ($addr, $chan, $temp, $hum, $pres) = ($2, $3, $4, $5, $6);
	    $date = $1;
	    # if you have more than one, use add/chan to differentiate them
	    if ($addr eq '230' or $addr eq '42' or $addr eq '66' or $addr eq '246' or $addr eq '21')
	    {
		$new_data{'fmr_temp'} = $temp;
		$new_data{'fmr_hum'} = $hum;
		$new_data{'fmr_pres'} = $pres;
	    } 
	    elsif ($addr eq '95' or $addr eq '219' or $addr eq '244') 
	    {
		$new_data{'mbr_temp'} = $temp;
		$new_data{'mbr_hum'} = $hum;
		$new_data{'mbr_pres'} = $pres;
	    }
	}
	# 2010/04/17 23:32:03: THGR918 addr: 225, chan: 1 temp: 17.40 °C / 63.32 °F hum: 70 % normal battery level 80% 
	elsif (/^(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d): THGR918 addr: (\d+), chan: (\d+).* (\d+.\d+) ?°F hum: (\d+) %/)
	{
	    my ($addr, $chan, $temp, $hum) = ($2, $3, $4, $5);
	    $date = $1;
	    # if you have more than one, use add/chan to differentiate them
	    $new_data{'out_temp'} = $temp;
	    $new_data{'out_hum'} = $hum;
        }
	# 2010/04/17 23:32:03: RGR126,RGR682,RGR918 addr: 147, chan: 0 rain: total 1163 mm, 0 mm/hr, count 1 
	elsif (/^(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d): RGR126,RGR682,RGR918 addr: (\d+), chan: (\d+) rain: total (\d+) mm, (\d+) mm\/hr/)
	{
	    my ($addr, $chan, $rain) = ($2, $3, $5);
	    $date = $1;
	    # if you have more than one, use add/chan to differentiate them
	    $new_data{'rain'} = $rain;
        }
	# 2010/04/17 23:32:03: STR918,WGR918 addr: 232, chan: 0 wind: 5 ° N, speed 1.80 m/s / 3 kts average 1.40 m/s / 2 kts battery level 100%
	# 2010/08/10 14:00:29: STR918,WGR918 addr: 232, chan: 0 wind: 14° NNE, speed 2.20 m/s / 4 kts average 2.80 m/s / 5 kts battery level 100%
	elsif (/^(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d): STR918,WGR918 addr: (\d+), chan: (\d+) wind: (\d+) ?° .* (\d+) kts.* (\d+) kts/)
	{
	    my ($addr, $chan, $winddir, $windpeakspd, $windavgspd) = ($2, $3, $4, $5, $6);
	    $date = $1;
	    # My local wind is mostly from the north, so the values are often in the 
	    # 330-30 degree range, which causes jumps up and down the graph (not pretty).
	    # I therefore offset from 0 -> 360 to -180 -> 180 to avoid the jumps.

	    $winddir -= 180;

	    # if you have more than one, use add/chan to differentiate them
	    $new_data{'windavgspd'} = $windavgspd;
	    $new_data{'windpeakspd'} = $windpeakspd;
	    # ignore wind direction unless there is some wind
	    $new_data{'winddir'} = $winddir if ($windpeakspd >= 1);
        }
	else
	{
	    warn("Did not match line $_");
	    next;
	}

	my $read_min = date_min($date);
	#warn("Got read_min: $read_min\n");

	# we gather data one minute at a time on the minute. Any data missing will
	# not be part of this sample
	if (not defined $process_min or $read_min != $process_min)
	{
	    $prev_datatime = $datatime if (defined $datatime);
	    $datatime = $date;
	    if (defined $prev_datatime)
	    {
		my $sample_delta = delta_mins($prev_datatime, $datatime);
		if (defined $prev_datatime and $sample_delta > $SAMPLE_TIME * 2)
		{
		    warn "Warning data log time jumped by $sample_delta mn at $datatime\n";
		}
		else
		{
		    #warn "accepted delta of $sample_delta mn between $datatime and $prev_datatime\n";
		}

	    }
	    # save data from the last run, if any
	    if (defined $process_min)
	    {
		$data{$prev_datatime} = $values;
		push (@timeslot, $prev_datatime);
	    }

	    # (re)set variables to 'U'
	    $values = reset_idx_hash();
	    $process_min = $read_min;
	}
	
	foreach my $key (keys %new_data)
	{
	    my $value = $new_data{$key};
	    die "Got unknown index $key for $datatime / $key:$value" unless (defined $values->{$key});
	    $values->{$key} = $value;
	}
    }
    # save data we last gathered before EOF
    $data{$datatime} = $values;
    push (@timeslot, $datatime);
}


sub cacti_dump
{
    my @lines;
    my $lineblock = 100000;
    my $pass = 1;
    my $exit = 0;
    my $from = $_[0] ? $_[0] : 0;
    my $tail = $_[1] ? $_[1] : 0;

    if ($tail)
    {
	verbose("Will gather stats from $from working on $tail lines");
	open(TEMPS, "tail -n $tail $LOGFILE |");
    }
    else
    {
	verbose("Will gather stats for $from (reading the whole file)");
	open(TEMPS, $LOGFILE);
    }

    while (1)
    {
	my $i;

	@lines = ();
	for ($i = 0; $i < $lineblock; $i++)
	{
	    $_ = <TEMPS>;
	    if (not $_)
	    {
		$exit = 1;
		last;
	    }
	    push(@lines, $_);
	}
	warn("Read block of $i lines, now parsing (pass $pass)\n");
	load_file_data(\@lines, $from, "now");
	@lines = [];

	warn("Parsed block of $i lines, now dumping to STDOUT (pass $pass) from ".$timeslot[0]."\n");
	my $hash = reset_idx_hash();
	foreach my $timeslot (0 .. $#timeslot)
	{
	    my $time = $timeslot[$timeslot];
	    my $values = "";
	    my $foundvalues = 0;

	    # get probe list to output fields in the right order.
	    foreach my $probe (@probes)
	    {
		if (defined $data{$time}->{$probe})
		{
		    $foundvalues++;
		    $_ = $data{$time}->{$probe};
		    $values .= ":$_";
		}
	    }
	    if (not $values)
	    {
		warn "No values for ".epoch($time).", skipping...\n";
	    }
	    elsif ($values =~  /[A-TV-z]/)
	    {
		warn "Illegal characters in values for ".epoch($time).": $values, skipping...\n";
	    }
	    else
	    {
		print epoch($time)."$values\n";
	    }
	}
	last if ($exit);
	$pass++;
    }
    close(TEMPS);
}

sub find_last_defined_value
{
    my ($var) = @_;
    my $date;
    my $datestr;
    my $timeslotoffset = $#timeslot;
    my $value = "U";

    while (1)
    {
	$date = $timeslot[$timeslotoffset];
	$datestr = ParseDate($date);

	# how old samples can be before we reject them
	my $acceptable_time = DateCalc("now", "-".($SAMPLE_TIME*2 + 1)." minutes");
	#warn("read: $date and acceptable: $acceptable_time with string $str\n");
	if ($acceptable_time gt $datestr)
	{
	    warn "FAIL: $acceptable_time > $date while looking for $var\n" unless (defined $unused_probes{$var});
	    last;
	}
	$value = $data{$date}->{$var};
	#warn("Got $value for data{$date}->{$var}\n");
	if ($value eq "U")
	{
	    $timeslotoffset--;
	    if ($timeslotoffset < 0)
	    {
		warn "FAIL: did not find defined value for $var\n" unless (defined $unused_probes{$var});
		last;
	    }
	}
	else
	{
	    last;
	}
    }

    return ($value);
}

sub data_tail
{
    my ($var) = @_;
    my @lines;

    open(TEMPS, "tail -n $REC_LINES $LOGFILE |");
    @lines=<TEMPS>;
    close(TEMPS);
    load_file_data(\@lines, 0, "now");

    my $str = "";
    #warn("will look for $var\n");
    if ($var)
    {
	$str = find_last_defined_value($var);
    }
    else
    {
	# we don't need to order for cacti, but it's nice for debugging and reading
	foreach my $probe ( @probes )
	{
	    $str .= "$probe:".find_last_defined_value($probe)." ";
	}
    }

    print "$str\n";
}

sub usage
{
    print STDERR "$_[0]\n\n" if ($_[0]);
    print STDERR <<EOF;

    $0 [--cacti | --cacti-dump | --cacti-dump-header | --value label]
EOF
    exit;
}

GetOptions(
    "cacti" => \$CACTI,
    "cacti-dump" => \$CACTI_DUMP,
    "cacti-dump-header" => \$CACTI_DUMP_HEADER,
    "value:s" => \$VAR,
) or usage;

if ($CACTI)
{
    data_tail();
}
elsif ($VAR)
{
    my $idx = $label_to_idx{$VAR};

    usage("$VAR is not a known value\nValid options are: @template_labels") if (not defined $idx);
    data_tail($VAR);
}
elsif ($CACTI_DUMP)
{
    cacti_dump(@ARGV);
}
elsif ($CACTI_DUMP_HEADER)
{
    print "$template_str\n";
}
else
{
    usage()
}


# vim:sts=4:sw=4