#!/usr/bin/perl
# This program will grab webpages, scrape some content and generate
# a JavaScript formatted version of the scraped output, and XHTML 1.0-Strict
# output of Wunderground and NWS local forecasts.
#
# Ken True - http://saratoga-weather.org/     
# webmaster@saratoga-weather.org
#
# This program resides in the WeatherLink Templates directory and runs once an hour via
# a Windows XP automatic task.
#
# the contents of the output JavaScript is processed by WeatherLink as a template with output
# file type of '.js' and uploaded as the weather station uploads are scheduled.
#
# the webpage just has a simple script to place on the page:
#
#  <script language="JavaScript" src="/forecast.js">

#  </script>

#  <noscript><p align=center>Note: JavaScript must be enabled to view WeatherUnderground and NWS forecasts.</p></noscript>

#
#  Perl V5.8.0 (806) with LWP library.
#
#  Version 1.09 adds two more outputs:
#    forecast-inc.htx (WU and NWS forecast icons)
#    forecast-full-inc.htx (NWS forecast icons with text description)
#
# Note: the NWS forecast works ONLY with the SF/Monterey website:  www.wrh.noaa.gov so it's not
# useful outside the West Coast area (probably).
#
# As with all free programs, use at your own risk ;-)
#
#  Ken True  webmaster@saratoga-weather.org
#
$version = 'Grab-forecast.pl V1.09 - 23-Feb-2006';

require LWP::UserAgent;
require HTTP::Headers;
$outfilename = './forecast.htx';
$outfilenameHTML = './forecast-inc.htx';  #XHTML 1.0 compliant (hopefully)
$outfilenameHTMLFULL = './forecast-full-inc.htx';  # Full NWS forecast

$quiet = 0;
$|=1;

my %CONDS = ( 
	'PartlyCloudy','PtCldy',
	'Partly Cloudy','PtCldy',
	'MostlyCloudy','MostCldy',
	'Clear','Clr'
        );

 $ua = new LWP::UserAgent;

 print "$version\n";
 
 open (OUTPUT, ">$outfilename") || die "..unable to open $outfilename: $!\n";
 open (HTML, ">$outfilenameHTML") || die "..unable to open $outfilenameHTML: $!\n";
 open (HTMLFULL, ">$outfilenameHTMLFULL") || die "..unable to open $outfilenameHTMLFULL: $!\n";
 open (LOG,">>forecast.log");

  &get_weather_WU;
  &get_weather_NWS;

 close OUTPUT; 
 close LOG;
 close HTML;
 close HTMLFULL;

print STDERR "..wrote to $outfilename and $outfilenameHTML.\n";


exit;
#----------------------------------------------------------------------------



sub get_weather_WU {
 my($req) = new HTTP::Request('GET', 'http://www.wunderground.com/cgi-bin/findweather/getForecast?query=95070');

 $req->header('Cache-control' => 'no-cache');
 $req->header('Pragma' => 'no-cache');

 my($content) = $ua->request($req)->content();
 my($hdrs) = $ua->request($req)->headers;


 my($dt) = scalar localtime(time);
 print "..$dt  X-Cache: ",$hdrs->header('X-Cache'),"\n";
 
 my(@lines) = split(/\n/,$content);

#  open (IN,"WUNewSample.txt");
#  while (<IN>) {
#    s/\n//;
#    push(@lines,$_) if $_;
#  }

 
 my($i) = 0;
 my($line) = "";
 my(@newlines);

 while ($i<=$#lines) {
   if ($lines[$i] =~ /5-Day Forecast/) {
#      $i = $i -2;
       last;
   }
   $i++;
 }

 return unless $i;
 print OUTPUT <<EOH1;
//generated by $version
//
//  <!--StationCity-->, <!--StationState-->, as of
//     <!--Date--> <!--Time--> (Pacific) 
//
var wtext = '';

wtext = '<style>'+
 'BODY { font-size: 12px; font-family: Arial, sans-serif; background-color: #FFF; color: #000; margin-top: 0px; margin-left: 0px; margin-right: 0px; }'+
 'H3 { padding-left: 5px; padding-top: 2px; padding-bottom: 2px; margin: 0px; font-size: 13px; font-weight: bold; background-color: #008; color: #FFF; border-top: 1px solid #4245AD; border-left: 1px solid #4245AD; border-right: 1px solid #333; border-bottom: 1px solid #333; }'+
 'H4 { padding: 2px; margin: 0px; font-size: 12px; font-weight: bold; background-color: #CCC; border: 1px solid #EEE; }'+
 'H5 { padding: 2px; margin: 0px; font-size: 12px; font-weight: normal; background-color: #E5E5E5; border-top: 1px dotted #CCC; text-align: center; }'+
 'DIV { margin: 0px; padding: 0px; }'+
 'TD.sortH { padding: 5px; background-color: #CCC; color: #000; font-size: 11px; font-weight: bold; font-family: Verdana; border-top: 1px solid #EEE; border-left: 1px solid #EEE; border-right: 1px solid #999; border-bottom: 1px solid #999; cursor: pointer; }'+
 'TD.sortC { color: #333; font-family: Verdana; font-size: 11px; padding: 1px; padding-left: 5px; padding: 5px; border-bottom: 1px solid #CCC; }'+
 '#message3 { text-align: center; background-color: #F5F5F5; color: #000; border: 1px solid #CCC; padding: 2px; }'+
 '#p5 { padding: 5px; }'+
 '#b { font-weight: bold; }'+
 '.taC { text-align: center; }'+
 '.boxB { border: 1px solid #008; background-color: #FFF; vertical-align: top; }'+
 '</style>'
document.write(wtext);


wtext = '<br clear="all"><center>'+
  '<table CELLPADDING=0 BORDER=0 CELLSPACING=1 class=smalltable>'+
  '  <tr valign=top align=center bgcolor=#ffffff>'+
EOH1
;

 print HTML <<EOH1A;
<!-- $version -->
<!-- Wunderground Forecast -->
<div class="WU">
  <table>
   <tr>
EOH1A
;

my @days = ();
my @conds = ();
my @temps = ();

 while ($i<=$#lines) {
  $line = $lines[$i];
  $line =~ s!href="/cgi-bin/findweather!href="http://www.wunderground.com/cgi-bin/findweather!;
  $line =~ s!'!''!g;
  $line =~ s!5-Day Forecast!<td><h3>Weather Underground 5-Day Forecast!;
  $line =~ s!</h3>!</h3></td></tr><TR><TD>!;
  $line =~ s!</table>!</table></TD></TR></TABLE>!;
  $line =~ s!<br>!<br />!gi;
  $line =~ s!id="b"!!;
  $line =~ s!<img ([^>]+)!<img $1 /!;
  $line =~ s!title="([^"]+)"!alt="$1" title="$1"!;
  $line =~ s!TR>!tr>!g;
  $line =~ s!TD>!td>!g;
  $line =~ s!TABLE>!table>!g;
  $line =~ s!bgcolor="#f6f6f6" style="!style="background-color: #F6F6F6; !;
  if ($line =~ m/href\="/ && $line =~ /\&/) {
    $line =~ s!\&!\&amp;!g;
  }
  print OUTPUT "  \'$line\'\+\n" unless $line =~ /DisplayMOS.asp|^\s+for$|ZIP|Today''s|DisplayPollen/;
  print HTML "$line\n" unless $line =~ /DisplayMOS.asp|^\s+for$|ZIP|Today''s|DisplayPollen/;

# Save our data from the WeatherUnderground forecast
  if ($line =~ m!<h4>([^\<]+)</h4>!) {
     push(@days,substr($1,0,3));
  }
  if ($line =~ m/align\=center class\=smalltable \>([^\<]+)\</) {
     my($t) = $1;
     $t = $CONDS{$t} if defined $CONDS{$t};
     push(@conds,$t);
  }
  if ($line =~ m!\&deg\;!) {
    my $t = $line;
    $t =~ s/\<[^\>]+\>//g;
    $t =~ s/\&deg\;//g;
    $t =~ s/\s+//g;
    push(@temps,$t);
  }
     
  
  last if $line =~ m|</table>|;
  $i++;
  }
#print STDERR join(":",@days), "\n";
#print STDERR join(":",@conds), "\n";
#print STDERR join(":",@temps), "\n";
$line = join("\t",join(":",@days),join(":",@conds),join(":",@temps));
#print LOG "$dt\tWUG\t$line\n";
print OUTPUT <<EOH2;
  '<br clear="all"><small>Forecast as of $dt Pacific</small></center><br>';
document.write(wtext);

EOH2
;
print HTML <<EOH2A;

</div>
<!-- end of Wunderground forecast -->

  <br /><div style="text-align: center"><h3>Forecast as of $dt Pacific</h3></div><br />

EOH2A
;


} # end --- get_weather

#----------------------------------------------------------------------------



sub get_weather_NWS {
 my($req) = new HTTP::Request('GET', 'http://www.wrh.noaa.gov/total_forecast/printable_forecast.php?wfo=mtr&zone=caz008&fire=&county=cac085&dgtl=1&lat=37.26389&lon=-122.02194');

 $req->header('Cache-control' => 'no-cache');
 $req->header('Pragma' => 'no-cache');

 my($content) = $ua->request($req)->content();
 my($hdrs) = $ua->request($req)->headers;
#  print $content;


# print $hdrs->as_string, "\n";
 my($dt) = scalar localtime(time);
 my($dow) = substr($dt,0,3);

 print "..$dt  X-Cache: ",$hdrs->header('X-Cache'),"\n";
 
 my(@lines) = split(/\n/,$content);

 
 my($i) = 0;
 my($line) = "";
 my(@newlines);

 while ($i<=$#lines) {
   if ($lines[$i] =~ s/<b>Forecast For: /<b>National Weather Service Forecast for: /) {
       last;
   }
   $i++;
 }

 return unless $i;
 print OUTPUT <<EOH1;
var wtextnws = '';

wtextnws = '<br clear="all"><center>'+
//  '<table CELLPADDING=2 BORDER=0 CELLSPACING=1 class=smalltable width="640"><tr><td>'+
  '<table width="640" BORDER=0>'+
  '  <tr valign=top align=center bgcolor=#FFFFFF>'+
EOH1
;
 print HTML <<EOH1A;
<!-- NOAA forecast -->
<div class="NOAA">

    <table width="100%" style="border: none;">
      <tr align="center" style="background-color: #FFFFFF;">
EOH1A
;
 print HTMLFULL <<EOH1B;
<!-- NOAA forecast -->
<div class="NOAA">

    <table width="100%" style="border: none;">
      <tr align="center" style="background-color: #FFFFFF;">
EOH1B
;
my @days = ();
my @conds = ();
my @temps = ();
my $toggle = 0;

$ntabs = 0;

 while ($i<=$#lines) {
  $line = $lines[$i];
  $line =~ s!/images/total_forecast!http://www.wrh.noaa.gov/images/total_forecast!;
  $line =~ s!'!"!g;
  $line =~ s/
//g;
  $line =~ s!alt="([^<]+)<br>!alt="$1 !;  #Fix NOAA alt= tags for XHTML
  $line =~ s!<br>!<br />!gi;
  $line =~ s!<img ([^>]+)!<img $1 /!;
  $line =~ s! colspan="2"!!;
  $line =~ s!<font color="([^"]+)">([^<]+)</font>!<span style="color: $1;">$2</span>!;
  $line =~ s!<font color="([^"]+)">([^<]+)!<span style="color: $1;">$2!;
  $line =~ s!</font>!</span>!;
  $line =~ s!<td width="([^"]+)">!<td style="width: $1;">!;
  $line =~ s!<tr vailgn="top" align="center">!<tr style="text-align: center; vertical-align: top;">!;
  $line =~ s!border=0!border="0"!;

  if ($line =~ m!\&deg\;!) {
    my $t = $line;
    $t =~ s/\<[^\>]+\>//g;
    $t =~ s/\&deg\;F//g;
    $t =~ s/Lo|Hi//g;
    $t =~ s/\s+//g;
    push(@temps,$t);
  }
  if ($line =~ m/<br><br>|<br>Night/) {
    my $t = $line;
    $t =~ s/\<[^\>]+\>//g;
    $t =~ s/\s+//g;
    if ($t) {
      if ($t =~ /Today|Tonight|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday/i) {
        $t = substr($t,0,3);
        $t = $dow if $t =~ /Tod|Ton/;
        $t .= "N" if $line =~ /Night/i;
      }
      $t = $CONDS{$t} if defined $CONDS{$t};

      if ($toggle) {
        push(@conds,$t);
        $toggle = 0;
      } else {
        push(@days,$t);
        $toggle = 1;
      }
    }
  }

  print OUTPUT "  \'$line\'\+\n";
  print HTML "$line\n";
  print HTMLFULL "$line\n";
  $ntabs++ if $line =~ '</table>';
  last if $ntabs >= 3;
  $i++;
  }
print OUTPUT <<EOH2;
  '</td></tr></table><br></center><br>';
document.write(wtextnws);

EOH2
;
print HTML <<EOH2A;
    </td></tr></table>
</div>
<!-- end of NOAA forecast -->
EOH2A
;
print HTMLFULL <<EOH2B;
  </td></tr></table>
</div>
<!-- end of NOAA summary forecast -->
<br /><div style="text-align: center"><h3>NWS Forecast Details</h3></div><br />
<div class="NOAA">
EOH2B
;
#print STDERR join(":",@days), "\n";
#print STDERR join(":",@conds), "\n";
#print STDERR join(":",@temps), "\n";
$line = join("\t",join(":",@days),join(":",@conds),join(":",@temps));
print LOG "$dt\tNWS\t$line\n";

# now process the rest of the forecast for the full page
 my $doit = 0;

 while ($i<=$#lines) {
  $line = $lines[$i];
  if ($line =~ m!<table!) { $doit = 1; }
  if (! $doit) { # skip over flotsom
    $i++;
    next;
   }
  $line =~ s!'!"!g;
  $line =~ s/
//g;
  $line =~ s!<br>!<br />!gi;
#  $line =~ s!<font \+2>!<span style="font-size: +2;">!;
#  $line =~ s!^</font>$!</span>!;
  $line =~ s!<font color="([^"]+)">([^<]+)!<span style="color: $1;">$2!;

  $line =~ s!<font \+2>!<p>!;
  $line =~ s!^</font><br!</span><br!;  #crazy HTML from NWS
  $line =~ s!^</font>$!</p>!;


  print HTMLFULL "$line\n";
  last if ($line =~ m!</table>! && $doit);
  $i++;
 }

print HTMLFULL <<EOF3C;
</div>
<!-- end of full NOAA forecast -->
EOF3C
;

} # end --- get_weather