#!/usr/bin/perl -w
#
# Copyright (c) Stewart Whitman, 2004-2006.
#                        All rights reserved
# Permission to use, copy, modify and distribute this material for
# any purpose and without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all
# copies, and that the name of Stewart Whitman not be used in advertising
# or publicity pertaining to this material without the specific,
# prior written permission of an authorized representative of
# Stewart Whitman.
#
# STEWART WHITMAN MAKES NO REPRESENTATIONS AND EXTENDS NO WARRANTIES, EX-
# PRESS OR IMPLIED, WITH RESPECT TO THE SOFTWARE, INCLUDING, BUT
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# FITNESS FOR ANY PARTICULAR PURPOSE, AND THE WARRANTY AGAINST IN-
# FRINGEMENT OF PATENTS OR OTHER INTELLECTUAL PROPERTY RIGHTS.  THE
# SOFTWARE IS PROVIDED "AS IS", AND IN NO EVENT SHALL STEWART WHITMAN OR
# ANY OF ITS AFFILIATES BE LIABLE FOR ANY DAMAGES, INCLUDING ANY
# LOST PROFITS OR OTHER INCIDENTAL OR CONSEQUENTIAL DAMAGES RELAT-
# ING TO THE SOFTWARE.
#
# File:    piotroski.pl
# Project: Stock Scans
# Desc:    Determine Piotroski Scores of stocks from various internet data sources
#
# $Header$
#

use Getopt::Long;
use File::Basename;
use LWP::UserAgent;
use LWP::Parallel::UserAgent;
use URI;
use Date::Manip;
use vars qw/$POINTS $RETRIES $UA $PUA $PROGRAM $TIMEOUT $TRUE $FALSE $HELP $VERBOSE $VERSION $DEBUG $QUIET %PAGES %REGISTERED_PAGES/;
use strict;

sub error($);
sub warning($);
sub runScan($);

$PROGRAM = basename( $0 );
$VERSION = '1.2';
$TRUE    = 1;
$FALSE   = 0;
$RETRIES = 4;
$TIMEOUT = 60;

#
#  Option Syntax
#
#   -d = debug
#   -h = help
#   -v = verbose
#   -q = quiet
#

$DEBUG   = 0;
$HELP    = $FALSE;
$VERBOSE = 0;
$QUIET   = 0;
$POINTS = 'google';
if( !GetOptions( '-debug+', \$DEBUG, '-help', \$HELP, '-verbose+', \$VERBOSE, '-quiet', \$QUIET, '-points=s', \$POINTS ) ) {
  exit( 1 );
}

#
# If help option specified
#
if( $HELP ) {
  print STDERR "$PROGRAM Version $VERSION\n\n";
  print STDERR "`$PROGRAM' is a utility to determine Piotroski scores\nof stocks from various internet data sources.\n\n";
  print STDERR "Usage: $PROGRAM [OPTION]... [TICKERS]...\n\n";
  print STDERR "Options:\n";
  print STDERR "  --points=<source>  source for financial data ('morningstar' or 'google')\n";
  print STDERR "  -v, --verbose      output the descriptions (repeat for more info)\n";
  print STDERR "  -d, --debug        output debug information\n";
  print STDERR "  -h, --help         print this output, then exit\n";
  print STDERR "  -q, --quiet        suppress some warnings output\n\n";
  exit 1;
}

#
# Check sources
#
error( "invalid finance source specified '$POINTS'." ) if $POINTS ne 'morningstar' && $POINTS ne 'google';

#
# If no input arguments
#
if( scalar(@ARGV) < 1 ) {
  error( 'no arguments' );
}

#
# Create the main user agent
#
if( 1 ) {
  $PUA = LWP::Parallel::UserAgent->new();
  $PUA->redirect( 0 ); # no redirects
  $PUA->timeout( $TIMEOUT ); # timeout
  $PUA->parse_head( 0 );
}
else {
  $UA = LWP::UserAgent->new( timeout => $TIMEOUT, keep_alive => 1 );
}

$| = 1;

foreach my $ticker ( @ARGV ) {
  $ticker =~ s/\r//g;
  $ticker =~ s/\s//g;
  runScan( $ticker );
}

exit 0;

# getNonParallelPage:
#
# Get a url. Use a non-parallel interface
#
sub getNonParallelPage($)
{
  my ( $url ) = @_;
  my $host = URI->new($url)->host;

  if( !defined($UA) ) { 
    $UA = LWP::Parallel::UserAgent->new( timeout => $TIMEOUT, keep_alive => 1 );
  }

  my $req = new HTTP::Request GET => $url;
  $req->user_agent('Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q342532)');
  $req->referer( 'http://' . $host );

  for( my $attempts = 0; $attempts < $RETRIES; $attempts++ ) {
    print STDERR "Getting $url (non-parallel)\n" if( $DEBUG > 1 );

    my $res = $UA->request( $req );

    print STDERR $res->status_line . "\n" if( $DEBUG > 1 );
    print STDERR $res->content . "\n" if( $DEBUG > 2 );

    if( $res->is_redirect ) {
      warning( "redirect response from $host: \"" . $res->status_line . '"' ) if $DEBUG;
      return '';
    }
    elsif( !$res->is_success ) {
      warning( "unexpected response from $host: \"" . $res->status_line . '"' . ($attempts < $RETRIES-1 ? ' (retrying)' : '') ) if $DEBUG || (!$QUIET && ($attempts == $RETRIES-1));
    }
    elsif( $res->content =~ /^[ \t\n\r]*$/ ) {
      warning( "unexpected empty response from $host" . ($attempts < $RETRIES-1 ? ' (retrying)' : '') ) if $DEBUG || (!$QUIET && ($attempts == $RETRIES-1));
    }
    else {
      return $res->content;
    }
    sleep( 2 );
  }

  return '';
} # end getNonParallelPage

# registerPage:
#
# Register a page to be fetched.
#
sub registerPage($)
{
  my $url = shift;

  if( defined($PUA) && !defined($REGISTERED_PAGES{$url}) ) {
    print STDERR "Registering $url\n" if( $DEBUG > 1 );

    my $req = new HTTP::Request GET => $url;
    $req->user_agent('Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q342532)');
    $req->referer( 'http://' . URI->new($url)->host );

    $PUA->register( $req, undef, 80000 );

    $REGISTERED_PAGES{$url} = 1;
  }
} # end registerPage

# getPage:
#
# Get a page.
#
sub getPage($)
{
  my ( $url ) = @_;

  print STDERR "Getting $url\n" if( $DEBUG > 1 );

  while( defined($PUA) && exists $REGISTERED_PAGES{$url} && ! exists $PAGES{$url} ) {
    my $entries = $PUA->wait( $TIMEOUT );

    return getNonParallelPage( $url ) if ! scalar(%$entries);

    foreach my $entry (keys %$entries) {
      my $res    = $entries->{$entry}->response;
      my $resurl = $res->request->url; 

      print STDERR "Received response for $resurl\n" if $DEBUG > 1;
      print STDERR $res->status_line . "\n" if( $DEBUG > 1 );
      print STDERR $res->content . "\n" if( $DEBUG > 2 );

      $PAGES{$resurl} = '';
      if( $res->is_redirect ) {
	warning( "parallel: redirect response from $resurl: \"" . $res->status_line . '"' ) if $DEBUG;
      }
      elsif( !$res->is_success ) {
	warning( "parallel: unexpected response from $resurl: \"" . $res->status_line . '"' ) if $DEBUG;
      }
      elsif( $res->content =~ /^[ \t\n\r]*$/ ) {
	warning( "parallel: unexpected empty response from $resurl" ) if $DEBUG;
      }
      else {
	warning( "parallel: got content from $resurl" ) if $DEBUG > 1;
	$PAGES{$resurl} = $res->content;
      }
    }
    foreach my $entry (keys %$entries) {
      $PUA->discard_entry( $entries->{$entry} );
    }
  }
  
  my $page = $PAGES{$url};
  return getNonParallelPage( $url ) if !defined($page) || $page eq '';
  return $page;
} # end getPage

sub displayField($$)
{
  if( $VERBOSE ) {
    my ( $fieldName, $value ) = @_;
    print "  $fieldName: " . (defined($value) ? $value : 'missing') . "\n";
  }
} # end displayField

sub getMorningstarRow($$)
{
  my ($fieldName, $page) = @_;

  foreach my $row ( split( /<tr/, $page ) ) {
    if( $row =~ />(<b>)?\Q$fieldName\E(<\/b>)?</ ) {
      $row = $';
      my @result = $row =~ /<(?:(?:td)|(?:font))\s+[^<]*class="(?:(?:msData)|(?:msBold)|(?:msSmall))">([^<]*)<\/(?:(?:font)|(?:td))>/g;
      @result = map { s/[\s\$%,]//g; $_ = -$1 if $_ =~ /^\((-?[\d\.]+)\)$/; $_ } @result;
      print STDERR "  $fieldName: " . join( ',', @result) . "\n" if $DEBUG;
      return @result;
    }
  }

  warning( "field '$fieldName' not found" ) if $DEBUG;
  return ();
} # end getMorningstarRow

sub getGoogleRowText($$)
{
  my ($fieldName,$fieldRegex) = @_;

  foreach my $row ( split( /<tr/i ) ) {
    if( $row =~ /<td[^>]*>\s*$fieldRegex\s*</i ) {
      $row = $';
      my @result = ($row =~ /<td[^>]*>\s*(.*?)\s*<\/td>/ig);
      print STDERR "  $fieldName: " . join( ',', @result) . "\n" if $DEBUG > 1;
      return @result;
    }
  }

  warning( "field '$fieldName' not found" ) if $DEBUG;
  return ();
} # end getGoogleRowText

sub getGoogleRow($)
{
  my ($fieldName) = @_;

  foreach my $row ( split( /<tr/ ) ) {
    if( $row =~ /<td[^>]*>\s*\Q$fieldName\E\s*</ ) {
      $row = $';
      # Just accept the table entries '-' which means 0, or '(+ve number)' or 'number'
      # Numbers may include decimal points or commas
      my @result = ($row =~ /<td[^>]*>\s*(-|\([\d\,\.]+\)|-?[\d\,\.]+)\s*<\/td>/g);
      @result = map { $_ = 0 if $_ eq '-'; s/,//g; $_ = -$1 if /^\((.*)\)/; $_ } @result;
      print STDERR "  $fieldName: " . join( ',', @result) . "\n" if $DEBUG > 1;
      return @result;
    }
  }

  warning( "field '$fieldName' not found" ) if $DEBUG;
  return ();
} # end getGoogleRow

sub getGoogleElement($$)
{
  my ( $fieldName, $index ) = @_;
  my @values = getGoogleRow( $fieldName );
  my $value  = $values[$index];
  displayField( $fieldName, $value ) if $DEBUG;
  return $value;
} # end getGoogleElement

sub getGoogleElement0($$)
{
  my ( $fieldName, $index ) = @_;
  my $value = getGoogleElement( $fieldName, $index );
  return defined($value) ? $value : 0;
} # end getGoogleElement0

sub getGoogleElementSum($$$)
{
  my ( $fieldName, $count, $lastWeight ) = @_;
  my @values = getGoogleRow( $fieldName );
  my $value;
  if( @values >= $count ) {
    $value = 0;
    for( my $i = 0; $i < $count; $i++ ) {
      my $x = $values[$i];
      $x *= $lastWeight if( $i == $count-1 ); 
      $value += $x;
    }
  }
  displayField( $fieldName . ' (TTM)', $value ) if $DEBUG;
  return $value;
} # end getGoogleElementSum

sub getGoogleTwo($$)
{
  my $fieldName = shift;
  my $count     = shift;
  my @values    = getGoogleRow( $fieldName );
  if( scalar(@values) > $count ) {
    my @result = ( $values[0], $values[$count] );
    print STDERR "  $fieldName: " . join( ',', @result) . "\n" if $DEBUG > 1;
    return @result;
  }
  return ();
} # end getGoogleTwo

sub getGoogleRatio($$$)
{
  my ( $fieldName, $numP, $divP ) = @_;
  my @num = @{$numP};
  my @div = @{$divP};

  my @result = ();

  $result[0] = defined($num[1]) && defined($div[1]) && $div[1] != 0 ? $num[1] / $div[1] : '---';
  $result[1] = defined($num[0]) && defined($div[0]) && $div[0] != 0 ? $num[0] / $div[0] : '---';
  $result[2] = '---';
  print STDERR "  $fieldName: " . join( ',', @result) . "\n" if $DEBUG;
  return @result;
} # end getGoogleRatio

sub getGoogleDiv($$)
{
  my ( $divName, $page ) = @_;
  $divName .= 'div';
  return ($page =~ /<div\s+id="$divName"(.*?)<\/div>/s) ? $& : '';
} # end getGoogleDiv

sub getGooglePage($)
{
  local $_ = getPage( shift );
  s/<\/?((small)|(b)|(span)|(br)|(wbr))[^>]*>//ig;
  s/\&nbsp;/ /g;
  s/\&amp;/\&/g;
  return $_;
} # end getGooglePage

sub check($$$)
{
  my( $test, $description, $calcdesc ) = @_;

  my $result = $test ? 1 : 0;

  if( $VERBOSE ) {
    print "  $description... " . ($test ? "yes" : "no" ) . ($VERBOSE > 1 ? " ($calcdesc)" : '') . "\n";
  }

  return $result;
} # end check

sub missing($$$)
{
  my( $test, $description, $calcdesc ) = @_;

  my $result = $test ? 0 : 1;

  if( $VERBOSE && !$test ) {
    print "  $description... missing data" . ($VERBOSE > 1 ? " ($calcdesc)" : '') . "\n";
  }

  return $result;
} # end missing

sub checknum($)
{
  my( $num ) = @_;
  my $test = (defined($num) && ($num =~ /-?\d+(\.\d*)?/)) ? $TRUE : $FALSE;

  if( $DEBUG && !$test ) {
    warning( "bad number " . (defined($num) ? $num : '<undefined>') );
  }

  return $test;
} # end checknum

sub runScan($)
{
  my($ticker)  = @_;
  my $points   = 0;
  my $misses   = 0;
  my $asOfDate = undef;
  local $_;

  print "$ticker:\n" if $VERBOSE;

  %PAGES            = ();
  %REGISTERED_PAGES = ();

  my ( @netincome, @cashflow, @finlev, @curratio, @assetturnover, @roa, @shares, @grossmargin );

  if( $POINTS eq 'google' ) {
    my $googleOverview = 'http://finance.google.com/finance?q=' . $ticker . '&hl=en';
    registerPage( $googleOverview );

    # Check the industry or sector for financial (or TBD:Utility)
    $_ = getGooglePage( $googleOverview );
    if( /Your search - \Q$ticker\E - produced no matches/s ) {
      warning( "cannot get $ticker Google Stock page" ) if !$QUIET;
      $_ = '';
    }

    # Most things in google are indexed off this variable
    my $cid = /var\s+_companyId\s*=\s*(\d+);/ ? $1 : undef;;
    displayField( 'Google Company ID', $cid ) if $DEBUG > 1;

    if( !defined($cid) ) {
      warning( "cannot get $ticker company id" ) if !$QUIET;
      goto SKIP;
    }

    my $googleFinancialStatements = 'http://finance.google.com/finance?fstype=ii&cid=' . $cid;
    $googleFinancialStatements    = 'http://finance.google.com/finance' . $1 if /<a\s+href=(\?fstype=ii\&cid=\d+)>Income\s+Statement<\/a>/;
    registerPage( $googleFinancialStatements );

    # Get finanacial statement page
    my $financialStatements = getGooglePage( $googleFinancialStatements );
    $_ = $financialStatements;
    if( !/Balance\s+Sheet/ || !/Income\s+Statement/ || !/Cash\s+Flow/ || /Your search - \Q$ticker\E - produced no matches/s ) {
      warning( "cannot get $ticker Google Financial Statements page" ) if !$QUIET;
      goto SKIP;
    }

    # Get data from set of quarterly income statement 
    $_ = getGoogleDiv( 'incinterim', $financialStatements );

    my $quarterCount      = 0;
    my $quarterAnomoly    = $FALSE;
    my $quarterLastWeight = 1;
    {
      my @periods = getGoogleRowText( 'Heading', 'In [\w ]+\(except for per share items\)' );
      my @durations = map { my $weeks = 13;
			    $weeks = $1*365.2/7/12 if /(\d+)\s+months/i;
			    $weeks = $1 if /(\d+)\s+weeks/i;
			    $weeks = $1/7 if /(\d+)\s+days/i;
			    $weeks } @periods;
      @periods = map { s/^.*Ending\s*//; s/^.*As\s*of\s*//; $_ } @periods;
      $asOfDate = @periods ? ParseDate( $periods[0] ) : undef;

      # Find the end date using the number of weeks (ideally - 3 months = 13 and 4*13 = 52)
      my $yearlyWeeks = 0;
      FINDEND:
      for( my $i = 0; $i < @durations; $i++ ) {
	displayField( 'Duration ' . $i, $durations[$i] ) if $DEBUG > 1;
	last FINDEND if $yearlyWeeks > 50.5;
	$yearlyWeeks += $durations[$i];
	$quarterCount++;
      }

      $quarterAnomoly = ($yearlyWeeks < 50.5 || $yearlyWeeks > 53.5);
      displayField( 'Yearly Weeks', $yearlyWeeks ) if $DEBUG > 1;

      # This extrapolates linearly based the income statement based on the last quarter
      if( $quarterAnomoly && $yearlyWeeks ) {
        $quarterLastWeight = (365.2/7 - ($yearlyWeeks - $durations[$quarterCount-1]))/$durations[$quarterCount-1];
      }
    }
    displayField( 'As Of', defined($asOfDate) ? UnixDate( $asOfDate, '%Y-%m-%d' ) : undef ) if $DEBUG > 1;
    displayField( 'Quarters in Year', $quarterCount ) if $DEBUG > 1;
    displayField( 'Last Quarter Weight', $quarterLastWeight) if $DEBUG > 1;

    #
    # Quarterly Stuff (TTM or Most Recent Quarter) 
    #

    # Get data from set of interim income statements
    $_ = getGoogleDiv( 'incinterim', $financialStatements );

    # Get TTM net income
    $netincome[0] = getGoogleElementSum( 'Net Income', $quarterCount, $quarterLastWeight );

    # Get data from set of interim cash flow statement
    $_ = getGoogleDiv( 'casinterim', $financialStatements );

    # Get TTM cash flow
    $cashflow[0] = getGoogleElementSum( 'Cash from Operating Activities', $quarterCount, $quarterLastWeight );
    
    # Get data from set of interim balance sheet statement
    $_ = getGoogleDiv( 'balinterim', $financialStatements );

    my @assetsTwo             = getGoogleTwo( 'Total Assets', $quarterCount );
    my @equityTwo             = getGoogleTwo( 'Total Equity', $quarterCount );
    my @currentAssetsTwo      = getGoogleTwo( 'Total Current Assets', $quarterCount );
    my @currentLiabilitiesTwo = getGoogleTwo( 'Total Current Liabilities', $quarterCount );
    my @sharesTwo             = getGoogleTwo( 'Total Common Shares Outstanding', $quarterCount );
    my @oneTwo                = ( 1, 1 );

    # Fix up the current assets and current liabilities if they are 0
    if( defined($currentAssetsTwo[0]) && $currentAssetsTwo[0] == 0 ) {
      $currentAssetsTwo[0] += getGoogleElement0( 'Cash and Short Term Investments', 0 );
      $currentAssetsTwo[0] += getGoogleElement0( 'Total Receivables, Net', 0 );
      $currentAssetsTwo[0] += getGoogleElement0( 'Total Inventory', 0 );
      $currentAssetsTwo[0] += getGoogleElement0( 'Prepaid Expenses', 0 );
      $currentAssetsTwo[0] += getGoogleElement0( 'Other Current Assets, Total', 0 );
    }
    if( defined($currentAssetsTwo[1]) && $currentAssetsTwo[1] == 0 ) {
      $currentAssetsTwo[1] += getGoogleElement0( 'Cash and Short Term Investments', $quarterCount );
      $currentAssetsTwo[1] += getGoogleElement0( 'Total Receivables, Net', $quarterCount );
      $currentAssetsTwo[1] += getGoogleElement0( 'Total Inventory', $quarterCount );
      $currentAssetsTwo[1] += getGoogleElement0( 'Prepaid Expenses', $quarterCount );
      $currentAssetsTwo[1] += getGoogleElement0( 'Other Current Assets, Total', $quarterCount );
    }
    if( defined($currentLiabilitiesTwo[0]) && $currentLiabilitiesTwo[0] == 0 ) {
      $currentLiabilitiesTwo[0] += getGoogleElement0( 'Accounts Payable', 0 );
      $currentLiabilitiesTwo[0] += getGoogleElement0( 'Accrued Expenses', 0 );
      $currentLiabilitiesTwo[0] += getGoogleElement0( 'Notes Payable/Short Term Debt', 0 );
      $currentLiabilitiesTwo[0] += getGoogleElement0( 'Current Port. of LT Debt/Capital Leases', 0 );
      $currentLiabilitiesTwo[0] += getGoogleElement0( 'Other Current liabilities, Total', 0 );
    }
    if( defined($currentLiabilitiesTwo[1]) && $currentLiabilitiesTwo[1] == 0 ) {
      $currentLiabilitiesTwo[1] += getGoogleElement0( 'Accounts Payable', $quarterCount );
      $currentLiabilitiesTwo[1] += getGoogleElement0( 'Accrued Expenses', $quarterCount );
      $currentLiabilitiesTwo[1] += getGoogleElement0( 'Notes Payable/Short Term Debt', $quarterCount );
      $currentLiabilitiesTwo[1] += getGoogleElement0( 'Current Port. of LT Debt/Capital Leases', $quarterCount );
      $currentLiabilitiesTwo[1] += getGoogleElement0( 'Other Current liabilities, Total', $quarterCount );
    }

    @finlev   = getGoogleRatio( 'Financial Leverage', \@assetsTwo, \@equityTwo );
    @curratio = getGoogleRatio( 'Current Ratio', \@currentAssetsTwo, \@currentLiabilitiesTwo );
    @shares   = getGoogleRatio( 'Shares', \@sharesTwo, \@oneTwo );

    #
    # Annual Stuff
    #

    # Get data from set of annual balance sheet statement
    $_ = getGoogleDiv( 'balannual', $financialStatements );

    my @assetsATwo = getGoogleTwo( 'Total Assets', 1 );

    # Get data from set of annual income statements
    $_ = getGoogleDiv( 'incannual', $financialStatements );

    my @revenuesATwo    = getGoogleTwo( 'Total Revenue', 1 );
    my @grossProfitATwo = getGoogleTwo( 'Gross Profit', 1 );
    my @netIncomeATwo   = getGoogleTwo( 'Net Income', 1 );

    @assetturnover = getGoogleRatio( 'Asset Turnover', \@revenuesATwo, \@assetsATwo );
    @roa           = getGoogleRatio( 'ROA', \@netIncomeATwo, \@assetsATwo );
    @grossmargin   = getGoogleRatio( 'Gross Margin', \@grossProfitATwo, \@revenuesATwo );
SKIP:
  }
  else {
    my $morningStar5YrFinancials = 'http://quicktake.morningstar.com/Stock/financials.asp?Country=USA&stocktab=finance&Symbol=' . $ticker;
    my $morningStarStockGrades   = 'http://quicktake.morningstar.com/Stock/MStockGrades.asp?Country=USA&Symbol=' . $ticker;
    my $morningStarIncome        = 'http://quicktake.morningstar.com/Stock/Income10.asp?Country=USA&stocktab=finance&Symbol=' . $ticker;

    registerPage( $morningStar5YrFinancials );
    registerPage( $morningStarStockGrades );
    registerPage( $morningStarIncome );

    #
    # Morningstar 5 Year Financials
    #
    $_ = getPage( $morningStar5YrFinancials );
    if( ! /Income\s+Statement/ || /We could not find the security.*you are looking for/s ) {
      warning( "cannot get $ticker Morningstar page" ) if !$QUIET;
      $_ = '';
    }

    @netincome     = getMorningstarRow( 'Net Income $Mil', $_ );
    @cashflow      = getMorningstarRow( 'Operating Cash Flow', $_ );

    # Sometimes the last row is undefined, so we use the previous row
    if( (scalar(@netincome) >= 1 && ($netincome[$#netincome] eq '---')) ||
	(scalar(@cashflow) >= 1 && ($cashflow[$#cashflow] eq '---')) ) {
      pop @netincome;
      pop @cashflow;
    }

    my $haveFinancials = $_ ne '';

    #
    # Morningstar Stock Grades Page
    #
    if( $haveFinancials ) {
      # Don't try the stock grades page if the financials was not present
      $_ = getPage( $morningStarStockGrades );
      if( ! /Financial\s+Health\s+Grade/ || /We could not find the security.*you are looking for/s ) {
	warning( "cannot get $ticker Morningstar stock grades page" ) if !$QUIET;
	$_ = '';
      }
    }

    @finlev        = getMorningstarRow( 'Financial Leverage', $_ );
    @curratio      = getMorningstarRow( 'Current Ratio', $_ );
    @assetturnover = getMorningstarRow( 'Asset Turnover', $_ );
    @roa           = getMorningstarRow( 'Return on Assets %', $_ );

    #
    # Morningstar 10 Year Income
    #
    if( $haveFinancials ) {
      # Don't try the 10-year page if the other pages were not present
      $_ = getPage( $morningStarIncome );
      if( ! /Income\s+Statement/ ) {
	warning( "cannot get $ticker Morningstar 10-Year Income Statement" ) if !$QUIET;
	$_ = '';
      }
    }

    @shares      = getMorningstarRow( 'Shares', $_ );
    my @revenue     = getMorningstarRow( 'Revenue', $_ );
    my @grossprofit = getMorningstarRow( 'Gross Profit', $_ );

    @grossmargin = ();
    if( scalar(@revenue) == scalar(@grossprofit) ) {
      for( my $i = 0; $i < scalar(@revenue); $i++ ) {
	if( checknum($revenue[$i]) && checknum($grossprofit[$i]) && $revenue[$i] != 0 ) {
	  $grossmargin[$i] = $grossprofit[$i] / $revenue[$i];
	}
	else {
	  $grossmargin[$i] = '---';
	}
      }
    }
    else {
      warning( "mismatch $ticker revenues and gross profit" ) if !$QUIET;
    }
  }

  undef $_;

  #
  # All data has been scraped
  #

  if( !missing( scalar(@netincome) >= 1 && checknum($netincome[$#netincome]), 'Positive Net Income', 'Net Income Positive' ) ) {
    $points += check( $netincome[$#netincome] > 0, 'Positive Net Income', 'Net Income Positive' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@cashflow) >= 1 && checknum($cashflow[$#cashflow]), 'Positive Cash Flow', 'Cash Flow Positive' ) ) {
    $points += check( $cashflow[$#cashflow] > 0, 'Positive Cash Flow', 'Cash Flow Positive' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@cashflow) >= 1 && checknum($cashflow[$#cashflow]) && scalar(@netincome) > 0 && checknum($netincome[$#netincome]), 'Earnings Quality', 'Cash Flow greater than Net Income' ) ) {
    $points += check( $cashflow[$#cashflow] > $netincome[$#netincome], 'Earnings Quality', 'Cash Flow greater than Net Income' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@finlev) >= 3 && checknum($finlev[$#finlev-1]) && checknum($finlev[$#finlev-2]), 'Decreasing Debt', 'Financial Leverage Decreasing' ) ) {
    $points += check( $finlev[$#finlev-1] < $finlev[$#finlev-2], 'Decreasing Debt', 'Financial Leverage Decreasing' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@curratio) >= 3 && checknum($curratio[$#curratio-1]) && checknum($curratio[$#curratio-2]), 'Increasing Working Capital', 'Current Ratio Increasing' ) ) {
    $points += check( $curratio[$#curratio-1] > $curratio[$#curratio-2], 'Increasing Working Capital', 'Current Ratio Increasing' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@assetturnover) >= 3 && checknum($assetturnover[$#assetturnover-1]) && checknum($assetturnover[$#assetturnover-2]), 'Improving Productivity', 'Asset Turnover Increasing' ) ) {
    $points += check( $assetturnover[$#assetturnover-1] > $assetturnover[$#assetturnover-2], 'Improving Productivity', 'Asset Turnover Increasing' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@roa) >= 3 && checknum($roa[$#roa-1]) && checknum($roa[$#roa-2]), 'Growing Profitability', 'ROA Increasing' ) ) {
    $points += check( $roa[$#roa-1] > $roa[$#roa-2], 'Growing Profitability', 'ROA Increasing' );
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@shares) >= 3 && checknum($shares[$#shares-1]) && checknum($shares[$#shares-2]), 'Not Issuing Stock', 'Shares Outstanding Decreasing' ) ) {
    if( check( $shares[$#shares-1] <= $shares[$#shares-2], 'Not Issuing Stock', 'Shares Outstanding Decreasing' ) ) {
      $points++;
    }
    elsif( $shares[$#shares-2] > 0 && (($shares[$#shares-1] - $shares[$#shares-2])/$shares[$#shares-2]) < 0.03 ) {
      $points += 0.5;
    }
    # There are no decimal points on the morningstar page, so if we cannot tell, we all .5
    elsif( !($shares[$#shares-1] =~ /\./ || $shares[$#shares-2] =~ /\./) && ($shares[$#shares-1] - $shares[$#shares-2]) == 1 ) {
      $points += 0.5;
    } 
  }
  else {
    $misses++;
  }

  if( !missing( scalar(@grossmargin) >= 3 && checknum($grossmargin[$#grossmargin-1]) && checknum($grossmargin[$#grossmargin-2]), 'Competitive Position', 'Gross Profit Margin Expanding' ) ) {
    $points += check( $grossmargin[$#grossmargin-1] > $grossmargin[$#grossmargin-2], 'Competitive Position', 'Gross Profit Margin Expanding' );
  }
  else {
    $misses++;
  }

  my $missingMsg = $misses > 0 ? " ($misses Misses)" : '';

  if( $VERBOSE ) {
    print "  $points Points$missingMsg\n";
  }
  else {
    print "$ticker: $points Points$missingMsg\n";
  }
} # end runScan

# error:
#
# error message
#
sub error($)
{
  my($message) = @_;
  print STDERR "$PROGRAM: error: $message\n";
  exit 1;
} # end error

# warning:
#
# warning message
#
sub warning($)
{
  my($message) = @_;
  print STDERR "$PROGRAM: warning: $message\n";
} # end warning