diff -u -r -N yasuck-0.0.2.orig/docs/ChangeLog yasuck-0.0.2/docs/ChangeLog --- yasuck-0.0.2.orig/docs/ChangeLog 2004-09-18 22:54:37.000000000 -0500 +++ yasuck-0.0.2/docs/ChangeLog 2004-09-19 14:33:03.308581128 -0500 @@ -2,6 +2,22 @@ # Copyright (c)2004 badpenguins.com; Distributed under the GPL v2 # +*yasuck-0.0.3 (19 Sep 2004) + +*yasuck-0.0.2 (18 Sep 2004) + + 19 Sep 2004; Mike Green docs/ChangeLog, + includes/app.functions.php, includes/globals.php, yasuck, + patch-0.0.2-01: + + Added code to detect yahoo unavailable messages since yasuck was + catching them as missing instead of unavailable. Added --dumphtml + for debugging, will create html files of each page retrieved into + the current working directory. Aligned output of helpMessage(). + Set FETCHSLEEP default to between 2 and 5 seconds. Turned off + RANDOMIZE_UA by default. Corrected perms on files. Updated ChangeLog. + Bumped version to 0.0.3. + *yasuck-0.0.1 (17 Sep 2004) 18 Sep 2004; Mike Green docs/ChangeLog, diff -u -r -N yasuck-0.0.2.orig/includes/app.functions.php yasuck-0.0.2/includes/app.functions.php --- yasuck-0.0.2.orig/includes/app.functions.php 2004-09-18 13:09:50.000000000 -0500 +++ yasuck-0.0.2/includes/app.functions.php 2004-09-19 14:19:38.556921976 -0500 @@ -42,6 +42,9 @@ case "--dryrun": define(DRYRUN,true); break; + case "--dumphtml": + define(DUMPHTML,true); + break; case "--end": $j++; $end = $args[$j]; @@ -334,13 +337,14 @@ $msg = "Usage: $fname action args [--start msg] [--end msg] --debug Turn on verbose debug messages - --dryrun Fetch only, NO database message insert/updates - --end MSG End with message number MSG (optional) + --dryrun Fetch only, NO database message insert/updates + --dumphtml Dump copy of pages as they are retrieved (in CWD) + --end MSG End with message number MSG (optional) --get BOARD Retrieve messages from board BOARD --help Display this help message --init Initialize database (destructive) - --poster POSTER Only retrieve messages posted by POSTER - --sanity Run sanity check and display results + --poster POSTER Only retrieve messages posted by POSTER + --sanity Run sanity check and display results --skipmsgs Don't import messages (subject/poster only) --start MSG Start with message number MSG (optional) --status BOARD Display status of board BOARD database @@ -793,6 +797,22 @@ /***************************************************************************** * * *****************************************************************************/ +function parseMsgUnavailable($page='') { + +if ($page == '' or !is_array($page)) { return false; } +$text = "999 This page is currently unavailable"; +$text = strtolower($text); +foreach($page as $line) { + $line = strtolower($line); + $pos = strpos($line,$text); + if (!($pos === false)) { return true; } + } +return false; +} + +/***************************************************************************** + * * + *****************************************************************************/ function parseTickerId($page='') { if ($page == '' or !is_array($page)) { return false; } diff -u -r -N yasuck-0.0.2.orig/includes/globals.php yasuck-0.0.2/includes/globals.php --- yasuck-0.0.2.orig/includes/globals.php 2004-09-17 23:14:26.000000000 -0500 +++ yasuck-0.0.2/includes/globals.php 2004-09-19 14:24:24.900391152 -0500 @@ -21,13 +21,13 @@ define(USER_AGENT,'Mozilla/5.0 (compatible; Konqueror/3.3)'); define(YAHOOHOST,'finance.messages.yahoo.com'); define(SOCKTIMEOUT,10); -//define(FETCHSLEEP,"1,3"); -define(FETCHSLEEP,1); +define(FETCHSLEEP,"2,5"); +//define(FETCHSLEEP,1); define(RETRIES,5); // misc settings define(RANDOM_UA,INCDIR . "/agents.txt"); -define(RANDOMIZE_UA,true); +define(RANDOMIZE_UA,false); define(MAXMISSES,50); // debug settings diff -u -r -N yasuck-0.0.2.orig/yasuck yasuck-0.0.2/yasuck --- yasuck-0.0.2.orig/yasuck 2004-09-18 22:49:34.000000000 -0500 +++ yasuck-0.0.2/yasuck 2004-09-19 14:22:04.130791384 -0500 @@ -44,13 +44,44 @@ } // start sucking -$maxmisses = 0; echo "Retrieving messages: Board($board) Start($start) End($end)\n"; for($j = $start; $j <= $end; $j++) { if (!$page = retrieveMsg($board_id,$ticker_id,$j)) { echo "$j(retrieval failure) "; continue; } + if (DUMPHTML === true) { + if (!$fd = fopen("$j.html","w")) { + echo "cannot open file for html dump "; + } else { + foreach($page as $line) { + if (!fwrite($fd,$line . "\n")) { + echo "could not write to $j.html "; + fclose($fd); + break; + } + } + fclose($fd); + } + } + // account for yahoo outages + $maxunavail = 0; + $sleepmax = 3600; + $sleep = 10; + while (parseMsgUnavailable($page) === true) { + echo "$j(unavailable) "; + $maxunavail++; + if ($maxunavail > MAXMISSES) { + toggleLock($board_id,0); + die("Hit MAXMISSES (" . MAXMISSES . ") - aborting\n"); + } + if ($sleep > $sleepmax) { + $sleep = 10; + } else { + $sleep = $sleep * 2; + } + sleep($sleep); + } if (!parseMsgExists($page)) { echo "$j(missing) "; $maxmisses++;