diff -u -r -N yasuck-0.1.0.orig/docs/ChangeLog yasuck-0.1.0/docs/ChangeLog --- yasuck-0.1.0.orig/docs/ChangeLog 2005-10-08 01:19:25.000000000 -0500 +++ yasuck-0.1.0/docs/ChangeLog 2005-10-08 22:24:19.000000000 -0500 @@ -4,6 +4,14 @@ *yasuck-0.1.0 (08 Oct 2005) + 08 Oct 2005; Mike Green docs/ChangeLog, + includes/app.functions.php, patch-0.1.0-01: + + Changed behavior of --exporthtml. Dumps to a directory structure based + on the message dates so that directory reads will not be out of control + when listing hundreds of thousands of files. Removed extraneous spaces + from line endings. + *yasuck-0.0.9 (17 Jul 2005) 08 Oct 2005; Mike Green docs/ChangeLog, @@ -11,7 +19,7 @@ patch-0.0.9-01: Added quoting to all define() statements. Added --exporthtml function that - will dump html copies of posts to a spool directory for spidering. + will dump html copies of posts to a spool directory for spidering. Converted nntpSpoolLog() and nntpSpoolUnlock to generic spoolLock() and spoolUnlock() functions. Fixed bug in runUpgrades(), was not returning false if checkDatabase() failed. Probably need to fix this elsewhere. @@ -27,7 +35,7 @@ authors or subjects were assumed to be non-existant. Actually these posts did have valid messages. Fixed it to enter !!UNKNOWN AUTHOR!! or !!NO SUBJECT!! in these cases. Removed --listgaps option since I - cannot recall what purpose it served and it did not appear to + cannot recall what purpose it served and it did not appear to produce useful results. Added --listmissing to list msgids that are missing from the database as a replacement. Bumped version to 0.0.9. @@ -36,7 +44,7 @@ patch-0.0.8-01: Added newsd compatibility. If NNTP_USE_NEWSD is defined as true, - when spool directories are created the .config file will be + when spool directories are created the .config file will be created, and when nntp messages are dumped the .info file will be updated with the correct counts. Created nntpSpoolLock and nntpSpoolUnlock functions to handle spool locking. Changed default @@ -45,12 +53,12 @@ *yasuck-0.0.7 (29 Sep 2004) 05 Jan 2005: Mike Green docs/ChangeLog, - docs/README, includes/app.functions.php, includes/globals.php, + docs/README, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.7-01: - + Added --exportnntp and --dumpnntp switches and code to dump messages - to nntp spool directory. Does not include text of posts by default, - only links to original posts due to a-hole groklaw supporter cdbaric + to nntp spool directory. Does not include text of posts by default, + only links to original posts due to a-hole groklaw supporter cdbaric threatening ckx board members with hints of lawsuits. Full text can be including by setting NNTP_COPY_TEXT to true in globals.php (would not suggest non-private access to this). Meant to be compatible with @@ -61,37 +69,37 @@ 04 Oct 2004: Mike Green docs/ChangeLog, includes/app.functions.php, patch-0.0.6-03: - + Got rid of the poorly implemented last_msg column and replaced it with num_msg column. This upgrade will break any version of yaview prior to version 0.0.5. Cleaned up the counter and poster statistics code. Started to modularize any code that performs database updates/inserts/deletes so that DRYRUN can be utilized easier. Bumped version up to 0.0.7. - + 29 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, yasuck, patch-0.0.6-02: - + Added --plugholes argument, to fetch messages in the database that are blank. Basically to fix the results of the bug that was patched by patch-0.0.6-01. - + 29 Sep 2004: Mike Green docs/ChangeLog, include/apps.functions.php, patch-0.0.6-01: - + Fixed bug that was skipping message text entries. *yasuck-0.0.5 (24 Sep 2004) 29 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, yasuck, patch-0.0.5-02: - + Added poster stats table and --refresh argument to go along with it. Keeps track of number of posts, number of recs, highest recs, average recs, etc.. Added code to upgrade tables automatically. Bumped version up to 0.0.6. 29 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.5-01: - + Added --merge to merge another database into the existing one. Started lots of code cleanups in preparation for an official 1.0.0 release. Rewrote argument checking, "functionalized" everything to flow more smoothly, @@ -103,7 +111,7 @@ 24 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.4-03: - + Added versioning information and arguement to check version (--version). Turned off the automatic initialization of boards when calling queryBoardInfo from arguments that shouldn't initialize boards. Reordered processing of @@ -115,14 +123,14 @@ 24 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, yasuck, patch-0.0.4-02: - + Fixed typo that was causing updates to fail, added call to sanityCheck() that should have been there all along! 24 Sep 2004: Mike Green docs/AUTHORS, - docs/ChangeLog, docs/README, includes/app.functions.php, + docs/ChangeLog, docs/README, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.4-01: - + Spam-obscured email addresses. Added new arguments: --quiet to quelch non error messages and --recs to update recommendations only. Sanitized text output. Added code to retrieve recommendations and created function @@ -131,13 +139,13 @@ which will check for upgrade tasks like adding table columns etc... Changed default values of FETCHSLEEP to 1,4, testing for yahoo lockout on those numbers. Created patch for all of this and updated ChangeLog. - + *yasuck-0.0.3 (19 Sep 2004) 20 Sep 2004: Mike Green docs/ChangeLog, includes/app.functions.php, patch-0.0.3-01: - + Oops, fixed bug that was hardcoding DRYRUN. Nothing was being saved to the database. Bumped version to 0.0.4. @@ -146,7 +154,7 @@ 19 Sep 2004; Mike Green docs/ChangeLog, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.2-01: - + Added code to detect yahoo unavailable messages since yasuck was catching them as missing instead of unavailable. Added --dumphtml for debugging, will create html files of each page retrieved into @@ -159,20 +167,20 @@ 18 Sep 2004; Mike Green docs/ChangeLog, includes/app.functions.php, patch-0.0.1-02: - + Added --dryrun and beginning of code to add previous/next message link information. Bumped version to 0.0.2. 17 Sep 2004; Mike Green docs/ChangeLog, includes/app.functions.php, includes/globals.php, yasuck, patch-0.0.1-01: - + Added --skipmsgs, --poster, and --subject arguments. These arguments skip adding the entire message text to the database and allow searchs on the poster or subject. Added MAXMISSES to abort if it is determined that the network may be down. Updated ChangeLog and created patch. 17 Sep 2004; Mike Green docs/ChangeLog: - + First rough cut release. - + diff -u -r -N yasuck-0.1.0.orig/includes/app.functions.php yasuck-0.1.0/includes/app.functions.php --- yasuck-0.1.0.orig/includes/app.functions.php 2005-10-08 02:31:29.000000000 -0500 +++ yasuck-0.1.0/includes/app.functions.php 2005-10-08 22:18:33.000000000 -0500 @@ -1,9 +1,9 @@ - * - *****************************************************************************/ + *****************************************************************************/ if (basename($PHP_SELF) == basename(__FILE__)) { die("Access Denied\n"); } /***************************************************************************** @@ -237,7 +237,7 @@ *****************************************************************************/ function checkLock($board_id='') { -if ($board_id == '' or !is_numeric($board_id)) { +if ($board_id == '' or !is_numeric($board_id)) { die("ABORT: invalid board_id: checkLock($board_id).\n"); } @@ -300,7 +300,7 @@ echo "checkStatus(): invalid board name: $board_name\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -321,7 +321,7 @@ } else { $locked = "unlocked"; } -if (retrieveMsg($board_id,$ticker_id,1)) { +if (retrieveMsg($board_id,$ticker_id,1)) { $getmsg = "ok"; } else { $getmsg = "failed"; @@ -396,7 +396,7 @@ *****************************************************************************/ function dropBoard($board_name='') { -if (defined('DRYRUN') and DRYRUN === true) { +if (defined('DRYRUN') and DRYRUN === true) { echo "DRYRUN: skipping dropBoard()\n"; return true; } @@ -405,7 +405,7 @@ echo "What board do you want to ditch?"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -647,7 +647,7 @@ echo "ERROR: invalid parent_id: $parent_id\n"; return false; } -if ($msg_date == '') { +if (($timestamp = strtotime($msg_date)) === -1) { echo "ERROR: invalid msg_date\n"; return false; } @@ -659,17 +659,17 @@ echo "ERROR: no board name specified\n"; return false; } -if (!htmlSpoolCheck($board_name)) { - echo "ABORT: htmlSpoolCheck() failed\n"; - return false; - } if (!$spool_dir = htmlSpoolDirName($board_name)) { echo "ABORT: could not determine spool directory\n"; return false; } -$date_str = date("d M Y H:i:s -0600",strtotime($msg_date)); +$output_dir = "$spool_dir/" . date('Y/m/d',$timestamp); +if (!htmlSpoolCheck($output_dir)) { + echo "ABORT: htmlSpoolCheck() failed on $output_dir\n"; + return false; + } -$outfile = "$spool_dir/$msg_id.html"; +$outfile = "$output_dir/$msg_id.html"; if (defined('LICENSE_TEXT') and LICENSE_TEXT !== '') { $line = str_repeat("-",60); @@ -739,11 +739,11 @@ if (defined('DRYRUN') and DRYRUN === true) { return true; } -if ($board_name == '') { +if ($board_name == '') { echo "What board do you want to export to html?\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkDatabase() failure\n"; return false; } @@ -756,17 +756,14 @@ $board_id = $info_ar['board_id']; $ticker_id = $info_ar['ticker_id']; } - -if (!htmlSpoolCheck($board_name)) { - echo "ABORT: htmlSpoolCheck() failed\n"; - return false; - } - if (! $spool_dir = htmlSpoolDirName($board_name)) { echo "ABORT: could not determine spool directory\n"; return false; } - +if (!htmlSpoolCheck($spool_dir)) { + echo "ABORT: htmlSpoolCheck() failed\n"; + return false; + } // determine start/stop if (!is_numeric($start)) { if (! $start = htmlGetLastMsgId($spool_dir)) { @@ -799,7 +796,7 @@ $max = db_result($qid,0,2); db_free_result($qid); -if ($hits < 1) { +if ($hits < 1) { echo "INFO: No messages to export\n"; return true; } @@ -892,25 +889,80 @@ echo "ERRROR: could not open spool: $spool_dir\n"; return false; } +// Get the latest year directory +$high_year = 0; +while ( false !== ($file = readdir($dh))) { + if (is_dir("$spool_dir/$file") and + is_numeric($file) and + $file > $high_year) { + $high_year = $file; + } + } +closedir($dh); +// Start at msgid 1 if no directories exist yet +if ($high_year < 1) { return "1"; } + +// Get the latest month directory +if (!$dh = opendir("$spool_dir/$high_year")) { + echo "ERRROR: could not open spool: $spool_dir/$high_year\n"; + return false; + } +$high_month = 0; +while ( false !== ($file = readdir($dh))) { + if (is_dir("$spool_dir/$high_year/$file") and + is_numeric($file) and + $file > $high_month) { + $high_month = $file; + } + } +closedir($dh); +// Start at msgid 1 if no directories exist yet +if ($high_month < 1) { return "1"; } + +// Get the latest day entry +if (!$dh = opendir("$spool_dir/$high_year/$high_month")) { + echo "ERRROR: could not open spool: $spool_dir/$high_year/$high_month\n"; + return false; + } +$high_day = 0; +while ( false !== ($file = readdir($dh))) { + if (is_dir("$spool_dir/$high_year/$high_month/$file") and + is_numeric($file) and + $file > $high_day) { + $high_day = $file; + } + } +closedir($dh); +// Start at msgid 1 if no directories exist yet +if ($high_month < 1) { return "1"; } + +// Get the latest msgid +if (!$dh = opendir("$spool_dir/$high_year/$high_month/$high_day")) { + echo "ERRROR: could not open spool: $spool_dir/$high_year/$high_month" + . "/$high_day\n"; + return false; + } +$high_msg = 0; while ( false !== ($file = readdir($dh))) { $junk_ar = explode('.',$file); - if (is_numeric($junk_ar[0]) and ($junk_ar[0] > $max)) { - $max = $junk_ar[0]; + if (is_numeric($junk_ar[0]) and ($junk_ar[0] > $high_msg)) { + $high_msg = $junk_ar[0]; } } closedir($dh); +if ($high_msg < 1) { return "1"; } -return $max + 1; +return $high_msg + 1; } /***************************************************************************** * * *****************************************************************************/ -function htmlSpoolCheck($board_name='') { +function htmlSpoolCheck($spool_dir='') { -if (! $spool_dir = htmlSpoolDirName($board_name)) { return false; } -if ($spool_dir == './') { return true; } +if ($spool_dir == '') { return false; } +if ($spool_dir == getcwd()) { return true; } $spool_dir = addslashes($spool_dir); @@ -945,7 +997,7 @@ } if (!defined('HTML_EXPORT_DIR') or HTML_EXPORT_DIR == '') { - return "./"; + return getcwd(); } return HTML_EXPORT_DIR . "/" . "${board_name}"; @@ -962,7 +1014,7 @@ if ($board_name == '') { return false; } // Don't initialize if it already exists, make the admin wipe it. -if (! queryBoardInfo($board_name,false) === false) { +if (!(queryBoardInfo($board_name,false) === false)) { echo "ERROR: queryBoardInfo($board_name).\n"; return false; } @@ -1087,7 +1139,7 @@ echo "What board do you want to list missing msgs for?"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -1104,12 +1156,12 @@ $sql = "select message_id from message_ids "; $sql .= "where board_id=$board_id order by message_id"; -if (!$qid = db_query($sql)) { +if (!$qid = db_query($sql)) { echo "ABORT: could not query message ids\n"; return false; } $num = db_num_rows($qid); -if ($num < 1) { +if ($num < 1) { echo "ABORT: no rows returned by query\n"; return false; } @@ -1142,7 +1194,7 @@ function lockOverride($board_name='') { $info_ar = queryBoardInfo($board_name,false); -if ($info_ar === false or !is_array($info_ar)) { +if ($info_ar === false or !is_array($info_ar)) { die("ABORT: can't unlock non-existent board: $board_name\n"); } @@ -1168,7 +1220,7 @@ echo "No merge database specified\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -1209,7 +1261,7 @@ echo "ABORT: cannot lock merge database.\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -1346,9 +1398,9 @@ if (QUIET !== true) { echo "ID: $mid P:$merge_poster_name S:$ms R:$mr\n"; } - + if (defined('DRYRUN') and DRYRUN === true) { continue; } - + $sql = "insert into message_ids(message_id,board_id,poster_id,parent_id,"; $sql .= "thread_id,message_date,message_subject,message_recs,"; $sql .= "message_text) values($mid,$board_id,$rpid,$parent,$tid,'$md',"; @@ -1504,7 +1556,7 @@ $thr_jref .= "('$thr_url','news','location=no,scrollbars=auto,"; $thr_jref .= "menubar=no,toolbar=no,resizable=yes');"; $thr_jref .= "nw.opener=self;return false;\">View Thread"; - + $thr_href_td = "$thr_href"; $thr_jref_td = "$thr_jref"; } else { @@ -1551,7 +1603,7 @@ $body .= " Due to legal threats by a groklaw supporter known as CDBaric, yahoo posts can no longer be displayed here without exposure to -the threat of legal expense. As a courtesy, links to the original +the threat of legal expense. As a courtesy, links to the original yahoo post are provided above/below."; } @@ -1583,11 +1635,11 @@ if (defined('DRYRUN') and DRYRUN === true) { return true; } -if ($board_name == '') { +if ($board_name == '') { echo "What board do you want to export to nntp?\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkDatabase() failure\n"; return false; } @@ -1643,7 +1695,7 @@ $max = db_result($qid,0,2); db_free_result($qid); -if ($hits < 1) { +if ($hits < 1) { echo "INFO: No messages to export\n"; return true; } @@ -2287,11 +2339,11 @@ return $val[1]; } } - + return false; } - + /***************************************************************************** * * *****************************************************************************/ @@ -2318,7 +2370,7 @@ echo "ERROR: ID: $message_id: board_id\n"; return false; } -if ($message_id == '' or !is_array($page)) { +if ($message_id == '' or !is_array($page)) { echo "ERROR: ID: $message_id: no page\n"; return false; } @@ -2431,7 +2483,7 @@ return true; } - + /***************************************************************************** * * *****************************************************************************/ @@ -2442,7 +2494,7 @@ if (!($initialize === true || $initialize === false)) { return false; } -$sql = "select board_id,ticker_id,low_msg,high_msg,num_msg,locked +$sql = "select board_id,ticker_id,low_msg,high_msg,num_msg,locked from board_ids where board_name='$board_name' limit 1"; if (!$qid = db_query($sql)) { die(mysql_error()); return false; } $num = db_num_rows($qid); @@ -2468,7 +2520,7 @@ $poster_name = addslashes($poster_name); -$sql = "select poster_id from poster_ids +$sql = "select poster_id from poster_ids where poster_name='$poster_name' limit 1"; if (!$qid = db_query($sql)) { return false; } if (db_num_rows($qid) == 1) { @@ -2492,7 +2544,7 @@ echo "What board do you want to refresh stats for?"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkStatus(): checkDatabase() failure\n"; return false; } @@ -2507,7 +2559,7 @@ $sql .= "order by poster_id"; if (!$qid = db_query($sql)) { return false; } $num = db_num_rows($qid); -if ($num < 1) { +if ($num < 1) { echo "ERROR: no messages for board $board_name?\n"; return false; } @@ -2530,7 +2582,7 @@ $posts = db_result($qid,0,0); $recs_total = db_result($qid,0,1); $recs_high = db_result($qid,0,2); - + if ($recs_total > 0) { $sql = "select count(message_id) from message_ids "; $sql .= "where board_id=$board_id and poster_id=$pid"; @@ -2549,7 +2601,7 @@ $msg .= "$recs_high RECS(avg): $recs_avg\n"; echo $msg; } - + $sql = "replace into poster_stats(board_id,poster_id,posts_total,"; $sql .= "recs_total,recs_high,recs_avg) "; $sql .= "values($board_id,$pid,$posts,$recs_total,$recs_high,$recs_avg)"; @@ -2572,11 +2624,11 @@ *****************************************************************************/ function resuckMessages($board_name='',$start='',$end='') { -if ($board_name == '') { +if ($board_name == '') { echo "What board do you want to suck?\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkDatabase() failure\n"; return false; } @@ -2808,7 +2860,7 @@ return true; } - + /***************************************************************************** * * *****************************************************************************/ @@ -2830,16 +2882,16 @@ defined('SOCKTIMEOUT') ? $to = true : $to = false; defined('MAXMISSES') ? $mm = true : $mm = false; -if (!$dbh or DBH == '') { +if (!$dbh or DBH == '') { $msg .= "ERROR: Database host not defined (DBH).\n"; } -if (!$dbn or DBN == '') { +if (!$dbn or DBN == '') { $msg .= "ERROR: Database name not defined (DBN).\n"; } -if (!$dbu or DBU == '') { +if (!$dbu or DBU == '') { $msg .= "ERROR: Database user not defined (DBU).\n"; } -if (!$dbp or DBP == '') { +if (!$dbp or DBP == '') { $msg .= "ERROR: Database password not defined (DBP).\n"; } if (!$rd or ROOTDIR == '' or !is_dir(ROOTDIR)) { @@ -2908,7 +2960,7 @@ *****************************************************************************/ function setRecs($board_id='',$message_id='',$msg_recs='') { -if (defined('DRYRUN') and DRYRUN === true) { +if (defined('DRYRUN') and DRYRUN === true) { echo "DRYRUN: skipping setRecs()\n"; return true; } @@ -2972,11 +3024,11 @@ *****************************************************************************/ function suckMessages($board_name='',$start='',$end='') { -if ($board_name == '') { +if ($board_name == '') { echo "What board do you want to suck?\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkDatabase() failure\n"; return false; } @@ -3097,7 +3149,7 @@ echo "What board name do you want to suck?\n"; return false; } -if (!checkDatabase()) { +if (!checkDatabase()) { echo "checkDatabase() failure\n"; return false; } @@ -3234,7 +3286,7 @@ *****************************************************************************/ function toggleLock($board_id='',$lock=1) { -if (defined('DRYRUN') and DRYRUN === true) { +if (defined('DRYRUN') and DRYRUN === true) { echo "DRYRUN: skipping toggleLock()\n"; return true; } @@ -3264,7 +3316,7 @@ *****************************************************************************/ function updateCounters($board_id='') { -if (defined('DRYRUN') and DRYRUN === true) { +if (defined('DRYRUN') and DRYRUN === true) { echo "DRYRUN: skipping updateCounters()\n"; return true; } @@ -3272,7 +3324,7 @@ if (!is_numeric($board_id)) { return false; } $board_id = addslashes($board_id); -$sql = "select min(message_id),max(message_id),count(message_id) +$sql = "select min(message_id),max(message_id),count(message_id) from message_ids where board_id=$board_id limit 1"; if (!$qid = db_query($sql)) { return false; } if (db_num_rows($qid) !== 1) { return false; } @@ -3280,11 +3332,14 @@ $low_msg = db_result($qid,0,0); $high_msg = db_result($qid,0,1); $num_msg = db_result($qid,0,2); +if ($low_msg == '') { $low_msg = 0; } +if ($high_msg == '') { $high_msg = 0; } +if ($num_msg == '') { $num_msg = 0; } $sql = "update board_ids set low_msg=$low_msg,high_msg=$high_msg, num_msg=$num_msg where board_id=$board_id limit 1"; -if (!db_query($sql)) { +if (!db_query($sql)) { $msg = "ABORT: SQL Error:\n" . mysql_error() . "\n"; echo $msg; return false; @@ -3295,5 +3350,5 @@ return true; } - + ?>