diff -u -r -N yasuck-0.0.7/docs/ChangeLog yasuck-0.0.8/docs/ChangeLog --- yasuck-0.0.7/docs/ChangeLog 2004-10-04 14:38:50.000000000 -0500 +++ yasuck-0.0.8/docs/ChangeLog 2005-01-06 11:22:57.322272144 -0600 @@ -2,6 +2,22 @@ # Copyright (c)2004 badpenguins.com; Distributed under the GPL v2 # +*yasuck-0.0.8 (5 Jan 2005) + +*yasuck-0.0.7 (29 Sep 2004) + 05 Jan 2005: Mike Green docs/ChangeLog, + docs/README, includes/app.functions.php, includes/globals.php, + yasuck, patch-0.0.7-01: + + Added --exportnntp and --dumpnntp switches and code to dump messages + to nntp spool directory. Does not include text of posts by default, + only links to original posts due to a-hole groklaw supporter cdbaric + threatening ckx board members with hints of lawsuits. Full text can + be including by setting NNTP_COPY_TEXT to true in globals.php (would + not suggest non-private access to this). Meant to be compatible with + newsd at http://www.easysw.com/~mike/newsd/. + Bumped version up to 0.0.8. + *yasuck-0.0.6 (29 Sep 2004) 04 Oct 2004: Mike Green docs/ChangeLog, diff -u -r -N yasuck-0.0.7/docs/README yasuck-0.0.8/docs/README --- yasuck-0.0.7/docs/README 2004-09-24 08:46:58.000000000 -0500 +++ yasuck-0.0.8/docs/README 2005-01-06 11:36:28.793909704 -0600 @@ -16,3 +16,33 @@ drop/create the database. Running yasuck without arguments will give you a help message. + +NOTES ON NNTP FUNCTIONALITY + +The ability to export database messages to newsgroup messages was added +on Jan 5, 2005. This functionality was developed to interact specifically +with the GPL'ed newsd personal nntp server which is available at: + +http://www.easysw.com/~mike/newsd/ + +To get newsd functioning do the usual ./configure --prefix=/usr, make, +make install. Verify the init script that is provided. Run newsd -newgroup +to initialize the groups. In yasuck, define NNTP_SPOOL_DIR to match the +SpoolDir setting in /etc/newsd.conf. For personal use only +(no non-private access), the full yahoo post can be included in the +newsgroup message by defining NNTP_COPY_TEXT to true in globals.php. + +The nntp command line arguments accept the standard --start and --end +arguments. + +yasuck --exportnntp board_name will dump all messages present in the +database not present in the spool directory. If message are present +in the spool directory, --exportnntp will start where the existing +messages left off. The export will be done in chunks of 10000 messages +to account for machines with low resources. + +Adding --dumpnntp will dump nntp messages during standard runs. + +Change ownership of the resulting messages created by yasuck in the spool +directory to whatever user newsd is running as (news by default). If +a cron script is used, run it under the same user as newsd. \ No newline at end of file diff -u -r -N yasuck-0.0.7/includes/app.functions.php yasuck-0.0.8/includes/app.functions.php --- yasuck-0.0.7/includes/app.functions.php 2004-10-04 14:22:55.000000000 -0500 +++ yasuck-0.0.8/includes/app.functions.php 2005-01-06 12:07:25.475651176 -0600 @@ -38,6 +38,9 @@ case "--dumphtml": define(DUMPHTML,true); break; + case "--dumpnntp": + define(DUMPNNTP,true); + break; case "--end": if (defined('END')) { return false; } $j++; @@ -47,6 +50,16 @@ if (!is_numeric($end)) { return false; } define(END,$end); break; + case "--exportnntp": + if (defined('ACTION')) { return false; } + if (defined('BOARD')) { return false; } + $j++; + $board = $args[$j]; + if (empty($board)) { return false; } + if (substr($board,0,2) == "--") { return false; } + define(ACTION,"exportnntp"); + define(BOARD,$board); + break; case "--get": if (defined('ACTION')) { return false; } if (defined('BOARD')) { return false; } @@ -453,6 +466,8 @@ return true; } + + /***************************************************************************** * * *****************************************************************************/ @@ -549,7 +564,9 @@ --drop BOARD Drop/Purge BOARD out of the database --dryrun Fetch only, NO database message insert/updates --dumphtml Dump copy of pages as they are retrieved (in CWD) + --dumpnntp Dump copy of pages to nntp spool --end MSG End with message number MSG (optional) + --exportnntp BOARD Export BOARD messages from db to NNTP --get BOARD Retrieve messages from board BOARD --help Display this help message --init Initialize database (destructive) @@ -1026,6 +1043,429 @@ /***************************************************************************** * * *****************************************************************************/ +function nntpCreateMessage($msg_ar='') { + +if (defined('DRYRUN') and DRYRUN === true) { return true; } + +if (!is_array($msg_ar)) { + echo "ERROR: invalid array passed to nntpCreateMessage()\n"; + return false; + } + +$msg_id = $msg_ar['message_id']; +$msg_poster = stripslashes($msg_ar['poster_name']); +$board_id = $msg_ar['board_id']; +$board_name = $msg_ar['board_name']; +$ticker_id = $msg_ar['ticker_id']; +$parent_id = $msg_ar['parent_id']; +$thread_id = $msg_ar['thread_id']; +$msg_date = $msg_ar['message_date']; +$msg_subject = stripslashes($msg_ar['message_subject']); +$msg_recs = $msg_ar['message_recs']; +if (NNTP_COPY_TEXT === true) { + $msg_text = stripslashes($msg_ar['message_text']); + } +if (!is_numeric($msg_id)) { + echo "ERROR: invalid msg_id: $msg_id\n"; + return false; + } +if ($msg_poster == '') { + echo "ERROR: invalid msg_poster\n"; + return false; + } +if (!is_numeric($board_id)) { + echo "ERROR: invalid board_id: $board_id\n"; + return false; + } +if (!is_numeric($parent_id)) { + echo "ERROR: invalid parent_id: $parent_id\n"; + return false; + } +if ($msg_date == '') { + echo "ERROR: invalid msg_date\n"; + return false; + } +if (!is_numeric($msg_recs)) { + echo "ERROR: invalid msg_recs: $msg_recs\n"; + return false; + } +if ($board_name == '') { + echo "ERROR: no board name specified\n"; + return false; + } +if (!nntpSpoolCheck($board_name)) { + echo "ABORT: nntpSpoolCheck() failed\n"; + return false; + } +if (!$spool_dir = nntpSpoolDirName($board_name)) { + echo "ABORT: could not determine spool directory\n"; + return false; + } + +$date_str = date("d M Y H:i:s -0600",strtotime($msg_date)); + +$org_url = "http://" . YAHOOHOST . "/bbs?.mm=FN&action=m"; +$org_url .= "&board=$board_id&tid=$ticker_id&sid=$board_id"; +$org_url .= "&mid=$msg_id"; +$org_href = "Original Post"; +$org_jref = "Original Post"; + +$org_href_td = "$org_href"; +$org_jref_td = "$org_jref"; + +$rep_url = "http://" . YAHOOHOST . "/bbs?.mm=FN&action=r"; +$rep_url .= "&board=$board_id&tid=$ticker_id&sid=$board_id"; +$rep_url .= "&mid=$msg_id"; +$rep_href = "Reply"; +$rep_jref = "Reply"; + +$rep_href_td = "$rep_href"; +$rep_jref_td = "$rep_jref"; + +$rec_url = "http://" . YAHOO_RED_HOST . "/mb/recommend?"; +$rec_url .= "http://" . YAHOO_POST_HOST . "/bbs?action=0"; +$rec_url .= "&board=$board_id&tid=$ticker_id&sid=$board_id"; +$rec_url .= "&mid=$msg_id"; +$rec_href = "Recommend"; +$rec_jref = "Recommend"; + +$rec_href_td = "$rec_href"; +$rec_jref_td = "$rec_jref"; + +if (is_numeric($thread_id) and $thread_id > 0) { + $thr_url = "http://" . YAHOO_RED_HOST . "/mb/view/?"; + $thr_url .= "http://" . YAHOOHOST . "/bbs?.mm=FN&action=m"; + $thr_url .= "&board=$board_id&tid=$ticker_id&sid=$board_id"; + $thr_url .= "&mid=$msg_id&thr=$thread_id&cur=$msg_id"; + $thr_href = "View Thread"; + $thr_jref = "View Thread"; + + $thr_href_td = "$thr_href"; + $thr_jref_td = "$thr_jref"; + } else { + $thr_href_td = ''; + $thr_jref_td = ''; + } + +$link_table = " +

+ + + "; +$link_table .= $org_href_td . $rep_href_td . $rec_href_td . $thr_href_td; +$link_table .= " + + + "; +$link_table .= $org_jref_td . $rep_jref_td . $rec_jref_td . $thr_jref_td; +$link_table .= " + \n"; + +$body = "From: \"$msg_poster\" <${msg_poster}@tld.always.invalid> +Date: $date_str +Message-ID: " . nntpGenerateMsgId($msg_id,$board_name) . " +Subject: " . html_entity_decode($msg_subject) . " +Newsgroups: " . NNTP_GROUP_PREFIX . ".${board_name} +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: 8bit +X-Recs: $msg_recs +Lines: $msg_recs\n"; + +if ($parent_id > 0) { + $pmid = nntpGenerateMsgId($parent_id,$board_name); + $body .= "References: $pmid\n"; + } + +$body .= "\nOriginal Yahoo Post: $org_url\n

\n"; + +if (NNTP_COPY_TEXT === true and $msg_text !== '') { + $msg_text = str_replace("h++p://", "http://", $msg_text); + $msg_text = wordwrap(str_replace("
","
\n",$msg_text)); + $body .= "$msg_text"; + } else { + $body .= " +Due to legal threats by a groklaw supporter known as CDBaric, +yahoo posts can no longer be displayed here without exposure to +the threat of legal expense. As a courtesy, links to the original +yahoo post are provided above/below."; + } + +$body .= $link_table; + +$outfile = "$spool_dir/$msg_id"; + +if (!$fd = fopen($outfile,'w')) { + echo "ERROR: cannot open file $outfile\n"; + return false; + } +if (!fwrite($fd,$body)) { + echo "ERROR: failed to write body of message\n"; + @fclose($fd); + return false; + } +@fclose($fd); + +echo "created: $outfile\n"; + +return true; + +} + +/***************************************************************************** + * * + *****************************************************************************/ +function nntpExport($board_name='',$start='',$end='') { + +if (defined('DRYRUN') and DRYRUN === true) { return true; } + +if ($board_name == '') { + echo "What board do you want to export to nntp?\n"; + return false; + } +if (!checkDatabase()) { + echo "checkDatabase() failure\n"; + return false; + } +// create the board if it does not exist +$info_ar = queryBoardInfo($board_name,true); +if ($info_ar === false or !is_array($info_ar)) { + echo "queryBoardInfo() failed\n"; + return false; + } else { + $board_id = $info_ar['board_id']; + $ticker_id = $info_ar['ticker_id']; + } + +if (!nntpSpoolCheck($board_name)) { + echo "ABORT: nntpSpoolCheck() failed\n"; + return false; + } + +if (! $spool_dir = nntpSpoolDirName($board_name)) { + echo "ABORT: could not determine spool directory\n"; + return false; + } + +// determine start/stop +if (!is_numeric($start)) { + if (! $start = nntpGetLastMsgId($spool_dir)) { + echo "ABORT: could not determine last msg id\n"; + return false; + } + if (!is_numeric($start)) { + echo "ABORT: could not determine last msg id\n"; + return false; + } + echo "INFO: starting at $start\n"; + } + +$sql = "select count(message_id), min(message_id), max(message_id) "; +$sql .= "from message_ids where board_id=$board_id "; +if (is_numeric($start)) { + $sql .= "and message_id >= $start "; + } +if (is_numeric($end)) { + $sql .= "and message_id <= $end "; + } +$sql .= "order by message_id"; +if (!$qid = db_query($sql)) { + echo "ABORT: cannot retrieve message list: $sql\n"; + return false; + } + +$hits = db_result($qid,0,0); +$min = db_result($qid,0,1); +$max = db_result($qid,0,2); +db_free_result($qid); + +if ($hits < 1) { + echo "INFO: No messages to export\n"; + return true; + } + +if (!is_numeric($start)) { $start = $min; } +if (!is_numeric($end)) { $end = $max; } + +if ($hits > 10000) { + while($hits >= 1) { + $hits -= 10000; + $end = $start + 10000; + if (!nntpExport($board_name,$start,$end)) { + echo "ABORT: failed board: $board_name start: $start end: $end\n"; + return false; + } + $start += 10000; + } + return true; + } + +// Figure out where to start/end if it was not specified on command line. +$sql = "select message_id from message_ids "; +$sql .= "where board_id=$board_id "; +$sql .= "and message_id >= $start "; +$sql .= "and message_id <= $end "; +$sql .= "order by message_id "; +$sql .= "limit 10000"; + +if (!$qid = db_query($sql)) { + echo "ABORT: cannot retrieve message list: $sql\n"; + return false; + } +$num = db_num_rows($qid); +if ($num < 1) { + echo "INFO: No messages to export\n"; + return true; + } + +// abort if a lock file exists +$LOCK = "${spool_dir}/.LCK"; +if (is_file($LOCK)) { + echo "ABORT: lock file exists: $LOCK\n"; + return false; + } else { + if (!$fd = fopen($LOCK,'w')) { + echo "ABORT: could not create lock file: $LOCK\n"; + return false; + } else { + @fclose($fd); + } + } + +$msg_ar = array(); +for($j=0; $j < $num; $j++) { + $msg_ar[] = db_result($qid,$j,0); + } +db_free_result($qid); + +$start = $msg_ar[0]; +$end = $msg_ar[(count($msg_ar) - 1)]; + +echo "Exporting messages to nntp: Board($board_name) Start($start) End($end)\n"; +foreach($msg_ar as $j) { + $sql = "select a.poster_name,b.message_id,b.board_id,b.parent_id,"; + $sql .= "b.thread_id,b.message_date,b.message_subject,b.message_recs "; + if (NNTP_COPY_TEXT === true) { + $sql .= ", b.message_text "; + } + $sql .= "from poster_ids as a, message_ids as b "; + $sql .= "where b.board_id=$board_id and b.message_id=$j and "; + $sql .= "b.poster_id=a.poster_id "; + $sql .= "limit 1"; + if (!$qid = db_query($sql)) { + echo "ABORT: query failure\n"; + @unlink($LOCK); + return false; + } + $row_ar = db_fetch_array($qid); + $row_ar['board_name'] = $board_name; + $row_ar['ticker_id'] = $ticker_id; + if (!nntpCreateMessage($row_ar)) { + echo "ABORT: failed to export msgid $j\n"; + @unlink($LOCK); + return false; + } + } + +@unlink($LOCK); + +return true; + +} + +/***************************************************************************** + * * + *****************************************************************************/ +function nntpGenerateMsgId($msg_id='',$board_name='') { + +return "<${msg_id}." . NNTP_GROUP_PREFIX . ".${board_name}@localhost>"; + +} + +/***************************************************************************** + * * + *****************************************************************************/ +function nntpGetLastMsgId($spool_dir='') { + +if (!is_dir($spool_dir)) { return false; } + +$max = 0; +if (!$dh = opendir($spool_dir)) { + echo "ERRROR: could not open spool: $spool_dir\n"; + return false; + } +while ( false !== ($file = readdir($dh))) { + if (is_numeric($file) and ($file > $max)) { + $max = $file; + } + } + +return $max + 1; + +} + +/***************************************************************************** + * * + *****************************************************************************/ +function nntpSpoolCheck($board_name='') { + +if (! $spool_dir = nntpSpoolDirName($board_name)) { return false; } + +if (!is_dir($spool_dir)) { + $tmp_ar = explode("/",$spool_dir); + $junk = array_shift($tmp_ar); + $dir_str = ''; + foreach($tmp_ar as $dir) { + $dir_str .= "/${dir}"; + if (!is_dir($dir_str)) { + if (!@mkdir($dir_str,0755)) { + echo "ERROR: could not create dir: $dir_str\n"; + return false; + } + } + } + echo "INFO: created spool directory: $spool_dir\n"; + } + +return true; + +} + +/***************************************************************************** + * * + *****************************************************************************/ +function nntpSpoolDirName($board_name='') { + +if ($board_name == '') { + echo "ERROR: no board name fed to nntpSpoolDirName()"; + return false; + } + +if (!defined('NNTP_SPOOL_DIR') or NNTP_SPOOL_DIR == '') { + echo "ERROR: NNTP_SPOOL_DIR not defined\n"; + return false; + } +if (!defined('NNTP_GROUP_PREFIX') or NNTP_GROUP_PREFIX == '') { + echo "ERROR: NNTP_GROUP_PREFIX not defined\n"; + return false; + } + +return NNTP_SPOOL_DIR . "/" . str_replace('.','/',NNTP_GROUP_PREFIX) + . "/${board_name}"; + +} + +/***************************************************************************** + * * + *****************************************************************************/ function parseBoardId($page='') { if ($page == '' or !is_array($page)) { return false; } @@ -1723,7 +2163,6 @@ return false; } - // Figure out where to start/end if it was not specified on command line. $sql = "select message_id from message_ids "; $sql .= "where board_id=$board_id and message_text='' "; @@ -1797,6 +2236,9 @@ $maxmisses = 0; } processMessage($board_id,$j,$page); + if (DUMPNNTP === true) { + nntpExport($board_name,$j,$j); + } } if (!refreshStats($board_name)) { @@ -1850,11 +2292,12 @@ *****************************************************************************/ function runUpgrades() { -if (defined('DRYRUN') and DRYRUN === true) { +if (defined('DRYRUN') and DRYRUN === true) { echo "DRYRUN: skipping runUpgrades()\n"; return true; } + // verify message_recs column exists when upgrading from // 0.0.4 checkDatabase(); @@ -2011,9 +2454,11 @@ $msg .= "ERROR: Cannot set user_agent: " . USER_AGENT . ".\n"; $msg .= "ERROR: Currently it is " . ini_get('user_agent') . ".\n"; } -if (function_exists(runUpgrades)) { - if (!runUpgrades()) { - $msg .= "ERROR: Upgrade attempt failed.\n"; +if (ACTION !== 'init') { + if (function_exists(runUpgrades)) { + if (!runUpgrades()) { + $msg .= "ERROR: Upgrade attempt failed.\n"; + } } } if (! $msg == '') { @@ -2160,6 +2605,9 @@ $maxmisses = 0; } processMessage($board_id,$j,$page); + if (DUMPNNTP === true) { + nntpExport($board_name,$j,$j); + } } if (!updateCounters($board_id)) { @@ -2297,6 +2745,10 @@ setRecs($board_id,$j,$msg_recs); + if (DUMPNNTP === true) { + nntpExport($board_name,$j,$j); + } + if (QUIET !== true) { echo "ID: $j R: $msg_recs\n"; } diff -u -r -N yasuck-0.0.7/includes/globals.php yasuck-0.0.8/includes/globals.php --- yasuck-0.0.7/includes/globals.php 2004-09-29 10:37:31.000000000 -0500 +++ yasuck-0.0.8/includes/globals.php 2005-01-06 11:39:14.096779848 -0600 @@ -20,6 +20,8 @@ // board settings define(USER_AGENT,'Mozilla/5.0 (compatible; Konqueror/3.3; phpcli)'); define(YAHOOHOST,'finance.messages.yahoo.com'); +define(YAHOO_RED_HOST,'us.rd.yahoo.com'); +define(YAHOO_POST_HOST,'post.messages.yahoo.com'); define(SOCKTIMEOUT,10); //define(FETCHSLEEP,"1,4"); define(FETCHSLEEP,1); @@ -29,10 +31,19 @@ define(RANDOM_UA,INCDIR . "/agents.txt"); define(RANDOMIZE_UA,false); define(MAXMISSES,50); -define(VERSION,"0.0.5"); +define(VERSION,"0.0.8"); define(PATCHLEVEL,""); define(GAPMAX,15); +// +// nntp export related settings +// NNTP_COPY_TEXT if true will copy entire post, +// false will provide links to article. +// +define(NNTP_SPOOL_DIR,"/var/spool/newsd"); +define(NNTP_GROUP_PREFIX,"alt.yahoo"); +define(NNTP_COPY_TEXT,false); + // debug settings define(DEBUG,false); define(DEBUGUA,false); diff -u -r -N yasuck-0.0.7/yasuck yasuck-0.0.8/yasuck --- yasuck-0.0.7/yasuck 2004-10-04 12:20:40.000000000 -0500 +++ yasuck-0.0.8/yasuck 2005-01-05 09:25:52.000000000 -0600 @@ -22,6 +22,17 @@ echo $status; exit(); break; + case "exportnntp": + $rc = nntpExport(BOARD,START,END); + if ($rc === true) { + $status = "Done (success)\n"; + toggleLock(BOARD,0); + } else { + $status = "Done (failure)\n"; + } + echo $status; + exit(); + break; case "get": $rc = suckMessages(BOARD,START,END); if ($rc === true) {
Standard Links
JS Popup Links