#! /usr/bin/perl use XML::RSS; use LWP::UserAgent; use LWP::ConnCache; use HTTP::Cookies; use HTTP::Date; use Data::Dumper; use File::stat; use LWP::Debug; $basedir="$ENV{HOME}/.bjhportal"; open URLS, "< $basedir/urls" || die "no urls"; @urls = ; chomp(@urls); @urls = grep($_ !~ /^#/, @urls); close URLS; dbmopen(%history, "$basedir/history", 0666) or die "history: $!"; $ua = new LWP::UserAgent(keep_alive => 5); $ua->cookie_jar(new HTTP::Cookies(file => "$basedir/cookies" , autosave => 1)); for $url (@urls) { $file = $url; $file =~ s,/,!,g; $file = "$basedir/cache/$file"; mkdir "$basedir/cache", 0777; $resp = $ua->mirror($url, $file); if ($resp->is_error) { warn "$url: ", $resp->status_line, "\n"; next; } if (stat($file)->size == 0) { warn "$url: no data\n"; unlink($file); next; } $rss = new XML::RSS(); eval { $rss->parsefile($file) }; next if $@; for $item (@{$rss->{items}}) { $item->{_src} = $rss; if (defined($item->{pubDate})) { $item->{_date} = str2time($item->{pubDate}); } else { if ($item->{link} && $history{$item->{link}}) { $item->{_date} = $history{$item->{link}}; } else { $item->{_date} = stat($file)->mtime; if ($item->{link}) { $history{$item->{link}} = $item->{_date}; } } } push(@items, $item); } } @items = sort { -($a->{_date} <=> $b->{_date}) } @items; @items = @items[0..99]; binmode STDOUT, ":utf8"; print < stuff EOF for $item (@items) { print qq[
\n]; print qq[
], $item->{_src}->channel('title'), "
\n"; for $attr (qw(pubDate title description)) { if ($item->{$attr}) { print qq[
], $item->{$attr}, "
\n"; } } if ($item->{link}) { print qq[
Link
]; } print "
\n"; } print < EOF