iamdoubz/pod_feeder
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge changes from master

Browse Source
This commit is contained in:
Brian O'Donnell
2015-05-27 10:46:02 -04:00
parent 1619049c86
commit d374ffd0d0
1 changed files with 90 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pod_feeder.pl
+90 -89
View File
@@ -26,7 +26,7 @@ use Getopt::Long;
my $opts = { my $opts = {
'database' => './pod_feeder.db', 'database' => './pod_feeder.db',
'limit' => 0, 'limit' => 0,
'timeout' => 72, # hours 'timeout' => 72, # hours
}; };
my @auto_tags = (); my @auto_tags = ();
@@ -70,13 +70,13 @@ if( $fetched ){
eval { eval {
# update the database # update the database
update_feed( update_feed(
$feed, $feed,
db_file => $opts->{'database'}, db_file => $opts->{'database'},
feed_id => $opts->{'feed-id'}, feed_id => $opts->{'feed-id'},
auto_tags => hashtagify( \@auto_tags ), auto_tags => hashtagify( \@auto_tags ),
extract_tags_from_url => $opts->{'url-tags'}, extract_tags_from_url => $opts->{'url-tags'},
extract_tags_from_title => $opts->{'title-tags'}, extract_tags_from_title => $opts->{'title-tags'},
tag_categories => $opts->{'category-tags'}, tag_categories => $opts->{'category-tags'},
); );
}; };
warn "$@" if $@; warn "$@" if $@;
@@ -84,15 +84,15 @@ if( $fetched ){
eval { eval {
# publish new feed items to the pod, unless the user specified --fetch-only # publish new feed items to the pod, unless the user specified --fetch-only
publish_feed_items( publish_feed_items(
db_file => $opts->{'database'}, db_file => $opts->{'database'},
feed_id => $opts->{'feed-id'}, feed_id => $opts->{'feed-id'},
timeout => $opts->{'timeout'}, timeout => $opts->{'timeout'},
pod_url => $opts->{'pod-url'}, pod_url => $opts->{'pod-url'},
username => $opts->{'username'}, username => $opts->{'username'},
password => $opts->{'password'}, password => $opts->{'password'},
aspect_ids => \@aspect_ids, aspect_ids => \@aspect_ids,
raw_link => $opts->{'post-raw-links'}, raw_link => $opts->{'post-raw-links'},
limit => $opts->{'limit'}, limit => $opts->{'limit'},
) unless $opts->{'fetch-only'}; ) unless $opts->{'fetch-only'};
}; };
warn "$@" if $@; warn "$@" if $@;
@@ -105,7 +105,7 @@ else {
sub publish_feed_items { sub publish_feed_items {
my ( %params ) = @_; my ( %params ) = @_;
my @updates = (); my @updates = ();
my $query_string = "SELECT guid, title, link, hashtags FROM feeds WHERE feed_id == ? AND posted == 0 AND timestamp > ? ORDER BY timestamp"; my $query_string = "SELECT guid, title, link, hashtags FROM feeds WHERE feed_id == ? AND posted == 0 AND timestamp > ? ORDER BY timestamp";
my $dbh = connect_to_db( $params{'db_file'} ); my $dbh = connect_to_db( $params{'db_file'} );
# limit the number of items published if limit is specified # limit the number of items published if limit is specified
@@ -122,12 +122,12 @@ sub publish_feed_items {
foreach my $update ( @updates ){ foreach my $update ( @updates ){
my $content = $update->{'hashtags'}; my $content = $update->{'hashtags'};
# to hyperlink the title or not to hyperlink the title... # to hyperlink the title or not to hyperlink the title...
if( $params{'raw_link'} ){ if( $params{'raw_link'} ){
$content = '### ' . $update->{'title'} . "\n\n" . $update->{'link'} . "\n" . $content; $content = '### ' . $update->{'title'} . "\n\n" . $update->{'link'} . "\n" . $content;
} }
else { else {
$content = '### [' . $update->{'title'} . '](' . $update->{'link'} . ")\n\n" . $content; $content = '### [' . $update->{'title'} . '](' . $update->{'link'} . ")\n\n" . $content;
} }
print "Publishing $params{'feed_id'}\t$update->{'guid'}\n"; print "Publishing $params{'feed_id'}\t$update->{'guid'}\n";
@@ -173,13 +173,13 @@ sub update_feed {
"INSERT INTO feeds( guid, feed_id, title, link, hashtags, posted, timestamp ) VALUES( ?, ?, ?, ?, ?, ?, ?)" "INSERT INTO feeds( guid, feed_id, title, link, hashtags, posted, timestamp ) VALUES( ?, ?, ?, ?, ?, ?, ?)"
) or die "Can't prepare statement: $DBI::errstr"; ) or die "Can't prepare statement: $DBI::errstr";
$sth->execute( $sth->execute(
$item->{'guid'}, $item->{'guid'},
$params{'feed_id'}, $params{'feed_id'},
$item->{'title'}, $item->{'title'},
$item->{'link'}, $item->{'link'},
join( ' ', @{$item->{'hashtags'}} ), join( ' ', @{$item->{'hashtags'}} ),
0, 0,
time, time,
) or die "Can't execute statement: $DBI::errstr"; ) or die "Can't execute statement: $DBI::errstr";
} }
} }
@@ -198,7 +198,7 @@ sub connect_to_db {
sub get_feed_items { sub get_feed_items {
my ( $feed, %params ) = @_; my ( $feed, %params ) = @_;
my @items = (); my @items = ();
my $list = decode_feed( $feed ); my $list = decode_feed( $feed );
$params{'auto_tags'} = 0 unless defined $params{'auto_tags'}; $params{'auto_tags'} = 0 unless defined $params{'auto_tags'};
$params{'extract_tags_from_url'} = 0 unless defined $params{'extract_tags_from_url'}; $params{'extract_tags_from_url'} = 0 unless defined $params{'extract_tags_from_url'};
@@ -206,12 +206,15 @@ sub get_feed_items {
foreach my $item ( @$list ){ foreach my $item ( @$list ){
my $link = $item->{'link'}; my $link = $item->{'link'};
my $title =
# no link, no go
next unless defined $link and ref $link ne 'HASH' and ref $link ne 'ARRAY';
my @hashtags = (); my @hashtags = ();
my $guid = ''; my $guid = undef;
# strip trailing / # strip trailing /
$link =~ s/\/+$//; $link =~ s/\/+$// if defined $link;
# add user-specified tags # add user-specified tags
push( @hashtags, @{$params{'auto_tags'}} ) if defined $params{'auto_tags'}; push( @hashtags, @{$params{'auto_tags'}} ) if defined $params{'auto_tags'};
@@ -233,18 +236,18 @@ sub get_feed_items {
# try to guess tags from the title # try to guess tags from the title
if( $params{'extract_tags_from_title'} ){ if( $params{'extract_tags_from_title'} ){
my $title = $item->{'title'}; my $title = $item->{'title'};
# strip apostrophes # strip apostrophes
$title =~ s/'//g; $title =~ s/'//g;
# split up string on non-alphanumerics # split up string on non-alphanumerics
my @parts = split( /([^(\p{Letter}|\p{Number})]|\p{Punctuation})/, $title ); my @parts = split( /([^(\p{Letter}|\p{Number})]|\p{Punctuation})/, $title );
my @tags = (); my @tags = ();
foreach my $part ( @parts ){ foreach my $part ( @parts ){
push( @tags, $part ) unless $part =~ m/^(\s+)?$/; push( @tags, $part ) unless $part =~ m/^(\s+)?$/;
} }
push( @hashtags, @tags ); push( @hashtags, @tags );
} }
@@ -269,20 +272,18 @@ sub get_feed_items {
if( ref $item->{'guid'} eq 'HASH' and defined $item->{'guid'}->{'content'} ){ if( ref $item->{'guid'} eq 'HASH' and defined $item->{'guid'}->{'content'} ){
$guid = $item->{'guid'}->{'content'}; $guid = $item->{'guid'}->{'content'};
} }
else { elsif( ref $item->{'guid'} ne 'HASH' ) {
$guid = $item->{'guid'}; $guid = $item->{'guid'};
} }
} }
elsif( defined $item->{'id'} ){ elsif( defined $item->{'id'} ){
$guid = $item->{'id'}; $guid = $item->{'id'};
} }
elsif( defined $item->{'link'} ){ else { $guid = $link }
$guid = $item->{'link'};
}
my $obj = { my $obj = {
guid => $guid, guid => $guid,
title => $item->{'title'}, title => $item->{'title'},
link => $link, link => $link,
hashtags => hashtagify( \@hashtags ), hashtags => hashtagify( \@hashtags ),
}; };
@@ -295,47 +296,47 @@ sub get_feed_items {
# extract the data we need based on feed type (RSS v. Atom) # extract the data we need based on feed type (RSS v. Atom)
sub decode_feed{ sub decode_feed{
my ( $feed ) = @_; my ( $feed ) = @_;
my @list = (); my @list = ();
# RSS # RSS
if( defined $feed->{'channel'} and defined $feed->{'channel'}->{'item'} and ref $feed->{'channel'}->{'item'} eq 'ARRAY' ){ if( defined $feed->{'channel'} and defined $feed->{'channel'}->{'item'} and ref $feed->{'channel'}->{'item'} eq 'ARRAY' ){
@list = @{$feed->{'channel'}->{'item'}}; @list = @{$feed->{'channel'}->{'item'}};
} }
# Atom # Atom
elsif( defined $feed->{'entry'} and ref $feed->{'entry'} eq 'HASH' ){ elsif( defined $feed->{'entry'} and ref $feed->{'entry'} eq 'HASH' ){
my $entries = $feed->{'entry'}; my $entries = $feed->{'entry'};
foreach my $guid ( keys %$entries ){ foreach my $guid ( keys %$entries ){
my $item = { my $item = {
guid => $guid, guid => $guid,
}; };
if( defined $entries->{$guid}->{'title'} ){ if( defined $entries->{$guid}->{'title'} ){
if( ref $entries->{$guid}->{'title'} eq 'HASH' and defined $entries->{$guid}->{'title'}->{'content'} ){ if( ref $entries->{$guid}->{'title'} eq 'HASH' and defined $entries->{$guid}->{'title'}->{'content'} ){
$item->{'title'} = $entries->{$guid}->{'title'}->{'content'}; $item->{'title'} = $entries->{$guid}->{'title'}->{'content'};
} }
elsif( ref $entries->{$guid}->{'title'} eq '' ){ elsif( ref $entries->{$guid}->{'title'} eq '' ){
$item->{'title'} = $entries->{$guid}->{'title'}; $item->{'title'} = $entries->{$guid}->{'title'};
} }
} }
if( defined $entries->{$guid}->{'link'} ){ if( defined $entries->{$guid}->{'link'} ){
if( ref $entries->{$guid}->{'link'} eq 'HASH' and defined $entries->{$guid}->{'link'}->{'href'} ){ if( ref $entries->{$guid}->{'link'} eq 'HASH' and defined $entries->{$guid}->{'link'}->{'href'} ){
$item->{'link'} = $entries->{$guid}->{'link'}->{'href'}; $item->{'link'} = $entries->{$guid}->{'link'}->{'href'};
} }
elsif( ref $entries->{$guid}->{'link'} eq '' ){ elsif( ref $entries->{$guid}->{'link'} eq '' ){
$item->{'link'} = $entries->{$guid}->{'link'}; $item->{'link'} = $entries->{$guid}->{'link'};
} }
} }
$item->{'category'} = $entries->{'category'} if defined $entries->{'category'}; $item->{'category'} = $entries->{'category'} if defined $entries->{'category'};
push( @list, $item ) if defined $item->{'link'} and defined $item->{'title'}; push( @list, $item ) if defined $item->{'link'} and defined $item->{'title'};
} }
} }
return \@list; return \@list;
} }
# fetch the feed and convert the XML to an data object # fetch the feed and convert the XML to an data object
@@ -499,17 +500,17 @@ sub post_message {
# create a new sqlite db file with a 'feeds' table if it does not exist already # create a new sqlite db file with a 'feeds' table if it does not exist already
sub init_database { sub init_database {
my ( $db_file ) = @_; my ( $db_file ) = @_;
unless( -e $db_file ){ unless( -e $db_file ){
my $dbh = connect_to_db( $db_file ); my $dbh = connect_to_db( $db_file );
my $sth = $dbh->prepare( my $sth = $dbh->prepare(
'CREATE TABLE feeds(guid VARCHAR(255) PRIMARY KEY,feed_id VARCHAR(127),title VARCHAR(255),link VARCHAR(255),hashtags VARCHAR(255),timestamp INTEGER(10),posted INTEGER(1))' 'CREATE TABLE feeds(guid VARCHAR(255) PRIMARY KEY,feed_id VARCHAR(127),title VARCHAR(255),link VARCHAR(255),hashtags VARCHAR(255),timestamp INTEGER(10),posted INTEGER(1))'
) or die "Can't prepare statement: $DBI::errstr"; ) or die "Can't prepare statement: $DBI::errstr";
$sth->execute() or die "Can't execute statement: $DBI::errstr"; $sth->execute() or die "Can't execute statement: $DBI::errstr";
$dbh->disconnect(); $dbh->disconnect();
} }
} }
sub usage { sub usage {
@@ -518,7 +519,7 @@ sub usage {
print " -a --aspect-id <id> Aspects to share with. May specify multiple times (default: 'public')\n"; print " -a --aspect-id <id> Aspects to share with. May specify multiple times (default: 'public')\n";
print " -c --category-tags Attempt to automatically hashtagify RSS item 'categories' (default: off)\n"; print " -c --category-tags Attempt to automatically hashtagify RSS item 'categories' (default: off)\n";
print " -d --database <sqlite file> The SQLite file to store feed data (default: 'feed.db')\n"; print " -d --database <sqlite file> The SQLite file to store feed data (default: 'feed.db')\n";
print " -e --title-tags Automatically hashtagify RSS item title\n"; print " -e --title-tags Automatically hashtagify RSS item title\n";
print " -f --feed-url <http://...> The feed URL\n"; print " -f --feed-url <http://...> The feed URL\n";
print " -g --user-agent <string> Use this to spoof the user-agent if the feed blocks bots (ex: 'Mozilla/5.0')\n"; print " -g --user-agent <string> Use this to spoof the user-agent if the feed blocks bots (ex: 'Mozilla/5.0')\n";
print " -i --feed-id <string> An arbitrary identifier to associate database entries with this feed\n"; print " -i --feed-id <string> An arbitrary identifier to associate database entries with this feed\n";
@@ -530,7 +531,7 @@ sub usage {
print " -t --auto-tag <#hashtag> Hashtags to add to all posts. May be specified multiple times (default: none)\n"; print " -t --auto-tag <#hashtag> Hashtags to add to all posts. May be specified multiple times (default: none)\n";
print " -u --username <user> The D* login username\n"; print " -u --username <user> The D* login username\n";
print " -w --post-raw-link Post the raw link instead of hyperlinking the article title (default: off)\n"; print " -w --post-raw-link Post the raw link instead of hyperlinking the article title (default: off)\n";
print " -x --limit <n> Only post n items per script run, to prevent post-spamming (default: no limit)\n"; print " -x --limit <n> Only post n items per script run, to prevent post-spamming (default: no limit)\n";
print "\n"; print "\n";
exit; exit;