Include original URL to repo

Added indentation

.gitignore

@@ -0,0 +1 @@
+*.db

README.md

@@ -1,4 +1,9 @@
-# pod_feeder
+# NOTICE: THIS PROJECT IS NO LONGER MAINTAINED
+Please migrate to
+[pod_feeder_v2](https://gitlab.com/brianodonnell/pod_feeder_v2) and let this
+version die with dignity 🙂
+
+# [pod_feeder](https://github.com/rev138/pod_feeder)
 Publishes RSS/Atom feeds to Diaspora*
 
 This is a lightweight, customizable "bot" script to harvest RSS/Atom feeds and re-publish them to the Diaspora social network. It is posted here without warranty, for public use.
@@ -13,8 +18,11 @@ This is a lightweight, customizable "bot" script to harvest RSS/Atom feeds and r
 - JSON
 - XML::Simple
 - DBD::SQLite
-- Unicode::Normalize
-- Getopt::Long
+- HTML::FormatMarkdown
+
+For instance, on a debian system you can install them like so:
+
+`sudo apt-get install libwww-perl libany-uri-escape-perl libhtml-parser-perl libjson-perl libxml-simple-perl libdbd-sqlite3-perl libhtml-format-perl`
 
 This script is intended to be run as a cron job, which might look something like this:
 
@@ -23,32 +31,30 @@ This script is intended to be run as a cron job, which might look something like
 ## Usage
 
     -a   --aspect-id <id>                Aspects to share with. May specify multiple times (default: 'public')
+    -b   --embed-image                   Embed an image in the post if a link exists (default: off)
+         --body                          Post the body of the feed (description or content:encoded item)
     -c   --category-tags                 Attempt to automatically hashtagify RSS item 'categories' (default: off)
     -d   --database <sqlite file>        The SQLite file to store feed data (default: 'feed.db')
     -e   --title-tags                    Automatically hashtagify RSS item title
     -f   --feed-url <http://...>         The feed URL
     -g   --user-agent <string>           Use this to spoof the user-agent if the feed blocks bots (ex: 'Mozilla/5.0')
     -i   --feed-id <string>              An arbitrary identifier to associate database entries with this feed
+    -j   --no-branding                   Do not include 'posted via pod_feeder' footer to posts
     -l   --pod-url <https://...>         The pod URL
     -m   --timeout <hours>               How long (in hours) to keep attempting failed posts (default 72)
+    -n   --ignore-tag <#hashtag>         Hashtags to filter out. May be specified multiple times (default: none)
     -o   --fetch-only                    Don't publish to Diaspora, just queue the new feed items for later
     -p   --password <********>           The D* user password
     -r   --url-tags                      Attempt to automatically hashtagify the RSS link URL (default: off)
     -t   --auto-tag <#hashtag>           Hashtags to add to all posts. May be specified multiple times (default: none)
+    -s   --insecure                      Allows the option to bypass any errors caused from self-signed certificates(default: off)
     -u   --username <user>               The D* login username
+    -v   --via <string>                  Sets the 'posted via' text (default: 'pod_feeder')
     -w   --post-raw-link                 Post the raw link instead of hyperlinking the article title (default: off)
     -x   --limit <n>                     Only post n items per script run, to prevent post-spamming (default: no limit)
 
 ## A Note on YouTube Feeds
 
-It is possible to publish a YouTube channel's feed, however YT makes it a little difficult:
+It is possible to publish a YouTube channel's feed, by using the following URL format:
 
-Get the URL of the YouTube Channel, ex:
-
-https://www.youtube.com/channel/UCQzdMyuz0Lf4zo4uGcEujFw
-
-Modify it thusly, then feed it to the script:
-
-[https://www.youtube.com/**rss**/channel/UCQzdMyuz0Lf4zo4uGcEujFw/**feed.rss**](https://www.youtube.com/rss/channel/UCQzdMyuz0Lf4zo4uGcEujFw/feed.rss)
-
-If you'd want Diaspora to automatically embed the video, you must also pass the `--post-raw-link` argument
+    https://www.youtube.com/feeds/videos.xml?channel_id=<channel id>

pod_feeder.pl

@@ -1,4 +1,5 @@
 #!/usr/bin/perl
+# vim:set sw=8 ts=8 sts=8 ft=perl expandtab:
 
 ##
 ## pod_feeder.pl
@@ -23,26 +24,34 @@ use XML::Simple;
 use DBI;
 use Unicode::Normalize 'normalize';
 use Getopt::Long;
+use HTML::FormatMarkdown;
 
 my $opts = {
-        'database'              => './pod_feeder.db',
-        'limit'			=> 0,
-        'timeout'               => 72,  # hours
+        'database' => './pod_feeder.db',
+        'limit'    => 0,
+        'timeout'  => 72,  # hours
+        'via'      => 'pod_feeder',
 };
 my @auto_tags = ();
+my @ignored_tags = ();
 my @aspect_ids = ();
 
 GetOptions(
         $opts,
-        'aspect-id|a=s'         => \@aspect_ids,
-        'auto-tag|t=s'          => \@auto_tags,
+        'aspect-id|a=s'     => \@aspect_ids,
+        'auto-tag|t=s'      => \@auto_tags,
+        'body',
         'category-tags|c',
         'database|d=s',
+        'embed-image|b',
         'feed-id|i=s',
         'feed-url|f=s',
         'fetch-only|o',
-        'help|h',               => \&usage,
+        'help|h',           => \&usage,
+        'ignore-tag|n=s',   => \@ignored_tags,
+        'insecure|s=s',
         'limit|x=i',
+        'no-branding',
         'password|p=s',
         'pod-url|l=s',
         'post-raw-links|w',
@@ -51,6 +60,7 @@ GetOptions(
         'url-tags|r',
         'user-agent|g=s',
         'username|u=s',
+        'via|v=s',
 );
 
 # defaults to 'public' if no aspect ids are specified
@@ -70,13 +80,14 @@ if( $fetched ){
         eval {
                 # update the database
                 update_feed(
-                	$feed,
-                	db_file 		=> $opts->{'database'},
-                	feed_id 		=> $opts->{'feed-id'},
-                	auto_tags		=> hashtagify( \@auto_tags ),
-                	extract_tags_from_url	=> $opts->{'url-tags'},
-                	extract_tags_from_title	=> $opts->{'title-tags'},
-                	tag_categories		=> $opts->{'category-tags'},
+                        $feed,
+                        db_file                 => $opts->{'database'},
+                        feed_id                 => $opts->{'feed-id'},
+                        auto_tags               => hashtagify( \@auto_tags ),
+                        ignored_tags            => hashtagify( \@ignored_tags ),
+                        extract_tags_from_url   => $opts->{'url-tags'},
+                        extract_tags_from_title => $opts->{'title-tags'},
+                        tag_categories          => $opts->{'category-tags'},
                 );
         };
         warn "$@" if $@;
@@ -84,15 +95,20 @@ if( $fetched ){
         eval {
                 # publish new feed items to the pod, unless the user specified --fetch-only
                 publish_feed_items(
-                	db_file		=> $opts->{'database'},
-                	feed_id		=> $opts->{'feed-id'},
-                	timeout		=> $opts->{'timeout'},
-                	pod_url		=> $opts->{'pod-url'},
-                	username	=> $opts->{'username'},
-                	password	=> $opts->{'password'},
-                	aspect_ids	=> \@aspect_ids,
-                	raw_link	=> $opts->{'post-raw-links'},
-                	limit		=> $opts->{'limit'},
+                        db_file     => $opts->{'database'},
+                        embed_image => $opts->{'embed-image'},
+                        feed_id     => $opts->{'feed-id'},
+                        timeout     => $opts->{'timeout'},
+                        pod_url     => $opts->{'pod-url'},
+                        username    => $opts->{'username'},
+                        password    => $opts->{'password'},
+                        aspect_ids  => \@aspect_ids,
+                        raw_link    => $opts->{'post-raw-links'},
+                        limit       => $opts->{'limit'},
+                        no_branding => $opts->{'no-branding'},
+                        via         => $opts->{'via'},
+                        insecure    => $opts->{'insecure'},
+                        body        => $opts->{'body'}
                 ) unless $opts->{'fetch-only'};
         };
         warn "$@" if $@;
@@ -105,12 +121,12 @@ else {
 sub publish_feed_items {
         my ( %params ) = @_;
         my @updates = ();
-	my $query_string = "SELECT guid, title, link, hashtags FROM feeds WHERE feed_id == ? AND posted == 0 AND timestamp > ? ORDER BY timestamp";
+        my $query_string = "SELECT guid, title, link, image, image_title, hashtags, body FROM feeds WHERE feed_id == ? AND posted == 0 AND timestamp > ? ORDER BY timestamp";
         my $dbh = connect_to_db( $params{'db_file'} );
-        
+
         # limit the number of items published if limit is specified
         $query_string .= " LIMIT $params{'limit'}" if $params{'limit'} > 0;
-        
+
         my $sth = $dbh->prepare( $query_string ) or die "Can't prepare statement: $DBI::errstr";
 
         $sth->execute( $params{'feed_id'}, time - ( $params{'timeout'} * 3600 ) ) or die "Can't execute statement: $DBI::errstr";
@@ -122,16 +138,24 @@ sub publish_feed_items {
         foreach my $update ( @updates ){
                 my $content = $update->{'hashtags'};
 
-		# to hyperlink the title or not to hyperlink the title...
+                if( $params{'embed_image'} and length $update->{'image'} ){
+                    my $image_link = '[![](' . $update->{'image'};
+                    $image_link .= ' "' . $update->{'image_title'} . '"' if length $update->{'image_title'};
+                    $image_link .= ')](' . $update->{'link'} . ')';
+                    $content = "$image_link\n$content";
+                }
+
+                # to hyperlink the title or not to hyperlink the title...
                 if( $params{'raw_link'} ){
-                	$content = '### ' . $update->{'title'} . "\n\n" . $update->{'link'} . "\n" . $content;
+                        $content = '### ' . $update->{'title'} . "\n\n" . $update->{'link'} . "\n" . $content;
                 }
                 else {
-                	$content = '### [' . $update->{'title'} . '](' . $update->{'link'} . ")\n\n" . $content;
+                        $content = '### [' . $update->{'title'} . '](' . $update->{'link'} . ")\n\n" . $content;
                 }
 
-                print "Publishing $params{'feed_id'}\t$update->{'guid'}\n";
+                $content .= "\n" . $update->{'body'} if $params{'body'};
 
+                print "Publishing $params{'feed_id'}\t$update->{'guid'}\n";
                 my $post = publish_post( $content, %params );
 
                 # mark the item as successfully posted
@@ -154,14 +178,23 @@ sub publish_feed_items {
 sub update_feed {
         my ( $feed, %params ) = @_;
 
-        $params{'auto_tags'} = 0 unless defined $params{'auto_tags'};
+        $params{'auto_tags'} = [] unless defined $params{'auto_tags'};
         $params{'extract_tags_from_url'} = 0 unless defined $params{'extract_tags_from_url'};
         $params{'tag_categories'} = 0 unless defined $params{'tag_categories'};
+        $params{'ignored_tags'} = [] unless defined $params{'ignored_tags'};
 
         my $items = get_feed_items( $feed, %params );
         my $dbh = connect_to_db( $params{'db_file'} );
 
         foreach my $item ( @$items ){
+                # strip junk
+                map { $item->{$_} =~ s/^\s+|\s+$//g } keys %$item;
+                map { $item->{$_} =~ s/^\n+|\n+$//g } keys %$item;
+
+                # decode uft8 strings before storing in the db
+                map { utf8::decode($item->{'title'}) } keys %$item;
+                map { utf8::decode($item->{'body'}) } keys %$item;
+
                 # check to see if it exists already
                 my $sth = $dbh->prepare("SELECT guid FROM feeds WHERE guid == ? LIMIT 1") or die "Can't prepare statement: $DBI::errstr";
                 $sth->execute( $item->{'guid'} ) or die "Can't execute statement: $DBI::errstr";
@@ -170,16 +203,19 @@ sub update_feed {
                 # and if not, insert it
                 unless( defined $row ){
                         $sth = $dbh->prepare(
-                                "INSERT INTO feeds( guid, feed_id, title, link, hashtags, posted, timestamp ) VALUES( ?, ?, ?, ?, ?, ?, ?)"
+                                "INSERT INTO feeds( guid, feed_id, title, body, link, image, image_title, hashtags, posted, timestamp ) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"
                         ) or die "Can't prepare statement: $DBI::errstr";
                         $sth->execute(
-                        	$item->{'guid'},
-                        	$params{'feed_id'},
-                        	$item->{'title'},
-                        	$item->{'link'},
-                        	join( ' ', @{$item->{'hashtags'}} ),
-                        	0,
-                        	time,
+                                $item->{'guid'},
+                                $params{'feed_id'},
+                                $item->{'title'},
+                                $item->{'body'},
+                                $item->{'link'},
+                                $item->{'image'},
+                                $item->{'image_title'},
+                                join( ' ', @{$item->{'hashtags'}} ),
+                                0,
+                                time,
                         ) or die "Can't execute statement: $DBI::errstr";
                 }
         }
@@ -189,7 +225,7 @@ sub update_feed {
 
 sub connect_to_db {
         my ( $db_file ) = @_;
-        my $dbh = DBI->connect("dbi:SQLite:dbname=$db_file", '', '', { RaiseError => 1 } ) or die $DBI::errstr;
+        my $dbh = DBI->connect("dbi:SQLite:dbname=$db_file", '', '', { RaiseError => 1, sqlite_unicode => 0 } ) or die $DBI::errstr;
 
         return $dbh;
 }
@@ -198,20 +234,26 @@ sub connect_to_db {
 sub get_feed_items {
         my ( $feed, %params ) = @_;
         my @items = ();
-	my $list = decode_feed( $feed );
+        my $list = decode_feed( $feed );
 
-        $params{'auto_tags'} = 0 unless defined $params{'auto_tags'};
+        $params{'auto_tags'} = [] unless defined $params{'auto_tags'};
         $params{'extract_tags_from_url'} = 0 unless defined $params{'extract_tags_from_url'};
         $params{'tag_categories'} = 0 unless defined $params{'tag_categories'};
+        $params{'ignored_tags'} = [] unless defined $params{'ignored_tags'};
 
         foreach my $item ( @$list ){
                 my $link = $item->{'link'};
-                my $title =
+
+                # no link, no go
+                next unless defined $link and ref $link ne 'HASH' and ref $link ne 'ARRAY';
+
                 my @hashtags = ();
-                my $guid = '';
+                my $guid = undef;
+                my $image = '';
+                my $image_title = '';
 
                 # strip trailing /
-                $link =~ s/\/+$//;
+                $link =~ s/\/+$// if defined $link;
 
                 # add user-specified tags
                 push( @hashtags, @{$params{'auto_tags'}} ) if defined $params{'auto_tags'};
@@ -226,25 +268,25 @@ sub get_feed_items {
                         $link_part =~ s/(\?.*)$//;
 
                         # split up string
-                        my @parts = split( /([^(\p{Letter}|\p{Number})]|\p{Punctuation})/, $link_part );
+                        my @parts = split( /([^[[:alnum:]]]|[[:blank:]]|[[:punct:]])/, $link_part );
 
                         push( @hashtags, @parts );
                 }
 
                 # try to guess tags from the title
                 if( $params{'extract_tags_from_title'} ){
-			my $title = $item->{'title'};
+                        my $title = $item->{'title'};
 
-			# strip apostrophes
-			$title =~ s/'//g;
+                        # strip apostrophes
+                        $title =~ s/('|`)//g;
 
                         # split up string on non-alphanumerics
-                        my @parts = split( /([^(\p{Letter}|\p{Number})]|\p{Punctuation})/, $title );
-			my @tags = ();
+                        my @parts = split( /([^[[:alnum:]]]|[[:blank:]]|[[:punct:]])/, $title );
+                        my @tags = ();
 
-			foreach my $part ( @parts ){
-				push( @tags, $part ) unless $part =~ m/^(\s+)?$/;
-			}
+                        foreach my $part ( @parts ){
+                                push( @tags, $part ) unless $part =~ m/^(\s+)?$/;
+                        }
 
                         push( @hashtags, @tags );
                 }
@@ -263,76 +305,162 @@ sub get_feed_items {
                         push ( @hashtags, @categories );
                 }
 
+                # extract image link and hover text from content:encoded if it exists
+                if( defined $item->{'content:encoded'} ){
+                        $item->{'content:encoded'} =~ /img .* ?src=\\?'(https?:\/\/[^']+)/ unless $item->{'content:encoded'} =~ /img .* ?src=\\?"(https?:\/\/[^"]+)/;
+
+                        if( defined $1 ){
+                                $image = $1;
+                                $item->{'content:encoded'} =~ / title='([^']+)/ unless $item->{'content:encoded'} =~ / title="([^"]+)/;
+                                $image_title = $1 if defined $1;
+                        }
+                }
+
+                # extract image link and hover text from description if it exists
+                if( not length $image and defined $item->{'description'} ){
+                        $item->{'description'} =~ /img .* ?src='(https?:\/\/[^']+)/ unless $item->{'description'} =~ /img .* ?src="(https?:\/\/[^"]+)/;
+
+                        if( defined $1 ){
+                                $image = $1;
+                                $item->{'description'} =~ / title='([^']+)/ unless $item->{'description'} =~ / title="([^"]+)/;
+                                $image_title = $1 if defined $1;
+                        }
+                }
+
+                # extract the image link from the enclosure tag if it exists
+                if( not length $image and defined $item->{'enclosure'} and defined $item->{'enclosure'}->{'type'} and $item->{'enclosure'}->{'type'} =~ /^image\// ){
+                        $image = $item->{'enclosure'}->{'url'} if defined $item->{'enclosure'}->{'url'};
+                }
+
+                # remove any query params from image link
+                $image =~ s/(\?.*)$//;
+
                 @hashtags = sort @hashtags;
 
                 if( defined $item->{'guid'} ){
                         if( ref $item->{'guid'} eq 'HASH' and defined $item->{'guid'}->{'content'} ){
                                 $guid = $item->{'guid'}->{'content'};
                         }
-                        else {
+                        elsif( ref $item->{'guid'} ne 'HASH' ) {
                                 $guid = $item->{'guid'};
                         }
                 }
                 elsif( defined $item->{'id'} ){
                         $guid = $item->{'id'};
                 }
+                else { $guid = $link }
+
+                @hashtags = @{ hashtagify( \@hashtags ) };
+
+                # filter out ignored tags
+                for( my $t = 0; $t < @hashtags; $t++ ){
+                        foreach my $ignored ( @{$params{'ignored_tags'}} ){
+                                splice( @hashtags, $t, 1 ) if $hashtags[$t] eq $ignored;
+                        }
+                }
+
+		my $body = '';
+                $body    = HTML::FormatMarkdown->format_from_string($item->{'description'}, rm => 100000) if ($item->{'description'});
+                $body    = HTML::FormatMarkdown->format_from_string($item->{'content:encoded'}, rm => 100000) if ($item->{'content:encoded'});
+                $body    = HTML::FormatMarkdown->format_from_string($item->{'content'}, rm => 100000) if ($item->{'content'});
 
                 my $obj = {
-                        guid            => $guid,
-                        title		=> $item->{'title'},
-                        link            => $link,
-                        hashtags        => hashtagify( \@hashtags ),
+                        guid        => $guid,
+                        title       => $item->{'title'},
+                        body        => $body,
+                        link        => $link,
+                        image       => $image,
+                        image_title => $image_title,
+                        hashtags    => \@hashtags,
                 };
 
                 $items[@items] = $obj;
         }
 
-        return \@items;
+        # the last shall be first and the first shall be last
+        my @reversed = ();
+        for( my $i = $#items; $i >= 0; $i-- ){
+                $reversed[@reversed] = $items[$i];
+        }
+
+        return \@reversed;
 }
 
 # extract the data we need based on feed type (RSS v. Atom)
 sub decode_feed{
-	my ( $feed ) = @_;
-	my @list = ();
+        my ( $feed ) = @_;
+        my @list = ();
+        # RSS
+        if( defined $feed->{'channel'} and defined $feed->{'channel'}->{'item'} ){
+                if( ref $feed->{'channel'}->{'item'} eq 'ARRAY' ){
+                        @list = @{$feed->{'channel'}->{'item'}};
+                }
+                elsif( ref $feed->{'channel'}->{'item'} eq 'HASH' ){
+                        if( length( keys %{$feed->{'channel'}->{'item'}} ) == 1 ){
+                                $list[@list] = $feed->{'channel'}->{'item'}
+                        }
+                        else{
+                                @list = values %{$feed->{'channel'}->{'item'}};
+                        }
+                }
+        }
+        elsif( defined $feed->{'item'} ){
+                if( ref $feed->{'item'} eq 'ARRAY' ){
+                        @list = @{$feed->{'item'}};
+                }
+                elsif( ref $feed->{'item'} eq 'HASH' ){
+                        @list = values %{$feed->{'item'}};
+                }
+        }
+        # Atom
+        elsif( defined $feed->{'entry'} and ref $feed->{'entry'} eq 'HASH' ){
+                my $entries = $feed->{'entry'};
 
-	# RSS
-	if( defined $feed->{'channel'} and defined $feed->{'channel'}->{'item'} and ref $feed->{'channel'}->{'item'} eq 'ARRAY' ){
-		@list = @{$feed->{'channel'}->{'item'}};
-	}
-	# Atom
-	elsif( defined $feed->{'entry'} and ref $feed->{'entry'} eq 'HASH' ){
-		my $entries = $feed->{'entry'};
+                foreach my $guid ( keys %$entries ){
+                        my $item = {
+                                guid    => $guid,
+                        };
 
-		foreach my $guid ( keys %$entries ){
-			my $item = {
-				guid	=> $guid,
-			};
+                        if( defined $entries->{$guid}->{'title'} ){
+                                if( ref $entries->{$guid}->{'title'} eq 'HASH' and defined $entries->{$guid}->{'title'}->{'content'} ){
+                                        $item->{'title'} = $entries->{$guid}->{'title'}->{'content'};
+                                }
+                                elsif( ref $entries->{$guid}->{'title'} eq '' ){
+                                        $item->{'title'} = $entries->{$guid}->{'title'};
+                                }
+                        }
 
-			if( defined $entries->{$guid}->{'title'} ){
-				if( ref $entries->{$guid}->{'title'} eq 'HASH' and defined $entries->{$guid}->{'title'}->{'content'} ){
-					$item->{'title'} = $entries->{$guid}->{'title'}->{'content'};
-				}
-				elsif( ref $entries->{$guid}->{'title'} eq '' ){
-					$item->{'title'} = $entries->{$guid}->{'title'};
-				}
-			}
+                        if( defined $entries->{$guid}->{'link'} ){
+                                if( ref $entries->{$guid}->{'link'} eq 'HASH' and defined $entries->{$guid}->{'link'}->{'href'} ){
+                                        $item->{'link'} = $entries->{$guid}->{'link'}->{'href'};
+                                }
+                                elsif( ref $entries->{$guid}->{'link'} eq '' ){
+                                        $item->{'link'} = $entries->{$guid}->{'link'};
+                                }
+                        }
 
-			if( defined $entries->{$guid}->{'link'} ){
-				if( ref $entries->{$guid}->{'link'} eq 'HASH' and defined $entries->{$guid}->{'link'}->{'href'} ){
-					$item->{'link'} = $entries->{$guid}->{'link'}->{'href'};
-				}
-				elsif( ref $entries->{$guid}->{'link'} eq '' ){
-					$item->{'link'} = $entries->{$guid}->{'link'};
-				}
-			}
+                        $item->{'category'} = $entries->{'category'} if defined $entries->{'category'};
 
-			$item->{'category'} = $entries->{'category'} if defined $entries->{'category'};
+                        if( defined $entries->{$guid}->{'summary'} ){
+                                if( ref($entries->{$guid}->{'summary'}) eq 'HASH' and defined $entries->{$guid}->{'summary'}->{'content'} ){
+                                        $item->{'description'} = $entries->{$guid}->{'summary'}->{'content'};
+                                }
+                                else {
+                                        $item->{'description'} = $entries->{$guid}->{'summary'};
+                                }
+                        }
 
-			push( @list,  $item ) if defined $item->{'link'} and defined $item->{'title'};
-		}
-	}
+                        if( defined $entries->{$guid}->{'content'} ){
+                                if( ref($entries->{$guid}->{'content'}) eq 'HASH' and defined $entries->{$guid}->{'content'}->{'content'} ){
+                                        $item->{'content'} = $entries->{$guid}->{'content'}->{'content'};
+                                }
+                        }
 
-	return \@list;
+                        push( @list,  $item ) if defined $item->{'link'} and defined $item->{'title'};
+                }
+        }
+
+        return \@list;
 }
 
 # fetch the feed and convert the XML to an data object
@@ -345,7 +473,13 @@ sub fetch_feed {
         my $response = $ua->get( $feed_url );
 
         if( $response->is_success ){
+            my $dc = $response->decoded_content;
+            if( Encode::is_utf8($dc) ){
                 return ( 1, XMLin normalize( 'D', $response->decoded_content ) );
+            }
+            else {
+                return ( 1, XMLin $dc );
+            }
         }
         else {
                 return ( 0, $response );
@@ -360,13 +494,15 @@ sub hashtagify {
 
         foreach my $item ( @list ){
                 # remove non-alphanumerics
-                $item =~ s/[^(\p{Letter}|\p{Number})]//g;
+                $item =~ s/[^[[:alnum:]]]//g;
 
                 # drop stop words
+                # TODO : make these overridable
                 next if length( $item ) < 3;
-                next if lc( $item ) =~ m/^(and|are|but|for|from|how|its|the|this)$/;
+                next if lc( $item ) =~ m/^(a(lso|nd|ny|re)|been|but|can(not|t)?|e(ach|tc|very)|for|from|g(e|o)t|ha(d|ve)|has(nt)?|hers?|hi(m|s)|how|its|no(r|t)|ours?|she|some|th(an|at|em?|eirs?|(e|o)se|ey|eyre|is)|too|very|was|wh(at|en|o)|with|you(r|rs)?)$/;
                 # hashtagify it
-                $item = '#' . $item;
+                $item = '#' . $item unless $item =~ m/^#/;
+
                 # use a hash here instead of an ordered list for auto-dedupe
                 $hashtags{ lc( $item ) } = undef;
         }
@@ -388,12 +524,20 @@ sub publish_post {
         # initialize an empty cookie jar
         $ua->cookie_jar( {} );
 
+        # allow option for insecure certs
+        if( $params{'insecure'} ){
+                $ua->ssl_opts( verify_hostname  => 0);
+        }
+
         # log in
         my $login_response = login( $ua, $params{'pod_url'}, $params{'username'}, $params{'password'} ) ;
 
         # if we've logged in successfully, post the message
         if( $login_response->is_success ){
-                my $post = post_message( $ua, $params{'pod_url'}, $content, $params{'aspect_ids'} );
+                # # encode utf-8 characters
+                # utf8::encode($content);
+
+                my $post = post_message( $ua, $params{'pod_url'}, $content, $params{'aspect_ids'}, %params );
                 #logout( $ua, $pod_url );
                 return $post;
         }
@@ -465,26 +609,26 @@ sub extract_token {
 
 # make any necessary string manipulations to play nice with markdown
 sub format_content {
-        my ( $content ) = @_;
+        my ( $content, %params ) = @_;
 
         $content =~ s/\n/\n\n/g;
-        $content .= "\nposted by [pod_feeder](https://github.com/rev138/pod_feeder)";
+        $content .= "\nposted by [pod_feeder](https://github.com/rev138/pod_feeder)" unless( $params{'no_branding'} );
 
         return $content;
 }
 
 # post a message
 sub post_message {
-        my ( $ua, $base_url, $content, $aspect_ids ) = @_;
+        my ( $ua, $base_url, $content, $aspect_ids, %params ) = @_;
         my ( $get_stream, $result ) = get_page( $ua, "$base_url/stream" );
 
         if( $get_stream ){
                 my $csrf = extract_token( $result );
                 my $post_url = "$base_url/status_messages";
-                my $message = { status_message => { text => format_content( $content ),  provider_display_name => 'pod_feeder' }, aspect_ids => $aspect_ids };
+                my $message = { status_message => { text => format_content( $content, %params ),  provider_display_name => $params{'via'} }, aspect_ids => $aspect_ids };
                 my $json = JSON->new->allow_nonref;
 
-                $json = $json->utf8(0) unless utf8::is_utf8( $message );
+                # $json = $json->utf8(1) unless utf8::is_utf8( $message );
 
                 my $json_message = $json->encode( $message );
 
@@ -496,38 +640,57 @@ sub post_message {
 
 # create a new sqlite db file with a 'feeds' table if it does not exist already
 sub init_database {
-	my ( $db_file ) = @_;
+        my ( $db_file ) = @_;
 
-	unless( -e $db_file ){
-		my $dbh = connect_to_db( $db_file );
-        	my $sth = $dbh->prepare(
-        		'CREATE TABLE feeds(guid VARCHAR(255) PRIMARY KEY,feed_id VARCHAR(127),title VARCHAR(255),link VARCHAR(255),hashtags VARCHAR(255),timestamp INTEGER(10),posted INTEGER(1))'
-        	) or die "Can't prepare statement: $DBI::errstr";
+        unless( -e $db_file ){
+	        my $dbh = connect_to_db( $db_file );
+                my $sth = $dbh->prepare(
+                        'CREATE TABLE feeds(guid VARCHAR(255) PRIMARY KEY,feed_id VARCHAR(127),title VARCHAR(255),link VARCHAR(255),image VARCHAR(255),image_title VARCHAR(255),hashtags VARCHAR(255),timestamp INTEGER(10),posted INTEGER(1),body VARCHAR(10000))'
+                ) or die "Can't prepare statement: $DBI::errstr";
 
-        	$sth->execute() or die "Can't execute statement: $DBI::errstr";
-        	$dbh->disconnect();
-	}
+                $sth->execute() or die "Can't execute statement: $DBI::errstr";
+                $dbh->disconnect();
+        }
+        else {
+                my $dbh = connect_to_db( $db_file );
+	        my $sth = $dbh->column_info(undef, undef, 'feeds', undef);
+                my $body_exists = 0;
+                while( my( $tcat, $tscheme, $tname, $column_name ) = $sth->fetchrow_array() ) {
+                        $body_exists = 1 if $column_name eq 'body';
+                }
+                unless( $body_exists ) {
+                        $sth = $dbh->prepare('ALTER TABLE feeds ADD body VARCHAR(10000)');
+                        $sth->execute() or die "Can't execute statement: $DBI::errstr";
+                }
+                $dbh->disconnect();
+        }
 }
 
 sub usage {
         print "$0\n";
         print "usage:\n";
         print "    -a   --aspect-id <id>                Aspects to share with. May specify multiple times (default: 'public')\n";
+        print "    -b   --embed-image                   Embed an image in the post if a link exists (default: off)\n";
         print "    -c   --category-tags                 Attempt to automatically hashtagify RSS item 'categories' (default: off)\n";
         print "    -d   --database <sqlite file>        The SQLite file to store feed data (default: 'feed.db')\n";
-        print "    -e	 --title-tags			 Automatically hashtagify RSS item title\n";
+        print "    -e   --title-tags                    Automatically hashtagify RSS item title\n";
         print "    -f   --feed-url <http://...>         The feed URL\n";
         print "    -g   --user-agent <string>           Use this to spoof the user-agent if the feed blocks bots (ex: 'Mozilla/5.0')\n";
         print "    -i   --feed-id <string>              An arbitrary identifier to associate database entries with this feed\n";
+        print "    -j   --no-branding                   Do not include 'posted via pod_feeder' footer to posts\n";
         print "    -l   --pod-url <https://...>         The pod URL\n";
         print "    -m   --timeout <hours>               How long (in hours) to keep attempting failed posts (default 72)\n";
+        print "    -n   --ignore-tag <#hashtag>         Hashtags to filter out. May be specified multiple times (default: none)\n";
         print "    -o   --fetch-only                    Don't publish to Diaspora, just queue the new feed items for later\n";
         print "    -p   --password <********>           The D* user password\n";
         print "    -r   --url-tags                      Attempt to automatically hashtagify the RSS link URL (default: off)\n";
         print "    -t   --auto-tag <#hashtag>           Hashtags to add to all posts. May be specified multiple times (default: none)\n";
+        print "    -s   --insecure                      Allows the option to bypass any errors caused from self-signed certificates(default: off)\n";
         print "    -u   --username <user>               The D* login username\n";
+        print "    -v   --via <string>                  Sets the 'posted via' text (default: 'pod_feeder')\n";
         print "    -w   --post-raw-link                 Post the raw link instead of hyperlinking the article title (default: off)\n";
-        print "    -x	 --limit <n>			 Only post n items per script run, to prevent post-spamming (default: no limit)\n";
+        print "    -x   --limit <n>                     Only post n items per script run, to prevent post-spamming (default: no limit)\n";
+        print "         --body                          Post the body of the feed (description or content:encoded item)\n";
         print "\n";
 
         exit;