Assign feed item's categories to post.

Changed htmlparser2 to node-feedparser

packages/telescope-post-by-feed/.npm/package/npm-shrinkwrap.json

@@ -1,22 +1,19 @@
 {
   "dependencies": {
-    "he": {
-      "version": "0.5.0"
-    },
-    "htmlparser2": {
-      "version": "3.8.2",
+    "feedparser": {
+      "version": "1.0.0",
       "dependencies": {
-        "domhandler": {
-          "version": "2.3.0"
+        "sax": {
+          "version": "0.6.1"
         },
-        "domutils": {
-          "version": "1.5.0"
+        "addressparser": {
+          "version": "0.1.3"
         },
-        "domelementtype": {
-          "version": "1.1.3"
+        "array-indexofobject": {
+          "version": "0.0.1"
         },
         "readable-stream": {
-          "version": "1.1.13",
+          "version": "1.0.33",
           "dependencies": {
             "core-util-is": {
               "version": "1.0.1"
@@ -31,12 +28,12 @@
               "version": "2.0.1"
             }
           }
-        },
-        "entities": {
-          "version": "1.0.0"
         }
       }
     },
+    "he": {
+      "version": "0.5.0"
+    },
     "to-markdown": {
       "version": "0.0.2"
     }

packages/telescope-post-by-feed/lib/feeds.js

@@ -13,7 +13,7 @@ var feedSchema = new SimpleSchema({
           return {
             value: user._id,
             label: getDisplayName(user)
-          }  
+          };
         });
         return users;
       }
@@ -28,11 +28,11 @@ var feedSchema = new SimpleSchema({
       noselect: true,
       editable: true,
       options: function () {
-          var categories = Categories.find().map(function (category) {
-            return {
-              value: category._id,
-              label: category.name
-            }  
+        var categories = Categories.find().map(function (category) {
+          return {
+            value: category._id,
+            label: category.name
+          };
         });
         return categories;
       }
@@ -40,7 +40,7 @@ var feedSchema = new SimpleSchema({
   }
 });
 
-Feeds = new Meteor.Collection("feeds");
+Feeds = new Meteor.Collection('feeds');
 Feeds.attachSchema(feedSchema);
 
 // used to keep track of which feed a post was imported from

packages/telescope-post-by-feed/lib/server/fetch_feeds.js

@@ -1,73 +1,120 @@
-var htmlParser = Npm.require('htmlparser2');
 var toMarkdown = Npm.require('to-markdown').toMarkdown;
-var he = Npm.require('he')
+var he = Npm.require('he');
+var FeedParser = Npm.require('feedparser');
+var Readable = Npm.require('stream').Readable;
 
 var getFirstAdminUser = function() {
   return Meteor.users.findOne({isAdmin: true}, {sort: {createdAt: 1}});
-}
-
-var handleFeed = function(error, feed) {
-  if (error) return;
-
-  var feedItems = _.first(feed.items, 20); // limit feed to 20 items just in case
-  var userId = this._parser._options.userId;
-  var feedId = this._parser._options.feedId;
-  var categories = this._parser._options.categories;
- 
-  clog('// Parsing RSS feed: '+ feed.title)
-
-  var newItemsCount = 0;
-
-  feedItems.forEach(function(item, index, array) {
-
-    // if item has no id, use the URL to give it one
-    if (!item.id)
-      item.id = item.link;
-
-    // check if post already exists
-    if (!!Posts.findOne({feedItemId: item.id})) {
-      // clog('// Feed item already imported')
-    } else {
-      newItemsCount++;
-
-      var post = {
-        title: he.decode(item.title),
-        url: item.link,
-        feedId: feedId,
-        feedItemId: item.id,
-        userId: userId,
-        categories: categories
-      }
-
-      if (item.description)
-        post.body = toMarkdown(he.decode(item.description));
-
-      // console.log(item)
-
-      // if RSS item link is a 301 or 302 redirect, follow the redirect
-      var get = HTTP.get(item.link, {followRedirects: false});
-      if (!!get.statusCode && (get.statusCode === 301 || get.statusCode === 302) && !!get.headers && !!get.headers.location) {
-        post.url = get.headers.location;
-      }
-
-      // if RSS item has a date, use it
-      if (item.pubDate)
-        post.postedAt = moment(item.pubDate).toDate();
-
-      try {
-        submitPost(post);
-      } catch (error) {
-        // catch errors so they don't stop the loop
-        clog(error);
-      }
-
-    }
-  });
-
-  clog('// Found ' + newItemsCount + ' new feed items')
 };
 
-fetchFeeds = function() {
+var feedHandler = {
+  getStream: function(content) {
+    var stream = new Readable();
+    stream.push(content);
+    stream.push(null);
+
+    return stream;
+  },
+
+  getItemCategories: function(categories, item) {
+    var itemCategories = [],
+    category;
+
+    if (item.categories && item.categories.length > 0) {
+      item.categories.forEach(function(name) {
+        category = Categories.findOne({name: name}, {fields: {_id: 1}});
+
+        if (category) {
+          itemCategories.push(category._id);
+        }
+      });
+    }
+
+    if (itemCategories.length > 0) {
+      if (categories) {
+        itemCategories = itemCategories.concat(categories);
+      }
+    } else {
+      itemCategories = categories;
+    }
+
+    return itemCategories;
+  },
+
+  handle: function(content, userId, categories, feedId) {
+    var stream = this.getStream(content),
+    feedParser = new FeedParser(),
+    newItemsCount = 0,
+    self = this;
+
+    stream.pipe(feedParser);
+
+    feedParser.on('meta', Meteor.bindEnvironment(function(meta) {
+      clog('// Parsing RSS feed: '+ meta.title);
+    }));
+
+    feedParser.on('error', Meteor.bindEnvironment(function(error) {
+      clog(error);
+    }));
+
+    feedParser.on('readable', Meteor.bindEnvironment(function() {
+      var s = this, item;
+
+      while (item = s.read()) {
+        // if item has no guid, use the URL to give it one
+        if (!item.guid) {
+          item.guid = item.link;
+        }
+
+        // check if post already exists
+        if (!!Posts.findOne({feedItemId: item.guid})) {
+          // clog('// Feed item already imported');
+          continue;
+        }
+
+        newItemsCount++;
+
+        var post = {
+          title: he.decode(item.title),
+          url: item.link,
+          feedId: feedId,
+          feedItemId: item.guid,
+          userId: userId,
+          categories: self.getItemCategories(categories, item)
+        };
+
+        if (item.description)
+          post.body = toMarkdown(he.decode(item.description));
+
+        // console.log(item)
+
+        // if RSS item link is a 301 or 302 redirect, follow the redirect
+        var get = HTTP.get(item.link, {followRedirects: false});
+        if (!!get.statusCode && (get.statusCode === 301 || get.statusCode === 302) &&
+            !!get.headers && !!get.headers.location) {
+              post.url = get.headers.location;
+            }
+
+        // if RSS item has a date, use it
+        if (item.pubdate)
+          post.postedAt = moment(item.pubdate).toDate();
+
+        try {
+          submitPost(post);
+        } catch (error) {
+          // catch errors so they don't stop the loop
+          clog(error);
+        }
+      }
+
+      // clog('// Found ' + newItemsCount + ' new feed items');
+    }, function() {
+      clog('Failed to bind environment');
+    }, feedParser));
+  }
+};
+
+var fetchFeeds = function() {
   var content;
 
   Feeds.find().forEach(function(feed) {
@@ -76,23 +123,16 @@ fetchFeeds = function() {
     var userId = !!feed.userId ? feed.userId : getFirstAdminUser()._id;
     var categories = feed.categories;
     var feedId = feed._id;
-    
+
     try {
-
       content = HTTP.get(feed.url).content;
-      var feedHandler = new htmlParser.FeedHandler(handleFeed);
-      var parser = new htmlParser.Parser(feedHandler, {xmlMode: true, userId: userId, categories:categories, feedId:feedId});
-      parser.write(content);
-      parser.end();
-
+      feedHandler.handle(content, userId, categories, feedId);
     } catch (error) {
-
       console.log(error);
       return true; // just go to next url
-      
     }
   });
-}
+};
 
 Meteor.methods({
   fetchFeeds: function () {
@@ -103,5 +143,5 @@ Meteor.methods({
   },
   testToMarkdown: function (text) {
     console.log(toMarkdown(text));
-  } 
+  }
 })

packages/telescope-post-by-feed/package.js

@@ -5,7 +5,7 @@ Package.describe({
 });
 
 Npm.depends({
-  'htmlparser2': '3.8.2',
+  'feedparser': '1.0.0',
   'to-markdown': '0.0.2',
   'he': '0.5.0'
 });
@@ -13,7 +13,7 @@ Npm.depends({
 Package.onUse(function(api) {
 
   api.use([
-    'telescope-base', 
+    'telescope-base',
     'telescope-tags',
     'aldeed:simple-schema',
     'aldeed:autoform',