diff --git a/.meteor/versions b/.meteor/versions index 525ded9bc..3265e7e7c 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -7,6 +7,7 @@ accounts-ui@1.1.5 accounts-ui-unstyled@1.1.7 aldeed:autoform@5.1.2 aldeed:collection2@2.3.3 +aldeed:http@0.2.2 aldeed:simple-schema@1.3.2 aldeed:template-extension@3.4.3 anti:i18n@0.4.3 diff --git a/client/views/posts/post_submit.js b/client/views/posts/post_submit.js index 8a348d0bd..d17aca4bb 100644 --- a/client/views/posts/post_submit.js +++ b/client/views/posts/post_submit.js @@ -2,7 +2,7 @@ AutoForm.hooks({ submitPostForm: { before: { - submitPost: function(doc) { + method: function(doc) { this.template.$('button[type=submit]').addClass('loading'); diff --git a/packages/telescope-post-by-feed/.npm/package/npm-shrinkwrap.json b/packages/telescope-post-by-feed/.npm/package/npm-shrinkwrap.json index d35307261..28deebea3 100644 --- a/packages/telescope-post-by-feed/.npm/package/npm-shrinkwrap.json +++ b/packages/telescope-post-by-feed/.npm/package/npm-shrinkwrap.json @@ -34,6 +34,9 @@ "he": { "version": "0.5.0" }, + "iconv-lite": { + "version": "0.4.7" + }, "to-markdown": { "version": "0.0.2" } diff --git a/packages/telescope-post-by-feed/lib/server/fetch_feeds.js b/packages/telescope-post-by-feed/lib/server/fetch_feeds.js index c97575287..1ba209379 100644 --- a/packages/telescope-post-by-feed/lib/server/fetch_feeds.js +++ b/packages/telescope-post-by-feed/lib/server/fetch_feeds.js @@ -2,11 +2,36 @@ var toMarkdown = Npm.require('to-markdown').toMarkdown; var he = Npm.require('he'); var FeedParser = Npm.require('feedparser'); var Readable = Npm.require('stream').Readable; +var iconv = Npm.require('iconv-lite'); var getFirstAdminUser = function() { return Meteor.users.findOne({isAdmin: true}, {sort: {createdAt: 1}}); }; +var normalizeEncoding = function (contentBuffer) { + // got from https://github.com/szwacz/sputnik/ + var encoding; + var content = contentBuffer.toString(); + + var xmlDeclaration = content.match(/^<\?xml .*\?>/); + if (xmlDeclaration) { + var encodingDeclaration = xmlDeclaration[0].match(/encoding=("|').*?("|')/); + if (encodingDeclaration) { + encoding = encodingDeclaration[0].substring(10, encodingDeclaration[0].length - 1); + } + } + + if (encoding && encoding.toLowerCase() !== 'utf-8') { + try { + content = iconv.decode(contentBuffer, encoding); + } catch (err) { + // detected encoding is not supported, leave it as it is + } + } + + return content; +}; + var feedHandler = { getStream: function(content) { var stream = new Readable(); @@ -17,9 +42,9 @@ var feedHandler = { }, getItemCategories: function(item, feedCategories) { - + var itemCategories = []; - + // loop over RSS categories for the current item if it has any if (item.categories && item.categories.length > 0) { item.categories.forEach(function(name) { @@ -41,7 +66,8 @@ var feedHandler = { return itemCategories; }, - handle: function(content, userId, feedCategories, feedId) { + handle: function(contentBuffer, userId, feedCategories, feedId) { + var content = normalizeEncoding(contentBuffer); var stream = this.getStream(content), feedParser = new FeedParser(), newItemsCount = 0, @@ -115,7 +141,7 @@ var feedHandler = { }; var fetchFeeds = function() { - var content; + var contentBuffer; Feeds.find().forEach(function(feed) { @@ -125,8 +151,8 @@ var fetchFeeds = function() { var feedId = feed._id; try { - content = HTTP.get(feed.url).content; - feedHandler.handle(content, userId, feedCategories, feedId); + contentBuffer = HTTP.get(feed.url, {responseType: 'buffer'}).content; + feedHandler.handle(contentBuffer, userId, feedCategories, feedId); } catch (error) { console.log(error); return true; // just go to next feed URL @@ -144,4 +170,4 @@ Meteor.methods({ testToMarkdown: function (text) { console.log(toMarkdown(text)); } -}) +}); diff --git a/packages/telescope-post-by-feed/package.js b/packages/telescope-post-by-feed/package.js index f635cf5e9..325e8ba10 100644 --- a/packages/telescope-post-by-feed/package.js +++ b/packages/telescope-post-by-feed/package.js @@ -7,7 +7,8 @@ Package.describe({ Npm.depends({ 'feedparser': '1.0.0', 'to-markdown': '0.0.2', - 'he': '0.5.0' + 'he': '0.5.0', + 'iconv-lite': '0.4.7' }); Package.onUse(function(api) { @@ -29,15 +30,16 @@ Package.onUse(function(api) { api.use([ 'http', + 'aldeed:http@0.2.2', 'momentjs:moment', 'percolatestudio:synced-cron' ], 'server'); - api.add_files([ + api.addFiles([ 'lib/feeds.js' ], ['client', 'server']); - api.add_files([ + api.addFiles([ 'lib/client/routes.js', 'lib/client/scss/feeds.scss', 'lib/client/templates/feeds.js', @@ -46,13 +48,13 @@ Package.onUse(function(api) { 'lib/client/templates/feed_item.html', ], 'client'); - api.add_files([ + api.addFiles([ 'lib/server/fetch_feeds.js', 'lib/server/cron.js', 'lib/server/publications.js' ], ['server']); - api.add_files([ + api.addFiles([ "i18n/en.i18n.json" ], ["client", "server"]);