From c5e813909a9d81a039515033596f9fc779833dd2 Mon Sep 17 00:00:00 2001 From: Delgermurun Date: Sat, 13 Dec 2014 22:55:10 +0800 Subject: [PATCH] Auto post via RSS urls. Fixes #453 --- .meteor/packages | 1 + .meteor/versions | 1 + packages/telescope-post-by-rss/.gitignore | 1 + .../.npm/package/.gitignore | 1 + .../telescope-post-by-rss/.npm/package/README | 7 + .../.npm/package/npm-shrinkwrap.json | 63 +++++ .../telescope-post-by-rss/i18n/en.i18n.json | 3 + .../lib/client/routes.js | 20 ++ .../lib/client/templates/rss_url_item.html | 6 + .../lib/client/templates/rss_url_item.js | 10 + .../lib/client/templates/rss_urls.html | 21 ++ .../lib/client/templates/rss_urls.js | 28 ++ packages/telescope-post-by-rss/lib/rssUrls.js | 35 +++ .../telescope-post-by-rss/lib/server/cron.js | 13 + .../lib/server/publications.js | 6 + .../telescope-post-by-rss/lib/server/utils.js | 98 +++++++ .../telescope-post-by-rss/package-tap.i18n | 5 + packages/telescope-post-by-rss/package.js | 56 ++++ packages/telescope-post-by-rss/versions.json | 239 ++++++++++++++++++ 19 files changed, 614 insertions(+) create mode 100644 packages/telescope-post-by-rss/.gitignore create mode 100644 packages/telescope-post-by-rss/.npm/package/.gitignore create mode 100644 packages/telescope-post-by-rss/.npm/package/README create mode 100644 packages/telescope-post-by-rss/.npm/package/npm-shrinkwrap.json create mode 100644 packages/telescope-post-by-rss/i18n/en.i18n.json create mode 100644 packages/telescope-post-by-rss/lib/client/routes.js create mode 100644 packages/telescope-post-by-rss/lib/client/templates/rss_url_item.html create mode 100644 packages/telescope-post-by-rss/lib/client/templates/rss_url_item.js create mode 100644 packages/telescope-post-by-rss/lib/client/templates/rss_urls.html create mode 100644 packages/telescope-post-by-rss/lib/client/templates/rss_urls.js create mode 100644 packages/telescope-post-by-rss/lib/rssUrls.js create mode 100644 packages/telescope-post-by-rss/lib/server/cron.js create mode 100644 packages/telescope-post-by-rss/lib/server/publications.js create mode 100644 packages/telescope-post-by-rss/lib/server/utils.js create mode 100644 packages/telescope-post-by-rss/package-tap.i18n create mode 100644 packages/telescope-post-by-rss/package.js create mode 100644 packages/telescope-post-by-rss/versions.json diff --git a/.meteor/packages b/.meteor/packages index 574edddee..4a89ec409 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -78,4 +78,5 @@ telescope-invites useraccounts:unstyled telescope-datetimepicker tsega:bootstrap3-datetimepicker@3.1.3_1 +telescope-post-by-rss diff --git a/.meteor/versions b/.meteor/versions index cf01a8629..d7064584b 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -123,6 +123,7 @@ telescope-lib@0.2.9 telescope-module-share@0.0.0 telescope-newsletter@0.1.0 telescope-notifications@0.1.0 +telescope-post-by-rss@0.0.1 telescope-rss@0.0.0 telescope-search@0.0.0 telescope-singleday@0.1.0 diff --git a/packages/telescope-post-by-rss/.gitignore b/packages/telescope-post-by-rss/.gitignore new file mode 100644 index 000000000..677a6fc26 --- /dev/null +++ b/packages/telescope-post-by-rss/.gitignore @@ -0,0 +1 @@ +.build* diff --git a/packages/telescope-post-by-rss/.npm/package/.gitignore b/packages/telescope-post-by-rss/.npm/package/.gitignore new file mode 100644 index 000000000..3c3629e64 --- /dev/null +++ b/packages/telescope-post-by-rss/.npm/package/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/packages/telescope-post-by-rss/.npm/package/README b/packages/telescope-post-by-rss/.npm/package/README new file mode 100644 index 000000000..3d492553a --- /dev/null +++ b/packages/telescope-post-by-rss/.npm/package/README @@ -0,0 +1,7 @@ +This directory and the files immediately inside it are automatically generated +when you change this package's NPM dependencies. Commit the files in this +directory (npm-shrinkwrap.json, .gitignore, and this README) to source control +so that others run the same versions of sub-dependencies. + +You should NOT check in the node_modules directory that Meteor automatically +creates; if you are using git, the .gitignore file tells git to ignore it. diff --git a/packages/telescope-post-by-rss/.npm/package/npm-shrinkwrap.json b/packages/telescope-post-by-rss/.npm/package/npm-shrinkwrap.json new file mode 100644 index 000000000..996faff49 --- /dev/null +++ b/packages/telescope-post-by-rss/.npm/package/npm-shrinkwrap.json @@ -0,0 +1,63 @@ +{ + "dependencies": { + "html-to-text": { + "version": "0.1.0", + "dependencies": { + "htmlparser": { + "version": "1.7.7" + }, + "underscore": { + "version": "1.7.0" + }, + "underscore.string": { + "version": "2.4.0" + }, + "optimist": { + "version": "0.6.1", + "dependencies": { + "wordwrap": { + "version": "0.0.2" + }, + "minimist": { + "version": "0.0.10" + } + } + } + } + }, + "htmlparser2": { + "version": "3.8.2", + "dependencies": { + "domhandler": { + "version": "2.3.0" + }, + "domutils": { + "version": "1.5.0" + }, + "domelementtype": { + "version": "1.1.3" + }, + "readable-stream": { + "version": "1.1.13", + "dependencies": { + "core-util-is": { + "version": "1.0.1" + }, + "isarray": { + "version": "0.0.1" + }, + "string_decoder": { + "version": "0.10.31" + }, + "inherits": { + "version": "2.0.1" + } + } + }, + "entities": { + "version": "1.0.0" + } + } + } + } +} diff --git a/packages/telescope-post-by-rss/i18n/en.i18n.json b/packages/telescope-post-by-rss/i18n/en.i18n.json new file mode 100644 index 000000000..d4818cc3a --- /dev/null +++ b/packages/telescope-post-by-rss/i18n/en.i18n.json @@ -0,0 +1,3 @@ +{ + "rss_url_is_already_exists": "Url is already exists." +} diff --git a/packages/telescope-post-by-rss/lib/client/routes.js b/packages/telescope-post-by-rss/lib/client/routes.js new file mode 100644 index 000000000..770722bb2 --- /dev/null +++ b/packages/telescope-post-by-rss/lib/client/routes.js @@ -0,0 +1,20 @@ +adminNav.push({ + route: 'rssUrls', + label: 'RSS Urls' +}); + +Meteor.startup(function () { + + Router.onBeforeAction(Router._filters.isAdmin, {only: ['rssUrls']}); + + // RSS Urls Admin + + Router.route('/rss-urls', { + name: 'rssUrls', + waitOn: function() { + return Meteor.subscribe('rssUrls'); + }, + template: getTemplate('rssUrls') + }); + +}); diff --git a/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.html b/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.html new file mode 100644 index 000000000..163063bad --- /dev/null +++ b/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.html @@ -0,0 +1,6 @@ + diff --git a/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.js b/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.js new file mode 100644 index 000000000..f01a39c37 --- /dev/null +++ b/packages/telescope-post-by-rss/lib/client/templates/rss_url_item.js @@ -0,0 +1,10 @@ +Meteor.startup(function () { + Template[getTemplate('rssUrlItem')].events({ + 'click .delete-url': function(e, instance){ + e.preventDefault(); + + if (confirm(i18n.t('are_you_sure'))) + RssUrls.remove(instance.data._id); + } + }); +}); diff --git a/packages/telescope-post-by-rss/lib/client/templates/rss_urls.html b/packages/telescope-post-by-rss/lib/client/templates/rss_urls.html new file mode 100644 index 000000000..5c70d8059 --- /dev/null +++ b/packages/telescope-post-by-rss/lib/client/templates/rss_urls.html @@ -0,0 +1,21 @@ + diff --git a/packages/telescope-post-by-rss/lib/client/templates/rss_urls.js b/packages/telescope-post-by-rss/lib/client/templates/rss_urls.js new file mode 100644 index 000000000..f23582edc --- /dev/null +++ b/packages/telescope-post-by-rss/lib/client/templates/rss_urls.js @@ -0,0 +1,28 @@ +Meteor.startup(function () { + Template[getTemplate('rssUrls')].helpers({ + rssUrls: function(){ + return RssUrls.find({}, {sort: {url: 1}}); + }, + rssUrlItem: function () { + return getTemplate('rssUrlItem'); + } + }); + + Template[getTemplate('rssUrls')].events({ + 'click input[type=submit]': function(e){ + e.preventDefault(); + + var url = $('#url').val(); + + Meteor.call('insertRssUrl', {url: url}, function(error, result) { + if(error){ + console.log(error); + flashMessage(error.reason, "error"); + clearSeenMessages(); + }else{ + $('#url').val(''); + } + }); + } + }); +}); diff --git a/packages/telescope-post-by-rss/lib/rssUrls.js b/packages/telescope-post-by-rss/lib/rssUrls.js new file mode 100644 index 000000000..b8c775dc0 --- /dev/null +++ b/packages/telescope-post-by-rss/lib/rssUrls.js @@ -0,0 +1,35 @@ +// rssUrl schema +rssUrlSchema = new SimpleSchema({ + _id: { + type: String, + optional: true + }, + url: { + type: String, + regEx: SimpleSchema.RegEx.Url + }, +}); + +RssUrls = new Meteor.Collection("rss_urls", { + schema: rssUrlSchema +}); + +Meteor.startup(function () { + RssUrls.allow({ + remove: isAdminById + }); + + Meteor.methods({ + insertRssUrl: function(rssUrl){ + check(rssUrl, rssUrlSchema); + + if (RssUrls.findOne({url: rssUrl.url})) + throw new Meteor.Error('already-exists', i18n.t('rss_url_is_already_exists')); + + if (!Meteor.user() || !isAdmin(Meteor.user())) + throw new Meteor.Error('login-required', i18n.t('you_need_to_login_and_be_an_admin_to_add_a_new_rss_url')); + + return RssUrls.insert(rssUrl); + } + }); +}); diff --git a/packages/telescope-post-by-rss/lib/server/cron.js b/packages/telescope-post-by-rss/lib/server/cron.js new file mode 100644 index 000000000..dac57f3ff --- /dev/null +++ b/packages/telescope-post-by-rss/lib/server/cron.js @@ -0,0 +1,13 @@ +SyncedCron.add({ + name: 'Post by RSS url', + schedule: function(parser) { + return parser.text('every 10 mins'); + }, + job: function() { + fetchUrls(); + } +}); + +Meteor.startup(function() { + SyncedCron.start(); +}); diff --git a/packages/telescope-post-by-rss/lib/server/publications.js b/packages/telescope-post-by-rss/lib/server/publications.js new file mode 100644 index 000000000..fbf8a5bac --- /dev/null +++ b/packages/telescope-post-by-rss/lib/server/publications.js @@ -0,0 +1,6 @@ +Meteor.publish('rssUrls', function() { + if(isAdminById(this.userId)){ + return RssUrls.find(); + } + return []; +}); diff --git a/packages/telescope-post-by-rss/lib/server/utils.js b/packages/telescope-post-by-rss/lib/server/utils.js new file mode 100644 index 000000000..ff7eeaa41 --- /dev/null +++ b/packages/telescope-post-by-rss/lib/server/utils.js @@ -0,0 +1,98 @@ +var htmlParser = Npm.require('htmlparser2'); +var htmlToText = Npm.require('html-to-text'); + +var getFirstAdminUser = function() { + return Meteor.users.findOne({isAdmin: true}, {sort: {createdAt: 1}}); +} + +var insertPost = function(feedItem) { + var post = { + title: feedItem.title, + body: htmlToText.fromString(feedItem.description), + url: feedItem.link + }; + + // check that there are no posts with the same URL + if (!!post.url) { + var sixMonthsAgo = moment().subtract(6, 'months').toDate(); + var postWithSameLink = Posts.findOne({url: post.url, postedAt: {$gte: sixMonthsAgo}}); + + if (typeof postWithSameLink !== 'undefined') { + return; + } + } + + var title = cleanUp(post.title), + body = post.body, + user = getFirstAdminUser(), + postId = ''; + + // ------------------------------ Checks ------------------------------ // + + // check that user provided a title + if(!post.title) + post.title = 'Untitled'; + + // ------------------------------ Properties ------------------------------ // + + // Basic Properties + properties = { + title: title, + body: body, + userId: user._id, + author: getDisplayNameById(user._id), + upvotes: 0, + downvotes: 0, + commentCount: 0, + clickCount: 0, + viewCount: 0, + baseScore: 0, + score: 0, + status: 2, + inactive: false, + createdAt: new Date(), + postedAt: new Date() + }; + + post = _.extend(post, properties); + + // ------------------------------ Insert ------------------------------ // + + post._id = Posts.insert(post); + + // ------------------------------ After Insert ------------------------------ // + + // increment posts count + Meteor.users.update({_id: user._id}, {$inc: {postCount: 1}}); + + Meteor.call('upvotePost', post, user); + + return post; +}; + +var handleFeed = function(error, feed) { + if (error) return; + + feed.items.forEach(function(item, index, array) { + insertPost(item); + }); +}; + +fetchUrls = function() { + var content; + + RssUrls.find().forEach(function(rssUrl) { + try { + content = HTTP.get(rssUrl.url).content; + } catch (e) { + // just go to next url + return true; + } + + var feedHandler = new htmlParser.FeedHandler(handleFeed); + + var parser = new htmlParser.Parser(feedHandler, {xmlMode: true}); + parser.write(content); + parser.end() + }); +} diff --git a/packages/telescope-post-by-rss/package-tap.i18n b/packages/telescope-post-by-rss/package-tap.i18n new file mode 100644 index 000000000..a52321d15 --- /dev/null +++ b/packages/telescope-post-by-rss/package-tap.i18n @@ -0,0 +1,5 @@ +{ + "translation_function_name": "__", + "helper_name": "_", + "namespace": "project" +} \ No newline at end of file diff --git a/packages/telescope-post-by-rss/package.js b/packages/telescope-post-by-rss/package.js new file mode 100644 index 000000000..1b5c593aa --- /dev/null +++ b/packages/telescope-post-by-rss/package.js @@ -0,0 +1,56 @@ +Package.describe({ + summary: 'Auto post via RSS to Telescope', + version: '0.0.1', + name: 'telescope-post-by-rss' +}); + +Npm.depends({ + 'htmlparser2': '3.8.2', + 'html-to-text': '0.1.0', +}); + +Package.onUse(function(api) { + + api.use([ + 'telescope-base', + 'aldeed:simple-schema', + 'tap:i18n' + ], ['client', 'server']); + + api.use([ + 'iron:router', + 'templating' + ], 'client'); + + api.use([ + 'http', + 'mrt:moment', + 'percolatestudio:synced-cron' + ], 'server'); + + api.add_files([ + 'lib/rssUrls.js' + ], ['client', 'server']); + + api.add_files([ + 'lib/client/routes.js', + 'lib/client/templates/rss_urls.js', + 'lib/client/templates/rss_urls.html', + 'lib/client/templates/rss_url_item.js', + 'lib/client/templates/rss_url_item.html', + ], 'client'); + + api.add_files([ + 'lib/server/utils.js', + 'lib/server/cron.js', + 'lib/server/publications.js' + ], ['server']); + + api.add_files([ + "i18n/en.i18n.json" + ], ["client", "server"]); +}); + +Package.onTest(function(api) { + api.use('tinytest'); +}); diff --git a/packages/telescope-post-by-rss/versions.json b/packages/telescope-post-by-rss/versions.json new file mode 100644 index 000000000..85d852394 --- /dev/null +++ b/packages/telescope-post-by-rss/versions.json @@ -0,0 +1,239 @@ +{ + "dependencies": [ + [ + "aldeed:simple-schema", + "1.1.0" + ], + [ + "application-configuration", + "1.0.3" + ], + [ + "base64", + "1.0.1" + ], + [ + "binary-heap", + "1.0.1" + ], + [ + "blaze", + "2.0.3" + ], + [ + "blaze-tools", + "1.0.1" + ], + [ + "boilerplate-generator", + "1.0.1" + ], + [ + "callback-hook", + "1.0.1" + ], + [ + "check", + "1.0.2" + ], + [ + "coffeescript", + "1.0.4" + ], + [ + "ddp", + "1.0.12" + ], + [ + "deps", + "1.0.5" + ], + [ + "ejson", + "1.0.4" + ], + [ + "follower-livedata", + "1.0.2" + ], + [ + "geojson-utils", + "1.0.1" + ], + [ + "html-tools", + "1.0.2" + ], + [ + "htmljs", + "1.0.2" + ], + [ + "http", + "1.0.8" + ], + [ + "id-map", + "1.0.1" + ], + [ + "iron:controller", + "1.0.3" + ], + [ + "iron:core", + "1.0.3" + ], + [ + "iron:dynamic-template", + "1.0.3" + ], + [ + "iron:layout", + "1.0.3" + ], + [ + "iron:location", + "1.0.3" + ], + [ + "iron:middleware-stack", + "1.0.3" + ], + [ + "iron:router", + "1.0.3" + ], + [ + "iron:url", + "1.0.3" + ], + [ + "jquery", + "1.0.1" + ], + [ + "json", + "1.0.1" + ], + [ + "logging", + "1.0.5" + ], + [ + "meteor", + "1.1.3" + ], + [ + "minifiers", + "1.1.2" + ], + [ + "minimongo", + "1.0.5" + ], + [ + "mongo", + "1.0.9" + ], + [ + "mrt:moment", + "2.8.1" + ], + [ + "observe-sequence", + "1.0.3" + ], + [ + "ordered-dict", + "1.0.1" + ], + [ + "percolatestudio:synced-cron", + "1.1.0" + ], + [ + "random", + "1.0.1" + ], + [ + "reactive-dict", + "1.0.4" + ], + [ + "reactive-var", + "1.0.3" + ], + [ + "retry", + "1.0.1" + ], + [ + "routepolicy", + "1.0.2" + ], + [ + "session", + "1.0.4" + ], + [ + "spacebars", + "1.0.3" + ], + [ + "spacebars-compiler", + "1.0.3" + ], + [ + "tap:http-methods", + "0.0.23" + ], + [ + "tap:i18n", + "1.2.1" + ], + [ + "telescope-base", + "0.0.0" + ], + [ + "telescope-i18n", + "0.0.0" + ], + [ + "telescope-lib", + "0.2.9" + ], + [ + "templating", + "1.0.9" + ], + [ + "tracker", + "1.0.3" + ], + [ + "ui", + "1.0.4" + ], + [ + "underscore", + "1.0.1" + ], + [ + "url", + "1.0.2" + ], + [ + "webapp", + "1.1.4" + ], + [ + "webapp-hashing", + "1.0.1" + ] + ], + "pluginDependencies": [], + "toolVersion": "meteor-tool@1.0.36", + "format": "1.0" +} \ No newline at end of file