Commit 31d74465 authored by Iván Sánchez Ortega's avatar Iván Sánchez Ortega

Update deshortifying filter; added (commented out) way to fetch public TW stream

parent 4224ee4f
......@@ -13,6 +13,7 @@
import getUrls from 'get-urls';
// import { extract } from 'article-parser';
import Deshortifier from '../../node_modules/deshortify/src/deshortify'
// import Deshortifier from '../../../deshortify/src/deshortify'
// console.log(Deshortifier);
......@@ -30,40 +31,47 @@ export default function deshortify(item) {
// console.log(urls);
var urlPromises = (Array.from(urls)).map( (url)=>{
return deshortifier.deshortify(url).then(deshortified=>[url, deshortified]);
return deshortifier.deshortify(url)
.then(deshortified=>[url, deshortified])
.catch(err=>console.error(err));
});
// var urlPromises = (Array.from(urls)).map( url=> {
// return deshortifier.deshortify(url);
// });
//
// var urlPromises = (Array.from(urls)).map( url=> {
// return Promise.all(urlPromises).then( resolvedUrls=>{
//
// if (url in cache) {
// return cache[url];
// }
// // console.log(resolvedUrls);
// resolvedUrls.forEach(([original, replacement])=>{
// if (original !== replacement) {
// console.log('Replacing: ', original, ' → ', replacement);
// item.str = item.str.replace(original, replacement);
// }
// });
//
// return extract(url)
// .then((extracted)=>{
// cache[url] = extracted.url;
// return [url, extracted.url]
// })
// .catch(()=>[url,url]);
// return item;
// } );
return Promise.all(urlPromises).then( resolvedUrls=>{
return Promise.race([
Promise.all(urlPromises).then( resolvedUrls=>{
// console.log(resolvedUrls);
resolvedUrls.forEach(([original, replacement])=>{
if (original !== replacement) {
// console.log('Replacing: ', original, ' → ', replacement);
item.str = item.str.replace(original, replacement);
}
});
return item;
} ),
new Promise(function (resolve, reject) { setTimeout(reject, 10000) })
]).catch((err=>{
// console.log(resolvedUrls);
resolvedUrls.forEach(([original, replacement])=>{
if (original !== replacement) {
console.log('Replacing: ', original, ' → ', replacement);
item.str = item.str.replace(original, replacement);
}
});
console.error('Resolving URLs took too long. URLs were:', urls);
return item;
} );
}));
}
......
......@@ -56,7 +56,7 @@ class TwitterAccount extends Account {
}, (stream)=>{
stream.on('data', (ev)=>{
// vorpal.log(ev);
this._normalizeAndEmitUpdate(ev);
this._normalizeAndEmitUpdate(ev, 'home');
});
stream.on('error', function(error) {
......@@ -64,16 +64,34 @@ class TwitterAccount extends Account {
});
});
// // Hackishly listen to the "samples" public timeline
// this._stream = this._tw.stream('statuses/sample', {
// // replies: "all", // Should explicitly listen to the "replies" channel
// // track: "foo,bar" // Should use this for listening to search terms channels
// stall_warnings: true
// }, (stream)=>{
// stream.on('data', (ev)=>{
// // vorpal.log(ev);
// this._normalizeAndEmitUpdate(ev, 'public');
// });
//
// stream.on('error', function(error) {
// vorpal.log(chalk.red(error));
// });
// });
// Get a few of the last statuses in the main timeline
this._tw.get('statuses/home_timeline', {}, (err, msgs, response)=>{
this._tw.get('statuses/home_timeline', {
count: 100
}, (err, msgs, response)=>{
if (err) {
vorpal.log(chalk.red(err));
}
for (let i=msgs.length-1; i>=0; i--) {
// console.log(msgs[i]);
this._normalizeAndEmitUpdate(msgs[i]);
this._normalizeAndEmitUpdate(msgs[i], 'home');
// vorpal.log('timelines/home', msgs);
}
});
......@@ -128,24 +146,24 @@ class TwitterAccount extends Account {
replace(/&lt;/g, "<");
}
// Given a stream descriptor, return a function that handles messages from the stream
_getOnStreamMessage(stream) {
return (msg)=>{
// this._vorpal.log(chalk.green(msg));
if (msg.event === 'update') {
this._normalizeAndEmitUpdate(stream, msg.data);
} else if (msg.event === 'notification') {
this._normalizeAndEmitNotification(stream, msg.data);
} else {
this._vorpal.log(chalk.green(msg));
}
}
}
// // Given a stream descriptor, return a function that handles messages from the stream
// _getOnStreamMessage(stream) {
// return (msg)=>{
// // this._vorpal.log(chalk.green(msg));
// if (msg.event === 'update') {
// this._normalizeAndEmitUpdate(stream, msg.data);
// } else if (msg.event === 'notification') {
// this._normalizeAndEmitNotification(stream, msg.data);
// } else {
// this._vorpal.log(chalk.green(msg));
// }
// }
// }
// Get a Twitter update, OR a part of an update (such a retweeted_status or a
// quoted_status), and normalize some stuff (to make different platforms
// see the same data from different platforms)
_normalize(msg) {
_normalize(msg, channel) {
if (('id_str' in msg) && ('contributors' in msg) && ('text' in msg)) {
// This looks like a tweet
......@@ -165,7 +183,7 @@ class TwitterAccount extends Account {
accountId: this._id,
sender: msg.user.screen_name,
str: '',
channel: 'home',
channel: channel,
media: [],
timestamp: Date.parse(msg.created_at),
echoed: this._normalize(msg.retweeted_status)
......@@ -198,7 +216,7 @@ class TwitterAccount extends Account {
accountId: this._id,
sender: msg.user.screen_name,
str: str,
channel: 'home',
channel: channel,
// realtime: true,
// media: msg.media_attachments.map(i=>i.text_url || i.remote_url),
media: allMedia,
......@@ -214,8 +232,8 @@ class TwitterAccount extends Account {
// Get a Twitter update, normalizes some stuff (to make different platforms
// see the same data from different platforms), and emits the update.
_normalizeAndEmitUpdate(msg) {
let normalized = this._normalize(msg);
_normalizeAndEmitUpdate(msg, channel) {
let normalized = this._normalize(msg, channel);
if (normalized) {
this.emit(normalized.timestamp, Promise.resolve(normalized));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment