Commit 4224ee4f authored by Iván Sánchez Ortega's avatar Iván Sánchez Ortega

Deshortify links in all updates

parent 2869000d
......@@ -9,7 +9,9 @@
"dependencies": {
"chalk256": "^5.1.0",
"configstore": "^3.0.0",
"deshortify": "^0.1.0",
"flatten-obj": "^3.1.0",
"get-urls": "^7.0.0",
"he": "^1.1.1",
"inquirer": "^3.0.6",
"json-stringify-safe": "^5.0.1",
......
......@@ -36,7 +36,7 @@ export function push(data) {
// fs.writeFile('debug/' + str + '.json', JSON.stringify(data, ' '));
// console.log(data);
fs.writeFile('debug/' + str + '.json', JSON.stringify(data.metadata, null, ' '));
fs.writeFile('debug/' + str + '.json', JSON.stringify(data, null, ' '));
return str;
}
// This is a filter. A filter takes in an item, and returns either:
// - an item
// - a promise for an item
// - a falsy value
// - a promise for a falsy value
// Items can be modified, kept the same, or filtered out altogether.
// Filters will be chained together by the soclial core, passed as parameters
// to itemPromise.then(...). This allows for both sync and async filters.
import getUrls from 'get-urls';
// import { extract } from 'article-parser';
import Deshortifier from '../../node_modules/deshortify/src/deshortify'
// console.log(Deshortifier);
let deshortifier = new Deshortifier({ verbose: false });
// let cache = {};
export default function deshortify(item) {
// This filter takes the text in the item, finds the URLs, and deshortifies them.
let urls = getUrls(item.str);
// console.log(urls);
var urlPromises = (Array.from(urls)).map( (url)=>{
return deshortifier.deshortify(url).then(deshortified=>[url, deshortified]);
});
// var urlPromises = (Array.from(urls)).map( url=> {
// return deshortifier.deshortify(url);
// });
//
// var urlPromises = (Array.from(urls)).map( url=> {
//
// if (url in cache) {
// return cache[url];
// }
//
// return extract(url)
// .then((extracted)=>{
// cache[url] = extracted.url;
// return [url, extracted.url]
// })
// .catch(()=>[url,url]);
// } );
return Promise.all(urlPromises).then( resolvedUrls=>{
// console.log(resolvedUrls);
resolvedUrls.forEach(([original, replacement])=>{
if (original !== replacement) {
console.log('Replacing: ', original, ' → ', replacement);
item.str = item.str.replace(original, replacement);
}
});
return item;
} );
}
......@@ -14,6 +14,8 @@ import TinyQueue from 'tinyqueue';
const flatten = Flatten();
const vorpal = Vorpal();
import deshortifyFilter from './filters/deshortify';
vorpal.history('soclial');
vorpal.log(chalk.purple('>>> Initializing soCLIal...'));
......@@ -57,6 +59,8 @@ function handle(timestamp, itemPromise) {
// console.log('handling: ', timestamp, itemPromise);
// console.log('handling: ', timestamp);
// Pass the item through all known filters
itemPromise = itemPromise.then(deshortifyFilter);
itemQueue.push({ timestamp: timestamp, itemPromise: itemPromise });
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment