tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

basically add declutter like capabilities.

Browse Source
This commit is contained in:
Jason Schwarzenberger
2020-11-11 17:16:04 +13:00
parent 3169af3002
commit 164b7e72c4
16 changed files with 694 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
readerserver/scraper/simple.js
+49 -31
View File
@@ -1,13 +1,8 @@
const request = require('request');
const JSDOM = require('jsdom').JSDOM;
const { Readability } = require('readability');
const fetch = require('node-fetch');
const { JSDOM } = require('jsdom');
const { Readability } = require('@mozilla/readability');
const { headers } = require('../constants');
const options = url => ({
url,
headers,
});
const { getUserAgent } = require('../utils/user-agent');
const extract = (url, body) => {
const doc = new JSDOM(body, { url: url });
@@ -15,27 +10,50 @@ const extract = (url, body) => {
return reader.parse();
};
module.exports.scrape = async (req, res) => {
try {
const { userAgent, headers } = getUserAgent(req.body.url);
const response = await fetch(req.body.url, {
headers: {
...headers,
'User-Agent': userAgent
}
});
if (!response.ok) {
return res.sendStatus(response.statusCode);
}
const html = await response.text();
const article = await extract(req.body.url, html);
if (article && article.content) {
return res.send(article.content);
}
return res.sendStatus(404);
} catch (e) {
console.error(e);
return res.sendStatus(500);
}
};
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => {
if (error || response.statusCode != 200) {
console.log('Response error:', error ? error.toString() : response.statusCode);
return res.sendStatus(response ? response.statusCode : 404);
module.exports.details = async (req, res) => {
try {
const { userAgent, headers } = getUserAgent(req.body.url);
const response = await fetch(req.body.url, {
headers: {
...headers,
'User-Agent': userAgent
}
});
if (!response.ok) {
return res.sendStatus(response.statusCode);
}
const html = await response.text();
const article = await extract(req.body.url, html);
if (article) {
return res.send(article);
}
return res.sendStatus(404);
} catch (e) {
console.error(e);
return res.sendStatus(500);
}
const article = extract(req.body.url, body);
if (article && article.content) {
return res.send(article.content);
}
return res.sendStatus(404);
});
module.exports.details = (req, res) => request(options(req.body.url), (error, response, body) => {
if (error || response.statusCode != 200) {
console.log('Response error:', error ? error.toString() : response.statusCode);
return res.sendStatus(response ? response.statusCode : 404);
}
const article = extract(req.body.url, body);
if (article) {
return res.send(article);
}
return res.sendStatus(404);
});
};