You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
303 lines
9.4 KiB
303 lines
9.4 KiB
// announcement.js
|
|
// Copyright (C) 2022 DTP Technologies, LLC
|
|
// License: Apache-2.0
|
|
|
|
'use strict';
|
|
|
|
const mongoose = require('mongoose');
|
|
const Feed = mongoose.model('Feed');
|
|
const FeedEntry = mongoose.model('FeedEntry');
|
|
|
|
const moment = require('moment');
|
|
const UserAgent = require('user-agents');
|
|
|
|
const { SiteService, SiteError, SiteAsync } = require('../../lib/site-lib');
|
|
const { read: feedReader } = require('feed-reader');
|
|
const { Feed: FeedGenerator } = require('feed');
|
|
const { getLinkPreview } = require('link-preview-js');
|
|
|
|
class FeedService extends SiteService {
|
|
|
|
constructor (dtp) {
|
|
super(dtp, module.exports);
|
|
|
|
this.populateFeedEntry = [
|
|
{
|
|
path: 'feed',
|
|
},
|
|
];
|
|
}
|
|
|
|
async start ( ) {
|
|
this.userAgent = new UserAgent();
|
|
this.jobQueue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom);
|
|
}
|
|
|
|
middleware (options) {
|
|
options = Object.assign({ maxEntryCount: 5 }, options);
|
|
return async (req, res, next) => {
|
|
if (this.isSystemRoute(req.path)) {
|
|
return next(); // don't load newsfeeds for non-content routes
|
|
}
|
|
res.locals.newsfeed = await this.getNewsfeed({ skip: 0, cpp: options.maxEntryCount });
|
|
return next();
|
|
};
|
|
}
|
|
|
|
async create (feedDefinition) {
|
|
feedDefinition.url = feedDefinition.url.trim();
|
|
const feedContent = await this.load(feedDefinition.url);
|
|
if (!feedContent) {
|
|
throw new SiteError(404, 'Feed failed to load');
|
|
}
|
|
|
|
const feed = new Feed();
|
|
feed.url = feedDefinition.url;
|
|
feed.title = feedDefinition.title || feedContent.title || 'New Feed';
|
|
feed.link = feedDefinition.link || feedContent.link;
|
|
feed.description = feedDefinition.description || feedContent.description;
|
|
feed.language = feedContent.language;
|
|
feed.generator = feedContent.generator;
|
|
feed.published = feedContent.published;
|
|
await feed.save();
|
|
|
|
this.jobQueue.add('update-feed', { feedId: feed._id });
|
|
|
|
return feed.toObject();
|
|
}
|
|
|
|
async update (feed, feedDefinition) {
|
|
feedDefinition.url = feedDefinition.url.trim();
|
|
const feedContent = await this.load(feedDefinition.url);
|
|
if (!feedContent) {
|
|
throw new SiteError(404, 'Feed failed to load');
|
|
}
|
|
|
|
const updateOp = { $set: { }, $unset: { } };
|
|
|
|
updateOp.$set.url = feedDefinition.url;
|
|
updateOp.$set.title = feedDefinition.title || feedContent.title || 'New Feed';
|
|
updateOp.$set.link = feedDefinition.link || feedContent.link;
|
|
updateOp.$set.description = feedDefinition.description || feedContent.description;
|
|
|
|
updateOp.$set.language = feedDefinition.language || feedContent.language;
|
|
updateOp.$set.generator = feedDefinition.generator || feedContent.generator;
|
|
|
|
await Feed.updateOne({ _id: feed._id }, updateOp);
|
|
|
|
this.jobQueue.add('update-feed', { feedId: feed._id });
|
|
}
|
|
|
|
async updateFavicon (feed) {
|
|
const linkPreview = await getLinkPreview(feed.link || feed.url, {
|
|
headers: {
|
|
'user-agent': this.userAgent.toString(),
|
|
'Accept-Language': 'en-US',
|
|
},
|
|
followRedirects: true,
|
|
resolveDNSHost: module.resolveHost,
|
|
timeout: 15000,
|
|
});
|
|
await Feed.updateOne(
|
|
{ _id: feed._id },
|
|
{
|
|
$set: {
|
|
favicons: linkPreview.favicons,
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
async getFeeds (pagination, options) {
|
|
options = Object.assign({ withEntries: false, entryCount: 3 }, options);
|
|
pagination = Object.assign({ skip: 0, cpp: 10 }, pagination);
|
|
const feeds = await Feed
|
|
.find()
|
|
.sort({ title: 1 })
|
|
.skip(pagination.skip)
|
|
.limit(pagination.cpp)
|
|
.lean();
|
|
|
|
if (options.withEntries) {
|
|
await SiteAsync.each(feeds, async (feed) => {
|
|
try {
|
|
feed.recent = await this.getFeedEntries(feed, { skip: 0, cpp: options.entryCount });
|
|
this.log.debug('feed entries', { count: feed.recent.entries.length });
|
|
} catch (error) {
|
|
this.log.error('failed to populate recent entries for feed', { feedId: feed._id, error });
|
|
// fall through
|
|
}
|
|
}, 2);
|
|
}
|
|
|
|
const totalFeedCount = await Feed.countDocuments();
|
|
|
|
return { feeds, totalFeedCount };
|
|
}
|
|
|
|
async getById (feedId) {
|
|
const feed = await Feed.findOne({ _id: feedId }).lean();
|
|
return feed;
|
|
}
|
|
|
|
async getFeedEntries (feed, pagination) {
|
|
pagination = Object.assign({ skip: 0, cpp: 10 }, pagination);
|
|
const entries = await FeedEntry
|
|
.find({ feed: feed._id })
|
|
.sort({ published: -1 })
|
|
.skip(pagination.skip)
|
|
.limit(pagination.cpp)
|
|
.populate(this.populateFeedEntry)
|
|
.lean();
|
|
const totalFeedEntryCount = await FeedEntry.countDocuments({ feed: feed._id });
|
|
return { entries, totalFeedEntryCount };
|
|
}
|
|
|
|
async getNewsfeed (pagination) {
|
|
pagination = Object.assign({ skip: 0, cpp: 5 }, pagination);
|
|
const entries = await FeedEntry
|
|
.find()
|
|
.sort({ published: -1 })
|
|
.skip(pagination.skip)
|
|
.limit(pagination.cpp)
|
|
.populate(this.populateFeedEntry)
|
|
.lean();
|
|
const totalFeedEntryCount = await FeedEntry.estimatedDocumentCount();
|
|
return { entries, totalFeedEntryCount };
|
|
}
|
|
|
|
async remove (feed) {
|
|
this.log.info('removing all feed entries', { feedId: feed._id, title: feed.title });
|
|
await FeedEntry.deleteMany({ feed: feed._id });
|
|
|
|
this.log.info('removing feed', { feedId: feed._id, title: feed.title });
|
|
await Feed.deleteOne({ _id: feed._id });
|
|
}
|
|
|
|
async load (url) {
|
|
const response = await feedReader(url);
|
|
return response;
|
|
}
|
|
|
|
async createEntry (feed, entryDefinition) {
|
|
const NOW = new Date();
|
|
const updateOp = { $setOnInsert: { }, $set: { }, $unset: { } };
|
|
|
|
updateOp.$setOnInsert.feed = feed._id;
|
|
updateOp.$setOnInsert.link = entryDefinition.link.trim();
|
|
updateOp.$setOnInsert.published = new Date(entryDefinition.published || NOW);
|
|
|
|
updateOp.$set.title = entryDefinition.title.trim();
|
|
updateOp.$set.description = entryDefinition.description.trim();
|
|
|
|
await FeedEntry.updateOne(
|
|
{
|
|
feed: feed._id,
|
|
link: updateOp.$setOnInsert.link,
|
|
},
|
|
updateOp,
|
|
{ upsert: true },
|
|
);
|
|
}
|
|
|
|
async getSiteFeed ( ) {
|
|
const { post: postService } = this.dtp.services;
|
|
const posts = await postService.getPosts({ skip: 0, cpp: 10 });
|
|
|
|
const siteDomain = this.dtp.config.site.domain;
|
|
const siteDomainKey = this.dtp.config.site.domainKey;
|
|
const siteUrl = `https://${siteDomain}`;
|
|
|
|
const cardUrl = `${siteUrl}/img/social-cards/${siteDomainKey}.png`;
|
|
const iconUrl = `${siteUrl}/img/icon/${siteDomainKey}/icon-512x512.png`;
|
|
|
|
const feed = new FeedGenerator({
|
|
title: this.dtp.config.site.name,
|
|
description: this.dtp.config.site.description,
|
|
id: siteUrl,
|
|
link: siteUrl,
|
|
language: 'en-US',
|
|
image: cardUrl,
|
|
favicon: iconUrl,
|
|
copyright: `Copyright © ${moment(new Date()).format('YYYY')}, ${this.dtp.config.site.company}`,
|
|
updated: posts[0].updated || posts[0].created,
|
|
generator: 'DTP Sites',
|
|
feedLinks: {
|
|
json: `${siteUrl}/feed/json`,
|
|
atom: `${siteUrl}/feed/json`,
|
|
},
|
|
});
|
|
|
|
const authorsAttributed = { };
|
|
posts.forEach((post) => {
|
|
const postUrl = `${siteUrl}/post/${post.slug}`;
|
|
if (!authorsAttributed[post.author._id]) {
|
|
authorsAttributed[post.author._id] = true;
|
|
feed.addContributor({
|
|
name: post.author.displayName || post.author.username,
|
|
link: `${siteUrl}/user/${post.author.username}`,
|
|
});
|
|
}
|
|
feed.addItem({
|
|
title: post.title,
|
|
id: postUrl,
|
|
link: postUrl,
|
|
date: post.updated || post.created,
|
|
description: post.summary,
|
|
image: post.image ? `${siteUrl}/image/${post.image._id}` : `${siteUrl}/img/default-poster.jpg`,
|
|
content: post.content,
|
|
author: [
|
|
{
|
|
name: post.author.displayName || post.author.username,
|
|
link: `${siteUrl}/user/${post.author.username}`,
|
|
},
|
|
],
|
|
});
|
|
});
|
|
|
|
return feed;
|
|
}
|
|
|
|
async resolveHost (url) {
|
|
const dns = require('dns');
|
|
return new Promise((resolve, reject) => {
|
|
const hostname = new URL(url).hostname;
|
|
module.log.info('resolving DNS hostname', { hostname });
|
|
dns.lookup(hostname, (err, address, family) => {
|
|
if (err) {
|
|
reject(err);
|
|
return;
|
|
}
|
|
if (['127.0.0.1','0.0.0.0'].includes(address)) {
|
|
/*
|
|
* TODO: maybe automatically add to domain blacklist? Want to get some
|
|
* miles on this service and observe results in prod before auto-
|
|
* blocking based on this result.
|
|
*/
|
|
module.log.alert('malicious intent link hostname detected', { hostname, address, family });
|
|
} else {
|
|
module.log.info('resolved link hostname', { hostname, address, family });
|
|
}
|
|
|
|
// dailyhodl resolves to 192.0.x.x, and the Page Preview library threw an
|
|
// SSRF exception on that. So, I'm just hacking the address to lie to the
|
|
// library as long as we're not mapping to 192.168.0.1
|
|
const tokens = address.split('.');
|
|
if (tokens[0] === '192' && tokens[1] === '0') {
|
|
if (tokens[2] !== '0' && tokens[3] !== '1') {
|
|
tokens[1] = '1';
|
|
}
|
|
}
|
|
address = tokens.join('.');
|
|
resolve(address);
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
slug: 'feed',
|
|
name: 'feed',
|
|
className: 'FeedService',
|
|
create: (dtp) => { return new FeedService(dtp); },
|
|
}; |