From 092cc2037c22c52a6879d00ec695ad522871fecf Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 21 Oct 2022 18:41:30 -0400 Subject: [PATCH] newsroom (RSS feeds for DTP) --- app/controllers/admin.js | 1 + app/controllers/admin/newsroom.js | 164 ++++++++++++++++++++++ app/controllers/home.js | 8 +- app/controllers/newsroom.js | 85 +++++++++++ app/models/feed-entry.js | 26 ++++ app/models/feed.js | 21 +++ app/services/feed.js | 148 +++++++++++++++++++ app/views/admin/components/menu.pug | 6 + app/views/admin/newsroom/editor.pug | 39 +++++ app/views/admin/newsroom/index.pug | 16 +++ app/views/components/off-canvas.pug | 4 + app/views/components/page-sidebar.pug | 19 +++ app/views/index.pug | 2 + app/views/newsroom/feed-view.pug | 35 +++++ app/views/newsroom/index.pug | 19 +++ app/workers/newsroom.js | 60 ++++++++ app/workers/newsroom/cron/update-feeds.js | 85 +++++++++++ app/workers/reeeper.js | 2 +- client/js/site-admin-app.js | 23 +++ config/limiter.js | 16 +++ package.json | 1 + yarn.lock | 31 +++- 22 files changed, 807 insertions(+), 4 deletions(-) create mode 100644 app/controllers/admin/newsroom.js create mode 100644 app/controllers/newsroom.js create mode 100644 app/models/feed-entry.js create mode 100644 app/models/feed.js create mode 100644 app/services/feed.js create mode 100644 app/views/admin/newsroom/editor.pug create mode 100644 app/views/admin/newsroom/index.pug create mode 100644 app/views/newsroom/feed-view.pug create mode 100644 app/views/newsroom/index.pug create mode 100644 app/workers/newsroom.js create mode 100644 app/workers/newsroom/cron/update-feeds.js diff --git a/app/controllers/admin.js b/app/controllers/admin.js index 0cdec58..ae3bc10 100644 --- a/app/controllers/admin.js +++ b/app/controllers/admin.js @@ -50,6 +50,7 @@ class AdminController extends SiteController { router.use('/job-queue', await this.loadChild(path.join(__dirname, 'admin', 'job-queue'))); router.use('/log', await this.loadChild(path.join(__dirname, 'admin', 'log'))); router.use('/newsletter', await this.loadChild(path.join(__dirname, 'admin', 'newsletter'))); + router.use('/newsroom', await this.loadChild(path.join(__dirname, 'admin', 'newsroom'))); router.use('/settings', await this.loadChild(path.join(__dirname, 'admin', 'settings'))); router.use('/service-node', await this.loadChild(path.join(__dirname, 'admin', 'service-node'))); router.use('/user', await this.loadChild(path.join(__dirname, 'admin', 'user'))); diff --git a/app/controllers/admin/newsroom.js b/app/controllers/admin/newsroom.js new file mode 100644 index 0000000..dea2a9e --- /dev/null +++ b/app/controllers/admin/newsroom.js @@ -0,0 +1,164 @@ +// admin/newsroom.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const express = require('express'); + +const { SiteController, SiteError } = require('../../../lib/site-lib'); + +class NewsroomAdminController extends SiteController { + + constructor (dtp) { + super(dtp, module.exports); + } + + async start ( ) { + const upload = this.createMulter('newsroom-admin'); + + const router = express.Router(); + router.use(async (req, res, next) => { + res.locals.currentView = 'admin'; + res.locals.adminView = 'newsroom'; + return next(); + }); + + router.param('feedId', this.populateFeedId.bind(this)); + router.param('feedEntryId', this.populateFeedEntryId.bind(this)); + + router.post('/resolve', upload.none(), this.postResolveFeed.bind(this)); + router.post('/:feedId', upload.none(), this.postUpdateFeed.bind(this)); + router.post('/', upload.none(), this.postCreateFeed.bind(this)); + + router.get('/create', this.getFeedEditor.bind(this)); + router.get('/:feedId', this.getFeedEditor.bind(this)); + + router.get('/', this.getHomeView.bind(this)); + + router.delete('/:feedId', this.deleteFeed.bind(this)); + + return router; + } + + async populateFeedId (req, res, next, feedId) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.feed = await feedService.getById(feedId); + if (!res.locals.feed) { + throw new SiteError(404, 'Feed not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate feedId', { feedId, error }); + return next(error); + } + } + + async populateFeedEntryId (req, res, next, feedEntryId) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.feedEntry = await feedService.getEntryById(feedEntryId); + if (!res.locals.feedEntry) { + throw new SiteError(404, 'Feed entry not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate feed entry', { feedEntryId, error }); + return next(error); + } + } + + async postResolveFeed (req, res) { + const { feed: feedService } = this.dtp.services; + try { + const feed = await feedService.load(req.body.feedUrl); + this.log.info('request body', { body: req.body, feed }); + + const displayList = this.createDisplayList('resolve-feed'); + displayList.setInputValue(`input#title`, feed.title); + displayList.setInputValue(`input#link`, feed.link); + displayList.setInputValue(`textarea#description`, feed.description); + + if (feed.generator) { + displayList.setInputValue(`input[type="hidden"][name="generator"]`, feed.generator); + } + if (feed.language) { + displayList.setInputValue(`input[type="hidden"][name="language"]`, feed.language); + } + if (feed.published) { + displayList.setInputValue(`input[type="hidden"][name="published"]`, feed.published); + } + + res.status(200).json({ success: true, displayList }); + } catch (error) { + this.log.error('failed to present the Newsroom Admin home', { error }); + return res.status(error.statusCode || 500).json({ + success: false, + message: error.message, + }); + } + } + + async postUpdateFeed (req, res, next) { + const { feed: feedService } = this.dtp.services; + try { + await feedService.update(res.locals.feed, req.body); + res.redirect('/admin/newsroom'); + } catch (error) { + this.log.error('failed to create feed', { error }); + return next(error); + } + } + + async postCreateFeed (req, res, next) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.feed = await feedService.create(req.body); + res.redirect(`/admin/newsroom/${res.locals.feed._id}`); + } catch (error) { + this.log.error('failed to create feed', { error }); + return next(error); + } + } + + async getFeedEditor (req, res) { + res.render('admin/newsroom/editor'); + } + + async getHomeView (req, res, next) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.pagination = this.getPaginationParameters(req, 20); + res.locals.newsroom = await feedService.getFeeds(res.locals.pagination); + res.render('admin/newsroom/index'); + } catch (error) { + this.log.error('failed to present the Newsroom Admin home', { error }); + return next(error); + } + } + + async deleteFeed (req, res) { + const { feed: feedService } = this.dtp.services; + try { + await feedService.remove(res.locals.feed); + + const displayList = this.createDisplayList('delete-feed'); + displayList.navigateTo('/admin/newsroom'); + + res.status(200).json({ success: true, displayList }); + } catch (error) { + this.log.error('failed to remove feed', { error }); + return res.status(error.statusCode || 500).json({ + success: false, + message: error.message, + }); + } + } +} + +module.exports = { + name: 'newsroomAdmin', + slug: 'newsroom-admin', + create: async (dtp) => { return new NewsroomAdminController(dtp); }, +}; \ No newline at end of file diff --git a/app/controllers/home.js b/app/controllers/home.js index 6c459cb..292704b 100644 --- a/app/controllers/home.js +++ b/app/controllers/home.js @@ -63,13 +63,19 @@ class HomeController extends SiteController { } async getHome (req, res, next) { - const { announcement: announcementService, hive: hiveService } = this.dtp.services; + const { + announcement: announcementService, + feed: feedService, + hive: hiveService, + } = this.dtp.services; try { res.locals.announcements = await announcementService.getLatest(req.user); res.locals.pagination = this.getPaginationParameters(req, 20); res.locals.constellationTimeline = await hiveService.getConstellationTimeline(req.user, res.locals.pagination); + res.locals.newsfeed = await feedService.getNewsfeed(); + res.render('index'); } catch (error) { this.log.error('failed to render home view', { error }); diff --git a/app/controllers/newsroom.js b/app/controllers/newsroom.js new file mode 100644 index 0000000..fb47135 --- /dev/null +++ b/app/controllers/newsroom.js @@ -0,0 +1,85 @@ +// newsroom.js +// Copyright (C) 2021 Digital Telepresence, LLC +// License: Apache-2.0 + +'use strict'; + +const express = require('express'); + +const { SiteController, SiteError } = require('../../lib/site-lib'); + +class NewsroomController extends SiteController { + + constructor (dtp) { + super(dtp, module.exports); + } + + async start ( ) { + const { dtp } = this; + const { limiter: limiterService } = dtp.services; + + const router = express.Router(); + dtp.app.use('/newsroom', router); + + router.use(async (req, res, next) => { + res.locals.currentView = module.exports.slug; + return next(); + }); + + router.param('feedId', this.populateFeedId.bind(this)); + + router.get('/:feedId', + limiterService.createMiddleware(limiterService.config.newsroom.getFeedView), + this.getFeedView.bind(this), + ); + + router.get('/', + limiterService.createMiddleware(limiterService.config.newsletter.getIndex), + this.getHome.bind(this), + ); + } + + async populateFeedId (req, res, next, feedId) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.feed = await feedService.getById(feedId); + if (!res.locals.feed) { + throw new SiteError(404, 'Feed not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate feedId', { feedId, error }); + return next(error); + } + } + + async getFeedView (req, res, next) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.pagination = this.getPaginationParameters(req, 10); + res.locals.newsroom = await feedService.getFeedEntries(res.locals.feed, res.locals.pagination); + res.render('newsroom/feed-view'); + } catch (error) { + this.log.error('failed to present newsroom home', { error }); + return next(error); + } + } + + async getHome (req, res, next) { + const { feed: feedService } = this.dtp.services; + try { + res.locals.pagination = this.getPaginationParameters(req, 10); + res.locals.newsroom = await feedService.getFeeds(res.locals.pagination); + res.render('newsroom/index'); + } catch (error) { + this.log.error('failed to present newsroom home', { error }); + return next(error); + } + } +} + +module.exports = { + slug: 'newsroom', + name: 'newsroom', + create: (dtp) => { return new NewsroomController(dtp); }, +}; \ No newline at end of file diff --git a/app/models/feed-entry.js b/app/models/feed-entry.js new file mode 100644 index 0000000..676888a --- /dev/null +++ b/app/models/feed-entry.js @@ -0,0 +1,26 @@ +// feed-entry.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const mongoose = require('mongoose'); + +const Schema = mongoose.Schema; + +const FeedEntrySchema = new Schema({ + feed: { type: Schema.ObjectId, required: true, index: 1, ref: 'Feed' }, + published: { type: Date }, + title: { type: String }, + description: { type: String }, + link: { type: String, index: 1 }, +}); + +FeedEntrySchema.index({ + feed: 1, + link: 1, +}, { + name: 'feed_entry_by_feed_idx', +}); + +module.exports = mongoose.model('FeedEntry', FeedEntrySchema); \ No newline at end of file diff --git a/app/models/feed.js b/app/models/feed.js new file mode 100644 index 0000000..84bc39e --- /dev/null +++ b/app/models/feed.js @@ -0,0 +1,21 @@ +// feed.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const mongoose = require('mongoose'); + +const Schema = mongoose.Schema; + +const FeedSchema = new Schema({ + url: { type: String, required: true, unique: true }, + title: { type: String }, + link: { type: String }, + description: { type: String }, + language: { type: String }, + generator: { type: String }, + published: { type: Date }, +}); + +module.exports = mongoose.model('Feed', FeedSchema); \ No newline at end of file diff --git a/app/services/feed.js b/app/services/feed.js new file mode 100644 index 0000000..f1b3f88 --- /dev/null +++ b/app/services/feed.js @@ -0,0 +1,148 @@ +// announcement.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const mongoose = require('mongoose'); +const Feed = mongoose.model('Feed'); +const FeedEntry = mongoose.model('FeedEntry'); + +const { SiteService, SiteError } = require('../../lib/site-lib'); +const { read: feedReader } = require('feed-reader'); + +class FeedService extends SiteService { + + constructor (dtp) { + super(dtp, module.exports); + + this.populateFeedEntry = [ + { + path: 'feed', + }, + ]; + } + + async create (feedDefinition) { + feedDefinition.url = feedDefinition.url.trim(); + const feedContent = await this.load(feedDefinition.url); + if (!feedContent) { + throw new SiteError(404, 'Feed failed to load'); + } + + const feed = new Feed(); + feed.url = feedDefinition.url; + feed.title = feedDefinition.title || feedContent.title || 'New Feed'; + feed.link = feedDefinition.link || feedContent.link; + feed.description = feedDefinition.description || feedContent.description; + feed.language = feedContent.language; + feed.generator = feedContent.generator; + feed.published = feedContent.published; + await feed.save(); + + return feed.toObject(); + } + + async update (feed, feedDefinition) { + feedDefinition.url = feedDefinition.url.trim(); + const feedContent = await this.load(feedDefinition.url); + if (!feedContent) { + throw new SiteError(404, 'Feed failed to load'); + } + + const updateOp = { $set: { }, $unset: { } }; + + updateOp.$set.url = feedDefinition.url; + updateOp.$set.title = feedDefinition.title || feedContent.title || 'New Feed'; + updateOp.$set.link = feedDefinition.link || feedContent.link; + updateOp.$set.description = feedDefinition.description || feedContent.description; + + updateOp.$set.language = feedDefinition.language || feedContent.language; + updateOp.$set.generator = feedDefinition.generator || feedContent.generator; + + await Feed.updateOne({ _id: feed._id }, updateOp); + } + + async getFeeds (pagination) { + pagination = Object.assign({ skip: 0, cpp: 10 }, pagination); + const feeds = await Feed + .find() + .sort({ title: 1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .lean(); + const totalFeedCount = await Feed.countDocuments(); + return { feeds, totalFeedCount }; + } + + async getById (feedId) { + const feed = await Feed.findOne({ _id: feedId }).lean(); + return feed; + } + + async getFeedEntries (feed, pagination) { + pagination = Object.assign({ skip: 0, cpp: 10 }, pagination); + const entries = await FeedEntry + .find({ feed: feed._id }) + .sort({ published: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .populate(this.populateFeedEntry) + .lean(); + const totalFeedEntryCount = await FeedEntry.countDocuments({ feed: feed._id }); + return { entries, totalFeedEntryCount }; + } + + async getNewsfeed (pagination) { + pagination = Object.assign({ skip: 0, cpp: 5 }, pagination); + const entries = await FeedEntry + .find() + .sort({ published: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .populate(this.populateFeedEntry) + .lean(); + const totalFeedEntryCount = await FeedEntry.estimatedDocumentCount(); + return { entries, totalFeedEntryCount }; + } + + async remove (feed) { + this.log.info('removing all feed entries', { feedId: feed._id, title: feed.title }); + await FeedEntry.deleteMany({ feed: feed._id }); + + this.log.info('removing feed', { feedId: feed._id, title: feed.title }); + await Feed.deleteOne({ _id: feed._id }); + } + + async load (url) { + const response = await feedReader(url); + return response; + } + + async createEntry (feed, entryDefinition) { + const NOW = new Date(); + const updateOp = { $setOnInsert: { }, $set: { }, $unset: { } }; + + updateOp.$setOnInsert.feed = feed._id; + updateOp.$setOnInsert.link = entryDefinition.link.trim(); + updateOp.$setOnInsert.published = new Date(entryDefinition.published || NOW); + + updateOp.$set.title = entryDefinition.title.trim(); + updateOp.$set.description = entryDefinition.description.trim(); + + await FeedEntry.updateOne( + { + feed: feed._id, + link: updateOp.$setOnInsert.link, + }, + updateOp, + { upsert: true }, + ); + } +} + +module.exports = { + slug: 'feed', + name: 'feed', + create: (dtp) => { return new FeedService(dtp); }, +}; \ No newline at end of file diff --git a/app/views/admin/components/menu.pug b/app/views/admin/components/menu.pug index 7dd7641..201eacc 100644 --- a/app/views/admin/components/menu.pug +++ b/app/views/admin/components/menu.pug @@ -33,6 +33,12 @@ ul(uk-nav).uk-nav-default i.fas.fa-ban span.uk-margin-small-left Content Reports + li(class={ 'uk-active': (adminView === 'newsroom') }) + a(href="/admin/newsroom") + span.nav-item-icon + i.fas.fa-newspaper + span.uk-margin-small-left Newsroom + li(class={ 'uk-active': (adminView === 'newsletter') }).uk-parent a span.nav-item-icon diff --git a/app/views/admin/newsroom/editor.pug b/app/views/admin/newsroom/editor.pug new file mode 100644 index 0000000..06700b3 --- /dev/null +++ b/app/views/admin/newsroom/editor.pug @@ -0,0 +1,39 @@ +extends ../layouts/main +block content + + - var formActionUrl = feed ? `/admin/newsroom/${feed._id}` : '/admin/newsroom'; + + form#add-feed-form(method="POST", action= formActionUrl).uk-form + + input(type="hidden", name="generator", value="DTP News") + input(type="hidden", name="language", value="en") + input(type="hidden", name="published") + + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title= feed ? 'Update Feed' : 'Add Feed' + .uk-card-body + .uk-margin + label(for="url").uk-form-label Feed URL + input(id="url", name="url", type="url", placeholder="Enter feed URL", value= feed ? feed.url : undefined).uk-input + .uk-margin-small + button(type="button", onclick="return dtp.adminApp.resolveNewsroomFeed(event);").uk-button.uk-button-default.uk-button-small.uk-border-rounded Resolve Feed + span.uk-margin-small-left to load the following information automatically. + + .uk-margin + label(for="title").uk-form-label Title + input(id="title", name="title", type="text", placeholder="Enter feed title", value= feed ? feed.title : undefined).uk-input + + .uk-margin + label(for="link").uk-form-label Link + input(id="link", name="link", type="url", placeholder="Enter feed website link", value= feed ? feed.link : undefined).uk-input + + .uk-margin + label(for="description").uk-form-label Description + textarea(id="description", name="description", rows="4", placeholder="Enter feed description").uk-textarea.uk-resize-vertical= feed ? feed.description : undefined + + .uk-card-footer + button(type="submit").uk-button.uk-button-primary.uk-border-rounded= feed ? 'Update Feed' : 'Add Feed' + + if feed + pre= JSON.stringify(feed, null, 2) \ No newline at end of file diff --git a/app/views/admin/newsroom/index.pug b/app/views/admin/newsroom/index.pug new file mode 100644 index 0000000..91644a0 --- /dev/null +++ b/app/views/admin/newsroom/index.pug @@ -0,0 +1,16 @@ +extends ../layouts/main +block content + + div(uk-grid) + .uk-width-expand + h1 Newsroom Feeds + .uk-width-auto + a(href='/admin/newsroom/create').uk-button.uk-button-primary #[i.fas.fa-plus]#[span.uk-margin-small-left Add Feed] + + if Array.isArray(newsroom.feeds) && (newsroom.feeds.length > 0) + ul.uk-list.uk-list-divider + each feed in newsroom.feeds + li + a(href=`/admin/newsroom/${feed._id}`)= feed.title + else + div There are no feeds. \ No newline at end of file diff --git a/app/views/components/off-canvas.pug b/app/views/components/off-canvas.pug index ffdf930..acd1a62 100644 --- a/app/views/components/off-canvas.pug +++ b/app/views/components/off-canvas.pug @@ -24,6 +24,10 @@ mixin renderMenuItem (iconClass, label) a(href='/announcement').uk-display-block +renderMenuItem('fa-bullhorn', 'Announcements') + li(class={ "uk-active": (currentView === 'newsroom') }) + a(href='/newsroom').uk-display-block + +renderMenuItem('fa-newspaper', 'Newsroom') + if user li.uk-nav-header Member Menu diff --git a/app/views/components/page-sidebar.pug b/app/views/components/page-sidebar.pug index cb8b97c..85b9383 100644 --- a/app/views/components/page-sidebar.pug +++ b/app/views/components/page-sidebar.pug @@ -7,6 +7,25 @@ mixin renderPageSidebar ( ) li +renderAnnouncement(announcement) + .uk-margin + +renderSectionTitle('Newsfeed', { + label: 'See All', + title: 'Browse all news feeds', + url: '/newsroom', + }) + if Array.isArray(newsfeed.entries) && (newsfeed.entries.length > 0) + ul.uk-list + each entry in newsfeed.entries + li + div + a(href= entry.link, target="_blank").uk-link-reset= entry.title + .uk-text-small + div(uk-grid).uk-grid-small + .uk-width-expand + a(href= entry.feed.link, target="_blank").uk-link-reset= entry.feed.title + .uk-width-auto + div= moment(entry.published).fromNow() + .uk-margin +renderSectionTitle('Widget', { label: 'Sample URL', diff --git a/app/views/index.pug b/app/views/index.pug index cc161cf..dc6287d 100644 --- a/app/views/index.pug +++ b/app/views/index.pug @@ -7,6 +7,8 @@ block content h1 Sample DTP Web Application p This application doesn't actually do anything. The Bobs would have questions. + pre= JSON.stringify(newsfeed, null, 2) + if user h2 Current User pre= JSON.stringify(user, null, 2) diff --git a/app/views/newsroom/feed-view.pug b/app/views/newsroom/feed-view.pug new file mode 100644 index 0000000..48eb681 --- /dev/null +++ b/app/views/newsroom/feed-view.pug @@ -0,0 +1,35 @@ +extends ../layouts/main +block content + + include ../components/pagination-bar + + section.uk-section.uk-section-default.uk-section-small + .uk-container + + article.uk-article + .uk-margin-large + .uk-margin + h1.uk-article-title.uk-margin-remove= feed.title + .uk-article-meta + div(uk-grid) + .uk-width-auto + a(href= feed.link, target="_blank") #[i.fas.fa-external-link-alt]#[span.uk-margin-small-left Visit site] + .uk-width-auto + div last updated #{moment(feed.published).fromNow()} + + .uk-text-lead= feed.description + + .uk-margin-large + if Array.isArray(newsroom.entries) && (newsroom.entries.length > 0) + ul.uk-list.uk-list-divider + each entry in newsroom.entries + li + .uk-text-bold + a(href= entry.link, target="_blank")= entry.title + .uk-text-small= entry.description + //- pre= JSON.stringify(entry, null, 2) + else + div There are no news feed entries. + + .uk-margin + +renderPaginationBar(`/newsroom/${feed._id}`, newsroom.totalFeedEntryCount) \ No newline at end of file diff --git a/app/views/newsroom/index.pug b/app/views/newsroom/index.pug new file mode 100644 index 0000000..5119ff8 --- /dev/null +++ b/app/views/newsroom/index.pug @@ -0,0 +1,19 @@ +extends ../layouts/main +block content + + section.uk-section.uk-section-default.uk-section-small + .uk-container + + h1 #{site.name} Newsroom + if Array.isArray(newsroom.feeds) && (newsroom.feeds.length > 0) + div(uk-grid).uk-grid-match + each feed in newsroom.feeds + .uk-width-1-3 + .uk-tile.uk-tile-secondary.uk-padding-small.uk-border-rounded + .uk-text-bold + a(href=`/newsroom/${feed._id}`)= feed.title + .uk-text-small.uk-text-muted + div last update #{moment(feed.published).fromNow()} + div= feed.description + else + div There are no configured news feeds. \ No newline at end of file diff --git a/app/workers/newsroom.js b/app/workers/newsroom.js new file mode 100644 index 0000000..ad4dc02 --- /dev/null +++ b/app/workers/newsroom.js @@ -0,0 +1,60 @@ +// newsroom.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); + +require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') }); + +const { + SiteLog, + SiteWorker, +} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib')); + +module.rootPath = path.resolve(__dirname, '..', '..'); +module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); + +module.config = { + environment: process.env.NODE_ENV, + root: module.rootPath, + component: { name: 'newsroom', slug: 'newsroom' }, +}; + +module.config.site = require(path.join(module.rootPath, 'config', 'site')); +module.config.http = require(path.join(module.rootPath, 'config', 'http')); + +class NewsroomWorker extends SiteWorker { + + constructor (dtp) { + super(dtp, dtp.config.component); + } + + async start ( ) { + await super.start(); + + await this.loadProcessor(path.join(__dirname, 'newsroom', 'cron', 'update-feeds.js')); + + await this.startProcessors(); + } + + async stop ( ) { + await super.stop(); + } +} + +(async ( ) => { + try { + module.log = new SiteLog(module, module.config.component); + + module.worker = new NewsroomWorker(module); + await module.worker.start(); + + module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`); + } catch (error) { + module.log.error('failed to start worker', { component: module.config.component, error }); + process.exit(-1); + } + +})(); \ No newline at end of file diff --git a/app/workers/newsroom/cron/update-feeds.js b/app/workers/newsroom/cron/update-feeds.js new file mode 100644 index 0000000..e2278da --- /dev/null +++ b/app/workers/newsroom/cron/update-feeds.js @@ -0,0 +1,85 @@ +// newsroom/cron/update-feeds.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); + +const mongoose = require('mongoose'); +const Feed = mongoose.model('Feed'); + +const { CronJob } = require('cron'); +const { read: feedReader } = require('feed-reader'); +const { SiteAsync } = require('../../../../lib/site-lib'); + +const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); + +class UpdateFeedsCron extends SiteWorkerProcess { + + static get COMPONENT ( ) { + return { + name: 'updateFeeds', + slug: 'update-feeds-cron', + }; + } + + constructor (worker) { + super(worker, UpdateFeedsCron.COMPONENT); + } + + async start ( ) { + await super.start(); + + await this.updateFeeds(); + + this.job = new CronJob( + '0 */15 * * * *', + this.updateFeeds.bind(this), + null, + true, + process.env.DTP_CRON_TIMEZONE || 'America/New_York', + ); + } + + async stop ( ) { + if (this.job) { + this.log.info('stopping feed update job'); + this.job.stop(); + delete this.job; + } + await super.stop(); + } + + async updateFeeds ( ) { + try { + await Feed + .find() + .lean() + .cursor() + .eachAsync(async (feed) => { + await this.updateFeed(feed); + }, 4); + } catch (error) { + this.log.error('failed to update feeds', { error }); + } + } + + async updateFeed (feed) { + const NOW = new Date(); + const { feed: feedService } = this.dtp.services; + try { + this.log.info('loading latest feed data', { feedId: feed._id, title: feed.title }); + const response = await feedReader(feed.url); + await SiteAsync.each(response.entries, async (entry) => { + await Feed.updateOne({ _id: feed._id }, { $set: { published: feed.published || NOW }}); + await feedService.createEntry(feed, entry); + }, 4); + this.log.info('feed updated', { entries: response.entries.length }); + } catch (error) { + this.log.error('failed to update feed', { feedId: feed._id, title: feed.title, error }); + } + } +} + +module.exports = UpdateFeedsCron; \ No newline at end of file diff --git a/app/workers/reeeper.js b/app/workers/reeeper.js index 6a0da5c..0236034 100644 --- a/app/workers/reeeper.js +++ b/app/workers/reeeper.js @@ -1,4 +1,4 @@ -// host-services.js +// reeeper.js // Copyright (C) 2022 DTP Technologies, LLC // License: Apache-2.0 diff --git a/client/js/site-admin-app.js b/client/js/site-admin-app.js index 57a1eb7..46c1910 100644 --- a/client/js/site-admin-app.js +++ b/client/js/site-admin-app.js @@ -344,6 +344,29 @@ export default class DtpSiteAdminHostStatsApp extends DtpApp { UIkit.modal.alert(`Failed to delete Service Node: ${error.message}`); } } + + async resolveNewsroomFeed (event) { + event.preventDefault(); + event.stopPropagation(); + + const form = document.querySelector('form#add-feed-form'); + const input = form.querySelector('input#url'); + const feedUrl = input.value; + try { + const response = await fetch(`/admin/newsroom/resolve`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ feedUrl }), + }); + await this.processResponse(response); + } catch (error) { + UIkit.modal.alert(`Failed to resolve feed: ${error.message}`); + } + + return false; + } } dtp.DtpSiteAdminHostStatsApp = DtpSiteAdminHostStatsApp; \ No newline at end of file diff --git a/config/limiter.js b/config/limiter.js index e9676fd..63397cf 100644 --- a/config/limiter.js +++ b/config/limiter.js @@ -227,6 +227,22 @@ module.exports = { */ newsletter: { getView: { + total: 15, + expire: ONE_MINUTE, + message: 'You are loading newsfeed views too quickly. Please try again later.', + }, + getIndex: { + total: 60, + expire: ONE_MINUTE, + message: 'You are loading the newsroom too quickly. Please try again later.', + }, + }, + + /* + * NewsroomController + */ + newsroom: { + getFeedView: { total: 5, expire: ONE_MINUTE, message: 'You are reading newsletters too quickly', diff --git a/package.json b/package.json index b2051aa..e8a40ad 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "express-limiter": "^1.6.1", "express-session": "^1.17.2", "feed": "^4.2.2", + "feed-reader": "^6.1.2", "geoip-lite": "^1.4.3", "glob": "^7.2.0", "highlight.js": "^11.4.0", diff --git a/yarn.lock b/yarn.lock index e8b8f5a..45a6a2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1762,6 +1762,11 @@ batch@0.6.1: resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16" integrity sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY= +bellajs@^11.0.7: + version "11.1.1" + resolved "https://registry.yarnpkg.com/bellajs/-/bellajs-11.1.1.tgz#1828dae65e396bf6c199fa8e0e402597b387ce29" + integrity sha512-Fjsx2ZVarl3UWeTq3YJbbPoQPyh4dWtduw+DMnDYhKya9agbEg/8eXP7yHOvv88zUEHoVl9O/XmgrNTMcMTVSQ== + binary-extensions@^1.0.0: version "1.13.1" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65" @@ -2689,7 +2694,7 @@ cropperjs@^1.5.12: resolved "https://registry.yarnpkg.com/cropperjs/-/cropperjs-1.5.12.tgz#d9c0db2bfb8c0d769d51739e8f916bbc44e10f50" integrity sha512-re7UdjE5UnwdrovyhNzZ6gathI4Rs3KGCBSc8HCIjUo5hO42CtzyblmWLj6QWVw7huHyDMfpKxhiO2II77nhDw== -cross-fetch@3.1.5: +cross-fetch@3.1.5, cross-fetch@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== @@ -3675,6 +3680,13 @@ fast-xml-parser@^3.17.5: dependencies: strnum "^1.0.4" +fast-xml-parser@^4.0.10: + version "4.0.11" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz#42332a9aca544520631c8919e6ea871c0185a985" + integrity sha512-4aUg3aNRR/WjQAcpceODG1C3x3lFANXRo8+1biqfieHmg9pyMt7qB4lQV/Ta6sJCTbA5vfD8fnA8S54JATiFUA== + dependencies: + strnum "^1.0.5" + fd-slicer@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" @@ -3682,6 +3694,16 @@ fd-slicer@~1.1.0: dependencies: pend "~1.2.0" +feed-reader@^6.1.2: + version "6.1.2" + resolved "https://registry.yarnpkg.com/feed-reader/-/feed-reader-6.1.2.tgz#6e6fb0c3d9bbdba85874676603fc86a50a1d3b5f" + integrity sha512-uvp5w3+mqNLFtdqQ89EJPWkLn/CKdxJkgEU4Erhft/5jGnjz3uepYlT5EWoijiFMO3rmK013/p6nKFqojke27g== + dependencies: + bellajs "^11.0.7" + cross-fetch "^3.1.5" + fast-xml-parser "^4.0.10" + html-entities "^2.3.3" + feed@^4.2.2: version "4.2.2" resolved "https://registry.yarnpkg.com/feed/-/feed-4.2.2.tgz#865783ef6ed12579e2c44bbef3c9113bc4956a7e" @@ -4384,6 +4406,11 @@ html-encoding-sniffer@^3.0.0: dependencies: whatwg-encoding "^2.0.0" +html-entities@^2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.3.3.tgz#117d7626bece327fc8baace8868fa6f5ef856e46" + integrity sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA== + html-filter@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/html-filter/-/html-filter-4.3.2.tgz#44bd2cee365699e8d0674d3253911b97d9381aa6" @@ -8069,7 +8096,7 @@ striptags@^3.2.0: resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.2.0.tgz#cc74a137db2de8b0b9a370006334161f7dd67052" integrity sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw== -strnum@^1.0.4: +strnum@^1.0.4, strnum@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==