Merge branch 'develop' of git.digitaltelepresence.com:digital-telepresence/dtp-base into develop

app/controllers/admin.js

@@ -42,17 +42,17 @@ class AdminController extends SiteController {
       }),
     );
 
-    router.use('/content-report',await this.loadChild(path.join(__dirname, 'admin', 'content-report')));
+    router.use('/content-report', await this.loadChild(path.join(__dirname, 'admin', 'content-report')));
-    router.use('/core-node',await this.loadChild(path.join(__dirname, 'admin', 'core-node')));
+    router.use('/core-node', await this.loadChild(path.join(__dirname, 'admin', 'core-node')));
-    router.use('/core-user',await this.loadChild(path.join(__dirname, 'admin', 'core-user')));
+    router.use('/core-user', await this.loadChild(path.join(__dirname, 'admin', 'core-user')));
-    router.use('/host',await this.loadChild(path.join(__dirname, 'admin', 'host')));
+    router.use('/host', await this.loadChild(path.join(__dirname, 'admin', 'host')));
     router.use('/job-queue', await this.loadChild(path.join(__dirname, 'admin', 'job-queue')));
     router.use('/log', await this.loadChild(path.join(__dirname, 'admin', 'log')));
     router.use('/newsletter', await this.loadChild(path.join(__dirname, 'admin', 'newsletter')));
     router.use('/page', await this.loadChild(path.join(__dirname, 'admin', 'page')));
     router.use('/post', await this.loadChild(path.join(__dirname, 'admin', 'post')));
     router.use('/settings', await this.loadChild(path.join(__dirname, 'admin', 'settings')));
-    router.use('/service-node',await this.loadChild(path.join(__dirname, 'admin', 'service-node')));
+    router.use('/service-node', await this.loadChild(path.join(__dirname, 'admin', 'service-node')));
     router.use('/user', await this.loadChild(path.join(__dirname, 'admin', 'user')));
 
     router.get('/diagnostics', this.getDiagnostics.bind(this));

app/controllers/admin/core-node.js

@@ -7,7 +7,7 @@
 const express = require('express');
 // const multer = require('multer');
 
-const { SiteController } = require('../../../lib/site-lib');
+const { SiteController, SiteError } = require('../../../lib/site-lib');
 
 class CoreNodeController extends SiteController {
 
@@ -25,16 +25,33 @@ class CoreNodeController extends SiteController {
       return next();
     });
 
+    router.param('coreNodeId', this.populateCoreNodeId.bind(this));
+
     router.post('/connect', this.postCoreNodeConnect.bind(this));
 
     router.get('/resolve', this.getCoreNodeResolveForm.bind(this));
     router.get('/connect', this.getCoreNodeConnectForm.bind(this));
 
+    router.get('/:coreNodeId', this.getCoreNodeView.bind(this));
     router.get('/', this.getIndex.bind(this));
 
     return router;
   }
 
+  async populateCoreNodeId (req, res, next, coreNodeId) {
+    const { coreNode: coreNodeService } = this.dtp.services;
+    try {
+      res.locals.coreNode = await coreNodeService.getCoreById(coreNodeId);
+      if (!res.locals.coreNode) {
+        throw new SiteError(404, 'Core Node not found');
+      }
+      return next();
+    } catch (error) {
+      this.log.error('failed to populate Core Node', { coreNodeId, error });
+      return next(error);
+    }
+  }
+
   async postCoreNodeConnect (req, res) {
     const { coreNode: coreNodeService } = this.dtp.services;
 
@@ -99,6 +116,18 @@ class CoreNodeController extends SiteController {
     res.render('admin/core-node/connect');
   }
 
+  async getCoreNodeView (req, res, next) {
+    const { coreNode: coreNodeService } = this.dtp.services;
+    try {
+      res.locals.pagination = this.getPaginationParameters(req, 20);
+      res.locals.requestHistory = await coreNodeService.getCoreRequestHistory(res.locals.coreNode, res.locals.pagination);
+      res.render('admin/core-node/view');
+    } catch (error) {
+      this.log.error('failed to render Core Node view', { error });
+      return next(error);
+    }
+  }
+
   async getIndex (req, res, next) {
     const { coreNode: coreNodeService } = this.dtp.services;
     try {

app/controllers/admin/newsletter.js

@@ -7,6 +7,7 @@
 const express = require('express');
 
 const { SiteController, SiteError } = require('../../../lib/site-lib');
+const Bull = require('bull');
 
 class NewsletterController extends SiteController {
 
@@ -15,6 +16,14 @@ class NewsletterController extends SiteController {
   }
 
   async start ( ) {
+    const { jobQueue: jobQueueService } = this.dtp.services;
+    const { config } = this.dtp;
+
+    this.newsletterQueue = await jobQueueService.getJobQueue(
+      'newsletter',
+      config.jobQueues.newsletter,
+    );
+
     const router = express.Router();
     router.use(async (req, res, next) => {
       res.locals.currentView = 'admin';
@@ -24,10 +33,13 @@ class NewsletterController extends SiteController {
 
     router.param('newsletterId', this.populateNewsletterId.bind(this));
 
+    router.post('/:newsletterId/transmit', this.postTransmitNewsletter.bind(this));
     router.post('/:newsletterId', this.postUpdateNewsletter.bind(this));
     router.post('/', this.postCreateNewsletter.bind(this));
 
     router.get('/compose', this.getComposer.bind(this));
+    router.get('/job-status', this.getJobStatusView.bind(this));
+
     router.get('/:newsletterId', this.getComposer.bind(this));
 
     router.get('/', this.getIndex.bind(this));
@@ -51,6 +63,22 @@ class NewsletterController extends SiteController {
     }
   }
 
+  async postTransmitNewsletter (req, res, next) {
+    try {
+      const displayList = this.createDisplayList('transmit-newsletter');
+      res.locals.jobData = {
+        newsletterId: res.locals.newsletter._id,
+      };
+      this.log.info('creating newsletter transmit job', { jobData: res.locals.jobData });
+      res.locals.job = await this.newsletterQueue.add('transmit', res.locals.jobData);
+      displayList.navigateTo(`/admin/job-queue/newsletter/${res.locals.job.id}`);
+      res.status(200).json({ success: true, displayList });
+    } catch (error) {
+      this.log.error('failed to create newsletter transmit job', { error });
+      return next(error);
+    }
+  }
+
   async postUpdateNewsletter (req, res, next) {
     const { newsletter: newsletterService } = this.dtp.services;
     try {
@@ -77,9 +105,32 @@ class NewsletterController extends SiteController {
     res.render('admin/newsletter/editor');
   }
 
+  async getJobStatusView (req, res, next) {
+    const { jobQueue: jobQueueService } = this.dtp.services;
+    try {
+      res.locals.newsletterView = 'job-status';
+      res.locals.queueName = 'newsletter';
+      res.locals.newsletterQueue = await jobQueueService.getJobQueue(res.locals.queueName);
+  
+      res.locals.newsletterQueue = this.newsletterQueue;
+      res.locals.jobCounts = await this.newsletterQueue.getJobCounts();
+      res.locals.jobs = {
+        waiting: await this.newsletterQueue.getWaiting(0, 5),
+        active: await this.newsletterQueue.getActive(0, 5),
+        delayed: await this.newsletterQueue.getDelayed(0, 5),
+        failed: await this.newsletterQueue.getFailed(0, 5),
+      };
+      res.render('admin/newsletter/job-status');
+    } catch (error) {
+      this.log.error('failed to render job status view', { error });
+      return next(error);
+    }
+  }
+
   async getIndex (req, res, next) {
     const { newsletter: newsletterService } = this.dtp.services;
     try {
+      res.locals.newsletterView = 'index';
       res.locals.pagination = this.getPaginationParameters(req, 20);
       res.locals.newsletters = await newsletterService.getNewsletters(res.locals.pagination, ['draft', 'published']);
       res.render('admin/newsletter/index');

app/models/core-node-request.js

@@ -22,7 +22,7 @@ const { RequestTokenSchema } = require('./lib/core-request-token');
  */
 
 const CoreNodeRequestSchema = new Schema({
-  created: { type: Date, default: Date.now, required: true, index: 1 },
+  created: { type: Date, default: Date.now, required: true, index: 1, expires: '7d' },
   core: { type: Schema.ObjectId, required: true, ref: 'CoreNode' },
   token: { type: RequestTokenSchema },
   method: { type: String, required: true },
@@ -30,6 +30,7 @@ const CoreNodeRequestSchema = new Schema({
   response: {
     received: { type: Date },
     elapsed: { type: Number },
+    statusCode: { type: String },
     success: { type: Boolean },
   },
 });

app/models/core-node.js

@@ -24,6 +24,9 @@ const CoreNodeSchema = new Schema({
     scopes: { type: [String] },
     redirectUri: { type: String },
   },
+  kaleidoscope: {
+    token: { type: String },
+  },
   meta: {
     name: { type: String },
     description: { type: String },

app/models/oauth2-client.js

@@ -21,6 +21,9 @@ const OAuth2ClientSchema = new Schema({
   secret: { type: String, required: true },
   scopes: { type: [String], required: true },
   callbackUrl: { type: String, required: true },
+  kaleidoscope: {
+    token: { type: String, required: true, index: 1 },
+  },
   flags: {
     isActive: { type: Boolean, default: true, required: true, index: 1 },
   },

app/services/core-node.js

@@ -59,6 +59,7 @@ class CoreNodeService extends SiteService {
 
   async start ( ) {
     const cores = await this.getConnectedCores(null, true);
+    this.log.info('Core Node service starting', { connectedCoreCount: cores.length });
     cores.forEach((core) => this.registerPassportCoreOAuth2(core));
   }
 
@@ -197,6 +198,13 @@ class CoreNodeService extends SiteService {
     return address.parse(host);
   }
 
+  async getCoreById (coreNodeId) {
+    const core = await CoreNode
+      .findOne({ _id: coreNodeId })
+      .lean();
+    return core;
+  }
+
   async getCoreByAddress (address) {
     const core = await CoreNode
       .findOne({
@@ -361,30 +369,17 @@ class CoreNodeService extends SiteService {
         try {
           json = await response.json();
         } catch (error) {
+          await this.setRequestResponse(req, response);
           throw new SiteError(response.status, response.statusText);
         }
         this.log.debug('received failure response', { json });
+        await this.setRequestResponse(req, response, json);
         throw new SiteError(response.status, json.message || response.statusText);
       }
 
       const json = await response.json();
 
-      /*
+      await this.setRequestResponse(req, response, json);
-       * capture a little inline health monitoring data, which can be used to
-       * generate health alerts.
-       */
-      const DONE = new Date();
-      const ELAPSED = DONE.valueOf() - req.created.valueOf();
-      await CoreNodeRequest.updateOne(
-        { _id: req._id },
-        {
-          $set: {
-            'response.received': DONE,
-            'response.elapsed': ELAPSED,
-            'response.success': json.success,
-          },
-        },
-      );
 
       this.log.info('Core node request complete', { request: req });
       return { request: req.toObject(), response: json };
@@ -394,6 +389,36 @@ class CoreNodeService extends SiteService {
     }
   }
 
+  async setRequestResponse (request, response, json) {
+    const DONE = new Date();
+    const ELAPSED = DONE.valueOf() - request.created.valueOf();
+    const updateOp = {
+      $set: {
+        'response.received': DONE,
+        'response.elapsed': ELAPSED,
+        'response.statusCode': response.status,
+      },
+    };
+    if (json) {
+      updateOp.$set['response.success'] = json.success;
+    }
+    await CoreNodeRequest.updateOne({ _id: request._id }, updateOp);
+  }
+
+  async getCoreRequestHistory (core, pagination) {
+    const requests = await CoreNodeRequest
+      .find({ core: core._id })
+      .sort({ created: -1 })
+      .skip(pagination.skip)
+      .limit(pagination.cpp)
+      .lean();
+    for (const req of requests) {
+      req.core = core;
+    }
+    const totalRequestCount = await CoreNodeRequest.countDocuments({ core: core._id });
+    return { requests, totalRequestCount };
+  }
+
   async connect (response) {
     const request = await CoreNodeRequest
       .findOne({ 'token.value': response.token })
@@ -481,6 +506,7 @@ class CoreNodeService extends SiteService {
     return request;
   }
 
+  
   async acceptServiceNode (requestToken, appNode) {
     const { oauth2: oauth2Service } = this.dtp.services;
     const response = { token: requestToken };

app/services/email.js

@@ -71,10 +71,12 @@ class EmailService extends SiteService {
   async send (message) {
     const NOW = new Date();
 
+    await this.checkEmailAddress(message.to);
+
     this.log.info('sending email', { to: message.to, subject: message.subject });
     const response = await this.transport.sendMail(message);
 
-    await EmailLog.create({
+    const log = await EmailLog.create({
       created: NOW,
       from: message.from,
       to: message.to,
@@ -82,6 +84,8 @@ class EmailService extends SiteService {
       subject: message.subject,
       messageId: response.messageId,
     });
+
+    return { response, log };
   }
 
   async checkEmailAddress (emailAddress) {

app/services/oauth2.js

@@ -286,6 +286,9 @@ class OAuth2Service extends SiteService {
           secret: clientDefinition.secret,
           scopes: clientDefinition.coreAuth.scopes,
           callbackUrl: clientDefinition.coreAuth.callbackUrl,
+          kaleidoscope: {
+            token: generatePassword(256, false),
+          },
         },
       },
       {

app/views/admin/components/menu.pug

@@ -1,4 +1,4 @@
-ul.uk-nav.uk-nav-default
+ul(uk-nav).uk-nav-default
   li.uk-nav-header Admin Menu
 
   li(class={ 'uk-active': (adminView === 'home') })
@@ -45,6 +45,35 @@ ul.uk-nav.uk-nav-default
         i.fas.fa-ban
       span.uk-margin-small-left Content Reports
 
+  li(class={ 'uk-active': (adminView === 'newsletter') }).uk-parent
+    a
+      span.nav-item-icon
+        i.fas.fa-newspaper
+      span.uk-margin-small-left Newsletter
+      span.uk-nav-parent-icon
+
+    ul.uk-nav-sub
+      li(class={ 'uk-active': (newsletterView === 'index') })
+        a(href="/admin/newsletter")
+          span.nav-item-icon
+            i.fas.fa-newspaper
+          span.uk-margin-small-left Home
+      li(class={ 'uk-active': (newsletterView === 'editor') })
+        a(href="/admin/newsletter/compose")
+          span.nav-item-icon
+            i.fas.fa-edit
+          span.uk-margin-small-left Composer
+      li(class={ 'uk-active': (newsletterView === 'recipient') })
+        a(href="/admin/newsletter/recipient")
+          span.nav-item-icon
+            i.fas.fa-user
+          span.uk-margin-small-left Recipients
+      li(class={ 'uk-active': (newsletterView === 'job-status') })
+        a(href="/admin/newsletter/job-status")
+          span.nav-item-icon
+            i.fas.fa-newspaper
+          span.uk-margin-small-left Job Status
+
   li.uk-nav-divider 
 
   li(class={ 'uk-active': (adminView === 'core-node') })
@@ -65,12 +94,6 @@ ul.uk-nav.uk-nav-default
         i.fas.fa-puzzle-piece
       span.uk-margin-small-left Service Nodes
 
-  li(class={ 'uk-active': (adminView === 'connect-queue') })
-    a(href="/admin/service-node/connect-queue")
-      span.nav-item-icon
-        i.fas.fa-plug
-      span.uk-margin-small-left Connect Queue
-
   li.uk-nav-divider 
 
   li(class={ 'uk-active': (adminView === 'host') })

app/views/admin/content-report/index.pug

@@ -2,7 +2,7 @@ extends ../layouts/main
 block content
 
   .uk-margin
-    +renderSectionTitle('Content Reports')
+    h1(style="line-height: 1em;").uk-margin-remove.uk-text-truncate Content Reports
 
   if Array.isArray(reports) && (reports.length > 0)
     .uk-overflow-auto

app/views/admin/core-node/components/list-item.pug

@@ -0,0 +1,29 @@
+mixin renderCoreNodeListItem (coreNode)
+  .uk-tile.uk-tile-default.uk-tile-small.uk-padding-small.dtp-border.uk-border-rounded
+    .uk-margin
+      div(uk-grid).uk-flex-between
+        .uk-width-auto
+          +renderCell('Name', coreNode.meta.name)
+        .uk-width-auto
+          +renderCell('Admin', coreNode.meta.admin)
+        .uk-width-auto
+          +renderCell('Domain', coreNode.meta.domain)
+        .uk-width-auto
+          +renderCell('Domain Key', coreNode.meta.domainKey)
+
+    .uk-margin
+      div(uk-grid).uk-flex-between
+        .uk-width-auto
+          +renderCell('Connected', moment(coreNode.created).format('MMM DD, YYYY'))
+        .uk-width-auto
+          +renderCell('Updated', moment(coreNode.updated).format('MMM DD, YYYY'))
+        .uk-width-auto
+          +renderCell('Version', coreNode.meta.version)
+        .uk-width-auto
+          +renderCell('Host', coreNode.address.host)
+        .uk-width-auto
+          +renderCell('Port', coreNode.address.port)
+        .uk-width-auto
+          +renderCell('Connected', coreNode.flags.isConnected)
+        .uk-width-auto
+          +renderCell('Blocked', coreNode.flags.isBlocked)

app/views/admin/core-node/connect.pug

@@ -15,4 +15,4 @@ block content
           input(id="host", name="host", placeholder="Enter host:port of Core to connect", required).uk-input
 
       .uk-card-footer
-        button(type="submit").uk-button.uk-button-primary Send Request
+        button(type="submit").uk-button.uk-button-primary.uk-border-rounded Send Request

app/views/admin/core-node/index.pug

@@ -1,43 +1,21 @@
 extends ../layouts/main
 block content
 
-  h1 Core Nodes
+  include components/list-item
-  a(href="/admin/core-node/connect").uk-button.uk-button-primary Connect Core
+
+  .uk-margin
+    div(uk-grid)
+      .uk-width-expand
+        h1(style="line-height: 1em;").uk-margin-remove.uk-text-truncate Core Nodes
+      .uk-width-auto
+        a(href="/admin/core-node/connect").uk-button.uk-button-primary.uk-border-rounded Connect Core
 
   p You can register with one or more Core nodes to exchange information with those nodes.
 
   if Array.isArray(coreNodes) && (coreNodes.length > 0)
     ul.uk-list
       each node in coreNodes
-        .uk-tile.uk-tile-default.uk-tile-small
+        a(href=`/admin/core-node/${node._id}`).uk-display-block.uk-link-reset
-          .uk-margin
+          +renderCoreNodeListItem(node)
-            div(uk-grid)
-              .uk-width-auto
-                +renderCell('Name', node.meta.name)
-              .uk-width-auto
-                +renderCell('Domain', node.meta.domain)
-              .uk-width-auto
-                +renderCell('Domain Key', node.meta.domainKey)
-              .uk-width-auto
-                +renderCell('Connected', moment(node.created).format('MMM DD, YYYY'))
-              .uk-width-auto
-                +renderCell('Updated', moment(node.updated).format('MMM DD, YYYY'))
-              .uk-width-auto
-                +renderCell('Version', node.meta.version)
-
-          .uk-margin
-            div(uk-grid)
-              .uk-width-auto
-                +renderCell('Host', node.address.host)
-              .uk-width-auto
-                +renderCell('Port', node.address.port)
-              .uk-width-auto
-                +renderCell('Connected', node.flags.isConnected)
-              .uk-width-auto
-                +renderCell('Blocked', node.flags.isBlocked)
-              .uk-width-auto
-                +renderCell('Admin', node.meta.admin)
-              .uk-width-auto
-                +renderCell('Support', node.meta.supportEmail)
   else
     p There are no registered core nodes.

app/views/admin/core-node/view.pug

@@ -0,0 +1,46 @@
+extends ../layouts/main
+block content
+
+  include ../../components/pagination-bar
+  include components/list-item
+  
+  .uk-margin
+    div(uk-grid).uk-grid-small.uk-flex-middle
+      div(class="uk-width-1-1 uk-width-expand@m")
+        h1(style="line-height: 1em;") Core Node
+      div(class="uk-width-1-1 uk-width-auto@m")
+        a(href=`mailto:${coreNode.meta.supportEmail}?subject=${encodeURIComponent(`Support request from ${site.name}`)}`)
+          span
+            i.fas.fa-envelope
+          span.uk-margin-small-left Email Support
+      div(class="uk-width-1-1 uk-width-auto@m")
+        span.uk-label(style="line-height: 1.75em;", class={
+          'uk-label-success': coreNode.flags.isConnected,
+          'uk-label-warning': !coreNode.flags.isConnected && !coreNode.flags.isBlocked,
+          'uk-label-danger': coreNode.flags.isBlocked,
+        }).no-select= coreNode.flags.isConnected ? 'Connected' : 'Pending'
+
+  +renderCoreNodeListItem(coreNode)
+
+  .uk-margin
+    table.uk-table.uk-table-small
+      thead
+        tr
+          th Timestamp
+          th Method
+          th URL
+          th Status
+          th Result
+          th Perf
+      tbody
+        each request in requestHistory.requests
+          tr
+            td= moment(request.created).format('YYYY-MM-DD HH:mm:ss.SSS')
+            td= request.method
+            td= request.url
+            td= (request.response && request.response.statusCode) ? request.response.statusCode : '- - -'
+            td= (request.response) ? ((request.response.success) ? 'success' : 'fail') : '- - -'
+            td= request.response ? `${numeral(request.response.elapsed).format('0,0')}ms` : '- - -'
+  
+  .uk-margin
+    +renderPaginationBar(`/admin/core-node/${coreNode._id}`, requestHistory.totalRequestCount)

app/views/admin/job-queue/components/job-list.pug

@@ -0,0 +1,18 @@
+mixin renderJobQueueJobList (jobQueue, jobList)
+  if !Array.isArray(jobList) || (jobList.length === 0)
+    div No jobs
+  else
+    table.uk-table.uk-table-small
+      thead
+        th ID
+        th Name
+        th Attempts
+        th Progress
+      tbody
+        each job in jobList
+          tr
+            td= job.id
+            td
+              a(href=`/admin/job-queue/${jobQueue.name}/${job.id}`)= job.name
+            td= job.attemptsMade
+            td #{job.progress()}%

app/views/admin/job-queue/queue-view.pug

@@ -1,24 +1,7 @@
 extends ../layouts/main
 block content
 
-  mixin renderJobList (jobList)
+  include components/job-list
-    if !Array.isArray(jobList) || (jobList.length === 0)
-      div No jobs
-    else
-      table.uk-table.uk-table-small
-        thead
-          th ID
-          th Name
-          th Attempts
-          th Progress
-        tbody
-          each job in jobList
-            tr
-              td= job.id
-              td
-                a(href=`/admin/job-queue/${queue.name}/${job.id}`)= job.name
-              td= job.attemptsMade
-              td #{job.progress()}%
 
   .uk-margin
     h1 Job Queue: #{queueName}
@@ -38,25 +21,25 @@ block content
         .uk-card-header
           h3.uk-card-title Active
         .uk-card-body
-          +renderJobList(jobs.active)
+          +renderJobQueueJobList(queue, jobs.active)
 
     div(class="uk-width-1-1 uk-width-1-2@l")
       .uk-card.uk-card-default.uk-card-small
         .uk-card-header
           h3.uk-card-title Waiting
         .uk-card-body
-          +renderJobList(jobs.waiting)
+          +renderJobQueueJobList(queue, jobs.waiting)
 
     div(class="uk-width-1-1 uk-width-1-2@l")
       .uk-card.uk-card-default.uk-card-small
         .uk-card-header
           h3.uk-card-title Delayed
         .uk-card-body
-          +renderJobList(jobs.delayed)
+          +renderJobQueueJobList(queue, jobs.delayed)
 
     div(class="uk-width-1-1 uk-width-1-2@l")
       .uk-card.uk-card-default.uk-card-small
         .uk-card-header
           h3.uk-card-title Failed
         .uk-card-body
-          +renderJobList(jobs.failed)
+          +renderJobQueueJobList(queue, jobs.failed)

app/views/admin/newsletter/editor.pug

@@ -6,8 +6,12 @@ block content
   form(method="POST", action= actionUrl).uk-form
     .uk-margin
       label(for="title").uk-form-label.sr-only Newsletter title
-      input(id="title", name="title", type="text", placeholder= "Enter newsletter title", value= newsletter ? newsletter.title : undefined).uk-input
+      input(id="title", name="title", type="text", placeholder= "Enter newsletter title", value= newsletter ? newsletter.title : undefined, required).uk-input
 
+    .uk-margin
+      label(for="summary").uk-form-label.sr-only Newsletter summary
+      textarea(id="summary", name="summary", rows="4", placeholder= "Enter newsletter summary (text only, no HTML)", required).uk-textarea= newsletter ? newsletter.summary : undefined
+    
     .uk-margin
       label(for="content-html").uk-form-label.sr-only Newsletter HTML body
       textarea(id="content-html", name="content.html", rows="4").uk-textarea= newsletter ? newsletter.content.html : undefined
@@ -17,31 +21,28 @@ block content
     
     .uk-margin
       label(for="content-text").uk-form-label.sr-only Newsletter text body
-      textarea(id="content-text", name="content.text", rows="4", placeholder= "Enter text-only version of newsletter.").uk-textarea= newsletter ? newsletter.content.text : undefined
+      textarea(id="content-text", name="content.text", rows="4", placeholder= "Enter text-only version of newsletter.", required).uk-textarea= newsletter ? newsletter.content.text : undefined
-    
-    .uk-margin
-      label(for="summary").uk-form-label.sr-only Newsletter summary
-      textarea(id="summary", name="summary", rows="4", placeholder= "Enter newsletter summary (text only, no HTML)").uk-textarea= newsletter ? newsletter.summary : undefined
     
     button(type="submit").uk-button.dtp-button-primary= newsletter ? 'Update newsletter' : 'Save newsletter'
 
 block viewjs
   script(src="/tinymce/tinymce.min.js")
   script.
+    const useDarkMode = document.body.classList.contains('dtp-dark');
     window.addEventListener('dtp-load', async ( ) => {
       const toolbarItems = [
         'undo redo',
-        'formatselect',
+        'blocks visualblocks',
         'bold italic backcolor',
         'alignleft aligncenter alignright alignjustify',
         'bullist numlist outdent indent removeformat',
-        'link image',
+        'link image media code',
         'help'
       ];
       const pluginItems = [
-        'advlist', 'autolink', 'lists', 'link', 'image', 'charmap', 'print',
+        'advlist', 'autolink', 'lists', 'link', 'image', 'charmap',
         'preview', 'anchor', 'searchreplace', 'visualblocks', 'code',
-        'fullscreen', 'insertdatetime', 'media', 'table', 'paste', 'code',
+        'fullscreen', 'insertdatetime', 'media', 'table',
         'help', 'wordcount',
       ]
 
@@ -58,8 +59,8 @@ block viewjs
           { title: 'Title Image', value: 'dtp-image-title' },
         ],
         convert_urls: false,
-        skin: "oxide-dark",
+        skin: useDarkMode ? "oxide-dark" : "oxide",
-        content_css: "dark",        
+        content_css: useDarkMode ? "dark" : "default",
       });
 
       window.dtp.app.editor = editors[0];

app/views/admin/newsletter/index.pug

@@ -2,20 +2,12 @@ extends ../layouts/main
 block content
 
   .uk-margin
-    div(uk-grid)
+    h1(style="line-height: 1em;").uk-text-truncate.uk-margin-remove Newsletters
-      .uk-width-expand
-        h1.uk-text-truncate Newsletters
-      .uk-width-auto
-        a(href="/admin/newsletter/compose").uk-button.dtp-button-primary
-          +renderButtonIcon('fa-plus', 'New Newsletter')
   
   .uk-margin
     if (Array.isArray(newsletters) && (newsletters.length > 0))
-
       ul.uk-list
-
         each newsletter in newsletters
-
           li(data-newsletter-id= newsletter._id)
             div(uk-grid).uk-grid-small.uk-flex-middle
               .uk-width-expand
@@ -29,11 +21,16 @@ block content
                       data-newsletter-id= newsletter._id,
                       data-newsletter-title= newsletter.title,
                       onclick="return dtp.adminApp.deleteNewsletter(event);",
-                    ).uk-button.dtp-button-danger
+                    ).uk-button.uk-button-danger
                       +renderButtonIcon('fa-trash', 'Delete')
 
                   .uk-width-auto
-                    button(type="button").uk-button.dtp-button-default
+                    button(
+                      type="button",
+                      data-newsletter-id= newsletter._id,
+                      data-newsletter-title= newsletter.title,
+                      onclick="return dtp.adminApp.sendNewsletter(event);",
+                    ).uk-button.uk-button-default
                       +renderButtonIcon('fa-paper-plane', 'Send')
     else
       div There are no newsletters at this time.

app/views/admin/newsletter/job-status.pug

@@ -0,0 +1,45 @@
+extends ../layouts/main
+block content
+
+  include ../job-queue/components/job-list
+
+  .uk-margin
+    h1 Job Queue: #{queueName}
+    div(uk-grid).uk-flex-between
+      - var pendingJobCount = jobCounts.waiting + jobCounts.delayed + jobCounts.paused + jobCounts.active
+      .uk-width-auto Total#[br]#{numeral(pendingJobCount).format('0,0')}
+      .uk-width-auto Waiting#[br]#{numeral(jobCounts.waiting).format('0,0')}
+      .uk-width-auto Delayed#[br]#{numeral(jobCounts.delayed).format('0,0')}
+      .uk-width-auto Paused#[br]#{numeral(jobCounts.paused).format('0,0')}
+      .uk-width-auto Active#[br]#{numeral(jobCounts.active).format('0,0')}
+      .uk-width-auto Completed#[br]#{numeral(jobCounts.completed).format('0,0')}
+      .uk-width-auto Failed#[br]#{numeral(jobCounts.failed).format('0,0')}
+
+  div(uk-grid)
+    div(class="uk-width-1-1 uk-width-1-2@l")
+      .uk-card.uk-card-default.uk-card-small
+        .uk-card-header
+          h3.uk-card-title Active
+        .uk-card-body
+          +renderJobQueueJobList(newsletterQueue, jobs.active)
+
+    div(class="uk-width-1-1 uk-width-1-2@l")
+      .uk-card.uk-card-default.uk-card-small
+        .uk-card-header
+          h3.uk-card-title Waiting
+        .uk-card-body
+          +renderJobQueueJobList(newsletterQueue, jobs.waiting)
+
+    div(class="uk-width-1-1 uk-width-1-2@l")
+      .uk-card.uk-card-default.uk-card-small
+        .uk-card-header
+          h3.uk-card-title Delayed
+        .uk-card-body
+          +renderJobQueueJobList(newsletterQueue, jobs.delayed)
+
+    div(class="uk-width-1-1 uk-width-1-2@l")
+      .uk-card.uk-card-default.uk-card-small
+        .uk-card-header
+          h3.uk-card-title Failed
+        .uk-card-body
+          +renderJobQueueJobList(newsletterQueue, jobs.failed)

app/views/admin/service-node/index.pug

@@ -3,7 +3,14 @@ block content
 
   include components/list-item
 
-  h1 Service Nodes
+  div(uk-grid)
+    .uk-width-expand
+      h1(style="line-height: 1em;").uk-margin-remove Service Nodes
+    .uk-width-auto
+      a(href='/admin/service-node/connect-queue').uk-button.uk-button-secondary.uk-border-rounded
+        span
+          i.fas.fa-plug
+        span.uk-margin-small-left Connect Queue
 
   if Array.isArray(serviceNodes) && (serviceNodes.length > 0)
     ul.uk-list.uk-list-divider

app/views/components/library.pug

@@ -16,7 +16,7 @@ include section-title
 
 mixin renderCell (label, value, className)
   div(title=`${label}: ${numeral(value).format('0,0')}`).uk-tile.uk-tile-default.uk-padding-remove.no-select
-    div(class=className)= value
+    div(class=className)!= value
     .uk-text-muted.uk-text-small= label
 
 mixin renderBackButton ( )

app/workers/newsletter.js

@@ -23,6 +23,7 @@ class NewsletterWorker extends SiteWorker {
 
   constructor (dtp) {
     super(dtp, dtp.config.component);
+    this.newsletters = this.newsletters || { };
   }
 
   async start ( ) {
@@ -32,7 +33,7 @@ class NewsletterWorker extends SiteWorker {
     this.jobQueue = await jobQueueService.getJobQueue('newsletter', {
       attempts: 3,
     });
-    this.jobQueue.process('email', this.sendNewsletter.bind(this));
+    this.jobQueue.process('transmit', this.transmitNewsletter.bind(this));
     this.jobQueue.process('email-send', this.sendNewsletterEmail.bind(this));
   }
 
@@ -45,13 +46,50 @@ class NewsletterWorker extends SiteWorker {
     await super.stop();
   }
 
-  async sendNewsletter (job) {
+  async loadNewsletter (newsletterId) {
+    const { newsletter: newsletterService } = this.dtp.services;
+    let newsletter = this.newsletters[newsletterId];
+    if (!newsletter) {
+      newsletter = await newsletterService.getById(newsletterId);
+      this.newsletters[newsletterId] = newsletter;
+    }
+    return newsletter;
+  }
+
+  async transmitNewsletter (job) {
+    const User = mongoose.model('User');
     const NewsletterRecipient = mongoose.model('NewsletterRecipient');
     this.log.info('newsletter email job received', { data: job.data });
-
     try {
       /*
-       * Create one Bull Queue job per email to be delivered.
+       * Transmit first to all local user accounts with verified email who've
+       * opted in for receiving marketing email.
+       */
+      await User
+        .find({
+          'flags.isEmailVerified': true,
+          'optIn.marketing': true,
+        })
+        .select('email displayName username username_lc')
+        .lean()
+        .cursor()
+        .eachAsync(async (user) => {
+          try {
+            const jobData = {
+              newsletterId: job.data.newsletterId,
+              recipient: user.email,
+              recipientName: user.displayName || user.username,
+            };
+            const jobOptions = { attempts: 3 };
+            await this.jobQueue.add('email-send', jobData, jobOptions);
+          } catch (error) {
+            this.log.error('failed to create newsletter email job', { error });
+          }
+        }, { parallel: 4 });
+
+      /*
+       * Transmit to all newsletter recipients on file who've joined through the
+       * widget on the site w/o signing up for an account.
        */
       await NewsletterRecipient
         .find({ 'flags.isVerified': true, 'flags.isOptIn': true, 'flags.isRejected': false })
@@ -63,47 +101,38 @@ class NewsletterWorker extends SiteWorker {
               newsletterId: job.data.newsletterId,
               recipient: recipient.address,
             };
-            const jobOptions = {
+            const jobOptions = { attempts: 3 };
-              attempts: 3,
-            };
             await this.jobQueue.add('email-send', jobData, jobOptions);
           } catch (error) {
             this.log.error('failed to create newsletter email job', { error });
-            // but continue
           }
         }, { parallel: 4 });
     } catch (error) {
       this.log.error('failed to send newsletter', { newsletterId: job.data.newsletterId, error });
-      throw error; // throw error to Bull so it can report in job reports
+      throw error;
     }
   }
 
   async sendNewsletterEmail (job) {
-    const { newsletter: newsletterService, email: emailService } = this.dtp.services;
+    const { email: emailService } = this.dtp.services;
     const { newsletterId, recipient } = job.data;
+
     try {
-  
+      let newsletter = await this.loadNewsletter(newsletterId);
-      let newsletter = this.newsletters[newsletterId];
-  
-      if (!newsletter) {
-        newsletter = await newsletterService.getById(newsletterId);
-        this.newsletters[newsletterId] = newsletter;
-        //TODO: clean up memory leak of newsletter (remove when all emails are sent)
-      }
-  
       if (!newsletter) {
         throw new Error('newsletter not found');
       }
   
-      const response = await emailService.send({
+      const result = await emailService.send({
-        from: 'demo@wherever.com',
+        from: process.env.DTP_EMAIL_SMTP_FROM || `noreply@${this.dtp.config.site.domainKey}`,
         to: recipient,
         subject: newsletter.title,
         html: newsletter.content.html,
         text: newsletter.content.text,
       });
   
-      job.log(`newsletter email sent: ${response}`);
+      job.log(`newsletter email sent: ${result}`);
+      this.log.info('newsletter email sent', { recipient, result });
     } catch (error) {
       this.log.error('failed to send newsletter email', { newsletterId, recipient, error });
       throw error; // throw error to Bull so it can report in job reports
@@ -111,7 +140,6 @@ class NewsletterWorker extends SiteWorker {
   }
 }  
 
-
 (async ( ) => {
   try {
     module.log = new SiteLog(module, module.config.component);

client/js/site-admin-app.js

@@ -238,6 +238,29 @@ export default class DtpSiteAdminHostStatsApp extends DtpApp {
     }
   }
 
+  async sendNewsletter (event) {
+    const newsletterId = event.currentTarget.getAttribute('data-newsletter-id');
+    const newsletterTitle = event.currentTarget.getAttribute('data-newsletter-title');
+    console.log(newsletterId, newsletterTitle);
+    try {
+      await UIkit.modal.confirm(`Are you sure you want to transmit "${newsletterTitle}"`);
+    } catch (error) {
+      this.log.info('sendNewsletter', 'aborted');
+      return;
+    }
+    try {
+      const response = await fetch(`/admin/newsletter/${newsletterId}/transmit`, {
+        method: 'POST',
+      });
+      if (!response.ok) {
+        throw new Error('Server error');
+      }
+      await this.processResponse(response);      
+    } catch (error) {
+      this.log.error('sendNewsletter', 'failed to send newsletter', { newsletterId, newsletterTitle, error });
+      UIkit.modal.alert(`Failed to send newsletter: ${error.message}`);
+    }
+  }
 
   async deleteNewsletter (event) {
     const newsletterId = event.currentTarget.getAttribute('data-newsletter-id');

config/job-queues.js

@@ -5,7 +5,8 @@
 'use strict';
 
 module.exports = {
-  // 'queue-name': {
+  'newsletter': {
-  //   attempts: 3,
+    attempts: 3,
-  // },
+    removeOnComplete: false,
+  },
 };

start-local

@@ -9,8 +9,10 @@ export MINIO_ROOT_USER MINIO_ROOT_PASSWORD MINIO_CI_CD
 
 forever start --killSignal=SIGINT app/workers/host-services.js
 forever start --killSignal=SIGINT app/workers/reeeper.js
+forever start --killSignal=SIGINT app/workers/newsletter.js
 
 minio server ./data/minio --address ":9020" --console-address ":9021"
 
+forever stop app/workers/newsletter.js
 forever stop app/workers/reeeper.js
 forever stop app/workers/host-services.js