diff --git a/.env.default b/.env.default index 913117f..f472529 100644 --- a/.env.default +++ b/.env.default @@ -121,6 +121,7 @@ DTP_LOG_HTTP_FORMAT=combined # DTP_LOGAN=disabled -DTP_LOGAN_SCHEME=http -DTP_LOGAN_HOST= -DTP_LOGAN_API_KEY= \ No newline at end of file +DTP_LOGAN_API_KEY=########-####-####-####-############ +DTP_LOGAN_SCHEME=https +DTP_LOGAN_HOST=logan.digitaltelepresence.com +DTP_LOGAN_QUEUE_NAME=logan \ No newline at end of file diff --git a/.jshintrc b/.jshintrc index 6dd1a7e..c3942c1 100644 --- a/.jshintrc +++ b/.jshintrc @@ -10,7 +10,7 @@ "undef": true, "unused": true, "futurehostile": true, - "esversion": 9, + "esversion": 11, "mocha": true, "globals": { "markdown": true, diff --git a/app/services/logan.js b/app/services/logan.js index 8b22ef0..10243a0 100644 --- a/app/services/logan.js +++ b/app/services/logan.js @@ -4,9 +4,7 @@ 'use strict'; -const os = require('os'); - -const { SiteService, SiteError } = require('../../lib/site-lib'); +const { SiteService } = require('../../lib/site-lib'); class LoganService extends SiteService { @@ -16,56 +14,56 @@ class LoganService extends SiteService { async start ( ) { await super.start(); - } - - async sendRequestEvent (component, req, event) { - if (req.user) { - event.data = event.data || { }; - event.data.user = { - _id: req.user._id, - username: req.user.username, - }; - } - event.ip = req.ip; - return this.sendEvent(component, event); - } - - async sendEvent (component, event) { - try { - event.host = os.hostname(); - event['component.slug'] = component.slug; - event['component.name'] = component.className || component.name; - this.log[event.level]('application event', { event }); + const { LoganClient } = await import('dtp-logan-api'); - if (process.env.DTP_LOGAN !== 'enabled') { - return; - } + this.log.info('creating Logan client'); + this.logan = new LoganClient(); - const loganScheme = process.env.DTP_LOGAN_SCHEME || 'http'; - const loganUrl = `${loganScheme}://${process.env.DTP_LOGAN_HOST}/api/event`; - const payload = JSON.stringify(event); - - const response = await fetch(loganUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': payload.length, - 'X-LogAn-Auth': process.env.DTP_LOGAN_API_KEY, + this.log.info('initializing Logan client'); + await this.logan.initialize({ + log: this.log, + api: { + enabled: process.env.DTP_LOGAN === 'enabled', + key: process.env.DTP_LOGAN_API_KEY, + scheme: process.env.DTP_LOGAN_SCHEME, + host: process.env.DTP_LOGAN_HOST, + }, + request: { + userField: 'user', + userIdField: '_id', + usernameField: 'username', + }, + flags: { + includeHostname: true, + includeClientIpAddress: true, + includeUser: true, + }, + queue: { + enabled: true, + name: process.env.DTP_LOGAN_QUEUE_NAME || 'logan', + redis: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + username: process.env.REDIS_USERNAME, // requires Redis >= 6 + password: process.env.REDIS_PASSWORD, + keyPrefix: process.env.REDIS_PREFIX, + }, + defaultJobOptions: { + attempts: 3, + removeOnComplete: true, + removeOnFail: true, }, - body: payload, - }); + }, + }); + } - const json = await response.json(); - if (!json.success) { - throw new SiteError(500, json.message); - } + async sendRequestEvent (component, req, event) { + return this.logan.sendRequestEvent(component, req, event); + } - return json; - } catch (error) { - this.log.error('failed to send LOGAN event', { event, error }); - // fall through - } + async sendEvent (component, event) { + return this.logan.sendEvent(component, event); } } diff --git a/app/workers/logan.js b/app/workers/logan.js new file mode 100644 index 0000000..79de615 --- /dev/null +++ b/app/workers/logan.js @@ -0,0 +1,92 @@ +'use strict'; + +require('dotenv').config(); + +const path = require('path'); + +const { + SiteLog, + SiteWorker, +} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib')); + +module.rootPath = path.resolve(__dirname, '..', '..'); +module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); + +module.config = { + environment: process.env.NODE_ENV, + root: module.rootPath, + component: { name: 'LoganSiteWorker', slug: 'logan-site-worker' }, + site: require(path.join(module.rootPath, 'config', 'site')), +}; + +class LoganSiteWorker extends SiteWorker { + + constructor (dtp) { + super(dtp, dtp.config.component); + } + + async start ( ) { + await super.start(); + + const { LoganWorker } = await import('dtp-logan-api'); + + this.log.info('creating Logan worker'); + this.loganWorker = new LoganWorker(); + + this.log.info('initializing Logan worker'); + await this.loganWorker.initialize({ + api: { + enabled: process.env.DTP_LOGAN === 'enabled', + key: process.env.DTP_LOGAN_API_KEY, + scheme: process.env.DTP_LOGAN_SCHEME, + host: process.env.DTP_LOGAN_HOST, + }, + queue: { + enabled: true, + name: 'logan', + redis: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + username: process.env.REDIS_USERNAME, // requires Redis >= 6 + password: process.env.REDIS_PASSWORD, + keyPrefix: process.env.REDIS_PREFIX, + }, + defaultJobOptions: { + attempts: 3, + priority: 10, + removeOnComplete: true, + removeOnFail: true, + }, + }, + }); + } + + async stop ( ) { + if (this.loganWorker) { + await this.loganWorker.close(); + delete this.loganWorker; + } + await super.stop(); + } +} + +(async ( ) => { + + module.log = new SiteLog(module, module.config.component); + + if (!process.env.DTP_LOGAN_API_KEY) { + console.log('Must define DTP_LOGAN_API_KEY environment variable to run test'); + process.exit(-1); + } + + try { + module.worker = new LoganSiteWorker(module); + await module.worker.start(); + + module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`); + } catch (error) { + module.log.error('failed to start worker', { component: module.config.component, error }); + process.exit(-1); + } + +})(); \ No newline at end of file diff --git a/package.json b/package.json index 7a70e54..1d3f0a2 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "disposable-email-provider-domains": "^1.0.9", "dotenv": "^16.0.0", "dtp-jshint-reporter": "git+https://git.digitaltelepresence.com/digital-telepresence/dtp-jshint-reporter.git#master", + "dtp-logan-api": "^0.3.2", "ein-validator": "^1.0.1", "email-domain-check": "^1.1.4", "email-validator": "^2.0.4", diff --git a/start-local b/start-local index bfd3a11..57c17dd 100755 --- a/start-local +++ b/start-local @@ -9,6 +9,7 @@ export MINIO_ROOT_USER MINIO_ROOT_PASSWORD MINIO_CI_CD forever start --killSignal=SIGINT app/workers/host-services.js forever start --killSignal=SIGINT app/workers/reeeper.js +forever start --killSignal=SIGINT app/workers/logan.js forever start --killSignal=SIGINT app/workers/newsletter.js forever start --killSignal=SIGINT app/workers/newsroom.js @@ -24,5 +25,6 @@ forever stop app/workers/media.js forever stop app/workers/newsroom.js forever stop app/workers/newsletter.js +forever stop app/workers/logan.js forever stop app/workers/reeeper.js forever stop app/workers/host-services.js \ No newline at end of file diff --git a/start-production b/start-production index e63207b..283fc67 100755 --- a/start-production +++ b/start-production @@ -3,6 +3,7 @@ sudo supervisorctl start \ sites-host-services:* \ sites-reeeper:* \ + sites-logan:* \ sites-newsletter:* \ sites-newsroom:* \ sites-media:* \ diff --git a/stop-production b/stop-production index f798ab6..d852794 100755 --- a/stop-production +++ b/stop-production @@ -6,5 +6,6 @@ sudo supervisorctl stop \ sites-media:* \ sites-newsroom:* \ sites-newsletter:* \ + sites-logan:* \ sites-reeeper:* \ - sites-host-services:* \ \ No newline at end of file + sites-host-services:* \ No newline at end of file diff --git a/supervisord/dtp-base-newsroom.conf b/supervisord/dtp-base-newsroom.conf deleted file mode 100644 index 50b4b8c..0000000 --- a/supervisord/dtp-base-newsroom.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:dtp-base:newsroom] -numprocs=1 -process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js -directory=/home/dtp/live/dtp-base -autostart=true -autorestart=true -startretries=3 -stopsignal=INT -stderr_logfile=/var/log/dtp-base/newsroom.err.log -stdout_logfile=/var/log/dtp-base/newsroom.out.log -user=dtp -environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsroom \ No newline at end of file diff --git a/supervisord/dtp-sites-chat.conf b/supervisord/dtp-sites-chat.conf new file mode 100644 index 0000000..d85e6bd --- /dev/null +++ b/supervisord/dtp-sites-chat.conf @@ -0,0 +1,13 @@ +[program:sites-chat] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/chat.js +directory=/home/dtp/live/dtp-sites +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp/dtp-sites-chat.err.log +stdout_logfile=/var/log/dtp/dtp-sites-chat.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-chat \ No newline at end of file diff --git a/supervisord/dtp-sites-host-services.conf b/supervisord/dtp-sites-host-services.conf index ef11c57..50a9e71 100644 --- a/supervisord/dtp-sites-host-services.conf +++ b/supervisord/dtp-sites-host-services.conf @@ -1,13 +1,13 @@ -[program:host-services] +[program:sites-host-services] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js directory=/home/dtp/live/dtp-sites autostart=true autorestart=true startretries=3 stopsignal=INT -stderr_logfile=/var/log/dtp-sites/host-services.err.log -stdout_logfile=/var/log/dtp-sites/host-services.out.log +stderr_logfile=/var/log/dtp/dtp-sites-host-services.err.log +stdout_logfile=/var/log/dtp/dtp-sites-host-services.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-host-services diff --git a/supervisord/dtp-sites-logan.conf b/supervisord/dtp-sites-logan.conf new file mode 100644 index 0000000..e70675d --- /dev/null +++ b/supervisord/dtp-sites-logan.conf @@ -0,0 +1,13 @@ +[program:dtp-sites:logan] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/logan.js +directory=/home/dtp/live/dtp-sites +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stdout_logfile=/var/log/dtp/dtp-sites-logan.out.log +stderr_logfile=/var/log/dtp/dtp-sites-logan.err.log +user=dtp +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-logan \ No newline at end of file diff --git a/supervisord/dtp-sites-media.conf b/supervisord/dtp-sites-media.conf new file mode 100644 index 0000000..a08bc0c --- /dev/null +++ b/supervisord/dtp-sites-media.conf @@ -0,0 +1,13 @@ +[program:sites-media] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/media.js +directory=/home/dtp/live/dtp-sites +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp/dtp-sites-media.err.log +stdout_logfile=/var/log/dtp/dtp-sites-media.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-media \ No newline at end of file diff --git a/supervisord/dtp-sites-newsletter.conf b/supervisord/dtp-sites-newsletter.conf index dbe583b..b9ebf21 100644 --- a/supervisord/dtp-sites-newsletter.conf +++ b/supervisord/dtp-sites-newsletter.conf @@ -1,13 +1,13 @@ -[program:dtp-sites:newsletter] +[program:sites-newsletter] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js directory=/home/dtp/live/dtp-sites autostart=true autorestart=true startretries=3 stopsignal=INT -stderr_logfile=/var/log/dtp-sites/newsletter.err.log -stdout_logfile=/var/log/dtp-sites/newsletter.out.log +stderr_logfile=/var/log/dtp/dtp-sites-newsletter.err.log +stdout_logfile=/var/log/dtp/dtp-sites-newsletter.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=sites-newsletter \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-newsletter \ No newline at end of file diff --git a/supervisord/dtp-sites-newsroom.conf b/supervisord/dtp-sites-newsroom.conf new file mode 100644 index 0000000..cadd5e7 --- /dev/null +++ b/supervisord/dtp-sites-newsroom.conf @@ -0,0 +1,13 @@ +[program:sites-newsroom] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js +directory=/home/dtp/live/dtp-sites +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp/dtp-sites-newsroom.err.log +stdout_logfile=/var/log/dtp/dtp-sites-newsroom.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-newsroom \ No newline at end of file diff --git a/supervisord/dtp-sites-reeeper.conf b/supervisord/dtp-sites-reeeper.conf new file mode 100644 index 0000000..a793f82 --- /dev/null +++ b/supervisord/dtp-sites-reeeper.conf @@ -0,0 +1,13 @@ +[program:sites-reeeper] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/reeeper.js +directory=/home/dtp/live/dtp-sites +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp/dtp-sites-reeeper.err.log +stdout_logfile=/var/log/dtp/dtp-sites-reeeper.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-sites',NODE_ENV=production,LOGNAME=dtp-sites-reeeper \ No newline at end of file diff --git a/supervisord/dtp-sites.conf b/supervisord/dtp-sites.conf index 2df7a0b..94c2265 100644 --- a/supervisord/dtp-sites.conf +++ b/supervisord/dtp-sites.conf @@ -1,13 +1,13 @@ -[program:dtp-sites] -numprocs=1 +[program:sites] +numprocs=2 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-sites.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-sites.js directory=/home/dtp/live/dtp-sites autostart=true autorestart=true startretries=3 stopsignal=INT -stderr_logfile=/var/log/dtp-sites/dtp-sites.err.log -stdout_logfile=/var/log/dtp-sites/dtp-sites.out.log +stderr_logfile=/var/log/dtp/dtp-sites-%(process_num)02d.err.log +stdout_logfile=/var/log/dtp/dtp-sites-%(process_num)02d.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=dtp-sites \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=34%(process_num)02d,HTTPS_BIND_PORT=35%(process_num)02d,NODE_ENV=production,LOGNAME=dtp-sites \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 9a56025..d9aff47 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2117,6 +2117,20 @@ builtin-modules@^3.1.0: resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.2.0.tgz#45d5db99e7ee5e6bc4f362e008bf917ab5049887" integrity sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA== +bull@^4.10.4: + version "4.10.4" + resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.4.tgz#db39ee0c3bfbe3b76f1f35db800501de5bba4f84" + integrity sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA== + dependencies: + cron-parser "^4.2.1" + debuglog "^1.0.0" + get-port "^5.1.1" + ioredis "^5.0.0" + lodash "^4.17.21" + msgpackr "^1.5.2" + semver "^7.3.2" + uuid "^8.3.0" + bull@^4.7.0: version "4.7.0" resolved "https://registry.yarnpkg.com/bull/-/bull-4.7.0.tgz#89442d4676117edd9f9a1359bb0edfb489595e70" @@ -3040,6 +3054,11 @@ denque@^2.0.1: resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -3255,6 +3274,14 @@ drange@^1.0.2: dependencies: chalk "^4.1.1" +dtp-logan-api@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/dtp-logan-api/-/dtp-logan-api-0.3.2.tgz#354c184ce9ea8a31c57bf0592278f5861fe672d2" + integrity sha512-7mORjmktZY0xpUNecXVLiHvRkBbJxQkjRom2QFzb+5NLQxml2u28cPegINNIlnFKjduXme7a+MQm0msuCmHYtA== + dependencies: + bull "^4.10.4" + ioredis "^5.3.2" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -4864,6 +4891,21 @@ ioredis@^4.28.5: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^5.0.0, ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ioredis@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.2.tgz#212467e04f6779b4e0e800cece7bb7d3d7b546d2"