From 26df84d0c84b21c3d24d06a14f5724b90678331e Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 23 Jun 2023 03:11:12 -0400 Subject: [PATCH 1/5] migrate to the use of dtp-logan-api - convert LoganService to use the API module - add logan worker to transmit Logan events from job queue - added logan worker config to supervisord directory - updated the other configs --- .env.default | 10 +++ .jshintrc | 2 +- app/services/logan.js | 92 ++++++++++++------------- app/workers/logan.js | 92 +++++++++++++++++++++++++ package.json | 1 + supervisord/dtp-base-host-services.conf | 4 +- supervisord/dtp-base-logan.conf | 13 ++++ supervisord/dtp-base-newsletter.conf | 4 +- supervisord/dtp-base-newsroom.conf | 4 +- supervisord/dtp-base.conf | 2 +- yarn.lock | 42 +++++++++++ 11 files changed, 211 insertions(+), 55 deletions(-) create mode 100644 app/workers/logan.js create mode 100644 supervisord/dtp-base-logan.conf diff --git a/.env.default b/.env.default index d789b37..2d47149 100644 --- a/.env.default +++ b/.env.default @@ -116,3 +116,13 @@ DTP_LOG_INFO=enabled DTP_LOG_WARN=enabled DTP_LOG_HTTP_FORMAT=combined + +# +# DTP Logan Integration +# + +DTP_LOGAN=disabled +DTP_LOGAN_API_KEY=########-####-####-####-############ +DTP_LOGAN_SCHEME=https +DTP_LOGAN_HOST=logan.digitaltelepresence.com +DTP_LOGAN_QUEUE_NAME=logan \ No newline at end of file diff --git a/.jshintrc b/.jshintrc index 6dd1a7e..c3942c1 100644 --- a/.jshintrc +++ b/.jshintrc @@ -10,7 +10,7 @@ "undef": true, "unused": true, "futurehostile": true, - "esversion": 9, + "esversion": 11, "mocha": true, "globals": { "markdown": true, diff --git a/app/services/logan.js b/app/services/logan.js index 8b22ef0..10243a0 100644 --- a/app/services/logan.js +++ b/app/services/logan.js @@ -4,9 +4,7 @@ 'use strict'; -const os = require('os'); - -const { SiteService, SiteError } = require('../../lib/site-lib'); +const { SiteService } = require('../../lib/site-lib'); class LoganService extends SiteService { @@ -16,56 +14,56 @@ class LoganService extends SiteService { async start ( ) { await super.start(); - } - - async sendRequestEvent (component, req, event) { - if (req.user) { - event.data = event.data || { }; - event.data.user = { - _id: req.user._id, - username: req.user.username, - }; - } - event.ip = req.ip; - return this.sendEvent(component, event); - } - - async sendEvent (component, event) { - try { - event.host = os.hostname(); - event['component.slug'] = component.slug; - event['component.name'] = component.className || component.name; - this.log[event.level]('application event', { event }); + const { LoganClient } = await import('dtp-logan-api'); - if (process.env.DTP_LOGAN !== 'enabled') { - return; - } + this.log.info('creating Logan client'); + this.logan = new LoganClient(); - const loganScheme = process.env.DTP_LOGAN_SCHEME || 'http'; - const loganUrl = `${loganScheme}://${process.env.DTP_LOGAN_HOST}/api/event`; - const payload = JSON.stringify(event); - - const response = await fetch(loganUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': payload.length, - 'X-LogAn-Auth': process.env.DTP_LOGAN_API_KEY, + this.log.info('initializing Logan client'); + await this.logan.initialize({ + log: this.log, + api: { + enabled: process.env.DTP_LOGAN === 'enabled', + key: process.env.DTP_LOGAN_API_KEY, + scheme: process.env.DTP_LOGAN_SCHEME, + host: process.env.DTP_LOGAN_HOST, + }, + request: { + userField: 'user', + userIdField: '_id', + usernameField: 'username', + }, + flags: { + includeHostname: true, + includeClientIpAddress: true, + includeUser: true, + }, + queue: { + enabled: true, + name: process.env.DTP_LOGAN_QUEUE_NAME || 'logan', + redis: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + username: process.env.REDIS_USERNAME, // requires Redis >= 6 + password: process.env.REDIS_PASSWORD, + keyPrefix: process.env.REDIS_PREFIX, + }, + defaultJobOptions: { + attempts: 3, + removeOnComplete: true, + removeOnFail: true, }, - body: payload, - }); + }, + }); + } - const json = await response.json(); - if (!json.success) { - throw new SiteError(500, json.message); - } + async sendRequestEvent (component, req, event) { + return this.logan.sendRequestEvent(component, req, event); + } - return json; - } catch (error) { - this.log.error('failed to send LOGAN event', { event, error }); - // fall through - } + async sendEvent (component, event) { + return this.logan.sendEvent(component, event); } } diff --git a/app/workers/logan.js b/app/workers/logan.js new file mode 100644 index 0000000..79de615 --- /dev/null +++ b/app/workers/logan.js @@ -0,0 +1,92 @@ +'use strict'; + +require('dotenv').config(); + +const path = require('path'); + +const { + SiteLog, + SiteWorker, +} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib')); + +module.rootPath = path.resolve(__dirname, '..', '..'); +module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); + +module.config = { + environment: process.env.NODE_ENV, + root: module.rootPath, + component: { name: 'LoganSiteWorker', slug: 'logan-site-worker' }, + site: require(path.join(module.rootPath, 'config', 'site')), +}; + +class LoganSiteWorker extends SiteWorker { + + constructor (dtp) { + super(dtp, dtp.config.component); + } + + async start ( ) { + await super.start(); + + const { LoganWorker } = await import('dtp-logan-api'); + + this.log.info('creating Logan worker'); + this.loganWorker = new LoganWorker(); + + this.log.info('initializing Logan worker'); + await this.loganWorker.initialize({ + api: { + enabled: process.env.DTP_LOGAN === 'enabled', + key: process.env.DTP_LOGAN_API_KEY, + scheme: process.env.DTP_LOGAN_SCHEME, + host: process.env.DTP_LOGAN_HOST, + }, + queue: { + enabled: true, + name: 'logan', + redis: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + username: process.env.REDIS_USERNAME, // requires Redis >= 6 + password: process.env.REDIS_PASSWORD, + keyPrefix: process.env.REDIS_PREFIX, + }, + defaultJobOptions: { + attempts: 3, + priority: 10, + removeOnComplete: true, + removeOnFail: true, + }, + }, + }); + } + + async stop ( ) { + if (this.loganWorker) { + await this.loganWorker.close(); + delete this.loganWorker; + } + await super.stop(); + } +} + +(async ( ) => { + + module.log = new SiteLog(module, module.config.component); + + if (!process.env.DTP_LOGAN_API_KEY) { + console.log('Must define DTP_LOGAN_API_KEY environment variable to run test'); + process.exit(-1); + } + + try { + module.worker = new LoganSiteWorker(module); + await module.worker.start(); + + module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`); + } catch (error) { + module.log.error('failed to start worker', { component: module.config.component, error }); + process.exit(-1); + } + +})(); \ No newline at end of file diff --git a/package.json b/package.json index 186689b..aecb003 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "disposable-email-provider-domains": "^1.0.9", "dotenv": "^16.0.0", "dtp-jshint-reporter": "git+https://git.digitaltelepresence.com/digital-telepresence/dtp-jshint-reporter.git#master", + "dtp-logan-api": "^0.3.2", "ein-validator": "^1.0.1", "email-domain-check": "^1.1.4", "email-validator": "^2.0.4", diff --git a/supervisord/dtp-base-host-services.conf b/supervisord/dtp-base-host-services.conf index c4754cb..14d11f7 100644 --- a/supervisord/dtp-base-host-services.conf +++ b/supervisord/dtp-base-host-services.conf @@ -1,7 +1,7 @@ [program:dtp-base:host-services] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js directory=/home/dtp/live/dtp-base autostart=true autorestart=true @@ -10,4 +10,4 @@ stopsignal=INT stderr_logfile=/var/log/dtp-base/host-services.err.log stdout_logfile=/var/log/dtp-base/host-services.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=host-services \ No newline at end of file diff --git a/supervisord/dtp-base-logan.conf b/supervisord/dtp-base-logan.conf new file mode 100644 index 0000000..3eae4cb --- /dev/null +++ b/supervisord/dtp-base-logan.conf @@ -0,0 +1,13 @@ +[program:dtp-base:logan] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/logan.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stdout_logfile=/var/log/dtp-base/logan.out.log +stderr_logfile=/var/log/dtp-base/logan.err.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=logan \ No newline at end of file diff --git a/supervisord/dtp-base-newsletter.conf b/supervisord/dtp-base-newsletter.conf index 7a710b8..f194a4a 100644 --- a/supervisord/dtp-base-newsletter.conf +++ b/supervisord/dtp-base-newsletter.conf @@ -1,7 +1,7 @@ [program:dtp-base:newsletter] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js directory=/home/dtp/live/dtp-base autostart=true autorestart=true @@ -10,4 +10,4 @@ stopsignal=INT stderr_logfile=/var/log/dtp-base/newsletter.err.log stdout_logfile=/var/log/dtp-base/newsletter.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsletter \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=newsletter \ No newline at end of file diff --git a/supervisord/dtp-base-newsroom.conf b/supervisord/dtp-base-newsroom.conf index 50b4b8c..a41051e 100644 --- a/supervisord/dtp-base-newsroom.conf +++ b/supervisord/dtp-base-newsroom.conf @@ -1,7 +1,7 @@ [program:dtp-base:newsroom] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js directory=/home/dtp/live/dtp-base autostart=true autorestart=true @@ -10,4 +10,4 @@ stopsignal=INT stderr_logfile=/var/log/dtp-base/newsroom.err.log stdout_logfile=/var/log/dtp-base/newsroom.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsroom \ No newline at end of file +environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=newsroom \ No newline at end of file diff --git a/supervisord/dtp-base.conf b/supervisord/dtp-base.conf index cea7c16..f834d38 100644 --- a/supervisord/dtp-base.conf +++ b/supervisord/dtp-base.conf @@ -1,7 +1,7 @@ [program:dtp-base] numprocs=1 process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js +command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js directory=/home/dtp/live/dtp-base autostart=true autorestart=true diff --git a/yarn.lock b/yarn.lock index dd5d577..2d05fad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2076,6 +2076,20 @@ builtin-modules@^3.1.0: resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.2.0.tgz#45d5db99e7ee5e6bc4f362e008bf917ab5049887" integrity sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA== +bull@^4.10.4: + version "4.10.4" + resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.4.tgz#db39ee0c3bfbe3b76f1f35db800501de5bba4f84" + integrity sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA== + dependencies: + cron-parser "^4.2.1" + debuglog "^1.0.0" + get-port "^5.1.1" + ioredis "^5.0.0" + lodash "^4.17.21" + msgpackr "^1.5.2" + semver "^7.3.2" + uuid "^8.3.0" + bull@^4.7.0: version "4.7.0" resolved "https://registry.yarnpkg.com/bull/-/bull-4.7.0.tgz#89442d4676117edd9f9a1359bb0edfb489595e70" @@ -2940,6 +2954,11 @@ denque@^2.0.1: resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -3106,6 +3125,14 @@ drange@^1.0.2: dependencies: chalk "^4.1.1" +dtp-logan-api@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/dtp-logan-api/-/dtp-logan-api-0.3.2.tgz#354c184ce9ea8a31c57bf0592278f5861fe672d2" + integrity sha512-7mORjmktZY0xpUNecXVLiHvRkBbJxQkjRom2QFzb+5NLQxml2u28cPegINNIlnFKjduXme7a+MQm0msuCmHYtA== + dependencies: + bull "^4.10.4" + ioredis "^5.3.2" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -4627,6 +4654,21 @@ ioredis@^4.28.5: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^5.0.0, ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ioredis@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.2.tgz#212467e04f6779b4e0e800cece7bb7d3d7b546d2" From 73843095f5c5b3cd16232c09d62e471bd6558dea Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 23 Jun 2023 03:11:55 -0400 Subject: [PATCH 2/5] v0.8.21 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aecb003..1d82d5a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.8.20", + "version": "0.8.21", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 4f23371be5099141d8be949fcb49f1c3873ffc23 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 23 Jun 2023 03:18:49 -0400 Subject: [PATCH 3/5] added Logan worker to start-local --- start-local | 2 ++ 1 file changed, 2 insertions(+) diff --git a/start-local b/start-local index 7a87f10..cce2711 100755 --- a/start-local +++ b/start-local @@ -9,6 +9,7 @@ export MINIO_ROOT_USER MINIO_ROOT_PASSWORD MINIO_CI_CD forever start --killSignal=SIGINT app/workers/host-services.js forever start --killSignal=SIGINT app/workers/reeeper.js +forever start --killSignal=SIGINT app/workers/logan.js forever start --killSignal=SIGINT app/workers/newsletter.js forever start --killSignal=SIGINT app/workers/newsroom.js @@ -24,5 +25,6 @@ forever stop app/workers/media.js forever stop app/workers/newsroom.js forever stop app/workers/newsletter.js +forever stop app/workers/logan.js forever stop app/workers/reeeper.js forever stop app/workers/host-services.js \ No newline at end of file From a8de1e15652f10e1c8c217bbb4c92b680ecd9c75 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 23 Jun 2023 03:19:41 -0400 Subject: [PATCH 4/5] added Logan worker to start-/stop-production --- start-production | 1 + stop-production | 1 + 2 files changed, 2 insertions(+) diff --git a/start-production b/start-production index e876e83..78322ae 100755 --- a/start-production +++ b/start-production @@ -3,6 +3,7 @@ sudo supervisorctl start \ base-host-services:* \ base-reeeper:* \ + base-logan:* \ base-newsletter:* \ base-newsroom:* \ base-media:* \ diff --git a/stop-production b/stop-production index 12b4516..e7152e0 100755 --- a/stop-production +++ b/stop-production @@ -6,5 +6,6 @@ sudo supervisorctl stop \ base-media:* \ base-newsroom:* \ base-newsletter:* \ + base-logan:* \ base-reeeper:* \ base-host-services:* \ \ No newline at end of file From 3a37d94e5d98e38b0679b102b2b003c89c9d58d8 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 23 Jun 2023 03:19:51 -0400 Subject: [PATCH 5/5] v0.8.22 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1d82d5a..b1bd5ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.8.21", + "version": "0.8.22", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC",