diff --git a/.env.default b/.env.default index 2516829..186c596 100644 --- a/.env.default +++ b/.env.default @@ -84,6 +84,8 @@ MINIO_PORT=9000 MINIO_USE_SSL=disabled MINIO_ACCESS_KEY=dtp-sites MINIO_SECRET_KEY= + +MINIO_ADMIN_BUCKET=site-admin MINIO_IMAGE_BUCKET=site-images MINIO_VIDEO_BUCKET=site-videos MINIO_ATTACHMENT_BUCKET=site-attachments diff --git a/app/controllers/admin.js b/app/controllers/admin.js index a348b28..fccd174 100644 --- a/app/controllers/admin.js +++ b/app/controllers/admin.js @@ -43,10 +43,12 @@ class AdminController extends SiteController { ); router.use('/announcement', await this.loadChild(path.join(__dirname, 'admin', 'announcement'))); + router.use('/attachment', await this.loadChild(path.join(__dirname, 'admin', 'attachment'))); router.use('/content-report', await this.loadChild(path.join(__dirname, 'admin', 'content-report'))); router.use('/core-node', await this.loadChild(path.join(__dirname, 'admin', 'core-node'))); router.use('/core-user', await this.loadChild(path.join(__dirname, 'admin', 'core-user'))); router.use('/host', await this.loadChild(path.join(__dirname, 'admin', 'host'))); + router.use('/image', await this.loadChild(path.join(__dirname, 'admin', 'image'))); router.use('/job-queue', await this.loadChild(path.join(__dirname, 'admin', 'job-queue'))); router.use('/log', await this.loadChild(path.join(__dirname, 'admin', 'log'))); router.use('/newsletter', await this.loadChild(path.join(__dirname, 'admin', 'newsletter'))); diff --git a/app/controllers/admin/attachment.js b/app/controllers/admin/attachment.js new file mode 100644 index 0000000..a521f30 --- /dev/null +++ b/app/controllers/admin/attachment.js @@ -0,0 +1,137 @@ +// admin/attachment.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const express = require('express'); + +const { SiteController } = require('../../../lib/site-lib'); + +class AttachmentAdminController extends SiteController { + + constructor (dtp) { + super(dtp, module.exports); + } + + async start ( ) { + const router = express.Router(); + router.use(async (req, res, next) => { + res.locals.currentView = 'admin'; + res.locals.adminView = 'attachment'; + return next(); + }); + + router.param('attachmentId', this.populateAttachmentId.bind(this)); + + router.post('/:attachmentId', this.postUpdateAttachment.bind(this)); + + router.get('/:attachmentId', this.getAttachmentView.bind(this)); + + router.get('/', this.getDashboard.bind(this)); + + router.delete('/:attachmentId', this.deleteAttachment.bind(this)); + + return router; + } + + async populateAttachmentId (req, res, next, attachmentId) { + const { + attachment: attachmentService, + logan: loganService, + } = this.dtp.services; + try { + res.locals.attachment = await attachmentService.getById(attachmentId); + return next(); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'populateAttachmentId', + message: `failed to populate attachment: ${error.message}`, + data: { attachmentId, error }, + }); + return next(error); + } + } + + async postUpdateAttachment (req, res, next) { + const { + attachment: attachmentService, + logan: loganService, + } = this.dtp.services; + try { + await attachmentService.update(res.locals.attachment, req.body); + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postUpdateAttachment', + data: { + attachment: { + _id: res.locals.attachment._id, + }, + }, + }); + res.redirect('/admin/attachment'); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'postUpdateAttachment', + message: `failed to update attachment: ${error.message}`, + data: { error }, + }); + return next(error); + } + } + + async getAttachmentView (req, res) { + res.render('admin/attachment/view'); + } + + async getDashboard (req, res, next) { + const { attachment: attachmentService } = this.dtp.services; + try { + res.locals.pagination = this.getPaginationParameters(req, 20); + res.locals.attachments = await attachmentService.getRecent(res.locals.pagination); + res.render('admin/attachment/index'); + } catch (error) { + return next(error); + } + } + + async deleteAttachment (req, res) { + const { + attachment: attachmentService, + logan: loganService, + } = this.dtp.services; + try { + const displayList = this.createDisplayList('delete-attachment'); + await attachmentService.remove(res.locals.attachment); + displayList.reload(); + + res.status(200).json({ success: true, displayList }); + + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'deleteAttachment', + data: { attachment: { _id: res.locals.attachment._id } }, + }); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'deleteAttachment', + message: `failed to delete attachment: ${error.message}`, + data: { error }, + }); + res.status(error.statusCode || 500).json({ + success: false, + message: error.message, + }); + } + } +} + +module.exports = { + name: 'adminAttachment', + slug: 'adminAttachment', + className: 'AttachmentAdminController', + create: async (dtp) => { return new AttachmentAdminController(dtp); }, +}; \ No newline at end of file diff --git a/app/controllers/admin/image.js b/app/controllers/admin/image.js new file mode 100644 index 0000000..af3c2c0 --- /dev/null +++ b/app/controllers/admin/image.js @@ -0,0 +1,121 @@ +// admin/image.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const express = require('express'); + +const { SiteController, SiteError } = require('../../../lib/site-lib'); + +class ImageAdminController extends SiteController { + + constructor (dtp) { + super(dtp, module.exports); + } + + async start ( ) { + const router = express.Router(); + router.use(async (req, res, next) => { + res.locals.currentView = 'admin'; + res.locals.adminView = 'image'; + return next(); + }); + + router.param('imageId', this.populateImageId.bind(this)); + + router.get('/:imageId/archive-user', this.getUserArchiveView.bind(this)); + router.get('/:imageId', this.getImageView.bind(this)); + + router.get('/', this.getDashboard.bind(this)); + + router.delete('/:imageId', this.deleteImage.bind(this)); + + return router; + } + + async populateImageId (req, res, next, imageId) { + const { + image: imageService, + logan: loganService, + } = this.dtp.services; + try { + res.locals.image = await imageService.getImageById(imageId); + if (!res.locals.image) { + throw new SiteError(404, 'Image not found'); + } + return next(); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'populateImageId', + message: `failed to populate image: ${error.message}`, + data: { imageId, error }, + }); + return next(error); + } + } + + async getUserArchiveView (req, res, next) { + const { image: imageService } = this.dtp.services; + try { + res.locals.imageHistory = await imageService.getRecentImagesForOwner(res.locals.image.owner, 10); + res.render('admin/image/archive-user'); + } catch (error) { + return next(error); + } + } + + async getImageView (req, res) { + res.render('admin/image/view'); + } + + async getDashboard (req, res, next) { + const { image: imageService } = this.dtp.services; + try { + res.locals.pagination = this.getPaginationParameters(req, 20); + res.locals.images = await imageService.getRecentImages(res.locals.pagination); + res.render('admin/image/index'); + } catch (error) { + return next(error); + } + } + + async deleteImage (req, res) { + const { + image: imageService, + logan: loganService, + } = this.dtp.services; + try { + const displayList = this.createDisplayList('delete-image'); + await imageService.deleteImage(res.locals.image); + displayList.reload(); + + res.status(200).json({ success: true, displayList }); + + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'deleteImage', + data: { image: { _id: res.locals.image._id } }, + }); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'deleteImage', + message: `failed to delete image: ${error.message}`, + data: { error }, + }); + res.status(error.statusCode || 500).json({ + success: false, + message: error.message, + }); + } + } +} + +module.exports = { + name: 'adminImage', + slug: 'adminImage', + className: 'ImageAdminController', + create: async (dtp) => { return new ImageAdminController(dtp); }, +}; \ No newline at end of file diff --git a/app/controllers/admin/user.js b/app/controllers/admin/user.js index b851b7e..dd1cf2d 100644 --- a/app/controllers/admin/user.js +++ b/app/controllers/admin/user.js @@ -15,6 +15,14 @@ class UserAdminController extends SiteController { } async start ( ) { + const { jobQueue: jobQueueService } = this.dtp.services; + + this.jobQueues = { }; + this.jobQueues.reeeper = await jobQueueService.getJobQueue( + 'reeeper', + this.dtp.config.jobQueues.reeeper, + ); + const router = express.Router(); router.use(async (req, res, next) => { res.locals.currentView = 'admin'; @@ -23,11 +31,26 @@ class UserAdminController extends SiteController { }); router.param('localUserId', this.populateLocalUserId.bind(this)); + router.param('archiveJobId', this.populateArchiveJobId.bind(this)); + router.param('archiveId', this.populateArchiveId.bind(this)); + + router.post('/local/:localUserId/archive', this.postArchiveLocalUser.bind(this)); router.post('/local/:localUserId', this.postUpdateLocalUser.bind(this)); + + router.get('/local/:localUserId/archive/confirm', this.getArchiveLocalUserConfirm.bind(this)); router.get('/local/:localUserId', this.getLocalUserView.bind(this)); + router.get('/archive/job/:archiveJobId', this.getUserArchiveJobView.bind(this)); + + router.post('/archive/:archiveId/action', this.postArchiveAction.bind(this)); + router.get('/archive/:archiveId/file', this.getUserArchiveFile.bind(this)); + router.get('/archive/:archiveId', this.getUserArchiveView.bind(this)); + + router.get('/archive', this.getUserArchiveIndex.bind(this)); + router.get('/', this.getHomeView.bind(this)); + return router; } @@ -44,6 +67,68 @@ class UserAdminController extends SiteController { } } + async populateArchiveJobId (req, res, next, archiveJobId) { + try { + res.locals.job = await this.jobQueues.reeeper.getJob(archiveJobId); + if (!res.locals.job) { + throw new SiteError(404, 'Job not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate Bull queue job', { archiveJobId, error }); + return next(error); + } + } + + async populateArchiveId (req, res, next, archiveId) { + const { user: userService } = this.dtp.services; + try { + res.locals.archive = await userService.getArchiveById(archiveId); + if (!res.locals.archive) { + throw new SiteError(404, 'Archive not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate UserArchive', { archiveId, error }); + return next(error); + } + } + + async postArchiveLocalUser (req, res, next) { + const { + logan: loganService, + user: userService, + } = this.dtp.services; + try { + const user = await userService.getLocalUserAccount(req.body.userId); + if (!user) { + throw new SiteError(404, 'User not found'); + } + res.locals.job = await userService.archiveLocalUser(user); + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveUser', + data: { + job: res.locals.job.id, + user: user, + }, + }); + res.redirect(`/admin/user/archive/job/${res.locals.job.id}`); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'postArchiveUser', + data: { + offender: { + _id: req.body.userId, + }, + error, + }, + }); + return next(error); + } + } + async postUpdateLocalUser (req, res, next) { const { logan: loganService, @@ -100,6 +185,133 @@ class UserAdminController extends SiteController { } } + async getUserArchiveJobView (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/job'); + } + + async getArchiveLocalUserConfirm (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/confirm'); + } + + async postArchiveAction (req, res, next) { + const { + logan: loganService, + user: userService, + } = this.dtp.services; + try { + switch (req.body.action) { + case 'update': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'updating user archive record', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.updateArchive(res.locals.archive, req.body); + return res.redirect(`/admin/user/archive/${res.locals.archive._id}`); + + case 'delete-file': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'removing user archive file', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.deleteArchiveFile(res.locals.archive); + return res.redirect(`/admin/user/archive/${res.locals.archive._id}`); + + case 'delete': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'removing user archive', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.deleteArchive(res.locals.archive); + return res.redirect(`/admin/user/archive`); + + default: + // unknown/invalid action + break; + } + + throw new SiteError(400, `Invalid user archive action: ${req.body.action}`); + + } catch (error) { + this.log.error('failed to delete archive file', { error }); + return next(error); + } + } + + async getUserArchiveFile (req, res, next) { + const { minio: minioService } = this.dtp.services; + try { + res.locals.adminView = 'user-archive'; + + this.log.debug('archive', { archive: res.locals.archive }); + const stream = await minioService.openDownloadStream({ + bucket: res.locals.archive.archive.bucket, + key: res.locals.archive.archive.key, + }); + + res.status(200); + res.set('Content-Type', 'application/zip'); + res.set('Content-Size', res.locals.archive.archive.size); + res.set('Content-Disposition', `attachment; filename="user-${res.locals.archive.user._id}.zip"`); + + stream.pipe(res); + } catch (error) { + this.log.error('failed to stream user archive file', { error }); + return next(error); + } + } + + async getUserArchiveView (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/view'); + } + + async getUserArchiveIndex (req, res, next) { + const { user: userService } = this.dtp.services; + try { + res.locals.adminView = 'user-archive'; + + res.locals.pagination = this.getPaginationParameters(req, 20); + res.locals.archive = await userService.getArchives(res.locals.pagination); + + res.render('admin/user/archive/index'); + } catch (error) { + this.log.error('failed to render the User archives index', { error }); + return next(error); + } + } + async getHomeView (req, res, next) { const { user: userService } = this.dtp.services; try { diff --git a/app/controllers/image.js b/app/controllers/image.js index 83a38f9..c8bb9ce 100644 --- a/app/controllers/image.js +++ b/app/controllers/image.js @@ -9,7 +9,7 @@ const fs = require('fs'); const express = require('express'); const mongoose = require('mongoose'); -const { SiteController/*, SiteError*/ } = require('../../lib/site-lib'); +const { SiteController, SiteError } = require('../../lib/site-lib'); class ImageController extends SiteController { @@ -60,6 +60,9 @@ class ImageController extends SiteController { try { res.locals.imageId = mongoose.Types.ObjectId(imageId); res.locals.image = await this.dtp.services.image.getImageById(res.locals.imageId); + if (!res.locals.image) { + throw new SiteError(404, 'Image not found'); + } return next(); } catch (error) { this.log.error('failed to populate image', { error }); diff --git a/app/controllers/user.js b/app/controllers/user.js index ebf8793..12fefa1 100644 --- a/app/controllers/user.js +++ b/app/controllers/user.js @@ -119,13 +119,13 @@ class UserController extends SiteController { ); router.get( - '/:userId/otp-setup', + '/:localUserId/otp-setup', limiterService.createMiddleware(limiterService.config.user.getOtpSetup), otpSetup, this.getOtpSetup.bind(this), ); router.get( - '/:userId/otp-disable', + '/:localUserId/otp-disable', limiterService.createMiddleware(limiterService.config.user.getOtpDisable), authRequired, this.getOtpDisable.bind(this), @@ -148,7 +148,7 @@ class UserController extends SiteController { ); router.delete( - '/:userId/profile-photo', + '/:localUserId/profile-photo', limiterService.createMiddleware(limiterService.config.user.deleteProfilePhoto), authRequired, checkProfileOwner, @@ -334,6 +334,12 @@ class UserController extends SiteController { }); } catch (error) { this.log.error('failed to create new user', { error }); + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'postCreateUser', + message: `failed to create user account: ${error.message}`, + data: { definition: req.body, error }, + }); return next(error); } } diff --git a/app/models/user-archive.js b/app/models/user-archive.js new file mode 100644 index 0000000..842fc67 --- /dev/null +++ b/app/models/user-archive.js @@ -0,0 +1,29 @@ +// user-archive.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const mongoose = require('mongoose'); + +const Schema = mongoose.Schema; + +const UserArchiveSchema = new Schema({ + created: { type: Date, default: Date.now, required: true, index: -1 }, + user: { + _id: { type: Schema.ObjectId, required: true, index: 1 }, + email: { type: String }, + username: { type: String, required: true }, + }, + archive: { + bucket: { type: String, required: true }, + key: { type: String, required: true }, + etag: { type: String, required: true }, + size: { type: Number, required: true }, + }, + notes: { type: String }, +}); + +module.exports = (conn) => { + return conn.model('UserArchive', UserArchiveSchema); +}; \ No newline at end of file diff --git a/app/services/attachment.js b/app/services/attachment.js index 8c086f9..4d9f1f9 100644 --- a/app/services/attachment.js +++ b/app/services/attachment.js @@ -126,6 +126,23 @@ class AttachmentService extends SiteService { return attachments; } + /** + * Access all attachments sorted by most recent with pagination. This is for + * use by Admin tools. + * @param {*} pagination required pagination parameters (skip and cpp) + * @returns Array of attachments + */ + async getRecent (pagination) { + const attachments = await Attachment + .find() + .sort({ created: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .populate(this.populateAttachment) + .lean(); + return attachments; + } + /** * * @param {mongoose.Types.ObjectId} attachmentId The ID of the attachment diff --git a/app/services/display-engine.js b/app/services/display-engine.js index 56296eb..66ba9b4 100644 --- a/app/services/display-engine.js +++ b/app/services/display-engine.js @@ -132,10 +132,6 @@ class DisplayEngineService extends SiteService { this.templates = { }; } - async start ( ) { } - - async stop ( ) { } - loadTemplate (name, pugScript) { const scriptFile = path.join(this.dtp.config.root, 'app', 'views', pugScript); this.templates[name] = pug.compileFile(scriptFile); diff --git a/app/services/image.js b/app/services/image.js index eb3a4f2..0864677 100644 --- a/app/services/image.js +++ b/app/services/image.js @@ -90,16 +90,37 @@ class ImageService extends SiteService { return image; } - async getRecentImagesForOwner(owner) { + async getRecentImagesForOwner(owner, limit = 10) { const images = await SiteImage .find({ owner: owner._id }) .sort({ created: -1 }) - .limit(10) + .limit(limit) .populate(this.populateImage) .lean(); return images; } + async getRecentImages (pagination) { + const images = await SiteImage + .find() + .sort({ created: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .populate(this.populateImage) + .lean(); + const totalImageCount = await SiteImage.estimatedDocumentCount(); + return { images, totalImageCount }; + } + + async downloadImage (image, filename) { + const { minio: minioService } = this.dtp.services; + return minioService.downloadFile({ + bucket: image.file.bucket, + key: image.file.key, + filePath: filename, + }); + } + async deleteImage(image) { const { minio: minioService } = this.dtp.services; diff --git a/app/services/logan.js b/app/services/logan.js index 94e78ef..8b22ef0 100644 --- a/app/services/logan.js +++ b/app/services/logan.js @@ -19,9 +19,6 @@ class LoganService extends SiteService { } async sendRequestEvent (component, req, event) { - if (process.env.DTP_LOGAN !== 'enabled') { - return; - } if (req.user) { event.data = event.data || { }; event.data.user = { @@ -34,20 +31,21 @@ class LoganService extends SiteService { } async sendEvent (component, event) { - if (process.env.DTP_LOGAN !== 'enabled') { - return; - } try { - const loganScheme = process.env.DTP_LOGAN_SCHEME || 'http'; - const loganUrl = `${loganScheme}://${process.env.DTP_LOGAN_HOST}/api/event`; - event.host = os.hostname(); event['component.slug'] = component.slug; event['component.name'] = component.className || component.name; - this.log[event.level]('sending Logan event', { event }); + this.log[event.level]('application event', { event }); + if (process.env.DTP_LOGAN !== 'enabled') { + return; + } + + const loganScheme = process.env.DTP_LOGAN_SCHEME || 'http'; + const loganUrl = `${loganScheme}://${process.env.DTP_LOGAN_HOST}/api/event`; const payload = JSON.stringify(event); + const response = await fetch(loganUrl, { method: 'POST', headers: { diff --git a/app/services/minio.js b/app/services/minio.js index d386cb2..60057ca 100644 --- a/app/services/minio.js +++ b/app/services/minio.js @@ -24,7 +24,6 @@ class MinioService extends SiteService { accessKey: process.env.MINIO_ACCESS_KEY, secretKey: process.env.MINIO_SECRET_KEY, }; - this.log.debug('MinIO config', { minioConfig }); this.minio = new Minio.Client(minioConfig); } diff --git a/app/services/session.js b/app/services/session.js index 785c8a5..0610978 100644 --- a/app/services/session.js +++ b/app/services/session.js @@ -18,7 +18,7 @@ class SessionService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); + passport.serializeUser(this.serializeUser.bind(this)); passport.deserializeUser(this.deserializeUser.bind(this)); } diff --git a/app/services/sms.js b/app/services/sms.js index 93c26c3..fa110e6 100644 --- a/app/services/sms.js +++ b/app/services/sms.js @@ -18,7 +18,6 @@ class SmsService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); } async stop ( ) { diff --git a/app/services/user.js b/app/services/user.js index 7f469c0..b64dcc7 100644 --- a/app/services/user.js +++ b/app/services/user.js @@ -11,6 +11,7 @@ const mongoose = require('mongoose'); const User = mongoose.model('User'); const CoreUser = mongoose.model('CoreUser'); const UserBlock = mongoose.model('UserBlock'); +const UserArchive = mongoose.model('UserArchive'); const passport = require('passport'); const PassportLocal = require('passport-local'); @@ -49,13 +50,20 @@ class UserService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); this.registerPassportLocal(); - if (process.env.DTP_ADMIN === 'enabled') { this.registerPassportAdmin(); } + + const { jobQueue: jobQueueService } = this.dtp.services; + this.jobQueues = { }; + + this.log.info('connecting to job queue', { name: 'reeeper', config: this.dtp.config.jobQueues.reeeper }); + this.jobQueues.reeeper = jobQueueService.getJobQueue( + 'reeeper', + this.dtp.config.jobQueues.reeeper, + ); } async stop ( ) { @@ -71,6 +79,7 @@ class UserService extends SiteService { } = this.dtp.services; try { + this.checkRestrictedKeys('create', userDefinition); userDefinition.email = userDefinition.email.trim().toLowerCase(); // strip characters we don't want to allow in username @@ -215,6 +224,8 @@ class UserService extends SiteService { throw SiteError(403, 'Invalid user account operation'); } + this.checkRestrictedKeys('create', userDefinition); + userDefinition.username = striptags(userDefinition.username.trim().replace(/[^A-Za-z0-9\-_]/gi, '')); const username_lc = userDefinition.username.toLowerCase(); @@ -781,7 +792,18 @@ class UserService extends SiteService { const { image: imageService } = this.dtp.services; this.log.info('remove profile photo', { user: user._id }); - user = await this.getUserAccount(user._id); + switch (user.type) { + case 'User': + user = await this.getLocalUserAccount(user._id); + break; + + case 'CoreUser': + user = await this.getCoreUserAccount(user._id); + break; + + default: + throw new SiteError(400, 'Invalid User type'); + } if (user.picture.large) { await imageService.deleteImage(user.picture.large); } @@ -939,6 +961,117 @@ class UserService extends SiteService { await stickerService.removeForUser(user); await userNotificationService.removeForUser(user); } + + checkRestrictedKeys (method, definition) { + const { logan: loganService } = this.dtp.services; + const restrictedKeys = [ + 'isAdmin', 'isModerator', 'isEmailVerified', + 'canLogin', 'canChat', 'canComment', 'canReport', + 'optInSystem', 'optInMarketing', + ]; + + const keys = Object.keys(definition); + for (const restrictedKey of restrictedKeys) { + if (keys.includes(restrictedKey)) { + loganService.sendEvent(module.exports, { + level: 'alert', + event: method, + message: 'malicious fields detected', + data: { definition }, + }); + throw new SiteError(403, 'invalid request'); + } + } + } + + /** + * Create a job to archive and ban a User (local). The job will immediately + * disable the specified user, create a .zip file of their content on storage. + * Once the worker confirms that the archive file is on storage, it creates a + * UserArchive record for it, then completely bans the User. That removes all + * of the User's content. + * + * It then removes the User record entirely. + * + * @param {User} user the User to be archived + * @returns the newly created Bull queue job + */ + async archiveLocalUser (user) { + return this.jobQueues.reeeper.add('archive-user-local', { userId: user._id }); + } + + /** + * Update a UserArchive document + * @param {UserArchive} archive the existing archive to be updated + * @param {*} archiveDefinition new values to be applied + */ + async updateArchive (archive, archiveDefinition) { + const update = { $set: { }, $unset: { } }; + + archiveDefinition.notes = archiveDefinition.notes.trim(); + if (archiveDefinition.notes && (archiveDefinition.notes.length > 0)) { + update.$set.notes = archiveDefinition.notes; + } else { + update.$unset.notes = 1; + } + + await UserArchive.updateOne({ _id: archive._id }, update); + } + + /** + * Fetch an Array of UserArchive documents with pagination. + * @param {DtpPagination} pagination self explanatory + * @returns Array of UserArchive documents (can be empty) + */ + async getArchives (pagination) { + const search = { }; + const archives = await UserArchive + .find(search) + .sort({ created: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .lean(); + const totalArchiveCount = await UserArchive.estimatedDocumentCount(); + return { archives, totalArchiveCount }; + } + + /** + * Fetch a UserArchive record. This does not fetch the archive file. + * @param {UserArchive} archiveId the ID of the archive to fetch + * @returns the requested UserArchive, or null/undefined. + */ + async getArchiveById (archiveId) { + const archive = await UserArchive.findOne({ _id: archiveId }).lean(); + return archive; + } + + /** + * Removes the .zip file attached to a UserArchive. + * @param {UserArchive} archive the archive for which an associated .zip file + * is to be removed + */ + async deleteArchiveFile (archive) { + const { minio: minioService } = this.dtp.services; + if (!archive.archive || !archive.archive.bucket || !archive.archive.key) { + return; // no archive file present, abort + } + await minioService.removeObject(archive.archive.bucket, archive.archive.key); + await UserArchive.updateOne( + { _id: archive._id }, + { + $unset: { archive: 1 }, + }, + ); + } + + /** + * Removes a UserArchive and any attached data. + * @param {UserArchive} archive the UserArchive to be removed. + */ + async deleteArchive (archive) { + await this.deleteArchiveFile(archive); + await UserArchive.deleteOne({ _id: archive._id }); + } } module.exports = { diff --git a/app/views/admin/attachment/index.pug b/app/views/admin/attachment/index.pug new file mode 100644 index 0000000..5120f13 --- /dev/null +++ b/app/views/admin/attachment/index.pug @@ -0,0 +1,21 @@ +extends ../layouts/main +block content + + h1 Attachments + + if Array.isArray(attachments) && (attachments.length > 0) + ul.uk-list.uk-list-divider + each attachment in attachments + li + div(uk-grid).uk-grid-small.uk-flex-middle + .uk-width-expand + //- had to abort while writing the renderer for an attachment. + //- will be back to finish this and have an attachment browser/manager. + pre= JSON.stringify(attachment, null, 2) + + .uk-width-auto + button(type="button", data-attachment-id= attachment._id, onclick="return dtp.adminApp.deleteAttachment(event);").uk-button.dtp-button-danger.uk-border-rounded + span + i.fas.fa-trash + else + div There are no attachments. \ No newline at end of file diff --git a/app/views/admin/components/menu.pug b/app/views/admin/components/menu.pug index 9573dbb..2f07774 100644 --- a/app/views/admin/components/menu.pug +++ b/app/views/admin/components/menu.pug @@ -32,6 +32,19 @@ ul(uk-nav).uk-nav-default span.nav-item-icon i.fas.fa-bullhorn span.uk-margin-small-left Announcements + + li(class={ 'uk-active': (adminView === 'attachment') }) + a(href="/admin/attachment") + span.nav-item-icon + i.fas.fa-file + span.uk-margin-small-left Attachments + + li(class={ 'uk-active': (adminView === 'image') }) + a(href="/admin/image") + span.nav-item-icon + i.fas.fa-image + span.uk-margin-small-left Images + li(class={ 'uk-active': (adminView === 'post') }) a(href="/admin/post") span.nav-item-icon @@ -97,13 +110,18 @@ ul(uk-nav).uk-nav-default i.fas.fa-user span.uk-margin-small-left Users + li(class={ 'uk-active': (adminView === 'user-archive') }) + a(href="/admin/user/archive") + span.nav-item-icon + i.fas.fa-file-archive + span.uk-margin-small-left User Archive + li(class={ 'uk-active': (adminView === 'content-report') }) a(href="/admin/content-report") span.nav-item-icon i.fas.fa-ban span.uk-margin-small-left Content Reports - li.uk-nav-divider li(class={ 'uk-active': (adminView === 'core-node') }) diff --git a/app/views/admin/image/archive-user.pug b/app/views/admin/image/archive-user.pug new file mode 100644 index 0000000..e52098b --- /dev/null +++ b/app/views/admin/image/archive-user.pug @@ -0,0 +1,28 @@ +extends ../layouts/main +block content + + include ../user/components/list-item + + form(method="POST", action=`/admin/user/local/${image.owner._id}/archive`).uk-form + input(type="hidden", name="userId", value= image.owner._id) + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title Archive Local User + .uk-card-body + p This action will pull all images from storage into an archive file, place the archive file on storage, delete all the image records and storage data, then ban the User. The archive is produced first because images would be deleted during the ban. So, the archive is made, then the user is banned. + + p These are the #{numeral(imageHistory.length).format('0,0')} most recent images uploaded by #{image.owner.username}. + + div(uk-grid) + each image in imageHistory + .uk-width-medium + .uk-margin-small(uk-lightbox) + a(href=`/image/${image._id}`, data-type="image", data-caption=`id: ${image._id}`) + div + img(src= `/image/${image._id}`).responsive + .uk-card-footer.uk-flex.uk-flex-middle + .uk-width-expand + +renderBackButton() + + .uk-width-auto + button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User \ No newline at end of file diff --git a/app/views/admin/image/index.pug b/app/views/admin/image/index.pug new file mode 100644 index 0000000..81f40d7 --- /dev/null +++ b/app/views/admin/image/index.pug @@ -0,0 +1,45 @@ +extends ../layouts/main +block content + + include ../user/components/list-item + include ../../components/pagination-bar + + h1.uk-text-center Image Manager + + if Array.isArray(images.images) && (images.images.length > 0) + div(uk-grid).uk-flex-center + each image in images.images + .uk-width-medium + .uk-margin-small(uk-lightbox) + a(href=`/image/${image._id}`, data-type="image", data-caption=`id: ${image._id}`) + div + img(src= `/image/${image._id}`).responsive + + if image.owner + .uk-margin-small + +renderUserListItem(image.owner) + + .uk-margin-small.uk-text-center + button(type="button").uk-button.uk-button-default.uk-button-small.uk-border-rounded + span Image Menu + + div(uk-drop={ mode: 'click', pos: 'top-center' }).uk-card.uk-card-default.uk-card-small.uk-border-rounded + .uk-card-header + .uk-text-small.uk-text-muted.uk-text-center id:#{image._id} + .uk-card-body + ul.uk-nav.uk-dropdown-nav + li + a(href="#", data-image-id= image._id, onclick="dtp.adminApp.deleteImage(event);") + span + i.fas.fa-trash + span.uk-margin-small-left Delete image + li + a(href=`/admin/image/${image._id}/archive-user`).uk-text-truncate + span + i.fas.fa-file-archive + span.uk-margin-small-left Archive and ban #[span.uk-text-bold= image.owner.username] + + +renderPaginationBar('/admin/image', images.totalImageCount) + + else + .uk-text-center There are no images. \ No newline at end of file diff --git a/app/views/admin/user/archive/confirm.pug b/app/views/admin/user/archive/confirm.pug new file mode 100644 index 0000000..9f83eaa --- /dev/null +++ b/app/views/admin/user/archive/confirm.pug @@ -0,0 +1,28 @@ +extends ../../layouts/main +block content + + include ../../user/components/list-item + + form(method="POST", action=`/admin/user/local/${userAccount._id}/archive`).uk-form + input(type="hidden", name="userId", value= userAccount._id) + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left Archive Local User + .uk-card-body + .uk-margin + +renderUserListItem(userAccount) + + .uk-margin + p This action will archive #{userAccount.displayName || userAccount.username}'s content to a .zip file, place the .zip file on storage, create a UserArchive record for this User account, ban this User account, and remove this User account from the database. + + p #{userAccount.displayName || userAccount.username}'s email address and username will become locked, and will remain unavailable for use for as long as this archive exists. + + .uk-card-footer.uk-flex.uk-flex-middle + .uk-width-expand + +renderBackButton() + + .uk-width-auto + button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User \ No newline at end of file diff --git a/app/views/admin/user/archive/index.pug b/app/views/admin/user/archive/index.pug new file mode 100644 index 0000000..798399d --- /dev/null +++ b/app/views/admin/user/archive/index.pug @@ -0,0 +1,41 @@ +extends ../../layouts/main +block content + + include ../components/list-item + include ../../../components/pagination-bar + + .uk-card.uk-card-default.uk-card-small.uk-margin + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left User Archives + + .uk-card-body + if Array.isArray(archive.archives) && (archive.archives.length > 0) + table.uk-table.uk-table-divider.uk-table-justify + thead + tr + th Username + th User ID + th Created + th Archive + tbody + each record in archive.archives + tr + td= record.user.username + td= record.user._id + td= moment(record.created).format('MMMM DD, YYYY, [at] h:mm a') + td + span + i.fas.fa-file-archive + a(href=`/admin/user/archive/${record._id}`).uk-margin-small-left View Archive + else + div There are no user archives. + + if Array.isArray(archive.archives) && (archive.archives.length > 0) + .uk-card-footer + +renderPaginationBar('/admin/user/archive', archive.totalArchiveCount) + + .uk-margin + .uk-text-small.uk-text-muted.uk-text-center User accounts referenced on this page have been removed from the database and are no longer able to use #{site.name}. \ No newline at end of file diff --git a/app/views/admin/user/archive/job.pug b/app/views/admin/user/archive/job.pug new file mode 100644 index 0000000..fbb33cc --- /dev/null +++ b/app/views/admin/user/archive/job.pug @@ -0,0 +1,7 @@ +extends ../../layouts/main +block content + + include ../components/list-item + + h1 User Archive Job + pre= JSON.stringify(job, null, 2) \ No newline at end of file diff --git a/app/views/admin/user/archive/view.pug b/app/views/admin/user/archive/view.pug new file mode 100644 index 0000000..a0122fe --- /dev/null +++ b/app/views/admin/user/archive/view.pug @@ -0,0 +1,83 @@ +extends ../../layouts/main +block content + + include ../components/list-item + + form(method="POST", action=`/admin/user/archive/${archive._id}/action`).uk-form + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left User Archive + + .uk-card-body + .uk-margin + div(uk-grid) + .uk-width-auto + .uk-form-label Archive ID + .uk-text-bold= archive._id + .uk-width-auto + .uk-form-label Created + .uk-text-bold= moment(archive.created).format('MMMM DD, YYYY, [at] h:mm:ss a') + .uk-width-auto + .uk-form-label User + .uk-text-bold= archive.user.username + .uk-width-auto + .uk-form-label User ID + .uk-text-bold= archive.user._id + .uk-width-auto + .uk-form-label User email + .uk-text-bold= archive.user.email + + if archive.archive + div(uk-grid) + .uk-width-auto + .uk-form-label Archive file + .uk-text-bold= archive.archive.key.replace(/\/user-archive\//, '') + .uk-width-auto + .uk-form-label Download size + .uk-text-bold= numeral(archive.archive.size).format('0,0.0a') + else + .uk-text-italic (archive file removed) + + .uk-margin + label(for="notes").uk-form-label Notes + textarea(id="notes", name="notes", rows="4", placeholder="Enter notes").uk-textarea.uk-resize-vertical= archive.notes + + .uk-card-footer + div(uk-grid) + .uk-width-expand + div(hidden= !archive.archive, uk-grid) + .uk-width-auto + a(href=`/admin/user/archive/${archive._id}/file`).uk-button.uk-button-default.uk-border-rounded + span + i.fas.fa-download + span.uk-margin-small-left Download#[span(class="uk-visible@s") File] + .uk-width-auto + button( + type="submit", + name="action", + value="delete-file", + uk-tooltip={ title: 'Remove the .zip file attached to the UserArchive' }, + ).uk-button.uk-button-danger.uk-border-rounded + span + i.fas.fa-trash + span.uk-margin-small-left Delete#[span(class="uk-visible@s") File] + + .uk-width-auto + button( + type="submit", + name="action", + value="delete", + uk-tooltip={ title: 'Remove the UserArchive from the database' }, + ).uk-button.uk-button-danger.uk-border-rounded + span + i.fas.fa-save + span.uk-margin-small-left Delete + + .uk-width-auto + button(type="submit", name="action", value="update").uk-button.uk-button-primary.uk-border-rounded + span + i.fas.fa-save + span.uk-margin-small-left Update \ No newline at end of file diff --git a/app/views/admin/user/components/list-item.pug b/app/views/admin/user/components/list-item.pug index 123f74b..ebdd631 100644 --- a/app/views/admin/user/components/list-item.pug +++ b/app/views/admin/user/components/list-item.pug @@ -7,7 +7,8 @@ mixin renderUserListItem (user) .uk-text-small.uk-text-muted a(href= getUserProfileUrl(user))= user.username .uk-text-small.uk-text-truncate= user.bio - .uk-text-small.uk-text-muted created #{moment(user.created).fromNow()} + if user.created + .uk-text-small.uk-text-muted created #{moment(user.created).fromNow()} .uk-width-auto a(href=`/admin/user/local/${user._id}`, uk-tooltip={ title: 'Manage user account' }).uk-button.uk-button-default.uk-button-small.uk-border-rounded span diff --git a/app/views/admin/user/form.pug b/app/views/admin/user/form.pug index 5d26940..9182898 100644 --- a/app/views/admin/user/form.pug +++ b/app/views/admin/user/form.pug @@ -93,6 +93,8 @@ block content div(uk-grid).uk-grid-small .uk-width-expand +renderBackButton() + .uk-width-auto + a(href=`/admin/user/local/${userAccount._id}/archive/confirm`).uk-button.uk-button-danger.uk-border-rounded Archive User .uk-width-auto button(type="submit", name="action", value="ban").uk-button.uk-button-danger.uk-border-rounded Ban User .uk-width-auto diff --git a/app/views/admin/user/index.pug b/app/views/admin/user/index.pug index 66c17c2..6afbfbc 100644 --- a/app/views/admin/user/index.pug +++ b/app/views/admin/user/index.pug @@ -3,6 +3,19 @@ block content include ../../components/pagination-bar + .uk-margin + div(uk-grid).uk-flex-middle + .uk-width-expand + h1 + span + i.fas.fa-user-cog + span.uk-margin-small-left User Manager + .uk-width-auto + a(href="/admin/user/archive").uk-button.uk-button-default.uk-border-rounded + span.nav-item-icon + i.fas.fa-file-archive + span.uk-margin-small-left Browse Archive + .uk-margin form(method="GET", action="/admin/user").uk-form div(uk-grid).uk-grid-collapse diff --git a/app/workers/reeeper.js b/app/workers/reeeper.js index 0236034..a3de7d7 100644 --- a/app/workers/reeeper.js +++ b/app/workers/reeeper.js @@ -36,6 +36,8 @@ class ReeeperWorker extends SiteWorker { await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-crashed-hosts.js')); await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-announcements.js')); + await this.loadProcessor(path.join(__dirname, 'reeeper', 'job', 'archive-user-local.js')); + await this.startProcessors(); } diff --git a/app/workers/reeeper/job/archive-user-local.js b/app/workers/reeeper/job/archive-user-local.js new file mode 100644 index 0000000..42a967c --- /dev/null +++ b/app/workers/reeeper/job/archive-user-local.js @@ -0,0 +1,381 @@ +// reeeper/job/archive-user-local.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +const util = require('util'); +const execFile = util.promisify(require('child_process').execFile); + +const mime = require('mime'); + +const mongoose = require('mongoose'); +const User = mongoose.model('User'); + +const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); + +/** + * A job to archive and ban a User (local). + * + * 1. Immediately disable the specified User + * 2. Create a .zip file of the User's content on storage + * 3. Creates a UserArchive record for the file and User + * 4. Ban the User (removes all of the User's content) + * 5. Remove the User record from the database + */ +class ArchiveUserLocalJob extends SiteWorkerProcess { + + static get COMPONENT ( ) { + return { + name: 'archiveUserLocalJob', + slug: 'archive-user-local-job', + }; + } + + static get JOB_NAME ( ) { return 'Local User Archive'; } + static get JOB_SLUG ( ) { return 'archive-user-local'; } + + constructor (worker) { + super(worker, ArchiveUserLocalJob.COMPONENT); + this.jobs = new Set(); + } + + async start ( ) { + await super.start(); + + this.queue = await this.getJobQueue('reeeper', this.dtp.config.jobQueues.reeeper); + + this.log.info('registering job processor', { queue: this.queue.name, job: ArchiveUserLocalJob.JOB_SLUG }); + this.queue.process(ArchiveUserLocalJob.JOB_SLUG, 1, this.processArchiveUserLocal.bind(this)); + } + + async stop ( ) { + try { + if (this.queue) { + this.log.info('halting job queue', { jobCount: this.jobs.size }); + await this.queue.pause(true, false); + delete this.queue; + } + } catch (error) { + this.log.error('failed to halt job queue', { error }); + // fall through + } finally { + await super.stop(); + } + } + + async processArchiveUserLocal (job) { + const { user: userService } = this.dtp.services; + try { + job.data.archivePath = path.join('/tmp', this.dtp.pkg.name, ArchiveUserLocalJob.JOB_SLUG); + this.jobs.add(job); + + job.data.userId = mongoose.Types.ObjectId(job.data.userId); + job.data.user = await userService.getLocalUserAccount(job.data.userId); + + job.data.workPath = path.join(job.data.archivePath, job.data.userId.toString()); + await fs.promises.mkdir(job.data.workPath, { recursive: true }); + + /* + * Save the User account data + */ + await this.archiveUserData(job); + + /* + * Disable the User account (which destroys their session and cookie(s)) + */ + // await this.disableUser(job); + + /* + * Archive the User's content to the workPath on the local file system. + */ + await this.archiveUserChat(job); + await this.archiveUserComments(job); + await this.archiveUserStickers(job); + await this.archiveUserImages(job); + + /* + * Create the .zip file archive, upload it to storage, and create the + * UserArchive record. + */ + await this.createArchiveFile(job); + + this.log.info('banning user', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + // await userService.ban(job.data.user); + + this.log.info('removing user', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + // await User.deleteOne({ _id: job.data.userId }); + } catch (error) { + this.log.error('failed to delete attachment', { attachmentId: job.data.attachmentId, error }); + throw error; + } finally { + if (job.data.workPath) { + this.log.info('cleaning up work directory'); + await fs.promises.rm(job.data.workPath, { force: true, recursive: true }); + + delete job.data.workPath; + } + this.jobs.delete(job); + this.log.info('job complete', { job: job.id, name: ArchiveUserLocalJob.JOB_NAME }); + } + } + + async archiveUserData (job) { + // fetch the entire User record (all fields) + job.data.fullUser = await User + .findOne({ _id: job.data.user._id }) + .select('+email +passwordSalt +password +flags +permissions +optIn') + .lean(); + if (!job.data.fullUser) { + throw new Error('user does not exist'); + } + + const userFilename = path.join(job.data.workPath, `user-${job.data.user._id}.json`); + await fs.promises.writeFile(userFilename, JSON.stringify(job.data.fullUser, null, 2)); + } + + async disableUser (job) { + this.log.info('disabling local User account', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + await User.updateOne( + { _id: job.data.user._id }, + { + $set: { + 'flags.isAdmin': false, + 'flags.isModerator': false, + 'flags.isEmailVerified': false, + 'permissions.canLogin': false, + 'permissions.canChat': false, + 'permissions.canComment': false, + 'permissions.canReport': false, + 'optIn.system': false, + 'optIn.marketing': false, + }, + }, + ); + } + + async archiveUserChat (job) { + const ChatMessage = mongoose.model('ChatMessage'); + const ChatRoom = mongoose.model('ChatRoom'); + + job.data.chatPath = path.join(job.data.workPath, 'chat'); + await fs.promises.mkdir(job.data.chatPath, { recursive: true }); + + this.log.info('archiving user chat', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await ChatRoom + .find({ owner: job.data.user._id }) + + .lean() + .cursor() + .eachAsync(async (room) => { + const roomFilename = path.join(job.data.workPath, 'chat', `room-${room._id}`); + await fs.promises.writeFile(roomFilename, JSON.stringify(room, null, 2)); + }); + + await ChatMessage + .find({ author: job.data.user._id }) + .lean() + .cursor() + .eachAsync(async (message) => { + const messageFilename = path.join(job.data.workPath, 'chat', `message-${message._id}.json`); + await fs.promises.writeFile(messageFilename, JSON.stringify(message, null, 2)); + }); + } + + async archiveUserComments (job) { + const Comment = mongoose.model('Comment'); + + job.data.commentPath = path.join(job.data.workPath, 'comments'); + await fs.promises.mkdir(job.data.commentPath, { recursive: true }); + + this.log.info('archiving user comments', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await Comment + .find({ author: job.data.userId }) + .cursor() + .eachAsync(async (comment) => { + const commentFilename = path.join(job.data.imagePath, `comment-${comment._id}.json`); + await fs.promises.writeFile(commentFilename, JSON.stringify(comment, null, 2)); + }); + } + + async archiveUserStickers (job) { + const Sticker = mongoose.model('Sticker'); + const { minio: minioService } = this.dtp.services; + + job.data.stickerPath = path.join(job.data.workPath, 'stickers'); + await fs.promises.mkdir(job.data.stickerPath, { recursive: true }); + + job.data.stickerMediaPath = path.join(job.data.stickerPath, 'media'); + await fs.promises.mkdir(job.data.stickerMediaPath, { recursive: true }); + + this.log.info('archiving user stickers', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await Sticker + .find({ owner: job.data.userId }) + .cursor() + .eachAsync(async (sticker) => { + const stickerFilename = path.join(job.data.stickerPath, `sticker-${sticker._id}.json`); + await fs.promises.writeFile(stickerFilename, JSON.stringify(sticker, null, 2)); + + if (sticker.original && sticker.original.bucket && sticker.orignal.key && sticker.encoded.type) { + const originalExt = mime.getExtension(sticker.original.type); + const originalFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.original.${originalExt}`); + await minioService.downloadFile({ + bucket: sticker.original.bucket, + key: sticker.original.key, + filePath: originalFilename, + }); + } + + if (sticker.encoded && sticker.encoded.bucket && sticker.encoded.key && sticker.encoded.type) { + const encodedExt = mime.getExtension(sticker.encoded.type); + const encodedFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.encoded.${encodedExt}`); + await minioService.downloadFile({ + bucket: sticker.encoded.bucket, + key: sticker.encoded.key, + filePath: encodedFilename, + }); + } + }); + } + + async archiveUserImages (job) { + const SiteImage = mongoose.model('Image'); + const { image: imageService } = this.dtp.services; + + job.data.imagePath = path.join(job.data.workPath, 'images'); + await fs.promises.mkdir(job.data.imagePath, { recursive: true }); + + this.log.info('archiving user images', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await SiteImage + .find({ owner: job.data.user._id }) + .cursor() + .eachAsync(async (image) => { + try { + let imageExt = mime.getExtension(image.type); + const imageFilename = path.join(job.data.imagePath, `image-${image._id}.${imageExt}`); + const metadataFilename = path.join(job.data.imagePath, `image-${image._id}.metadata.json`); + + await imageService.downloadImage(image, imageFilename); + await fs.promises.writeFile(metadataFilename, JSON.stringify(image.metadata, null, 2)); + + } catch (error) { + this.log.error('failed to download image', { + image: { _id: image._id }, + error, + }); + } + }); + } + + async createArchiveFile (job) { + const { minio: minioService } = this.dtp.services; + try { + job.data.zipFilename = path.join(job.data.archivePath, `user-${job.data.userId}.zip`); + const zipArgs = [ + '-r', '-9', + job.data.zipFilename, + `${job.data.userId}`, + ]; + const options = { + cwd: job.data.archivePath, + encoding: 'utf8', + }; + await execFile('/usr/bin/zip', zipArgs, options); + + const zipFileStat = await fs.promises.stat(job.data.zipFilename); + this.log.info('zip archive created', { size: zipFileStat.size }); + + job.data.archiveFile = { + bucket: process.env.MINIO_ADMIN_BUCKET, + key: `/user-archive/user-${job.data.userId}.zip`, + }; + + const response = await minioService.uploadFile({ + bucket: job.data.archiveFile.bucket, + key: job.data.archiveFile.key, + filePath: job.data.zipFilename, + metadata: { + job: { + id: job.id, + }, + user: job.data.user, + } + }); + + this.log.info('creating user archive record', { etag: response.etag, size: zipFileStat.size }); + const UserArchive = mongoose.model('UserArchive'); + await UserArchive.create({ + created: job.data.startTime, + user: { + _id: job.data.userId, + username: job.data.user.username, + email: job.data.fullUser.email, + }, + archive: { + bucket: job.data.archiveFile.bucket, + key: job.data.archiveFile.key, + etag: response.etag, + size: zipFileStat.size, + } + }); + } catch (error) { + this.log.error('failed to create archive .zip file', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + throw error; + } finally { + try { + await fs.promises.rm(job.data.zipFilename, { force: true }); + } catch (error) { + this.log.error('failed to remove temp .zip file', { error }); + } + } + } +} + +module.exports = ArchiveUserLocalJob; \ No newline at end of file diff --git a/client/js/site-admin-app.js b/client/js/site-admin-app.js index 89e608a..9e38e35 100644 --- a/client/js/site-admin-app.js +++ b/client/js/site-admin-app.js @@ -544,6 +544,13 @@ export default class DtpSiteAdminHostStatsApp extends DtpApp { return; } + + async deleteImage (event) { + const target = event.currentTarget || event.target; + const imageId = target.getAttribute('data-image-id'); + const response = await fetch(`/admin/image/${imageId}`, { method: 'DELETE' }); + return this.processResponse(response); + } } dtp.DtpSiteAdminHostStatsApp = DtpSiteAdminHostStatsApp; \ No newline at end of file