You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
445 lines
14 KiB
445 lines
14 KiB
// reeeper/job/archive-user-local.js
|
|
// Copyright (C) 2022 DTP Technologies, LLC
|
|
// License: Apache-2.0
|
|
|
|
'use strict';
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
|
|
const util = require('util');
|
|
const execFile = util.promisify(require('child_process').execFile);
|
|
|
|
const mime = require('mime');
|
|
|
|
const mongoose = require('mongoose');
|
|
const User = mongoose.model('User');
|
|
|
|
const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
|
|
|
|
/**
|
|
* A job to archive and ban a User (local).
|
|
*
|
|
* 1. Immediately disable the specified User
|
|
* 2. Create a .zip file of the User's content on storage
|
|
* 3. Creates a UserArchive record for the file and User
|
|
* 4. Ban the User (removes all of the User's content)
|
|
* 5. Remove the User record from the database
|
|
*/
|
|
class ArchiveUserLocalJob extends SiteWorkerProcess {
|
|
|
|
static get COMPONENT ( ) {
|
|
return {
|
|
logId: 'wrk:reeeper:archive-user-local:job',
|
|
index: 'archiveUserLocalJob',
|
|
className: 'ArchiveUserLocalJob',
|
|
};
|
|
}
|
|
|
|
constructor (worker) {
|
|
super(worker, ArchiveUserLocalJob.COMPONENT);
|
|
this.jobs = new Set();
|
|
}
|
|
|
|
async start ( ) {
|
|
await super.start();
|
|
|
|
this.queue = await this.getJobQueue('reeeper', this.dtp.config.jobQueues.reeeper);
|
|
|
|
this.log.info('registering job processor', { queue: this.queue.name });
|
|
this.queue.process('archive-user-local', 1, this.processArchiveUserLocal.bind(this));
|
|
}
|
|
|
|
async stop ( ) {
|
|
try {
|
|
if (this.queue) {
|
|
this.log.info('halting job queue', { jobCount: this.jobs.size });
|
|
await this.queue.pause(true, false);
|
|
delete this.queue;
|
|
}
|
|
} catch (error) {
|
|
this.log.error('failed to halt job queue', { error });
|
|
// fall through
|
|
} finally {
|
|
await super.stop();
|
|
}
|
|
}
|
|
|
|
async processArchiveUserLocal (job) {
|
|
const { user: userService } = this.dtp.services;
|
|
try {
|
|
job.data.archivePath = path.join('/tmp', this.dtp.pkg.name, 'archive-user-local');
|
|
this.jobs.add(job);
|
|
|
|
job.data.userId = mongoose.Types.ObjectId(job.data.userId);
|
|
job.data.user = await userService.getLocalUserAccount(job.data.userId);
|
|
|
|
job.data.workPath = path.join(job.data.archivePath, job.data.userId.toString());
|
|
await fs.promises.mkdir(job.data.workPath, { recursive: true });
|
|
|
|
/*
|
|
* Save the User account data
|
|
*/
|
|
await this.archiveUserData(job);
|
|
|
|
/*
|
|
* Disable the User account (which destroys their session and cookie(s))
|
|
*/
|
|
await this.disableUser(job);
|
|
|
|
/*
|
|
* Archive the User's content to the workPath on the local file system.
|
|
*/
|
|
await this.archiveUserChat(job);
|
|
await this.archiveUserComments(job);
|
|
await this.archiveUserStickers(job);
|
|
await this.archiveUserImages(job);
|
|
await this.archiveUserAttachments(job);
|
|
|
|
/*
|
|
* Create the .zip file archive, upload it to storage, and create the
|
|
* UserArchive record.
|
|
*/
|
|
await this.createArchiveFile(job);
|
|
|
|
this.log.info('banning user', {
|
|
user: {
|
|
_id: job.data.userId,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
await userService.ban(job.data.user);
|
|
|
|
this.log.info('removing user', {
|
|
user: {
|
|
_id: job.data.userId,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
await User.deleteOne({ _id: job.data.userId });
|
|
} catch (error) {
|
|
this.log.error('failed to archive user', { userId: job.data.userId, error });
|
|
throw error;
|
|
} finally {
|
|
if (job.data.workPath) {
|
|
this.log.info('cleaning up work directory');
|
|
await fs.promises.rm(job.data.workPath, { force: true, recursive: true });
|
|
|
|
delete job.data.workPath;
|
|
}
|
|
this.jobs.delete(job);
|
|
this.log.info('job complete', { job: job.id });
|
|
}
|
|
}
|
|
|
|
async archiveUserData (job) {
|
|
// fetch the entire User record (all fields)
|
|
job.data.fullUser = await User
|
|
.findOne({ _id: job.data.user._id })
|
|
.select('+email +passwordSalt +password +flags +permissions +optIn')
|
|
.lean();
|
|
if (!job.data.fullUser) {
|
|
throw new Error('user does not exist');
|
|
}
|
|
|
|
const userFilename = path.join(job.data.workPath, `user-${job.data.user._id}.json`);
|
|
await fs.promises.writeFile(userFilename, JSON.stringify(job.data.fullUser, null, 2));
|
|
}
|
|
|
|
async disableUser (job) {
|
|
this.log.info('disabling local User account', {
|
|
user: {
|
|
_id: job.data.userId,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
await User.updateOne(
|
|
{ _id: job.data.user._id },
|
|
{
|
|
$set: {
|
|
'flags.isAdmin': false,
|
|
'flags.isModerator': false,
|
|
'flags.isEmailVerified': false,
|
|
'permissions.canLogin': false,
|
|
'permissions.canChat': false,
|
|
'permissions.canComment': false,
|
|
'permissions.canReport': false,
|
|
'optIn.system': false,
|
|
'optIn.marketing': false,
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
async archiveUserChat (job) {
|
|
const ChatMessage = mongoose.model('ChatMessage');
|
|
const ChatRoom = mongoose.model('ChatRoom');
|
|
|
|
job.data.chatPath = path.join(job.data.workPath, 'chat');
|
|
await fs.promises.mkdir(job.data.chatPath, { recursive: true });
|
|
|
|
this.log.info('archiving user chat', {
|
|
user: {
|
|
_id: job.data.user._id,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
|
|
await ChatRoom
|
|
.find({ owner: job.data.user._id })
|
|
|
|
.lean()
|
|
.cursor()
|
|
.eachAsync(async (room) => {
|
|
const roomFilename = path.join(job.data.workPath, 'chat', `room-${room._id}`);
|
|
await fs.promises.writeFile(roomFilename, JSON.stringify(room, null, 2));
|
|
});
|
|
|
|
await ChatMessage
|
|
.find({ author: job.data.user._id })
|
|
.lean()
|
|
.cursor()
|
|
.eachAsync(async (message) => {
|
|
const messageFilename = path.join(job.data.workPath, 'chat', `message-${message._id}.json`);
|
|
await fs.promises.writeFile(messageFilename, JSON.stringify(message, null, 2));
|
|
});
|
|
}
|
|
|
|
async archiveUserComments (job) {
|
|
const Comment = mongoose.model('Comment');
|
|
|
|
job.data.commentPath = path.join(job.data.workPath, 'comments');
|
|
await fs.promises.mkdir(job.data.commentPath, { recursive: true });
|
|
|
|
this.log.info('archiving user comments', {
|
|
user: {
|
|
_id: job.data.user._id,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
|
|
await Comment
|
|
.find({ author: job.data.userId })
|
|
.cursor()
|
|
.eachAsync(async (comment) => {
|
|
const commentFilename = path.join(job.data.commentPath, `comment-${comment._id}.json`);
|
|
await fs.promises.writeFile(commentFilename, JSON.stringify(comment, null, 2));
|
|
});
|
|
}
|
|
|
|
async archiveUserStickers (job) {
|
|
const Sticker = mongoose.model('Sticker');
|
|
const { minio: minioService } = this.dtp.services;
|
|
|
|
job.data.stickerPath = path.join(job.data.workPath, 'stickers');
|
|
await fs.promises.mkdir(job.data.stickerPath, { recursive: true });
|
|
|
|
job.data.stickerMediaPath = path.join(job.data.stickerPath, 'media');
|
|
await fs.promises.mkdir(job.data.stickerMediaPath, { recursive: true });
|
|
|
|
this.log.info('archiving user stickers', {
|
|
user: {
|
|
_id: job.data.user._id,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
|
|
await Sticker
|
|
.find({ owner: job.data.userId })
|
|
.cursor()
|
|
.eachAsync(async (sticker) => {
|
|
const stickerFilename = path.join(job.data.stickerPath, `sticker-${sticker._id}.json`);
|
|
await fs.promises.writeFile(stickerFilename, JSON.stringify(sticker, null, 2));
|
|
|
|
if (sticker.original && sticker.original.bucket && sticker.orignal.key && sticker.encoded.type) {
|
|
const originalExt = mime.getExtension(sticker.original.type);
|
|
const originalFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.original.${originalExt}`);
|
|
await minioService.downloadFile({
|
|
bucket: sticker.original.bucket,
|
|
key: sticker.original.key,
|
|
filePath: originalFilename,
|
|
});
|
|
}
|
|
|
|
if (sticker.encoded && sticker.encoded.bucket && sticker.encoded.key && sticker.encoded.type) {
|
|
const encodedExt = mime.getExtension(sticker.encoded.type);
|
|
const encodedFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.encoded.${encodedExt}`);
|
|
await minioService.downloadFile({
|
|
bucket: sticker.encoded.bucket,
|
|
key: sticker.encoded.key,
|
|
filePath: encodedFilename,
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async archiveUserImages (job) {
|
|
const SiteImage = mongoose.model('Image');
|
|
const { image: imageService } = this.dtp.services;
|
|
|
|
job.data.imagePath = path.join(job.data.workPath, 'images');
|
|
await fs.promises.mkdir(job.data.imagePath, { recursive: true });
|
|
|
|
this.log.info('archiving user images', {
|
|
user: {
|
|
_id: job.data.user._id,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
|
|
await SiteImage
|
|
.find({ owner: job.data.user._id })
|
|
.cursor()
|
|
.eachAsync(async (image) => {
|
|
try {
|
|
let imageExt = mime.getExtension(image.type);
|
|
const imageFilename = path.join(job.data.imagePath, `image-${image._id}.${imageExt}`);
|
|
const metadataFilename = path.join(job.data.imagePath, `image-${image._id}.metadata.json`);
|
|
|
|
await imageService.downloadImage(image, imageFilename);
|
|
await fs.promises.writeFile(metadataFilename, JSON.stringify(image.metadata, null, 2));
|
|
|
|
} catch (error) {
|
|
this.log.error('failed to download image', {
|
|
image: { _id: image._id },
|
|
error,
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async archiveUserAttachments (job) {
|
|
const Attachment = mongoose.model('Attachment');
|
|
const { minio: minioService } = this.dtp.services;
|
|
|
|
job.data.attachmentPath = path.join(job.data.workPath, 'attachments');
|
|
await fs.promises.mkdir(job.data.attachmentPath, { recursive: true });
|
|
|
|
job.data.originalAttachmentPath = path.join(job.data.attachmentPath, 'original');
|
|
await fs.promises.mkdir(job.data.originalAttachmentPath, { recursive: true });
|
|
|
|
job.data.encodedAttachmentPath = path.join(job.data.attachmentPath, 'encoded');
|
|
await fs.promises.mkdir(job.data.encodedAttachmentPath, { recursive: true });
|
|
|
|
this.log.info('archiving user attachments', {
|
|
user: {
|
|
_id: job.data.user._id,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
|
|
await Attachment
|
|
.find({ owner: job.data.user._id })
|
|
.cursor()
|
|
.eachAsync(async (attachment) => {
|
|
try {
|
|
/*
|
|
* Write the JSON record archive
|
|
*/
|
|
const metadataFilename = path.join(job.data.attachmentPath, `attachment-${attachment._id}.metadata.json`);
|
|
await fs.promises.writeFile(metadataFilename, JSON.stringify(attachment, null, 2));
|
|
|
|
/*
|
|
* Download and save the original file (if present)
|
|
*/
|
|
if (attachment.original && attachment.original.bucket && attachment.original.key) {
|
|
let originalExt = mime.getExtension(attachment.original.mime);
|
|
const originalFilename = path.join(job.data.originalAttachmentPath, `attachment-${attachment._id}.${originalExt}`);
|
|
await minioService.downloadFile({
|
|
bucket: attachment.original.bucket,
|
|
key: attachment.original.key,
|
|
filePath: originalFilename,
|
|
});
|
|
}
|
|
|
|
/*
|
|
* Download and save the encoded file (if present)
|
|
*/
|
|
if (attachment.encoded && attachment.encoded.bucket && attachment.encoded.key) {
|
|
let encodedExt = mime.getExtension(attachment.encoded.mime);
|
|
const encodedFilename = path.join(job.data.encodedAttachmentPath, `attachment-${attachment._id}.${encodedExt}`);
|
|
await minioService.downloadFile({
|
|
bucket: attachment.encoded.bucket,
|
|
key: attachment.encoded.key,
|
|
filePath: encodedFilename,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
this.log.error('failed to archive attachment', {
|
|
attachment: { _id: attachment._id },
|
|
error,
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async createArchiveFile (job) {
|
|
const { minio: minioService } = this.dtp.services;
|
|
try {
|
|
job.data.zipFilename = path.join(job.data.archivePath, `user-${job.data.userId}.zip`);
|
|
const zipArgs = [
|
|
'-r', '-9',
|
|
job.data.zipFilename,
|
|
`${job.data.userId}`,
|
|
];
|
|
const options = {
|
|
cwd: job.data.archivePath,
|
|
encoding: 'utf8',
|
|
};
|
|
await execFile('/usr/bin/zip', zipArgs, options);
|
|
|
|
const zipFileStat = await fs.promises.stat(job.data.zipFilename);
|
|
this.log.info('zip archive created', { size: zipFileStat.size });
|
|
|
|
job.data.archiveFile = {
|
|
bucket: process.env.MINIO_ADMIN_BUCKET,
|
|
key: `/user-archive/user-${job.data.userId}.zip`,
|
|
};
|
|
|
|
const response = await minioService.uploadFile({
|
|
bucket: job.data.archiveFile.bucket,
|
|
key: job.data.archiveFile.key,
|
|
filePath: job.data.zipFilename,
|
|
metadata: {
|
|
job: {
|
|
id: job.id,
|
|
},
|
|
user: job.data.user,
|
|
}
|
|
});
|
|
|
|
this.log.info('creating user archive record', { etag: response.etag, size: zipFileStat.size });
|
|
const UserArchive = mongoose.model('UserArchive');
|
|
await UserArchive.create({
|
|
created: job.data.startTime,
|
|
user: {
|
|
_id: job.data.userId,
|
|
username: job.data.user.username,
|
|
email: job.data.fullUser.email,
|
|
},
|
|
archive: {
|
|
bucket: job.data.archiveFile.bucket,
|
|
key: job.data.archiveFile.key,
|
|
etag: response.etag,
|
|
size: zipFileStat.size,
|
|
}
|
|
});
|
|
} catch (error) {
|
|
this.log.error('failed to create archive .zip file', {
|
|
user: {
|
|
_id: job.data.userId,
|
|
username: job.data.user.username,
|
|
},
|
|
});
|
|
throw error;
|
|
} finally {
|
|
try {
|
|
await fs.promises.rm(job.data.zipFilename, { force: true });
|
|
} catch (error) {
|
|
this.log.error('failed to remove temp .zip file', { error });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = ArchiveUserLocalJob; |