diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49fccdb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.vscode +.git +.env + +node_modules +npm-debug.log + +data/minio \ No newline at end of file diff --git a/.env.docker.default b/.env.docker.default new file mode 100644 index 0000000..8b32275 --- /dev/null +++ b/.env.docker.default @@ -0,0 +1,115 @@ +# +# Site Configuration +# + +DTP_SITE_NAME=DTP Base +DTP_SITE_DESCRIPTION=Enabling the success of the Next economy +DTP_SITE_DOMAIN=localhost +DTP_SITE_DOMAIN_KEY=webapp.digitaltelepresence.com +DTP_SITE_COMPANY=DTP Technologies, LLC +DTP_PASSWORD_SALT=b63ebaf1-3afb-4aee-9138-c42059a58a69 + +# +# Redis +# + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=1f9936ce-9d21-43dd-8638-908088f06a00 +REDIS_KEY_PREFIX=soapy + +# +# DTP Host Cache +# + +DTP_HOST_CACHE_PORT=8000 +DTP_HOST_CACHE_PATH=/tmp/dtp/host-cache +DTP_HOST_CACHE_AUTH_KEY=c071fa18-3c48-4337-a6af-6459ebdd8537 +DTP_HOST_CACHE_CLEAN_CRON=*/30 * * * * * + +# +# MongoDB +# + +MONGODB_HOST=mongodb:27017 +MONGODB_DATABASE=dtp-base +MONGODB_POOL_SIZE=10 + +# +# MinIO/S3 +# + +MINIO_ENDPOINT=nginx +MINIO_PORT=9000 +MINIO_USE_SSL=disabled +MINIO_ACCESS_KEY=dtp-base +MINIO_SECRET_KEY=44710206-9115-443b-b02d-2ee74afe156d + +MINIO_IMAGE_BUCKET=base-images +MINIO_VIDEO_BUCKET=base-videos + +# +# Web Application Server +# + +HTTP_BIND_ADDRESS=0.0.0.0 +HTTP_BIND_PORT=3000 +HTTP_SESSION_SECRET=1c7ea8dc-f92b-49b9-8f35-da040e8f7313 + +# +# Bitcoin Average API +# + +BITCOINAVERAGE_KEY=YjUzZjgzNTI3OWY2NDhhZTk3NDc1YTNjZjQ5YWZhYjE +BITCOINAVERAGE_CRON=0 */30 * * * * + +# +# SMTP/email configuration +# + +DTP_EMAIL_SERVICE=disabled + +DTP_EMAIL_SMTP_HOST= +DTP_EMAIL_SMTP_PORT=465 +DTP_EMAIL_SMTP_SECURE=enabled + +DTP_EMAIL_SMTP_FROM= +DTP_EMAIL_SMTP_USER= +DTP_EMAIL_SMTP_PASS= + +DTP_EMAIL_SMTP_POOL_ENABLED=enabled +DTP_EMAIL_SMTP_POOL_MAX_CONN=5 +DTP_EMAIL_SMTP_POOL_MAX_MSGS=100 + +# +# Work Paths +# + +DTP_IMAGE_WORK_PATH=/tmp/dtp/image-work +DTP_VIDEO_WORK_PATH=/tmp/dtp/video-work + +# +# FFMPEG options +# + +DTP_FFMPEG_PATH=/usr/bin/ffmpeg +DTP_FFPROBE_PATH=/usr/bin/ffprobe +DTP_ENABLE_GPU=disabled + +# +# System Logger options +# + +DTP_LOG_CONSOLE=enabled +DTP_LOG_MONGODB=enabled +DTP_LOG_FILE=enabled + +DTP_LOG_FILE_PATH=/tmp/dtp/logs +DTP_LOG_FILE_NAME_APP=base-app.log +DTP_LOG_FILE_NAME_HTTP=base-access.log + +DTP_LOG_DEBUG=enabled +DTP_LOG_INFO=enabled +DTP_LOG_WARN=enabled + +DTP_LOG_HTTP_FORMAT=combined \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2347915..00c8845 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .env +.env.docker +ssl/*crt +ssl/*key data/minio node_modules dist -data/minio diff --git a/.vscode/launch.json b/.vscode/launch.json index acce262..166b1b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "web:dtp-sites", "skipFiles": [ @@ -18,7 +18,7 @@ } }, { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "cli:dtp-sites", "skipFiles": [ @@ -29,7 +29,7 @@ "args": ["--action=reset-indexes", "all"] }, { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "worker:sample", "skipFiles": [ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..46ffe9e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM node:16.16-alpine + +WORKDIR /home/dtp/live/dtp-base +ENV NODE_ENV=production + +# Populate git deploy keys +COPY docker/.ssh /root/.ssh +RUN chown -R root:root /root/.ssh +RUN chmod go-rwx /root/.ssh +RUN chmod 644 /root/.ssh/config +RUN chmod 600 /root/.ssh/id_rsa + +# Install system dependencies +RUN apk update +RUN apk add ffmpeg git openrc openssh + +# Populate Node.js application +COPY package.json yarn.lock ./ +RUN yarn + +COPY . . \ No newline at end of file diff --git a/README.md b/README.md index df67f7b..f034a09 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ DTP Sites is an open source blogging platform built on Node.js and the DTP ecosy The only qualified operated system for hosting a DTP Sites suite is [Ubuntu 20.04 LTS](https://releases.ubuntu.com/20.04/). It is acceptable to run it in a virtual machine for development and testing, but it should be run as close to bare metal as can be had for production environments. +## Install Data Tier Components + You will need MongoDB and MinIO installed and running before you can start DTP Sites web services. 1. [Install MongoDB](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) @@ -16,17 +18,43 @@ Install redis: sudo apt-get install redis ``` -## Environment Configuration +## Install Node Version Manager (NVM) and Node.js -On a new host or host image, copy `.env.default` to `.env` and edit it as necessary. +```sh +wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash +``` -For password salt and service passwords, the `uuidgen` tool may be useful to help generate hard-to-guess passwords for use in a development environment. Something more sophisticated than a UUID should be used in production. +## Resolve Global Node Dependencies + +Yarn, Gulp, and Forever are required to be installed globally. This should be done once per developer workstation and repeated when updates are required. + +```sh +nvm install --lts +npm install -g yarn gulp forever +``` -Install application dependencies by running Yarn. You should do this on initial install and after every update or `git pull` as dependencies do change often, become upgraded to new versions, receive security fixes, etc. +## Preparing a Fresh Install +### Clone the DTP Base Repository ```sh +cd ~/live +git clone git@git.digitaltelepresence.com:digital-telepresence/dtp-base.git +``` +### Install Required Dependencies +```sh +cd ~/live/dtp-base yarn ``` +### Generate SSL/TLS Certificate +```sh +cd ~/live/dtp-base/ssl +./mkcert +``` +### Environment Configuration + +On a new host or host image, copy `.env.default` to `.env` and edit it as necessary. + +For password salt and service passwords, the `uuidgen` tool may be useful to help generate hard-to-guess passwords for use in a development environment. Something more sophisticated than a UUID should be used in production. ## Starting DTP Sites In Development Mode @@ -48,6 +76,8 @@ DTP Sites is a multi-tier web hosting engine built on: ## Production Environment Information +The only qualified operated system for hosting a DTP Base suite is [Ubuntu 20.04 LTS](https://releases.ubuntu.com/20.04/). It is acceptable to run it in a virtual machine for development and testing, but it should be run as close to bare metal as possible in production environments. + It's impossible to give 100% generic advice here, but it all depends on how large your audience is and how much they use your website. Larger, more active audiences will require a different kind of production server deployment than smaller and less active audiences. Generally, it's possible to stack all components on one host and operate a fast site for small-to-medium audiences. The size of the host may vary, but it's possible to keep the system stacked and handle quite a large audience. diff --git a/app/services/core-node.js b/app/services/core-node.js index ca57bc3..04666b1 100644 --- a/app/services/core-node.js +++ b/app/services/core-node.js @@ -313,6 +313,9 @@ class CoreNodeService extends SiteService { .cursor() .eachAsync(async (core) => { try { + if (!core.kaleidoscope || !core.kaleidoscope.token) { + throw new Error('Core has not provided a Kaleidoscope token'); + } const response = await this.sendRequest(core, request); results.push({ coreId: core._id, request, response }); } catch (error) { @@ -344,6 +347,10 @@ class CoreNodeService extends SiteService { }, }; + if (core.kaleidoscope && core.kaleidoscope.token) { + options.headers.Authorization = `Bearer ${core.kaleidoscope.token}`; + } + req.created = new Date(); req.core = core._id; if (request.tokenized) { @@ -529,6 +536,7 @@ class CoreNodeService extends SiteService { 'oauth.clientSecret': client.secret, 'oauth.scopes': client.scopes, 'oauth.redirectUri': client.redirectUri, + 'kaleidoscope.token': client.kaleidoscope.token, }, }, ); diff --git a/app/services/oauth2.js b/app/services/oauth2.js index 1f00130..00f0820 100644 --- a/app/services/oauth2.js +++ b/app/services/oauth2.js @@ -57,6 +57,7 @@ class OAuth2Service extends SiteService { registerPassport ( ) { const verifyClient = this.verifyClient.bind(this); const verifyHttpBearer = this.verifyHttpBearer.bind(this); + const verifyKaleidoscopeBearer = this.verifyKaleidoscopeBearer.bind(this); const basicStrategy = new BasicStrategy(verifyClient); this.log.info('registering Basic strategy', { name: basicStrategy.name }); @@ -69,6 +70,10 @@ class OAuth2Service extends SiteService { const httpBearerStrategy = new BearerStrategy(verifyHttpBearer); this.log.info('registering Bearer strategy', { name: httpBearerStrategy.name }); passport.use(httpBearerStrategy); + + const kaleidoscopeBearerStrategy = new BearerStrategy(verifyKaleidoscopeBearer); + this.log.info('registering Kaleidoscope Bearer strategy'); + passport.use(kaleidoscopeBearerStrategy); } async serializeClient (client, done) { @@ -392,6 +397,7 @@ class OAuth2Service extends SiteService { async verifyHttpBearer (accessToken, done) { const token = await this.getAccessToken(accessToken); if (!token) { + this.log.error('no bearer token for client', { accessToken }); return done(null, false); } return done(null, token.user, { scope: token.scope }); @@ -411,6 +417,45 @@ class OAuth2Service extends SiteService { .lean(); return tokens; } + + async getKaleidoscopeClient (accessToken) { + const client = await OAuth2Client + .findOne({ 'kaleidoscope.token': accessToken }) + .select('-secret -kaleidoscope -admin') // don't fetch them + .lean(); + if (!client) { + return; // we don't have one, be undefined + } + /* + * extreme paranoia also serializes the object to absolutely prevent leaking + * a secret even if the underlying Mongoose library has a bug today. + */ + return { + _id: client._id, + created: client.created, + updated: client.updated, + site: client.site, + scopes: client.scopes, + flags: client.flags, + }; + } + + async verifyKaleidoscopeBearer (accessToken, done) { + const client = await this.getKaleidoscopeClient(accessToken); + if (!client) { + this.log.error('no Kaleidoscope token for client', { accessToken }); + return done(null, false); + } + /* + * Minor hack here. You don't get a User or CoreUser for use with + * Kaleidoscope. This is machine-to-machine, there simply is no "user" in + * this transaction. Instead, you get a Client - the machine. + * + * So, up in controller space, req.user isn't a User or CoreUser for + * Kaleidoscope APIs. It is the OAuth2 Client or Service Node. + */ + return done(null, client); + } } module.exports = { diff --git a/app/views/layouts/main.pug b/app/views/layouts/main.pug index 1320560..09926c1 100644 --- a/app/views/layouts/main.pug +++ b/app/views/layouts/main.pug @@ -3,7 +3,7 @@ include ../components/page-sidebar include ../user/components/profile-icon doctype html html(lang='en') - - var DEFAULT_THEME = 'dtp-dark'; + - var DEFAULT_THEME = 'dtp-light'; head meta(charset='UTF-8') meta(name='viewport', content='width=device-width, initial-scale=1.0') diff --git a/docker-build b/docker-build new file mode 100755 index 0000000..034a464 --- /dev/null +++ b/docker-build @@ -0,0 +1,6 @@ +#!/bin/sh + +yarn --production=false +gulp build + +docker build . -t dtp/base \ No newline at end of file diff --git a/docker-clean b/docker-clean new file mode 100755 index 0000000..1007656 --- /dev/null +++ b/docker-clean @@ -0,0 +1,15 @@ +#!/bin/bash + +echo "stopping all containers" +docker stop $(docker ps -a -q) + +echo "removing all containers" +docker rm $(docker ps -a -q) + +echo "removing all volumes" +docker volume rm $(docker volume ls -q) + +echo "removing all images" +docker rmi $(docker images -a -q) + +echo "Docker cleanup done" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6c434af --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,195 @@ +version: '3.7' + +# Settings common for all MinIO containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2022-03-08T22-28-51Z + command: server --console-address ":9001" http://minio{1...4}/data{1...2} + expose: + - 9000 + - 9001 + environment: + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} + restart: always + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + +# Settings common for all DTP Base containers +x-base-common: &base-common + image: dtp/base + restart: always + stop_signal: SIGINT + env_file: + - .env.docker + environment: + - NODE_ENV=production + volumes: + - /tmp/dtp:/tmp/dtp + depends_on: + - redis + - mongodb + - minio1 + - minio2 + - minio3 + - minio4 + +services: + + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + + redis: + image: redis:6.2-alpine + hostname: redis + restart: always + expose: + - 6379 + command: redis-server --save 20 1 --loglevel warning --requirepass ${REDIS_PASSWORD} + volumes: + - redis_data:/data + + mongodb: + image: mongo:latest + hostname: mongodb + restart: always + expose: + - 27017 + volumes: + - mongodb_data:/data/db + + base-host-services: + <<: *base-common + hostname: base-host-services + command: node app/workers/host-services.js + environment: + - DTP_HOST_CACHE_HOST=0.0.0.0 + expose: + - "8000/udp" + + base-reeeper: + <<: *base-common + hostname: base-reeeper + command: node app/workers/reeeper.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + depends_on: + - base-host-services + + # base-analytics: + # <<: *base-common + # hostname: base-analytics + # command: node app/workers/base-analytics.js + # environment: + # - DTP_HOST_CACHE_HOST=base-host-services + + # Base Crypto + # Makes calls to a Cryptocurrency exchange API to fetch the current exchange + # rate versus the US Dollar. + # base-crypto: + # <<: *base-common + # hostname: base-crypto + # command: node app/workers/base-crypto.js + # environment: + # - DTP_HOST_CACHE_HOST=base-host-services + + # Base Newsletter Worker + # Processes newsletter transmit jobs and their email jobs. + base-newsletter: + <<: *base-common + hostname: base-newsletter + command: node app/workers/newsletter.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + depends_on: + - base-host-services + + # Base Web Application + # Provides the DTP Base Node.js web application + base-app-001: + <<: *base-common + hostname: base-app-001 + command: node dtp-webapp.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + expose: + - 3000 + depends_on: + - base-host-services + - base-newsletter + - base-reeeper + + # Nginx + # Provides an HTTP front-end to MinIO and the DTP Base application components + # running in their various containers in the stack. + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + restart: always + volumes: + - ./docker/etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "3000:3000" + - "8000:8000" + - "8001:8001" + - "9000:9000" + - "9001:9001" + depends_on: + - base-app-001 + - minio1 + - minio2 + - minio3 + - minio4 + + dns-proxy-server: + image: defreitas/dns-proxy-server + hostname: dns.mageddo + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /etc/resolv.conf:/etc/resolv.conf + ports: + - "5380:5380" + depends_on: + - nginx + - base-app-001 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + mongodb_data: + redis_data: + data1-1: + data1-2: + data2-1: + data2-2: + data3-1: + data3-2: + data4-1: + data4-2: \ No newline at end of file diff --git a/docker-console b/docker-console new file mode 100755 index 0000000..47ba7a0 --- /dev/null +++ b/docker-console @@ -0,0 +1,2 @@ +#!/bin/bash +docker-compose run base-host-services sh \ No newline at end of file diff --git a/docker-proxy b/docker-proxy new file mode 100755 index 0000000..5d208bb --- /dev/null +++ b/docker-proxy @@ -0,0 +1,5 @@ +#!/bin/bash +docker run --rm --hostname dns.mageddo --name dns-proxy-server -p 5380:5380 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /etc/resolv.conf:/etc/resolv.conf \ + defreitas/dns-proxy-server diff --git a/docker/.ssh/config b/docker/.ssh/config new file mode 100644 index 0000000..cf31d23 --- /dev/null +++ b/docker/.ssh/config @@ -0,0 +1,3 @@ +Host git.digitaltelepresence.com + User git + CertificateFile /root/.ssh/id_rsa \ No newline at end of file diff --git a/docker/.ssh/id_rsa b/docker/.ssh/id_rsa new file mode 100644 index 0000000..32c5b74 --- /dev/null +++ b/docker/.ssh/id_rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAs6z571mCUPSki/q8EupgNSm1hGNrjkahAxUrBCdcgIKnAXwSvWNw +xIj5fpetMQeappVbJHm62qqFC4kjV5wLTL+oNxDW/SAGOjO0dmSK7Uq2TiWoD5LkPA4ASg +Q/casnK7klJpQUozwAPx/HBK42sLyI2E6TmWjhCbf+ul2OrRQk3YzrZpj2bySCn2363QOA +8DNVfvaxQ8CoK6xkMmZXNLkIoEZs3CL6Qv1BJqiJ/6Ad6jPiyKEN7RAmNjlfrc/J9D0U8K +Af68LPer1Bbm5Sb2RQ65vL9HGSxvl7NFyviQo45WluBrWdnMPTLR7aNl61MWPYwjGoO11f +5jAHN/qmYN9NX9j3U8fdvWd9Dzryb8KHAOUkYFrCmx1z94uudyIrQn8gySNNEezoyTTUu+ +AcBQclob/SiYwiRGq13EldzazIdqbcavEm7xYKFqEShUv74oqm+NVDgbGWay1ZzmwfnfS1 +UoBBMA2XW23T2vM3l+wMzI6GIz9NXypHE9m8rSafAAAFmEVSTaJFUk2iAAAAB3NzaC1yc2 +EAAAGBALOs+e9ZglD0pIv6vBLqYDUptYRja45GoQMVKwQnXICCpwF8Er1jcMSI+X6XrTEH +mqaVWyR5utqqhQuJI1ecC0y/qDcQ1v0gBjoztHZkiu1Ktk4lqA+S5DwOAEoEP3GrJyu5JS +aUFKM8AD8fxwSuNrC8iNhOk5lo4Qm3/rpdjq0UJN2M62aY9m8kgp9t+t0DgPAzVX72sUPA +qCusZDJmVzS5CKBGbNwi+kL9QSaoif+gHeoz4sihDe0QJjY5X63PyfQ9FPCgH+vCz3q9QW +5uUm9kUOuby/Rxksb5ezRcr4kKOOVpbga1nZzD0y0e2jZetTFj2MIxqDtdX+YwBzf6pmDf +TV/Y91PH3b1nfQ868m/ChwDlJGBawpsdc/eLrnciK0J/IMkjTRHs6Mk01LvgHAUHJaG/0o +mMIkRqtdxJXc2syHam3GrxJu8WChahEoVL++KKpvjVQ4GxlmstWc5sH530tVKAQTANl1tt +09rzN5fsDMyOhiM/TV8qRxPZvK0mnwAAAAMBAAEAAAGBAJ5C2IuI7g5czqs1mX+XzlJFgM +sxv0U81QogFRm9ie7wJiZaoS8MbgHPHOPyueyb5A6jM/q29m0xEauNigHbQuDBWGvYe2XO +VW3iqSw3d194FYhUBupg2oRDWOHZSADZXtFWbXDDskHt0dHILcbjpOybxHWzELNp+/5RgR ++q1MeDHM3BGH/wb/CJ2jyP5Q3TzjjFHqOMq6sgAYEmgcMa/elEqmJyZeSpMoLbDA3/Gj59 +TJxEQ5ewtNGBDgdpFevFP4XFz1wT4FHzuKzNPxygqhp9lPwp4Q8iHOYYIFCDY1AYcji+jc +YSLD4M5AnRFfzn+Vdb0zIAlnnjmxqcd0mDGXUfLeP7VxKR/H+RrKGzxVN3tiYrValX7eA0 +ZVVFi3OIvuAEb+CCBDb/yb5AT+Z3067f/4XhEg0wgFKnbJC3ZlffFQYIEWpvtSRoJAjMEW +1qDaKnMW665cJw/euqgLvYLUZ2mqDlAX12uwAmOYTCc/UWYoUCLLa+aDLwczjHEBoYAQAA +AMAYQ2OPyXoRGNMZwYMNjFG9oeZu9sDJDgP4vjGMgk6Wg5+7mHGYc4xIio3R1c5FTYl99s +25FeNtDX/DtybBi+8gSl4I+2Z5RIX7/VgU0TlfqZB/PjBKLMiAkZCb81ueURc4Zbl+jl7O +G1lbNZsITYmWrxk9evGTgoCN05TSJD2f/CtDeLyxUedX1YZgW+bRRsNPCILBksBH6Gm4Qv +moo1RC8MQVvw9Ry6MspkmGXL0thddlUQsSxLCsG5jaCw9iIIwAAADBAN3q1XJBQcxhwCeD +Ro3NKFT73ZVywF3KGMBaPspUEZl+eqnF+o8Y6kqijS9xIpfwfmaWOtzIA7QPghWEwAAICU +lZmjlQGqmPAfzI6OII+m5EE74AmCP78/ijgZwW8zPmQibrXXymBYBtjI+8+szBMuEV/jQS +hO4OY5aY/3bwpiECvqgMmppyIU25tkc656oG7e/95zFdfLGxd441HmlNckctLU6S9J3bch +wFJtVtNf2OBMKk2eZzdN1d8mIm7/smAQAAAMEAz0VR9d0a9YqlZ0NULQ8UraAJtgJQz3L+ +6p3S7TJratwCszswHA15f+jD+FEBYKebfY28fN516IyB2vWmOD9f74/5nQsR+tUJVa3lQV +qPUpnOMRpVRwiE9tCqbVwNqgxZvy/SbQp1+lW299DkM5OfpQw0RaDr8b6XgZaS/0SNbGhG +pZOLmziXXUXVGLR3W685+PMscIlc1XzDCNGKoioa20f9+1M+R76S81Is6ktU2F0OuFrSc7 +hUJEaB3q7m6IyfAAAAH3N1cHBvcnRAZGlnaXRhbHRlbGVwcmVzZW5jZS5jb20BAgM= +-----END OPENSSH PRIVATE KEY----- diff --git a/docker/.ssh/id_rsa.pub b/docker/.ssh/id_rsa.pub new file mode 100644 index 0000000..01e9282 --- /dev/null +++ b/docker/.ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzrPnvWYJQ9KSL+rwS6mA1KbWEY2uORqEDFSsEJ1yAgqcBfBK9Y3DEiPl+l60xB5qmlVskebraqoULiSNXnAtMv6g3ENb9IAY6M7R2ZIrtSrZOJagPkuQ8DgBKBD9xqycruSUmlBSjPAA/H8cErjawvIjYTpOZaOEJt/66XY6tFCTdjOtmmPZvJIKfbfrdA4DwM1V+9rFDwKgrrGQyZlc0uQigRmzcIvpC/UEmqIn/oB3qM+LIoQ3tECY2OV+tz8n0PRTwoB/rws96vUFublJvZFDrm8v0cZLG+Xs0XK+JCjjlaW4GtZ2cw9MtHto2XrUxY9jCMag7XV/mMAc3+qZg301f2PdTx929Z30POvJvwocA5SRgWsKbHXP3i653IitCfyDJI00R7OjJNNS74BwFByWhv9KJjCJEarXcSV3NrMh2ptxq8SbvFgoWoRKFS/viiqb41UOBsZZrLVnObB+d9LVSgEEwDZdbbdPa8zeX7AzMjoYjP01fKkcT2bytJp8= support@digitaltelepresence.com diff --git a/docker/.ssh/known_hosts b/docker/.ssh/known_hosts new file mode 100644 index 0000000..07effca --- /dev/null +++ b/docker/.ssh/known_hosts @@ -0,0 +1,2 @@ +|1|5jczd9sZCMV0zY+B2tHMNweYEhY=|Nh/+S/QWhYLG+vfiwY0J8VuBKfA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLNxeRruHUCy2HUyFVnOKv17vQbFth6iqc8T1i1DyRKC2nrS15VIEnwfHPo0qR7merRJEa/50CyunYY4PvbeYiU= +|1|Dy5o3mQ4SXKNMhgMYNzYVdi/KAs=|vg+t7yhXLjtkX3yd54VRzRROJXo= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLNxeRruHUCy2HUyFVnOKv17vQbFth6iqc8T1i1DyRKC2nrS15VIEnwfHPo0qR7merRJEa/50CyunYY4PvbeYiU= \ No newline at end of file diff --git a/docker/etc/nginx/nginx.conf b/docker/etc/nginx/nginx.conf new file mode 100644 index 0000000..df8d8c1 --- /dev/null +++ b/docker/etc/nginx/nginx.conf @@ -0,0 +1,138 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream base_app { + least_conn; + server base-app-001:3000; + } + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } + + # DTP Base Application Server + # Provides load-balanced front-end to the Base web app + server { + server_name localhost; + + listen [::]:3000 ipv6only=on; + listen 3000; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 20m; + + location / { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_pass http://base_app; + } + } +} \ No newline at end of file diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..9980948 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,26 @@ +# Docker Support + +Docker is only supported for use in development/local environments to emulate a production environment with scaled-out infrastructure. The Docker environment is defined and described in `docker-compose.yml` + +The host services worker is required to be operating on each host. And, we tend to think of a Docker container as a "host" - but it's not. And, they can easily map a path to a path on the host's file system. So, we map /tmp/dtp on the host into each DTP container, and we point all DTP processes at one container running the host services worker. Do not emulate that topology in a production environment, it would defeat the purpose of the host services worker as an edge cache. + +## Initialize the Environment +```sh +docker-compose run sites-host-services node dtp-sites-cli.js --action=create-domain shing.tv +``` + +## Grand Admin Access +```sh +docker-compose run sites-host-services node dtp-sites-cli.js --action=grant --permission=admin [email address] +``` + +## Start the Environment + +```sh +docker-compose up -d +``` + +## Stop the Environment +```sh +docker-compose down +``` \ No newline at end of file diff --git a/static/502.html b/static/502.html new file mode 100644 index 0000000..0c172c0 --- /dev/null +++ b/static/502.html @@ -0,0 +1,61 @@ + + + + + + DTP Soapbox Community Engine + + + + + +
+
+ DTP Core Logo +

The system is down for maintenance and will be back online shortly...

+
+
+ + + \ No newline at end of file diff --git a/static/img/dtp-core.svg b/static/img/dtp-core.svg new file mode 100644 index 0000000..9b1f0be --- /dev/null +++ b/static/img/dtp-core.svg @@ -0,0 +1,400 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/supervisord/dtp-base-newsletter.conf b/supervisord/dtp-base-newsletter.conf new file mode 100644 index 0000000..7a710b8 --- /dev/null +++ b/supervisord/dtp-base-newsletter.conf @@ -0,0 +1,13 @@ +[program:dtp-base:newsletter] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/newsletter.err.log +stdout_logfile=/var/log/dtp-base/newsletter.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsletter \ No newline at end of file diff --git a/supervisord/dtp-base.conf b/supervisord/dtp-base.conf new file mode 100644 index 0000000..cea7c16 --- /dev/null +++ b/supervisord/dtp-base.conf @@ -0,0 +1,13 @@ +[program:dtp-base] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/dtp-base.err.log +stdout_logfile=/var/log/dtp-base/dtp-base.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=dtp-base \ No newline at end of file diff --git a/supervisord/host-services.conf b/supervisord/dtp-sites-host-services.conf similarity index 90% rename from supervisord/host-services.conf rename to supervisord/dtp-sites-host-services.conf index 6f4f63f..ef11c57 100644 --- a/supervisord/host-services.conf +++ b/supervisord/dtp-sites-host-services.conf @@ -10,4 +10,4 @@ stopsignal=INT stderr_logfile=/var/log/dtp-sites/host-services.err.log stdout_logfile=/var/log/dtp-sites/host-services.out.log user=dtp -environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services +environment=HOME='/home/dtp/live/dtp-sites',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services \ No newline at end of file