From c4489f6385df56220330b9b796c8b664a33e991a Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 17 Jul 2022 22:15:01 -0400 Subject: [PATCH 1/7] Docker integration; cleanup of config file names --- .dockerignore | 8 + .env.docker.default | 115 +++++++ .gitignore | 4 +- Dockerfile | 21 ++ README.md | 40 ++- docker-build | 6 + docker-clean | 15 + docker-compose.yml | 195 ++++++++++++ docker-console | 2 + docker-proxy | 5 + docker/.ssh/config | 3 + docker/.ssh/id_rsa | 38 +++ docker/.ssh/id_rsa.pub | 1 + docker/.ssh/known_hosts | 2 + docker/etc/nginx/nginx.conf | 138 ++++++++ docs/docker.md | 26 ++ static/502.html | 61 ++++ static/img/dtp-core.svg | 400 ++++++++++++++++++++++++ supervisord/dtp-base-host-services.conf | 13 + supervisord/dtp-base-newsletter.conf | 13 + supervisord/dtp-base.conf | 13 + supervisord/dtp-webapp.conf | 13 - supervisord/host-services.conf | 13 - 23 files changed, 1111 insertions(+), 34 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.docker.default create mode 100644 Dockerfile create mode 100755 docker-build create mode 100755 docker-clean create mode 100644 docker-compose.yml create mode 100755 docker-console create mode 100755 docker-proxy create mode 100644 docker/.ssh/config create mode 100644 docker/.ssh/id_rsa create mode 100644 docker/.ssh/id_rsa.pub create mode 100644 docker/.ssh/known_hosts create mode 100644 docker/etc/nginx/nginx.conf create mode 100644 docs/docker.md create mode 100644 static/502.html create mode 100644 static/img/dtp-core.svg create mode 100644 supervisord/dtp-base-host-services.conf create mode 100644 supervisord/dtp-base-newsletter.conf create mode 100644 supervisord/dtp-base.conf delete mode 100644 supervisord/dtp-webapp.conf delete mode 100644 supervisord/host-services.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49fccdb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.vscode +.git +.env + +node_modules +npm-debug.log + +data/minio \ No newline at end of file diff --git a/.env.docker.default b/.env.docker.default new file mode 100644 index 0000000..8b32275 --- /dev/null +++ b/.env.docker.default @@ -0,0 +1,115 @@ +# +# Site Configuration +# + +DTP_SITE_NAME=DTP Base +DTP_SITE_DESCRIPTION=Enabling the success of the Next economy +DTP_SITE_DOMAIN=localhost +DTP_SITE_DOMAIN_KEY=webapp.digitaltelepresence.com +DTP_SITE_COMPANY=DTP Technologies, LLC +DTP_PASSWORD_SALT=b63ebaf1-3afb-4aee-9138-c42059a58a69 + +# +# Redis +# + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=1f9936ce-9d21-43dd-8638-908088f06a00 +REDIS_KEY_PREFIX=soapy + +# +# DTP Host Cache +# + +DTP_HOST_CACHE_PORT=8000 +DTP_HOST_CACHE_PATH=/tmp/dtp/host-cache +DTP_HOST_CACHE_AUTH_KEY=c071fa18-3c48-4337-a6af-6459ebdd8537 +DTP_HOST_CACHE_CLEAN_CRON=*/30 * * * * * + +# +# MongoDB +# + +MONGODB_HOST=mongodb:27017 +MONGODB_DATABASE=dtp-base +MONGODB_POOL_SIZE=10 + +# +# MinIO/S3 +# + +MINIO_ENDPOINT=nginx +MINIO_PORT=9000 +MINIO_USE_SSL=disabled +MINIO_ACCESS_KEY=dtp-base +MINIO_SECRET_KEY=44710206-9115-443b-b02d-2ee74afe156d + +MINIO_IMAGE_BUCKET=base-images +MINIO_VIDEO_BUCKET=base-videos + +# +# Web Application Server +# + +HTTP_BIND_ADDRESS=0.0.0.0 +HTTP_BIND_PORT=3000 +HTTP_SESSION_SECRET=1c7ea8dc-f92b-49b9-8f35-da040e8f7313 + +# +# Bitcoin Average API +# + +BITCOINAVERAGE_KEY=YjUzZjgzNTI3OWY2NDhhZTk3NDc1YTNjZjQ5YWZhYjE +BITCOINAVERAGE_CRON=0 */30 * * * * + +# +# SMTP/email configuration +# + +DTP_EMAIL_SERVICE=disabled + +DTP_EMAIL_SMTP_HOST= +DTP_EMAIL_SMTP_PORT=465 +DTP_EMAIL_SMTP_SECURE=enabled + +DTP_EMAIL_SMTP_FROM= +DTP_EMAIL_SMTP_USER= +DTP_EMAIL_SMTP_PASS= + +DTP_EMAIL_SMTP_POOL_ENABLED=enabled +DTP_EMAIL_SMTP_POOL_MAX_CONN=5 +DTP_EMAIL_SMTP_POOL_MAX_MSGS=100 + +# +# Work Paths +# + +DTP_IMAGE_WORK_PATH=/tmp/dtp/image-work +DTP_VIDEO_WORK_PATH=/tmp/dtp/video-work + +# +# FFMPEG options +# + +DTP_FFMPEG_PATH=/usr/bin/ffmpeg +DTP_FFPROBE_PATH=/usr/bin/ffprobe +DTP_ENABLE_GPU=disabled + +# +# System Logger options +# + +DTP_LOG_CONSOLE=enabled +DTP_LOG_MONGODB=enabled +DTP_LOG_FILE=enabled + +DTP_LOG_FILE_PATH=/tmp/dtp/logs +DTP_LOG_FILE_NAME_APP=base-app.log +DTP_LOG_FILE_NAME_HTTP=base-access.log + +DTP_LOG_DEBUG=enabled +DTP_LOG_INFO=enabled +DTP_LOG_WARN=enabled + +DTP_LOG_HTTP_FORMAT=combined \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2347915..00c8845 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .env +.env.docker +ssl/*crt +ssl/*key data/minio node_modules dist -data/minio diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..46ffe9e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM node:16.16-alpine + +WORKDIR /home/dtp/live/dtp-base +ENV NODE_ENV=production + +# Populate git deploy keys +COPY docker/.ssh /root/.ssh +RUN chown -R root:root /root/.ssh +RUN chmod go-rwx /root/.ssh +RUN chmod 644 /root/.ssh/config +RUN chmod 600 /root/.ssh/id_rsa + +# Install system dependencies +RUN apk update +RUN apk add ffmpeg git openrc openssh + +# Populate Node.js application +COPY package.json yarn.lock ./ +RUN yarn + +COPY . . \ No newline at end of file diff --git a/README.md b/README.md index 0103145..2e1d88d 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ The base project from which all others are forked when working with my framework and system. You don't have to start from this project at all. BUT, it can save you a lot of time by simply being a 100% compatible base on which you can build your apps and sites. -## Requirements - -The only qualified operated system for hosting a DTP Base suite is [Ubuntu 20.04 LTS](https://releases.ubuntu.com/20.04/). It is acceptable to run it in a virtual machine for development and testing, but it should be run as close to bare metal as can be had for production environments. +## Install Data Tier Components You will need MongoDB and MinIO installed and running before you can start DTP Base web services. @@ -16,17 +14,43 @@ Install redis: sudo apt-get install redis ``` -## Environment Configuration +## Install Node Version Manager (NVM) and Node.js -On a new host or host image, copy `.env.default` to `.env` and edit it as necessary. +```sh +wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash +``` -For password salt and service passwords, the `uuidgen` tool may be useful to help generate hard-to-guess passwords for use in a development environment. Something more sophisticated than a UUID should be used in production. +## Resolve Global Node Dependencies + +Yarn, Gulp, and Forever are required to be installed globally. This should be done once per developer workstation and repeated when updates are required. + +```sh +nvm install --lts +npm install -g yarn gulp forever +``` -Install application dependencies by running Yarn. You should do this on initial install and after every update or `git pull` as dependencies do change often, become upgraded to new versions, receive security fixes, etc. +## Preparing a Fresh Install +### Clone the DTP Base Repository ```sh +cd ~/live +git clone git@git.digitaltelepresence.com:digital-telepresence/dtp-base.git +``` +### Install Required Dependencies +```sh +cd ~/live/dtp-base yarn ``` +### Generate SSL/TLS Certificate +```sh +cd ~/live/dtp-base/ssl +./mkcert +``` +### Environment Configuration + +On a new host or host image, copy `.env.default` to `.env` and edit it as necessary. + +For password salt and service passwords, the `uuidgen` tool may be useful to help generate hard-to-guess passwords for use in a development environment. Something more sophisticated than a UUID should be used in production. ## Starting DTP Base In Development Mode @@ -48,6 +72,8 @@ DTP Base is a multi-tier web hosting engine built on: ## Production Environment Information +The only qualified operated system for hosting a DTP Base suite is [Ubuntu 20.04 LTS](https://releases.ubuntu.com/20.04/). It is acceptable to run it in a virtual machine for development and testing, but it should be run as close to bare metal as possible in production environments. + It's impossible to give 100% generic advice here, but it all depends on how large your audience is and how much they use your website. Larger, more active audiences will require a different kind of production server deployment than smaller and less active audiences. Generally, it's possible to stack all components on one host and operate a fast site for small-to-medium audiences. The size of the host may vary, but it's possible to keep the system stacked and handle quite a large audience. diff --git a/docker-build b/docker-build new file mode 100755 index 0000000..034a464 --- /dev/null +++ b/docker-build @@ -0,0 +1,6 @@ +#!/bin/sh + +yarn --production=false +gulp build + +docker build . -t dtp/base \ No newline at end of file diff --git a/docker-clean b/docker-clean new file mode 100755 index 0000000..1007656 --- /dev/null +++ b/docker-clean @@ -0,0 +1,15 @@ +#!/bin/bash + +echo "stopping all containers" +docker stop $(docker ps -a -q) + +echo "removing all containers" +docker rm $(docker ps -a -q) + +echo "removing all volumes" +docker volume rm $(docker volume ls -q) + +echo "removing all images" +docker rmi $(docker images -a -q) + +echo "Docker cleanup done" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6c434af --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,195 @@ +version: '3.7' + +# Settings common for all MinIO containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2022-03-08T22-28-51Z + command: server --console-address ":9001" http://minio{1...4}/data{1...2} + expose: + - 9000 + - 9001 + environment: + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} + restart: always + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + +# Settings common for all DTP Base containers +x-base-common: &base-common + image: dtp/base + restart: always + stop_signal: SIGINT + env_file: + - .env.docker + environment: + - NODE_ENV=production + volumes: + - /tmp/dtp:/tmp/dtp + depends_on: + - redis + - mongodb + - minio1 + - minio2 + - minio3 + - minio4 + +services: + + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + + redis: + image: redis:6.2-alpine + hostname: redis + restart: always + expose: + - 6379 + command: redis-server --save 20 1 --loglevel warning --requirepass ${REDIS_PASSWORD} + volumes: + - redis_data:/data + + mongodb: + image: mongo:latest + hostname: mongodb + restart: always + expose: + - 27017 + volumes: + - mongodb_data:/data/db + + base-host-services: + <<: *base-common + hostname: base-host-services + command: node app/workers/host-services.js + environment: + - DTP_HOST_CACHE_HOST=0.0.0.0 + expose: + - "8000/udp" + + base-reeeper: + <<: *base-common + hostname: base-reeeper + command: node app/workers/reeeper.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + depends_on: + - base-host-services + + # base-analytics: + # <<: *base-common + # hostname: base-analytics + # command: node app/workers/base-analytics.js + # environment: + # - DTP_HOST_CACHE_HOST=base-host-services + + # Base Crypto + # Makes calls to a Cryptocurrency exchange API to fetch the current exchange + # rate versus the US Dollar. + # base-crypto: + # <<: *base-common + # hostname: base-crypto + # command: node app/workers/base-crypto.js + # environment: + # - DTP_HOST_CACHE_HOST=base-host-services + + # Base Newsletter Worker + # Processes newsletter transmit jobs and their email jobs. + base-newsletter: + <<: *base-common + hostname: base-newsletter + command: node app/workers/newsletter.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + depends_on: + - base-host-services + + # Base Web Application + # Provides the DTP Base Node.js web application + base-app-001: + <<: *base-common + hostname: base-app-001 + command: node dtp-webapp.js + environment: + - DTP_HOST_CACHE_HOST=base-host-services + expose: + - 3000 + depends_on: + - base-host-services + - base-newsletter + - base-reeeper + + # Nginx + # Provides an HTTP front-end to MinIO and the DTP Base application components + # running in their various containers in the stack. + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + restart: always + volumes: + - ./docker/etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "3000:3000" + - "8000:8000" + - "8001:8001" + - "9000:9000" + - "9001:9001" + depends_on: + - base-app-001 + - minio1 + - minio2 + - minio3 + - minio4 + + dns-proxy-server: + image: defreitas/dns-proxy-server + hostname: dns.mageddo + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /etc/resolv.conf:/etc/resolv.conf + ports: + - "5380:5380" + depends_on: + - nginx + - base-app-001 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + mongodb_data: + redis_data: + data1-1: + data1-2: + data2-1: + data2-2: + data3-1: + data3-2: + data4-1: + data4-2: \ No newline at end of file diff --git a/docker-console b/docker-console new file mode 100755 index 0000000..47ba7a0 --- /dev/null +++ b/docker-console @@ -0,0 +1,2 @@ +#!/bin/bash +docker-compose run base-host-services sh \ No newline at end of file diff --git a/docker-proxy b/docker-proxy new file mode 100755 index 0000000..5d208bb --- /dev/null +++ b/docker-proxy @@ -0,0 +1,5 @@ +#!/bin/bash +docker run --rm --hostname dns.mageddo --name dns-proxy-server -p 5380:5380 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /etc/resolv.conf:/etc/resolv.conf \ + defreitas/dns-proxy-server diff --git a/docker/.ssh/config b/docker/.ssh/config new file mode 100644 index 0000000..cf31d23 --- /dev/null +++ b/docker/.ssh/config @@ -0,0 +1,3 @@ +Host git.digitaltelepresence.com + User git + CertificateFile /root/.ssh/id_rsa \ No newline at end of file diff --git a/docker/.ssh/id_rsa b/docker/.ssh/id_rsa new file mode 100644 index 0000000..32c5b74 --- /dev/null +++ b/docker/.ssh/id_rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAs6z571mCUPSki/q8EupgNSm1hGNrjkahAxUrBCdcgIKnAXwSvWNw +xIj5fpetMQeappVbJHm62qqFC4kjV5wLTL+oNxDW/SAGOjO0dmSK7Uq2TiWoD5LkPA4ASg +Q/casnK7klJpQUozwAPx/HBK42sLyI2E6TmWjhCbf+ul2OrRQk3YzrZpj2bySCn2363QOA +8DNVfvaxQ8CoK6xkMmZXNLkIoEZs3CL6Qv1BJqiJ/6Ad6jPiyKEN7RAmNjlfrc/J9D0U8K +Af68LPer1Bbm5Sb2RQ65vL9HGSxvl7NFyviQo45WluBrWdnMPTLR7aNl61MWPYwjGoO11f +5jAHN/qmYN9NX9j3U8fdvWd9Dzryb8KHAOUkYFrCmx1z94uudyIrQn8gySNNEezoyTTUu+ +AcBQclob/SiYwiRGq13EldzazIdqbcavEm7xYKFqEShUv74oqm+NVDgbGWay1ZzmwfnfS1 +UoBBMA2XW23T2vM3l+wMzI6GIz9NXypHE9m8rSafAAAFmEVSTaJFUk2iAAAAB3NzaC1yc2 +EAAAGBALOs+e9ZglD0pIv6vBLqYDUptYRja45GoQMVKwQnXICCpwF8Er1jcMSI+X6XrTEH +mqaVWyR5utqqhQuJI1ecC0y/qDcQ1v0gBjoztHZkiu1Ktk4lqA+S5DwOAEoEP3GrJyu5JS +aUFKM8AD8fxwSuNrC8iNhOk5lo4Qm3/rpdjq0UJN2M62aY9m8kgp9t+t0DgPAzVX72sUPA +qCusZDJmVzS5CKBGbNwi+kL9QSaoif+gHeoz4sihDe0QJjY5X63PyfQ9FPCgH+vCz3q9QW +5uUm9kUOuby/Rxksb5ezRcr4kKOOVpbga1nZzD0y0e2jZetTFj2MIxqDtdX+YwBzf6pmDf +TV/Y91PH3b1nfQ868m/ChwDlJGBawpsdc/eLrnciK0J/IMkjTRHs6Mk01LvgHAUHJaG/0o +mMIkRqtdxJXc2syHam3GrxJu8WChahEoVL++KKpvjVQ4GxlmstWc5sH530tVKAQTANl1tt +09rzN5fsDMyOhiM/TV8qRxPZvK0mnwAAAAMBAAEAAAGBAJ5C2IuI7g5czqs1mX+XzlJFgM +sxv0U81QogFRm9ie7wJiZaoS8MbgHPHOPyueyb5A6jM/q29m0xEauNigHbQuDBWGvYe2XO +VW3iqSw3d194FYhUBupg2oRDWOHZSADZXtFWbXDDskHt0dHILcbjpOybxHWzELNp+/5RgR ++q1MeDHM3BGH/wb/CJ2jyP5Q3TzjjFHqOMq6sgAYEmgcMa/elEqmJyZeSpMoLbDA3/Gj59 +TJxEQ5ewtNGBDgdpFevFP4XFz1wT4FHzuKzNPxygqhp9lPwp4Q8iHOYYIFCDY1AYcji+jc +YSLD4M5AnRFfzn+Vdb0zIAlnnjmxqcd0mDGXUfLeP7VxKR/H+RrKGzxVN3tiYrValX7eA0 +ZVVFi3OIvuAEb+CCBDb/yb5AT+Z3067f/4XhEg0wgFKnbJC3ZlffFQYIEWpvtSRoJAjMEW +1qDaKnMW665cJw/euqgLvYLUZ2mqDlAX12uwAmOYTCc/UWYoUCLLa+aDLwczjHEBoYAQAA +AMAYQ2OPyXoRGNMZwYMNjFG9oeZu9sDJDgP4vjGMgk6Wg5+7mHGYc4xIio3R1c5FTYl99s +25FeNtDX/DtybBi+8gSl4I+2Z5RIX7/VgU0TlfqZB/PjBKLMiAkZCb81ueURc4Zbl+jl7O +G1lbNZsITYmWrxk9evGTgoCN05TSJD2f/CtDeLyxUedX1YZgW+bRRsNPCILBksBH6Gm4Qv +moo1RC8MQVvw9Ry6MspkmGXL0thddlUQsSxLCsG5jaCw9iIIwAAADBAN3q1XJBQcxhwCeD +Ro3NKFT73ZVywF3KGMBaPspUEZl+eqnF+o8Y6kqijS9xIpfwfmaWOtzIA7QPghWEwAAICU +lZmjlQGqmPAfzI6OII+m5EE74AmCP78/ijgZwW8zPmQibrXXymBYBtjI+8+szBMuEV/jQS +hO4OY5aY/3bwpiECvqgMmppyIU25tkc656oG7e/95zFdfLGxd441HmlNckctLU6S9J3bch +wFJtVtNf2OBMKk2eZzdN1d8mIm7/smAQAAAMEAz0VR9d0a9YqlZ0NULQ8UraAJtgJQz3L+ +6p3S7TJratwCszswHA15f+jD+FEBYKebfY28fN516IyB2vWmOD9f74/5nQsR+tUJVa3lQV +qPUpnOMRpVRwiE9tCqbVwNqgxZvy/SbQp1+lW299DkM5OfpQw0RaDr8b6XgZaS/0SNbGhG +pZOLmziXXUXVGLR3W685+PMscIlc1XzDCNGKoioa20f9+1M+R76S81Is6ktU2F0OuFrSc7 +hUJEaB3q7m6IyfAAAAH3N1cHBvcnRAZGlnaXRhbHRlbGVwcmVzZW5jZS5jb20BAgM= +-----END OPENSSH PRIVATE KEY----- diff --git a/docker/.ssh/id_rsa.pub b/docker/.ssh/id_rsa.pub new file mode 100644 index 0000000..01e9282 --- /dev/null +++ b/docker/.ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzrPnvWYJQ9KSL+rwS6mA1KbWEY2uORqEDFSsEJ1yAgqcBfBK9Y3DEiPl+l60xB5qmlVskebraqoULiSNXnAtMv6g3ENb9IAY6M7R2ZIrtSrZOJagPkuQ8DgBKBD9xqycruSUmlBSjPAA/H8cErjawvIjYTpOZaOEJt/66XY6tFCTdjOtmmPZvJIKfbfrdA4DwM1V+9rFDwKgrrGQyZlc0uQigRmzcIvpC/UEmqIn/oB3qM+LIoQ3tECY2OV+tz8n0PRTwoB/rws96vUFublJvZFDrm8v0cZLG+Xs0XK+JCjjlaW4GtZ2cw9MtHto2XrUxY9jCMag7XV/mMAc3+qZg301f2PdTx929Z30POvJvwocA5SRgWsKbHXP3i653IitCfyDJI00R7OjJNNS74BwFByWhv9KJjCJEarXcSV3NrMh2ptxq8SbvFgoWoRKFS/viiqb41UOBsZZrLVnObB+d9LVSgEEwDZdbbdPa8zeX7AzMjoYjP01fKkcT2bytJp8= support@digitaltelepresence.com diff --git a/docker/.ssh/known_hosts b/docker/.ssh/known_hosts new file mode 100644 index 0000000..07effca --- /dev/null +++ b/docker/.ssh/known_hosts @@ -0,0 +1,2 @@ +|1|5jczd9sZCMV0zY+B2tHMNweYEhY=|Nh/+S/QWhYLG+vfiwY0J8VuBKfA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLNxeRruHUCy2HUyFVnOKv17vQbFth6iqc8T1i1DyRKC2nrS15VIEnwfHPo0qR7merRJEa/50CyunYY4PvbeYiU= +|1|Dy5o3mQ4SXKNMhgMYNzYVdi/KAs=|vg+t7yhXLjtkX3yd54VRzRROJXo= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLNxeRruHUCy2HUyFVnOKv17vQbFth6iqc8T1i1DyRKC2nrS15VIEnwfHPo0qR7merRJEa/50CyunYY4PvbeYiU= \ No newline at end of file diff --git a/docker/etc/nginx/nginx.conf b/docker/etc/nginx/nginx.conf new file mode 100644 index 0000000..df8d8c1 --- /dev/null +++ b/docker/etc/nginx/nginx.conf @@ -0,0 +1,138 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream base_app { + least_conn; + server base-app-001:3000; + } + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } + + # DTP Base Application Server + # Provides load-balanced front-end to the Base web app + server { + server_name localhost; + + listen [::]:3000 ipv6only=on; + listen 3000; + + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 20m; + + location / { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_pass http://base_app; + } + } +} \ No newline at end of file diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..9980948 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,26 @@ +# Docker Support + +Docker is only supported for use in development/local environments to emulate a production environment with scaled-out infrastructure. The Docker environment is defined and described in `docker-compose.yml` + +The host services worker is required to be operating on each host. And, we tend to think of a Docker container as a "host" - but it's not. And, they can easily map a path to a path on the host's file system. So, we map /tmp/dtp on the host into each DTP container, and we point all DTP processes at one container running the host services worker. Do not emulate that topology in a production environment, it would defeat the purpose of the host services worker as an edge cache. + +## Initialize the Environment +```sh +docker-compose run sites-host-services node dtp-sites-cli.js --action=create-domain shing.tv +``` + +## Grand Admin Access +```sh +docker-compose run sites-host-services node dtp-sites-cli.js --action=grant --permission=admin [email address] +``` + +## Start the Environment + +```sh +docker-compose up -d +``` + +## Stop the Environment +```sh +docker-compose down +``` \ No newline at end of file diff --git a/static/502.html b/static/502.html new file mode 100644 index 0000000..0c172c0 --- /dev/null +++ b/static/502.html @@ -0,0 +1,61 @@ + + + + + + DTP Soapbox Community Engine + + + + + +
+
+ DTP Core Logo +

The system is down for maintenance and will be back online shortly...

+
+
+ + + \ No newline at end of file diff --git a/static/img/dtp-core.svg b/static/img/dtp-core.svg new file mode 100644 index 0000000..9b1f0be --- /dev/null +++ b/static/img/dtp-core.svg @@ -0,0 +1,400 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/supervisord/dtp-base-host-services.conf b/supervisord/dtp-base-host-services.conf new file mode 100644 index 0000000..c4754cb --- /dev/null +++ b/supervisord/dtp-base-host-services.conf @@ -0,0 +1,13 @@ +[program:dtp-base:host-services] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/host-services.err.log +stdout_logfile=/var/log/dtp-base/host-services.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services \ No newline at end of file diff --git a/supervisord/dtp-base-newsletter.conf b/supervisord/dtp-base-newsletter.conf new file mode 100644 index 0000000..7a710b8 --- /dev/null +++ b/supervisord/dtp-base-newsletter.conf @@ -0,0 +1,13 @@ +[program:dtp-base:newsletter] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/newsletter.err.log +stdout_logfile=/var/log/dtp-base/newsletter.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsletter \ No newline at end of file diff --git a/supervisord/dtp-base.conf b/supervisord/dtp-base.conf new file mode 100644 index 0000000..cea7c16 --- /dev/null +++ b/supervisord/dtp-base.conf @@ -0,0 +1,13 @@ +[program:dtp-base] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/dtp-base.err.log +stdout_logfile=/var/log/dtp-base/dtp-base.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=dtp-base \ No newline at end of file diff --git a/supervisord/dtp-webapp.conf b/supervisord/dtp-webapp.conf deleted file mode 100644 index 08e1971..0000000 --- a/supervisord/dtp-webapp.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:dtp-webapp] -numprocs=1 -process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js -directory=/home/dtp/live/dtp-webapp -autostart=true -autorestart=true -startretries=3 -stopsignal=INT -stderr_logfile=/var/log/dtp-webapp/dtp-webapp.err.log -stdout_logfile=/var/log/dtp-webapp/dtp-webapp.out.log -user=dtp -environment=HOME='/home/dtp/live/dtp-webapp',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=dtp-webapp \ No newline at end of file diff --git a/supervisord/host-services.conf b/supervisord/host-services.conf deleted file mode 100644 index 1be12d6..0000000 --- a/supervisord/host-services.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:host-services] -numprocs=1 -process_name=%(program_name)s_%(process_num)02d -command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js -directory=/home/dtp/live/dtp-webapp -autostart=true -autorestart=true -startretries=3 -stopsignal=INT -stderr_logfile=/var/log/dtp-webapp/host-services.err.log -stdout_logfile=/var/log/dtp-webapp/host-services.out.log -user=dtp -environment=HOME='/home/dtp/live/dtp-webapp',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services \ No newline at end of file From f6f1bc380393b239c3b1cb5b46e5ce4801817d97 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 16:33:31 -0400 Subject: [PATCH 2/7] switch back to light mode default --- app/views/layouts/main.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/main.pug b/app/views/layouts/main.pug index 7bf97be..2d93af1 100644 --- a/app/views/layouts/main.pug +++ b/app/views/layouts/main.pug @@ -3,7 +3,7 @@ include ../components/page-sidebar include ../user/components/profile-icon doctype html html(lang='en') - - var DEFAULT_THEME = 'dtp-dark'; + - var DEFAULT_THEME = 'dtp-light'; head meta(charset='UTF-8') meta(name='viewport', content='width=device-width, initial-scale=1.0') From 40b03a5d1ee886ff447fbedb8853ec6da8fd69b1 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 16:33:47 -0400 Subject: [PATCH 3/7] store and use Kaleidoscope Bearer token --- app/services/core-node.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/services/core-node.js b/app/services/core-node.js index cb13e93..37bc706 100644 --- a/app/services/core-node.js +++ b/app/services/core-node.js @@ -344,6 +344,10 @@ class CoreNodeService extends SiteService { }, }; + if (core.kaleidoscope && core.kaleidoscope.token) { + options.headers.Bearer = core.kaleidoscope.token; + } + req.created = new Date(); req.core = core._id; if (request.tokenized) { @@ -529,6 +533,7 @@ class CoreNodeService extends SiteService { 'oauth.clientSecret': client.secret, 'oauth.scopes': client.scopes, 'oauth.redirectUri': client.redirectUri, + 'kaleidoscope.token': client.kaleidoscope.token, }, }, ); From c414885964721187bbd231c824b8675ad9265a08 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 16:49:41 -0400 Subject: [PATCH 4/7] corrected form for compliance --- app/services/core-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/core-node.js b/app/services/core-node.js index 37bc706..75de2a4 100644 --- a/app/services/core-node.js +++ b/app/services/core-node.js @@ -345,7 +345,7 @@ class CoreNodeService extends SiteService { }; if (core.kaleidoscope && core.kaleidoscope.token) { - options.headers.Bearer = core.kaleidoscope.token; + options.headers.Authorization = `Bearer ${core.kaleidoscope.token}`; } req.created = new Date(); From f0471e2d2087e88cd9a2d5f66bd3a2ef6b3ccc07 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 16:55:15 -0400 Subject: [PATCH 5/7] switch from pwa-node to node as type --- .vscode/launch.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ca2f26a..2727e2f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "web:dtp-webapp", "skipFiles": [ @@ -18,7 +18,7 @@ } }, { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "cli:dtp-webapp", "skipFiles": [ @@ -29,7 +29,7 @@ "args": ["--action=reset-indexes", "all"] }, { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "worker:sample", "skipFiles": [ From 8e29963eeb9555fc3e3f6dc29d24082e84d8078d Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 18:18:09 -0400 Subject: [PATCH 6/7] don't send request if we don't have a Kaleidoscope token --- app/services/core-node.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/core-node.js b/app/services/core-node.js index 75de2a4..a5963de 100644 --- a/app/services/core-node.js +++ b/app/services/core-node.js @@ -313,6 +313,9 @@ class CoreNodeService extends SiteService { .cursor() .eachAsync(async (core) => { try { + if (!core.kaleidoscope || !core.kaleidoscope.token) { + throw new Error('Core has not provided a Kaleidoscope token'); + } const response = await this.sendRequest(core, request); results.push({ coreId: core._id, request, response }); } catch (error) { From cc47c52a9de4a605cd07b30bc9db60e5979a0954 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 20 Jul 2022 18:21:42 -0400 Subject: [PATCH 7/7] verifyKaleidoscopeToken and support --- app/services/oauth2.js | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/app/services/oauth2.js b/app/services/oauth2.js index 1f00130..00f0820 100644 --- a/app/services/oauth2.js +++ b/app/services/oauth2.js @@ -57,6 +57,7 @@ class OAuth2Service extends SiteService { registerPassport ( ) { const verifyClient = this.verifyClient.bind(this); const verifyHttpBearer = this.verifyHttpBearer.bind(this); + const verifyKaleidoscopeBearer = this.verifyKaleidoscopeBearer.bind(this); const basicStrategy = new BasicStrategy(verifyClient); this.log.info('registering Basic strategy', { name: basicStrategy.name }); @@ -69,6 +70,10 @@ class OAuth2Service extends SiteService { const httpBearerStrategy = new BearerStrategy(verifyHttpBearer); this.log.info('registering Bearer strategy', { name: httpBearerStrategy.name }); passport.use(httpBearerStrategy); + + const kaleidoscopeBearerStrategy = new BearerStrategy(verifyKaleidoscopeBearer); + this.log.info('registering Kaleidoscope Bearer strategy'); + passport.use(kaleidoscopeBearerStrategy); } async serializeClient (client, done) { @@ -392,6 +397,7 @@ class OAuth2Service extends SiteService { async verifyHttpBearer (accessToken, done) { const token = await this.getAccessToken(accessToken); if (!token) { + this.log.error('no bearer token for client', { accessToken }); return done(null, false); } return done(null, token.user, { scope: token.scope }); @@ -411,6 +417,45 @@ class OAuth2Service extends SiteService { .lean(); return tokens; } + + async getKaleidoscopeClient (accessToken) { + const client = await OAuth2Client + .findOne({ 'kaleidoscope.token': accessToken }) + .select('-secret -kaleidoscope -admin') // don't fetch them + .lean(); + if (!client) { + return; // we don't have one, be undefined + } + /* + * extreme paranoia also serializes the object to absolutely prevent leaking + * a secret even if the underlying Mongoose library has a bug today. + */ + return { + _id: client._id, + created: client.created, + updated: client.updated, + site: client.site, + scopes: client.scopes, + flags: client.flags, + }; + } + + async verifyKaleidoscopeBearer (accessToken, done) { + const client = await this.getKaleidoscopeClient(accessToken); + if (!client) { + this.log.error('no Kaleidoscope token for client', { accessToken }); + return done(null, false); + } + /* + * Minor hack here. You don't get a User or CoreUser for use with + * Kaleidoscope. This is machine-to-machine, there simply is no "user" in + * this transaction. Instead, you get a Client - the machine. + * + * So, up in controller space, req.user isn't a User or CoreUser for + * Kaleidoscope APIs. It is the OAuth2 Client or Service Node. + */ + return done(null, client); + } } module.exports = {