Then reload this page. We're upgrading the server.
\n\n\n\n\n" }, { "path": "conf/web/sites-enabled/talkyard-servers.conf", "content": "## Nginx `server {}` blocks for your Talkyard forum.\n##\n## There's 1) a HTTP server, 2) a HTTPS server with auto generated LetsEncrypt certs.\n##\n## To redirect HTTP to HTTPS:\n## Comment out the 'include /etc/nginx/...' lines in the HTTP server (not the HTTPS server).\n## Comment in the 'return 302 ...' line.\n##\n## There's also 3) an out commented server block that shows how you can\n## add your own server with a custom cert, e.g. a wildcard cert.\n##\n## Note: If you comment-in/add-more server blocks, don't include 'backlog=8192'\n## in their listen directives — otherwise there'll be a \"duplicate listen options\"\n## Nginx error. The backlog should be the same as net.core.somaxconn in /etc/sysctl.conf,\n## namely 8192, set in /opt/talkyard-v1/scripts/prepare-os.sh [BACKLGSZ]\n## — but one may specify this in only one place.\n##\n\n\n## HTTP Server.\n## Replies to HTTPS cert challenges, can redirect to HTTPS.\n##\nserver {\n listen 80 backlog=8192; # about backlog: see above [BACKLGSZ]\n ## Using ipv6 here, can prevent Nginx from starting, if the host OS has disabled ipv6,\n ## Nginx then won't start and says: [ipv6_probl]\n # [emerg] socket() [::]:80 failed (97: Address family not supported by protocol)\n #listen [::]:80 backlog=8192;\n\n server_name _;\n\n ## For generating HTTPS certs via LetsEncrypt, HTTP-01 challenge.\n location /.well-known/acme-challenge {\n content_by_lua_block {\n ngx.log(ngx.INFO, \"Replying to ACME HTTP-01 challenge\" ..\n \", server name: \" .. ngx.var.server_name ..\n \", host: \" .. ngx.var.http_host .. \" [TyNACMEHTTP01]\")\n require(\"resty.acme.autossl\").serve_http_challenge()\n }\n }\n\n ## To redirect to HTTPS, comment out these two includes, and comment in\n ## \"location / { return 302 ... }\" below.\n include /etc/nginx/server-limits.conf;\n include /etc/nginx/server-locations.conf;\n\n ## Redirect from HTTP to HTTPS.\n ## Use temp redirects (302) not permanent (301) in case you'll want to allow\n ## http in the future, for some reason.\n #location / {\n # return 302 https://$http_host$request_uri;\n #}\n}\n\n\n## HTTPS Server with LetsEncrypt auto generated certs.\n##\nserver {\n listen 443 ssl default_server backlog=8192; # [BACKLGSZ]\n #listen [::]:443 ssl default_server backlog=8192; # [ipv6_probl]\n http2 on;\n\n server_name _;\n\n ## Required, or Nginx won't start. Gets used until we've gotten a LetsEncrypt cert\n ## (sth like 10 seconds after first HTTPS request to the server addr).\n ssl_certificate /etc/nginx/generated/https-cert-self-signed-fallback.pem;\n ssl_certificate_key /etc/nginx/generated/https-cert-self-signed-fallback.key;\n\n ssl_certificate_by_lua_block {\n require(\"resty.acme.autossl\").ssl_certificate()\n }\n\n ## For generating HTTPS certs via LetsEncrypt, TLS-ALPN-01 challenge\n ## (which works over HTTPS, unlike the HTTP-01 challenge).\n ## Disabled in nginx.conf, because experimental in the lua-resty-acme plugin.\n #location /.well-known/acme-challenge {\n # content_by_lua_block {\n # ngx.log(ngx.INFO, \"Replying to ACME TLS-ALPN-01 challenge\")\n # -- Cannot access here?:\n # -- \", server name: \" .. ngx.var.server_name .. \" [TyNACMEALPN01]\")\n # require(\"resty.acme.autossl\").serve_tls_alpn_challenge()\n # }\n #}\n\n include /etc/nginx/server-ssl.conf;\n include /etc/nginx/server-limits.conf;\n include /etc/nginx/server-locations.conf;\n}\n\n\n## HTTPS Server with custom (e.g. wildcard) HTTPS cert\n## ----------------------------------------------------\n\n## Redirect port 80 to 443: (without generating any LetsEncrypt cert, not needed)\n##\n#server {\n# listen 80; # backlog=8192; # [BACKLGSZ]\n# server_name ~^(.*)\\.example\\.com$;\n# return 302 https://$http_host$request_uri;\n#}\n\n## HTTPS Server with custom cert\n## Docs: http://nginx.org/en/docs/http/configuring_https_servers.html\n##\n#server {\n# ## Comment out 'backlog=...' if you also use a LetsEncrypt auto cert server (above).\n# ## Nginx won't start if 'backlog=...' is present at two places.\n# listen 443 ssl; # backlog=8192; # [BACKLGSZ]\n# #listen [::]:443 ssl; # backlog=8192; # [ipv6_probl]\n# http2 on;\n#\n# server_name ~^(.*)\\.example\\.com$;\n#\n# ssl_certificate /etc/certbot/live/example.com-0001/fullchain.pem;\n# ssl_certificate_key /etc/certbot/live/example.com-0001/privkey.pem;\n#\n# include /etc/nginx/server-ssl.conf;\n# include /etc/nginx/server-limits.conf;\n# include /etc/nginx/server-locations.conf;\n#}\n\n" }, { "path": "debug.yml", "content": "# Opens ports to Docker containers so you can connect and debug.\n# Only reachable locally (since listens on 127.0.0.1), so, access via an SSH tunnel.\n\nservices:\n app:\n ports:\n - '127.0.0.1:9000:9000' # Play Framework's HTTP listen port, for bypasssing Nginx (to debug)\n - '127.0.0.1:9443:9443' # Play Framework's HTTPS port\n - '127.0.0.1:9999:9999' # Java debugger port\n - '127.0.0.1:3333:3333' # JMX\n\n cache:\n ports:\n - '127.0.0.1:6379:6379'\n\n rdb:\n ports:\n - '127.0.0.1:5432:5432'\n\n search:\n ports:\n - '127.0.0.1:9200:9200'\n\n" }, { "path": "docker-compose.yml", "content": "# Talkyard, discussion forum software, v1. Production installation on a single server.\n#\n# Dockerfiles for the Docker images are in another Git repo:\n# https://github.com/debiki/talkyard, at: images/(service-name)/Dockerfile\n#\n# There's an image build script: https://github.com/debiki/talkyard/blob/main/Makefile,\n# the `prod-images` and `tag-and-push-latest-images` targets.\n\n# Docker Compose file spec:\n# https://github.com/compose-spec/compose-spec/blob/main/spec.md\n\n\n# You can override this in docker-compose.override.yml.\n# Volume names get prefixed by the COMPOSE_PROJECT_NAME = 'talkyard-v1_', from .env.\nvolumes:\n web-generated:\n pub-files:\n priv-files:\n redis-data:\n pg-data:\n es-data:\n es-logs:\n\n# See: https://docs.docker.com/compose/how-tos/use-secrets/\nsecrets:\n postgres_password:\n file: ./secrets/postgres_password.txt\n #backup_password:\n # file: ./secrets/backup_password.txt\n\n# The netw names get prefixed with the COMPOSE_PROJECT_NAME.\nnetworks:\n # Frontend public network. Connects Nginx to the Internet.\n fe_pub_net:\n ipam:\n config:\n - subnet: ${FE_PUB_NET_SUBNET}\n # So Nginx can talk to Redis and the app server.\n fe_int_net:\n internal: true\n ipam:\n config:\n - subnet: ${FE_INT_NET_SUBNET}\n # Backend network: the app server and various databases. Is an internal network,\n # not directly connected to the Internet.\n # See: https://docs.docker.com/reference/compose-file/networks/#internal\n be_int_net:\n internal: true\n ipam:\n config:\n - subnet: ${BE_INT_NET_SUBNET}\n # So the app server can connect to the egress proxy.\n eg_int_net:\n internal: true\n ipam:\n config:\n - subnet: ${EG_INT_NET_SUBNET}\n # So the egress proxy can access the Internet, e.g. to send webhooks or fetch link previews.\n eg_pub_net:\n ipam:\n config:\n - subnet: ${EG_PUB_NET_SUBNET}\n\n# See: https://github.com/compose-spec/compose-spec/blob/main/spec.md#logging\nx-logging: &default_logging\n # This driver rotates logs, so won't run out of disk.\n driver: local\n\n\nservices:\n web:\n image: ${DOCKER_REG_ORG}/talkyard-web:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/web/Dockerfile\n restart: always\n volumes:\n # Nginx `server {...}` blocks for your Talkyard forum.\n - ./conf/web/sites-enabled:/etc/nginx/sites-enabled:ro\n # A LetsEncrypt ACME account key gets generated on startup and saved here.\n # Once done, you could make this volume read-only: append ':ro' to the next line.\n - web-generated:/etc/nginx/generated\n # Don't, however, mount priv-files here.\n - pub-files:/var/talkyard/v1/pub-files:ro\n\n # Optionally, add a wildcard cert volume here:\n # talkyard-wildcard-certs:/etc/wildcard-certs:ro\n # Fetch certs in a Docker container using e.g. LEGO https://github.com/go-acme/lego\n # that mounts the same volume, run in a cron job,\n # and edit: ./conf/web/sites-enabled/talkyard-servers.conf\n # add e.g.: ssl_certificate /etc/wildcard-certs/...something.crt;\n # and: ssl_certificate_key ...\n\n ports:\n - '80:80'\n - '443:443'\n networks:\n fe_pub_net:\n ipv4_address: ${FE_PUB_NET_WEB_IP}\n fe_int_net:\n ipv4_address: ${FE_INT_NET_WEB_IP}\n depends_on:\n - app\n - cache\n logging: *default_logging\n environment:\n TY_NGX_ERROR_LOG_LEVEL: 'info' # or 'notice' or 'debug'\n # TY_NGX_ACCESS_LOG_CONFIG: 'tyalogfmt'\n ## Max uploaded file size, e.g. uploaded images or backups to restore:\n # TY_NGX_LIMIT_REQ_BODY_SIZE: '25m'\n ## To let any CDN of yours bypass Ty's Nginx rate limits:\n # X_PULL_KEY: '...'\n # CDN_PULL_KEY: '...'\n\n security_opt:\n - no-new-privileges=true\n # When cap_drop is ALL, it gets processed before cap_add, see:\n # https://stackoverflow.com/a/63219871\n # and:\n # https://github.com/moby/moby/blob/1c39b1c44c973f18f39bd684c6aba57bb96510fe/oci/caps/utils.go#L120\n cap_drop:\n - ALL\n cap_add:\n # Without CHOWN:\n # nginx: [emerg] chown(\"/opt/nginx/proxy-cache\", 100) failed (1: Operation not permitted)\n - CHOWN\n # To bypass file read, write, and execute permission checks:\n # (DAC means \"discretionary access control\", and DAC_OVERRIDE)\n # Without DAC_OVERRIDE:\n # nginx: [emerg] mkdir() \"/var/cache/nginx/client_temp\" failed (13: Permission denied)\n - DAC_OVERRIDE\n # To change the group and user id of a process: (so Nginx won't need to run as root)\n - SETGID\n - SETUID\n # To bind to lower ports. Maybe listen on 8080 instead, and map port 80:8080?\n - NET_BIND_SERVICE\n\n app:\n image: ${DOCKER_REG_ORG}/talkyard-app:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/app/Dockerfile.prod\n restart: always\n stdin_open: true # otherwise Play Framework exits\n volumes:\n - ./conf/app/play-framework.conf:/opt/talkyard/app/conf/app-prod-override.conf:ro # see [4WDKPU2] in github.com/debiki/talkyard\n - pub-files:/var/talkyard/v1/pub-files\n - priv-files:/var/talkyard/v1/priv-files\n # So backups can be downloaded via the admin web interface. But read-only,\n # so evil bugs cannot destroy all backups.\n # - /var/opt/backups/talkyard/v1/:/var/opt/backups/talkyard/v1/:ro\n secrets:\n - postgres_password\n networks:\n fe_int_net:\n ipv4_address: ${FE_INT_NET_APP_IP}\n be_int_net:\n ipv4_address: ${BE_INT_NET_APP_IP}\n eg_int_net:\n ipv4_address: ${EG_INT_NET_APP_IP}\n depends_on:\n # Start also if other services are unhealthy, so can show admin troubleshooting tips.\n - rendr\n - cache\n - rdb\n - search\n - egressp\n logging: *default_logging\n environment:\n # The `TALKYARD_SECURE: '${TALKYARD_SECURE}'` syntax doesn't work — results in\n # the env vars getting set to an empty string, but Play Framework wants a bool\n # and refuses to start.\n - TALKYARD_SECURE\n # The app always looks here, not configurable. [postgres_pw_path]\n # POSTGRES_PASSWORD_FILE=/tmp/postgres_password\n - TALKYARD_HOSTNAME\n - BECOME_OWNER_EMAIL_ADDRESS\n # [egressp_conf]\n - 'http_proxy=http://egressp:4750'\n - 'https_proxy=http://egressp:4750'\n - \"no_proxy=''\"\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n cap_add:\n - CHOWN # to make secrets readable to 'appuser'\n - FOWNER #\n - DAC_OVERRIDE\n - SETUID # to switch to 'appuser'\n - SETGID # 'appgroup'\n\n rendr:\n image: ${DOCKER_REG_ORG}/talkyard-rendr:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/rendr/Dockerfile\n networks:\n be_int_net:\n ipv4_address: ${BE_INT_NET_RENDR_IP}\n logging: *default_logging\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n\n cache:\n image: ${DOCKER_REG_ORG}/talkyard-cache:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/cache/Dockerfile\n restart: always\n volumes:\n - redis-data:/data\n networks:\n fe_int_net:\n ipv4_address: ${FE_INT_NET_CACHE_IP}\n logging: *default_logging\n sysctls:\n net.core.somaxconn: 511\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n cap_add: # [enough_caps]? See: https://github.com/docker-library/postgres/issues/649\n - CHOWN # [redis_cap_chown]\n - SETGID\n - SETUID\n - SETPCAP\n\n rdb:\n image: ${DOCKER_REG_ORG}/talkyard-rdb:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/rdb/Dockerfile\n restart: always\n secrets:\n - postgres_password\n volumes:\n - pg-data:/var/lib/postgresql\n networks:\n be_int_net:\n ipv4_address: ${BE_INT_NET_RDB_IP}\n logging: *default_logging\n environment:\n POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n cap_add: # [enough_caps]? See: https://github.com/docker-library/postgres/issues/649\n - CHOWN\n - DAC_READ_SEARCH\n - DAC_OVERRIDE # to copy Postgres secret\n - FOWNER\n - SETGID\n - SETUID\n - SYS_NICE\n\n search:\n image: ${DOCKER_REG_ORG}/talkyard-search:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/search/Dockerfile\n restart: always\n volumes:\n - es-data:/usr/share/elasticsearch/data\n # Deprecation logs aren't written to stdout, but to files in this dir,\n # auto rotated, max 5G in total.\n # See: https://www.elastic.co/docs/deploy-manage/monitor/logging-configuration/elasticsearch-deprecation-logs\n - es-logs:/usr/share/elasticsearch/logs\n networks:\n be_int_net:\n ipv4_address: ${BE_INT_NET_SEARCH_IP}\n logging: *default_logging\n environment:\n bootstrap.memory_lock: 'true'\n ES_JAVA_OPTS: '-Xms512m -Xmx512m'\n ulimits:\n memlock:\n soft: -1\n hard: -1\n nofile:\n soft: 65536\n hard: 65536\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n cap_add: # [enough_caps]?\n # SYS_CHROOT? See https://github.com/nasa-jpl/ASSESS/blob/master/docker-compose.yml\n - SETGID\n - SETUID\n\n # Egress proxy.\n # Stops Server Side Request Forgery (SSRF).\n egressp:\n image: ${DOCKER_REG_ORG}/talkyard-egressp:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n restart: always\n networks:\n eg_int_net:\n ipv4_address: ${EG_INT_NET_EGRESSP_IP}\n eg_pub_net:\n ipv4_address: ${EG_PUB_NET_EGRESSP_IP}\n logging: *default_logging\n read_only: true\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n\n backup:\n image: ${DOCKER_REG_ORG}/talkyard-backup:${PINNED_VERSION_TAG:-${VERSION_TAG}}\n # dockerfile: https://github.com/debiki/talkyard/blob/main/images/backup/Dockerfile\n profiles: [backup] # prevents auto-start on 'up'\n # If under heavy load, run the backup script at 25% priority. (Default is 1024 shares.)\n cpu_shares: 256\n # Don't let the backup container use more than 60% CPU (even if the system is idle),\n # otherwise on a small VM the kernel might panic [100_kernel_panic].\n cpu_quota: 60000\n cpu_period: 100000\n secrets:\n # Will appear at /run/secrets/postgres_password. [backup_pg_client_secret]\n - postgres_password\n #- backup_password\n volumes:\n # Here we save the backups.\n - /var/opt/backups/talkyard/v1:/var/opt/backups/talkyard/v1\n # So can backup uploaded files and Redis.\n - pub-files:/var/talkyard/v1/pub-files:ro\n - priv-files:/var/talkyard/v1/priv-files:ro\n - redis-data:/var/talkyard/v1/redis-data:ro\n # So can backup config files.\n - .:/opt/talkyard-v1:ro\n logging: *default_logging\n depends_on:\n # So can backup database.\n - rdb\n networks:\n be_int_net:\n security_opt:\n - no-new-privileges=true\n cap_drop:\n - ALL\n cap_add:\n # So 'root' can access Redis' dump.rdb file, which is owned by user 'redis' 999.\n - CAP_DAC_READ_SEARCH\n\n# vim: et ts=2 sw=2\n" }, { "path": "docs/copy-backups-elsewhere.md", "content": "Take regular off-site backups\n======================\n\nAfter you've installed Talkyard, you should regularly copy the backups\nto an off-site backup server. Here's a way to do that.\n\n**Note:** Not yet tested in Talkyard v1, but should work — only the\nbackup archives path has changed (from `/opt/talkyard-backups/archives/`\nto `/var/opt/backups/talkyard/v1/archives/`).\n\n### Create SSH key\n\nOn the backup server, preferably located in another datacenter, create a SSH\nkey:\n\n ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_remotebackup -C \"Automated remote backup\"\n\nYou can perhaps skip the passphrase, since the backup server will have\nread-only rsync access only, all backups will be available on the backup server\nanyway. So a passhprase doesn't give any additional security.\n\n\n### Enable restricted rsync, rrsync\n\nOn the Talkyard server, enable rrsync:\n\n zcat /usr/share/doc/rsync/scripts/rrsync.gz > /usr/local/bin/rrsync\n chmod ugo+x /usr/local/bin/rrsync\n\n\n### rsync keys\n\nThen create a backup user with an `authorized_keys` file that allows restricted rsync:\n\n # (still on the Talkyard server)\n useradd --create-home remotebackup\n su - remotebackup\n mkdir .ssh\n echo 'command=\"/usr/local/bin/rrsync -ro /var/opt/backups/talkyard/v1/archives/\",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding' >> .ssh/authorized_keys\n\nCopy the public key on the backup server:\n\n # on the backup server:\n cat ~/.ssh/id_remotebackup.pub\n\n # copy the output\n\nAppend the public key to the last line in `authorized_keys` on the Talkyard server:\n\n # as user remotebackup: (!)\n nano ~/.ssh/authorized_keys\n\n # append a space and then the stuff you just copied to the last line (which is the only line, if the file was just created).\n # Do not paste it on a new line.\n\nThe result should be that the `authorized_keys` file looks like: (and it's a really long line)\n\n command=..... ssh-rsa AAAA................ Automated remote backup\n\n\n### Test\n\nNow, on the backup server, test copying backups:\n\n # replace 'SERVERADDRESS' with your Talkyard server address\n rsync -e \"ssh -i $HOME/.ssh/id_remotebackup\" -av remotebackup@SERVERADDRESS:/ $HOME/talkyard-backups/\n\n\n### Schedule copying-of-backups\n\nIf the above test works, then schedule a cron job to copy backups regularly. Do this on the backup server:\n\n # again, replace 'SERVERADDRESS' with your Talkyard server address\n crontab -l | { cat; echo '@hourly rsync -e \"ssh -i .ssh/id_remotebackup\" -av remotebackup@SERVERADDRESS:/ talkyard-backups/ >> cron.log 2>&1'; } | crontab -\n\nNow you'll have fresh backups of your forum in ~/talkyard-backups/, in case the Talkyard\nserver disappears.\n\n(Why do we run the rsync client read-only on the backup server? Well, because\nif we were to let the Talkyard server connect and write to the backup server, then\nsomeone who breaks in to the Talkyard server could ransomware-encrypt all backups\n(that is, encrypt everything and tell you \"give me money, only then will I\nunencrypt your data so you can read it again\"). But when the Talkyard server doesn't\nhave access to the backup server, this cannot happen. Note that it should be\neasier to make the backup server safe, because it doesn't need to run the whole\nTalkyard tech stack.)\n\n\n### Get an email, if backups stop working\n\n*NOT YET IMPLEMENTED* ` [BADBKPEML]`, the following does not yet work:\n\nOn the remote backup server, copy the contents of the script\n[scripts/check-talkyard-backups.sh](../scripts/check-talkyard-backups.sh)\nto your home directory. Edit the script and fill in email server (SMTP)\ncredentials.\n\nThen, test run the script:\n\n cd $HOME\n ./check-talkyard-backups.sh --send-email-if-bad talkyard-backups/\n\nAnd test send an email:\n\n ./check-talkyard-backups.sh --send-test-email\n\nIf seems to work, run daily via Cron:\n\n crontab -l | { cat; echo '@daily ./check-talkyard-backups.sh --send-email-if-bad talkyard-backups/ >> cron.log 2>&1'; } | crontab -\n\n" }, { "path": "docs/how-restore-backup.md", "content": "\nHow restore a Talkyard backup\n=============================================\n\n**Notice:** Not yet updated for Talkyard v1,\nfor example, steps for decrypting encrypted backups are missing.\n\nIf your server disappeared, and you want to restore a backup on a new server.\nOr if you're upgrading from Talkyard epoch v0 to v1. Then you can do as follows.\n\nStart installing Talkyard on that new server, following the instructions in\nhttps://github.com/debiki/talkyard-prod-one/blob/ty-prod-one-v1/README.md\n— but stop at step 8: \"Edit config values\".\nInstead, we'll copy config files from the backup:\n\nOn the new server, as root, run the commands below:\n\nReplace `BACKUP_ARCHIVES_DIR` and `DB_BACKUP_FILE` etcetera below, with\nthe actual path and file names.\n\n```\nsudo -i # become root\ncd /opt/talkyard-v1\n\necho \"$(date -I): Restoring backup ...\" >> talkyard-maint.log\n\n\n# Restore config files and HTTPS certs\n# ------------------------------\n\n# First, let's \"backup\" the new conf, in case you'd like to diff old vs default.\nmkdir -p default-conf/data\nmv conf docker-compose.* .env default-conf/\nmv data/certbot data/sites-enabled-auto-gen default-conf/data/\n\n# Then restore the old config.\nmkdir old-conf\nmkdir data\ntar xf /BACKUP_ARCHIVES_DIR/CONFIG_BACKUP_FILE.tar.gz -C old-conf\nmv old-conf/.env ./\nmv old-conf/docker-compose.* ./\nmv old-conf/conf ./conf\n\n\n# Stop any App server\n# ------------------------------\n\n# This shouldn't be needed — you didn't start the Talkyard server yet?\n# You stopped at step 8 as mentioned above?\n# Anyway, if the Talkyard app server is running, stop it:\n# (Otherwise the restore will fail because of active database connections.)\ndocker compose stop app\n\n\n# Restore the database, PostgreSQL\n# ------------------------------\n\n# First, start PostgreSQL.\ndocker compose up -d rdb\n\n# NOTE: Overwrites any existing database (!).\nzcat /BACKUP_ARCHIVES_DIR/DB_BACKUP_FILE.sql.gz \\\n | docker exec -i $(docker compose ps -q rdb) psql postgres postgres \\\n | tee -a talkyard-maint.log\n\n\n# Restore Redis?\n# ------------------------------\n\n# Not needed, it's a cache. (Maybe write something about Redis later.)\n\n\n# Restore uploaded files\n# ------------------------------\n\n# Test if works! [ty_v1]\ndocker compose run --rm \\\n -v /BACKUP_ARCHIVES_DIR/UPLOADS_BACKUP_DIR.d:/uploads:ro \\\n app \\\n rsync -a /uploads/ /var/talkyard/v1/pub-files/uploads/\n```\n\n### Memory\n\nNext, configure memory: Run `free -m` to find out how many megabytes\nmemory your machine has. Look at docker-compose.override.yml to see how\nmuch memory Talkyard has been configure to use — and optionally,\nreplace that file with another more suitable one from `./mem/*`,\ne.g.: `cp mem/4g.yml docker-compose.override.yml`.\n\n\n### Start Talkyard\n\nNow, time to start everything:\n\n```\ndocker compose up -d\ndocker compose logs -f --tail 999\n```\n\nAlso, think about if you need to 1) update your DNS server with the IP address to\nyour new Talkyard server. Or maybe 2) change the hostname of the Talkyard server\n— you'd then edit Nginx config in `conf/app/play-framework.conf`,\nand `conf/web/sites-enabled/talkyard-servers.conf`, plus\ngenerate a LetsEncrypt cert\n(see: `https://github.com/debiki/talkyard-prod-one/blob/ty-prod-one-v1/docs/setup-https.md`).\n\n\n### Backups and automatic upgrades\n\nContinue with step ?? in the installation instructions in README.md,\nhttps://github.com/debiki/talkyard-prod-one/blob/ty-prod-one-v1/README.md,\nthat is, this step:\n*\"Schedule deletion of old log files, daily backups and deletion old backups, and automatic upgrades\"*.\n\nAlso, look at the *Next steps* just below, in README.md — you'll want to configure off-site backups?\n" }, { "path": "docs/multisite-talkyard.adoc", "content": "\nMultisite Talkyard\n======================================================================\n\n\nOne Talkyard server can host many Talkyard sites:\ndifferent forums and blogs, with different owners and admins.\n\nIf you've installed Talkyard as usual, for a single site,\nthen, you can enable Multisite Talkyard.\nLet's say your main Talkyard site is at: `main-talkyard.example.com`. Then,\n\n. Add DNS record(s) for the new sites you'll create.\nCould be a wildcard A or CNAME record:\n\n```\nmain-talkyard.example.com 3600 IN A 11.22.33.44\n*.multi-ty.example.com 3600 IN A main-talkyard.example.com.\n```\n\nIn `/opt/talkyard/conf/play-framework.conf`,\nscroll down to the Advanced section; add these settings:\n\n```\ntalkyard.createSiteHostname=\"main-talkyard.example.com\"\ntalkyard.baseDomain=\"multi-ty.example.com\"\n```\n\nwhere `main-talkyard.example.com` is the address to your already working Talkyard site,\nthe firt site you installed.\n\nNow, you can go to: `https://main-talkyard.example.com/-/create-site`, or\n`https://main-talkyard.example.com/-/create-site/blog-comments`,\nand create a new Talkyard site.\n\nIts address will be: `https://something.multi-ty.example.com`.\n\nHTTPS should work automatically — Talkyard and LetsEncrypt generates certs for you.\nOnly the very first time someone (you) accesses a new site,\nthere'll be a connection-not-secure error,\nand you'll need to wait 10 – 20 seconds and reload the page.\n\nOnce you've created an additional Talkyard site,\nyou can, if you want to, change its address to something else,\nby going here: `https://something.multi-ty.example.com/-/admin/settings/site`\nand clicking *Change address*.\n" }, { "path": "docs/release-channels.md", "content": "\nRelease Channels\n======================================================================\n\nLater, you can choose between:\n1. Getting new features and bug fixes more often — the `regular` release branch.\n2. Getting only important bug fixes — the `lts` (Long Term Stable) branch.\n\nYou choose this by editing `/opt/talkyard-v1/.env`\nand setting `RELEASE_BRANCH=regular` or `...=lts`.\n\nCurrently only the `regular` branch exists.\nIt's the default, so you don't need to do anything.\n\n(This is inspired by Kubernetes' release channels:\nhttps://cloud.google.com/kubernetes-engine/docs/concepts/release-channels)\n\n" }, { "path": "docs/risk-free-upgrades.md", "content": "Risk-Free Upgrades (Blue-Green, Google Cloud)\n=========================\n\nYou can use this method for major OS upgrades or major Talkyard migrations, e.g. v0 to v1.\n\nFor routine updates, we recommend just following the installation instructions,\nthat is, a daily cron job that calls `scripts/upgrade-if-needed.sh`.\n\n**Important:** Read/skim all steps before you start.\nEspecially read the **Make backups work again**\nand **Rolling back** sections at the end.\n\n\n\n\n\n\n\nPrerequisites\n-------------------------\n\n\n\n- These docs are for Google Cloud. (No docs written for AWS or anything else.)\n\n- Familiarity with Google Cloud, e.g. how to SSH into a VM.\n\n- SSH access to your off-site backup server, if any.\n\n- Know how to edit your hosts file (on your laptop).\n\n- Know how to open your browser's Dev Tools and switch to the Network tab to check\n IP addresses.\n\n\nInstructions\n-------------------------\n\n1. **Before**\n\n 1. **Enable Maintenance Mode** which also makes the server read-only.\n SSH into your server, and:\n\n ```\n cd /opt/talkyard\n sudo docker-compose exec rdb psql talkyard talkyard -c \\\n 'update system_settings_t set maintenance_until_unix_secs_c = 1;'\n ```\n\n Now the forum should show an Under Maintenance message:\n\n (screenshot)\n\n1. **Clone the server**\n 1. In Google Cloud, go to **Virtual Machines > VM Instances**, and find your VM.\n\n 1. Click the three dots **⋮** next to your VM, then click **Create new machine image**.\n\n (screenshot)\n\n 1. Once created, go to **Machine Images**, find the new image,\n click **⋮** then click **Create instance**.\n\n 1. Launch the VM in same region and zone as the old VM, with the same machine type.\n\n1. **Verify & Upgrade**\n\n 1. Let's see if the new VM works. Find the IP of the new VM, and\n add it to your laptop's `/etc/hosts`:\n\n ```\n 11.22.33.44 your-forum.example.com\n ```\n\n 1. In a browser, go to `https://your-forum.example.com`\n and open Dev Tools. Switch to the Network tab. Verify that you're\n hitting the IP of the new VM (and not the _old_ VM).\n\n 1. **Upgrade.** SSH into the new VM. Upgrade the forum or the OS.\n\n 1. Do some manual testing in the browser, including:\n - Visit: `your-forum.example.com/-/build-info`\n do you see the new (upgraded) Talkyard version number?\n - Post a test topic in a hidden category (e.g. staff-only).\n - Visit: `your-forum.example.com/-/last-errors` — you see any errors?\n\n1. **Switch Traffic**\n\n 1. Move your forum's IP address from the old VM to the new VM:\n Go to **VPC Network > IP addresses**, find the forum's public IP,\n click **⋮** and **Reassign to another resource**, select the new VM.\n\n 1. Remove the `/etc/hosts` entry (on your laptop).\n Reload web page, look in Dev Tools, the Network tab, and verify you're\n hitting the public IP address of the forum — which now points to the _new_ VM.\n\n1. **Afterwards**\n\n 1. **Disable Maintenance Mode.** On the new VM:\n\n ```\n cd /opt/talkyard\n sudo docker-compose exec rdb psql talkyard talkyard -c \\\n 'update system_settings_t set maintenance_until_unix_secs_c = null;'\n ```\n\n Reload the web page. Did the maintenance message disappear?\n\n 1. **Shut down** the old VM, but don't delete it (wait a month).\n\n Reload the web page — still works?\n\n \n\n1. **Make backups work again**\n\n If you copy backups off-site using rsync (as described in `copy-backups-elsewhere.md`),\n this now fails with a _\"REMOTE HOST IDENTIFICATION HAS CHANGED\"_ warning,\n because Google Cloud will have given the new VM a different SSH host key\n (even though it's a machine image of the old). Therefore:\n\n 1. SSH into the remote backup server. Remove the old key, accept the new:\n\n ```\n ssh-keygen -R your-forum.example.com # removes old key\n ssh your-forum.example.com # accepts new. Type 'yes' when prompted\n ```\n\n 1. Type `crontab -l` to list cron jobs (on the remote backup server).\n Copy-paste the line that `rsync`s the Talkyard backups,\n and run it manually, verify works fine.\n\n Also, make sure you've configured Google Cloud to backup the new VM regularly\n (if you want to do that in addition to Talkyard's own backups).\n\n### Rolling back\n\nIf there's any problem, either don't reassign the IP, or move it back to the old VM.\nDisable Maintenance Mode on the **old** VM using the SQL command from the\n_Disable Maintenance Mode_ step above.\n\nYou can figure out what went wrong on the new VM without any stress.\n(Unless, of course, something bad happens _after_ you've completely migrated to\nthe new VM. That's why it's good to run some tests before switching over.)\n\n\nRelated reading\n-------------------------\n\n- About machine images:\n https://cloud.google.com/compute/docs/machine-images#disk-backup\n- Reassigning external IP addresses:\n https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address#IP_assign\n- Reassigning an external IP programatically:\n - https://cloud.google.com/sdk/gcloud/reference/compute/instances/delete-access-config\n - https://cloud.google.com/sdk/gcloud/reference/compute/instances/add-access-config\n\n" }, { "path": "docs/troubleshooting.md", "content": "\nTroubleshooting Talkyard\n========================\n\nInstallation Problems\n---------------------\n\n\n### Error: talkyard_web_1, Read timed out\n\nIf, when you run:\n\n /scripts/upgrade-if-needed.sh 2>&1 | tee -a talkyard-maint.log\n\nyou're getting this error:\n\n Creating talkyard_web_1 ...\n\n ERROR: for talkyard_web_1 UnixHTTPConnectionPool(host='localhost', port=None): Read timed out. (read timeout=...)\n\n ERROR: for web UnixHTTPConnectionPool(host='localhost', port=None): Read timed out. (read timeout=...)\n An HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n If you encounter this issue regularly because of slow network conditions, consider setting COMPOSE_HTTP_TIMEOUT to a higher value (current value: 240).\n\nthen the reason can be that the server has too little memory — which apparently can cause\nNginx (OpenResty) to run out of memory and crash. Now you might wonder, why would Nginx use\nthat much memory? — I think it's OpenResty (an Nginx distribution) that just-in-time compiles\nlots of Lua code, and then uses lots of memory.\n\n\n\nOld: Troubleshooting and debugging\n----------------\n\n(Ignore this section; it's not completed and hard to understand.)\n\n? save Java crash dumps in ./play-crash\n+ tips about how to run jmap? or view in jvisualvm + Idea? jmap -heap PID\n\nHow to connect VisualVM\n\nTips about how to view logs: all logs, app specific logs.\n\nHow to jump into a Docker container.\n\nHow to connect a debugger: open Docker port, then connect via SSH tunnel (assuming a firewall blocks the port on the host).\nIf using Google Compute Engine, then ssh tunnel:\n\n gcloud compute ssh server-name --ssh-flag=-L9999:127.0.0.1:9999 --ssh-flag=-N\n\n\nHow to open console in Chrome, view messages & post to the E.D. help forum.\n\nView CPU & memory usage: `./scripts/stats.sh`\n\n\n" }, { "path": "docs/upgrade-v0-to-v1.md", "content": "Upgrading from Talkyard v0 to v1\n================================\n\n**NOTICE**: This is a **draft**, not yet finished or tested.\n\n---\n\nTalkyrad v1 is a major new version of Talkyard — a new epoch.\nPrevious versions have been v0.YYYY.NNN,\nnewer versions will be v1.YYYY.NNN (e.g. v1.2025.001).\n\nTo upgrade, you'll install Talkyrad v1 side-by-side with v0, backup v0,\nshut down v0, restore the backup to v1, and start v1.\n\n\nWhy upgrade?\n-------------------------\n\n- Talkyard v1 upgrades all components to more recent versions\n (upgrades to PostgreSQL 18, ElasticSearch 9 (or 8), Redis 8, Debian 12 or 13).\n This is good to do, so you'll be using supported versions of the software.\n\n- Improvements to the maintenance scripts, e.g. optionally encrypted backups.\n\n- We'll start using Docker named volumes, instead of bind mounts.\n\n- We'll start using the Linux Filesystem Hierarch Standard, e.g.\n backups in `/var/opt/backups/talkyard/` instead of `/opt/talkyard-backups/`.\n\n- Talkyard v0 will stop getting new features, only bug fixes.\n\n\nHow to upgrade\n-------------------------\n\n### Preparations\n\nUpgrade your Operating System to Debian 12 or 13\n(Ubuntu 22 or 24 LTS should work too — they're based on Debian 12 and 13).\n\nUpgrade Docker to >= ???. Install Docker Compose v2, if you haven't already:\n\n apt-get install docker-compose-plugin\n\n(Talkyard v0 uses Docker Compose v1, but Talkyard v1 uses Docker Compose v2.)\n\n\n### Phase 1: Backup and shut down v0\n\nMake your Talkyard v0 server read-only:\n\n```\ncd /opt/talkyard # this is v0\n\ndocker-compose exec rdb psql ...\n```\n\nTake a backup, let's name it `beforeV1Upgrade`:\n\n```\n./scripts/backup.sh beforeV1Upgrade\n```\n\nNow, shut down Talkyrad v0 — but don't delete anything! Just leave it as-is:\n\n```\ndocker-compose down # (using Docker-Compose v1)\n```\n\n#### Verification\n\nOpen a web browser and check that you the Talkyard site is inaccessible.\n\n\n### Phase 2: Install and configure v1\n\n\nInstall Talkyard v1, as per the installation instructions in ../README.md .\n\nCopy configuration files from your v0 installation to v1 — with one change:\n\n```\n# Go to the Talkyard v1 installation dir\ncd /opt/talkyard-v1\n\n# Back up default config\nmv conf conf.v1.default\n\n# Copy config from v0 to v1:\ncp -a /opt/talkyard/conf ./\n\n# Move play-framework.conf to 'app/' sub dir: (for consistency with other containers)\nmkdir conf/app\nmv conf/play-framework.conf conf/app/play-framework.conf\n```\n\nEdit the config files:\n\n```\n... TBD ...\n```\n\n\n#### Verification\n\nStart the new Talkyard v1 site:\n\n```\ncd /opt/talkyard-v1\ndocker compose up -d\n```\n\nSee if you can access it in a browser. It'll be empty, since you haven't restored the\ndatabase yet.\n\n\n### Phase 3: Data migration\n\n#### Restore database\n\nRestore the database backup into Talkyrad v1: (and replace `...beforeV1Upgrade...sql.g`\nwith the backup file name.)\n\n```\ncd /opt/talkyard-v1/ # note: v1\ndocker compose up -d rdb # (this is Docker Compose v2)\n\n# NOTE: Overwrites any existing database (!).\nzcat /opt/talkyard-backups/archives/...beforeV1Upgrade...sql.gz \\\n | docker exec -i $(docker compose ps -q rdb) psql postgres postgres \\\n | tee -a talkyard-maint.log\n```\n\n#### Copy uploaded files\n\nCopy uploaded files from v0 (located at `/opt/talkyard/data/uploads`)\ninto the v1 named volume, using a temporary 'app' container. This container\nwhich automatically mounts the volume, as specified in docker-compose.yml.\n\n```\n# In /opt/talkyard-v1:\n\ndocker compose run --rm \\\n -v /opt/talkyard/data/uploads:/uploads-v0:ro \\\n app \\\n rsync -a /uploads-v0/public/ /var/talkyard/v1/pub-files/uploads/\n```\n\n\n#### Does it work?\n\nStart Talkyard v1 with your data restored:\n\n```\n# In /opt/talkyard-v1:\n\ndocker compose up -d\ndocker compose logs -f\n```\n\nOpen a web browser and see if you can access your Talkyard site again.\n\n\n### Phase 4: Last steps\n\n#### Make the site read-write\n\n```\ndocker compose exec rdb psql ...\n```\n\n\n#### Reconfigure backups\n\nReconfigure the off-site backup script so it backups `/var/opt/backups/`\n(instead of `/opt/talkyard-backups/`) — see the end of ../README.md.\n\nWait a month or two. All fine? You can delete old v0.\n\n\n" }, { "path": "mem/1.7g.yml", "content": "# For servers with 1.7 GB RAM.\n# E.g. a Google Compute Engine g1-small instance: 1 shared CPU & 1.7 GB mem.\n\n# (We don't make use of all RAM here, because ElasticSearch and Postgres wants\n# fairly much memory in the operating system file system cache, e.g.\n# ElasticSearch wants as much mem for the OS cache as for its own heap.\n# Also, there're Nginx and Redis containers too.)\n\nservices:\n app:\n environment:\n # There's also stack memory and permanent-generation memory.\n PLAY_HEAP_MEMORY_MB: 256\n\n search:\n environment:\n ES_JAVA_OPTS: '-Xms192m -Xmx192m'\n deploy:\n resources:\n limits:\n memory: 0.9G\n\n\n" }, { "path": "mem/2g.yml", "content": "# For servers with 2 GB RAM.\n\n# (We don't make use of all RAM here, because ElasticSearch and Postgres wants\n# fairly much memory in the operating system file system cache, e.g.\n# ElasticSearch wants as much mem for the OS cache as for its own heap.\n# Also, there're Nginx and Redis containers too.)\n\nservices:\n app:\n environment:\n # There's also stack memory and permanent-generation memory.\n PLAY_HEAP_MEMORY_MB: 512\n\n search:\n environment:\n ES_JAVA_OPTS: '-Xms320m -Xmx320m'\n deploy:\n resources:\n limits:\n memory: 1G\n\n" }, { "path": "mem/4g.yml", "content": "# For servers with 4 GB RAM.\n\n# (We don't make use of all RAM here, because ElasticSearch and Postgres wants\n# fairly much memory in the operating system file system cache, e.g.\n# ElasticSearch wants as much mem for the OS cache as for its own heap.\n# Also, there're Nginx and Redis containers too.)\n\nservices:\n app:\n environment:\n # There's also stack memory and permanent-generation memory.\n PLAY_HEAP_MEMORY_MB: 1024\n\n search:\n environment:\n ES_JAVA_OPTS: '-Xms512m -Xmx512m'\n deploy:\n # Let's restrict the container to 40% of total mem.\n resources:\n limits:\n memory: 1.6G\n\n" }, { "path": "mem/7.5G.yml", "content": "# For servers with 7.5 GB RAM.\n\n# (We don't make use of all RAM here, because ElasticSearch and Postgres wants\n# fairly much memory in the operating system file system cache, e.g.\n# ElasticSearch wants as much mem for the OS cache as for its own heap.\n# Also, there's a Nginx and a Redis container too.)\n\nservices:\n app:\n environment:\n # There's also stack memory and permanent-generation memory.\n PLAY_HEAP_MEMORY_MB: 1300\n\n search:\n environment:\n ES_JAVA_OPTS: '-Xms1100m -Xmx1100m'\n deploy:\n # Let's restrict the container to 40% of total mem.\n resources:\n limits:\n memory: 3G\n\n" }, { "path": "mem/8G.yml", "content": "# For servers with 8 GB RAM.\n\n# (We don't make use of all RAM here, because ElasticSearch and Postgres wants\n# fairly much memory in the operating system file system cache, e.g.\n# ElasticSearch wants as much mem for the OS cache as for its own heap.\n# Also, there's a Nginx and a Redis container too.)\n\nservices:\n app:\n environment:\n # There's also stack memory and permanent-generation memory.\n PLAY_HEAP_MEMORY_MB: 1800\n\n search:\n environment:\n ES_JAVA_OPTS: '-Xms1800m -Xmx1800m'\n deploy:\n # Let's restrict the container to 40% of total mem.\n resources:\n limits:\n memory: 3.2G\n\n" }, { "path": "scripts/backup.sh", "content": "#!/bin/bash\n\nlabel=\"${1:-manual}\"\n\nexec /usr/bin/docker compose run --rm backup \\\n /ty/backup.sh \"$label\" \"$(hostname)\" \"$(date '+%FT%H%MZ' --utc)\"\n\n\n" }, { "path": "scripts/delete-old-backups.sh", "content": "#!/bin/bash\n\nexec /usr/bin/docker compose run --rm backup \\\n /ty/delete-old-backups.sh\n\n" }, { "path": "scripts/find-admin-login-link.sh", "content": "#!/bin/bash\n\n# This prints admin one-time login links, and admin reset password links,\n# generated via: http://ty-server/-/admin-login\n# or via the Reset Password buttons.\n#\n# This is useful, if email hasn't yet been configured. Then one\n# can login as root, and run this script instead.\n#\n# Sync with this: [GETADMLNK] and [RSTPWDLNK].\n\ndc=\"/usr/bin/docker compose\"\n\ndb_user=\"$1\"\n\nif [ -z \"$db_user\" ]; then\n db_user=\"talkyard\"\nfi\n\n# Set pager=off otherwise psql prints \"More...\" and waits for you to\n# hit Space.\npsql=\"psql -P pager=off $db_user $db_user\"\n\n\n# Print admin emails, in case one doesn't remember one's admin email — e.g.\n# after migrating from Talkyard.net to self hosted:\n\nadmin_addrs=$($dc exec rdb $psql -c \"\n select site_id, primary_email_addr, username, full_name\n from users3\n where\n is_admin\n -- Exclude System and Sysbot.\n and user_id >= 100\n order by site_id asc, username asc\n \")\n\necho\necho \"First, a tips:\"\necho \"To generate admin login links, go to: https://your-talkyard-server/-/admin-login\"\necho \"and type your admin email.\"\necho\necho \"Here're the admin email addresses:\"\necho\necho \"$admin_addrs\"\necho\n\n\n# Print admin one time login links.\n\necho \"Looking in $db_user's database for admin login link emails\"\necho \"and reset password emails ...\"\n\nemails=$($dc exec rdb $psql -c \"\n select\n -- Remove newlines, so can count and grep properly.\n -- (Need \\\\ not \\, because Bash eats one.)\n regexp_replace(body_html, '[\\\\n\\\\r]+', ' ', 'g' ) || '\\n'\n from emails_out3\n where\n -- This is EmailType.ResetPassword = 22 and OneTimeLoginLink = 23.\n type in (22, 23)\n -- One day should be enough?\n and sent_on > now_utc() - interval '1 day'\n order by sent_on asc\n limit 22\n \")\n\n\n# Sync with the email generating code [ADMLOGINEML].\nname_urls=$(echo \"$emails\" | \\\n sed -nr 's#.*>(Hi [^<]+).*(https?://[^\"]+).*$# \\1 \\2#p')\n\nif [ -z \"$name_urls\" ]; then\n echo\n echo \"Found nothing.\"\nelse\n how_many=$(echo \"$name_urls\" | wc --lines)\n echo\n echo \"Found $how_many recent admin login or reset password links, most recent last:\"\n echo\n echo \"$name_urls\"\n echo \" ^---- this last link is the most recent\"\n echo\n echo \"Copy-paste the links into your browser address bar.\"\n echo \"Each link works only once.\"\nfi\n\necho\n" }, { "path": "scripts/impl/check-talkyard-backups.sh", "content": "#!/bin/bash\n\n# Will finish this script later. Placed here in impl/ for now, so it won't\n# distract others who look in scripts/.\n\n\nfunction log_message {\n echo \"`date --iso-8601=seconds --utc` check-backups: $1\"\n}\n\nif [ $# -ne 2 ]; then\n echo \"Usage: $0 --send-email-if-bad BACKUP_DIR\"\n echo \"Or: $0 --send-test-email\"\n exit 1\nfi\n\nbackup_dir=\"$2\"\n\necho \"Not yet implemented. Bye. [BADBKPEML]\"\nexit\n\n# echo \"Checking daily backups in $2:\"\n\n\n\n# Find the most recent Postgres backup.\n# if [ older than two days — check both date in file name, and unix ctime? ]\n# then\n# problems=\" .....\"\n# fi\n\n\n# Get the backup's random value.\n# We can just look at the textual contents of the backup, to find out if they're\n# most likely ok — no need to restore the database into a real PostgreSQL\n# server. It'd be nice to do this too, optionally, though.\n#\n# good_row=$(zcat | grep \"$random_value\" | grep \"$hostname\" | grep 'postgres.sql')\n\n# if [ -z \"$good_row\" ]\n# then\n# problems=\" .....\"\n# fi\n\n\n# Have look in the uploads dir.\n# There should be this file:\n# touch $backup_test_dir/$when--$(hostname)--$random_value\n# if [ no such file ]\n# then\n# problems=\"$problems\\n ... more problems.....\"\n# fi\n\n# if [ uploads backup is older than two days ]\n# then\n# problems=\" .....\"\n# fi\n\n\n# if [ \"$problems\" ]\n# then\n# Send an email with the \"$problems\". [BADBKPEML]\n# Need SMTP server addr, username, pwd, send-to address.\n#\n# Websearch for \"how send email from linux server\" to find out how.\n#\n# fi\n" }, { "path": "scripts/impl/docker-compose.wrong-app-ip.yml", "content": "# Changes the 'app' container IP to the wrong IP [maint_app_ip], so 'web'\n# cannot connect to it. This makes 'web' respond quickly with a status 502,\n# which is good when in maintenance mode where we want to show a\n# \"We're upgrading the server\" message — instead of 'web' being able to connect\n# to 'app'; then it won't respond to the end user until half a minute (?)\n# has elapsed.\n\nservices:\n web:\n extra_hosts:\n # Try connecting to a non-existing IP on a subnet we *can* access.\n # Because if 'web' tries to connect to something outside the subnet, seems\n # it justs \"hang\", maybe the connection would eventually timeout?\n #\n # 'web' is on the fe_int_net subnet: FE_INT_NET_SUBNET = 172.26.2.0/24\n # and the correct app IP is: FE_INT_NET_APP_IP = 172.26.2.41\n # see ../.env.\n #\n app: 172.26.2.127 # intentionally wrong\n\n" }, { "path": "scripts/impl/recreate-web.sh", "content": "#/bin/bash\n\n# This makes 'web' container config changes in docker-compose.yml take effect\n# (need to recreate the container).\n\ndocker compose kill web\ndocker compose rm -f web\ndocker compose up -d web\n" }, { "path": "scripts/impl/unjson.sh", "content": "#!/bin/bash\n\n# Makes json log messages human readable, by parsing the json and pretty-printing\n# the app specific interesting fields.\n\n\nif [ -z `which jq` ]; then\n echo \"Please install 'jq' for Json pretty print, e.g.: sudo apt install jq\"\n exit 1\nfi\n\n# -r preserves backslashes, otherwise '\\n' gets converted to just 'n', and we can no longer\n# pretty-print e.g. stacktraces with newlines.\nwhile read -r line\ndo\n # Remove Docker's color codes.\n line=$(echo \"$line\" | sed -r \"s/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]//g\")\n\n # We'll add our own colors, here're some codes:\n # (from http://misc.flogisoft.com/bash/tip_colors_and_formatting)\n # \\e[34m = blue\n # \\e[95m = light magenta\n # \\e[32m = green\n # \\e[33m = yellow\n # \\e[90m = dark gray\n # \\e[92m = light green\n # \\e[39m = default\n\n if [[ \"$line\" =~ ^web_ ]] ; then\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[34m\\1\\\\\\\\e[39m\\2/'`\"\n elif [[ \"$line\" =~ ^rdb_ ]] ; then\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[95m\\1\\\\\\\\e[39m\\2/'`\"\n elif [[ \"$line\" =~ ^cache_ ]] ; then\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[32m\\1\\\\\\\\e[39m\\2/'`\"\n elif [[ \"$line\" =~ ^search_ ]] ; then\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[33m\\1\\\\\\\\e[39m\\2/'`\"\n elif [[ \"$line\" =~ ^gulp_ ]] ; then\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[90m\\1\\\\\\\\e[39m\\2/'`\"\n elif [[ \"$line\" =~ ^app_ ]] ; then\n # The program 'jq' extracts the timestamp, severity, message etc fields from a json log message.\n # The -j flag removes surrounding quotes.\n json=$( egrep -o '\\{\".*\\}' <<< \"$line\" )\n if [ -n \"$json\" ]; then\n pretty_json=$(echo \"$json\" | jq -j '.severity, \" \", .message, \" kvs: \", .kvs' )\n app=\"$(echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\1/')\"\n echo -e \"\\e[92m$app\\e[39m $pretty_json\"\n else\n # In dev mode, when compiling & reloading, Play Framework logs non-json messages.\n echo -e \"`echo \"$line\" | sed -r 's/^([^|]+\\|)(.*)$/\\\\\\\\e[92m\\1\\\\\\\\e[39m\\2/'`\"\n fi\n else\n echo \"$line\"\n fi\ndone\n\n" }, { "path": "scripts/old/Vagrantfile", "content": "# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\n#===========\n# What is this? This file tells a program named Vagrant how a\n# Virtual Machine (VM) can be created and configured on your computer,\n# in which you can test install Talkyard.\n#\n# ***\n# MIGHT NO LONGER WORK? Was last tested with Ubuntu 18.04, but now we're\n# at Ubuntu 22.04 — \"ubuntu/jammy64\" below. Don't know if today's Vagrant\n# is compatible with this 4 years old file?\n# ***\n#\n# Tips: You'll need to add talkyard.port=8080 to play-framework.conf\n# (mentioned in README.md).\n#\n# Go and read about Vagrant here: https://www.vagrantup.com/ — click\n# \"Getting Started\" and read that page. Download and install Vagrant.\n# Also install VirtualBox (or some othervirtualization system), so that\n# there'll be something that can run the VM Vagrant will download for you.\n# Then:\n#\n# 1) Create an empty folder named `talkyard-prod-test`. Copy this file into it.\n# 2) cd into that folder, and type `vagrant up`\n# 3) Wait for Vagrant do download stuff, and the VM to start.\n# 4) Then type `vagrant ssh`, to open a shelll inside the VM.\n# 5) Follow the instructions in README.md (but now you have a server already,\n# i.e. the Vagrant VM which you are inside right now).\n# Note that, when editing /opt/talkyard/conf/play-framework.conf, you'll need to\n# comment in: talkyard.port=8080\n# and set: talkyard.secure=false\n# since we'll access Talkyard via http://localhost:8080, no https cert.\n# 6) To stop the VM, type CTRL-D to exit Vagrant, then type `vagrant halt`.\n#\n# Advanced tips: To ssh into the VM and also expose a port on the host inside the VM,\n# do e.g.: vagrant ssh -- -R 5000:localhost:5000\n# This is useful if you have a local test Docker registry running at localhost:5000\n# on the host, and want to make it available in the VM also at localhost:5000.\n#===========\n\n# All Vagrant configuration is done below. The \"2\" in Vagrant.configure\n# configures the configuration version (we support older styles for\n# backwards compatibility). Please don't change it unless you know what\n# you're doing.\nVagrant.configure(2) do |config|\n # The most common configuration options are documented and commented below.\n # For a complete reference, please see the online documentation at\n # https://docs.vagrantup.com.\n\n # Every Vagrant development environment requires a box. You can search for\n # boxes at https://atlas.hashicorp.com/search.\n config.vm.box = \"ubuntu/jammy64\"\n\n # Disable automatic box update checking. If you disable this, then\n # boxes will only be checked for updates when the user runs\n # `vagrant box outdated`. This is not recommended.\n # config.vm.box_check_update = false\n\n # Create a forwarded port mapping which allows access to a specific port\n # within the machine from a port on the host machine. In the example below,\n # accessing \"localhost:8080\" will access port 80 on the guest machine.\n config.vm.network \"forwarded_port\", guest: 80, host: 8080\n\n # Create a private network, which allows host-only access to the machine\n # using a specific IP.\n # config.vm.network \"private_network\", ip: \"192.168.33.10\"\n\n # Create a public network, which generally matched to bridged network.\n # Bridged networks make the machine appear as another physical device on\n # your network.\n # config.vm.network \"public_network\"\n\n # Share an additional folder to the guest VM. The first argument is\n # the path on the host to the actual folder. The second argument is\n # the path on the guest to mount the folder. And the optional third\n # argument is a set of non-required options.\n # config.vm.synced_folder \"../data\", \"/vagrant_data\"\n\n # Provider-specific configuration so you can fine-tune various\n # backing providers for Vagrant. These expose provider-specific options.\n # Example for VirtualBox:\n #\n config.vm.provider \"virtualbox\" do |vb|\n vb.customize [\"modifyvm\", :id, \"--natdnshostresolver1\", \"on\"]\n # # Display the VirtualBox GUI when booting the machine\n # vb.gui = true\n #\n # # Customize the amount of memory on the VM:\n vb.memory = \"3000\"\n end\n #\n # View the documentation for the provider you are using for more\n # information on available options.\n\n # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies\n # such as FTP and Heroku are also available. See the documentation at\n # https://docs.vagrantup.com/v2/push/atlas.html for more information.\n # config.push.define \"atlas\" do |push|\n # push.app = \"YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME\"\n # end\n\n # Enable provisioning with a shell script. Additional provisioners such as\n # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the\n # documentation for more information about their specific syntax and use.\n # config.vm.provision \"shell\", inline: <<-SHELL\n # sudo apt-get update\n # sudo apt-get install -y apache2\n # SHELL\nend\n" }, { "path": "scripts/prepare-os.sh", "content": "#!/bin/bash\n\n# This script makes ElasticSearch work, simplifies troubleshooting,\n# and configures automatic security updates, with reboots.\n\nfunction log_message {\n echo \"`date --iso-8601=seconds --utc` prepare-os: $1\"\n}\n\necho\necho\nlog_message 'Configuring this Operating System:'\n\ndid_what=''\n\n# Avoid harmless \"warning: Setting locale failed\" warnings from Perl:\n# (https://askubuntu.com/questions/162391/how-do-i-fix-my-locale-issue)\nlocale-gen 'en_US.UTF-8'\nif ! grep -q 'LC_ALL=' /etc/default/locale; then\n echo 'Setting LC_ALL to en_US.UTF-8...'\n echo 'LC_ALL=en_US.UTF-8' >> /etc/default/locale\n export LC_ALL='en_US.UTF-8'\n did_what=\"Configured LC_ALL=en_US.UTF-8.\"\nfi\nif ! grep -q 'LANG=' /etc/default/locale; then\n echo 'Setting LANG to en_US.UTF-8...'\n echo 'LANG=en_US.UTF-8' >> /etc/default/locale\n export LANG='en_US.UTF-8'\n did_what=\"$did_what Configured LANG=en_US.UTF-8.\"\nfi\n\n\n# Append system config settings, so the ElasticSearch Docker container will work,\n# and so Nginx can handle more connections. [BACKLGSZ]\n\nif ! grep -q 'Talkyard' /etc/sysctl.conf; then\n log_message 'Amending the /etc/sysctl.conf config...'\n cat <<-EOF >> /etc/sysctl.conf\n\t\t\n\t\t###################################################################\n\t\t# Talkyard settings\n\t\t#\n\t\t# Turn off swap, default = 60.\n\t\tvm.swappiness=0\n\t\t# Up the max backlog queue size (num connections per port), default = 128.\n\t\t# Sync with conf/web/sites-enabled/talkyard-servers.conf.\n\t\tnet.core.somaxconn=8192\n\t\t# ElasticSearch wants this, default = 65530\n\t\t# See: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html\n\t\tvm.max_map_count=262144\n\t\tEOF\n\n log_message 'Reloading the system config...'\n sysctl --system\n did_what=\"$did_what Added Talkyard settings to /etc/sysctl.conf.\"\nelse\n log_message 'Talkyard settings found in /etc/sysctl.conf, leaving as is.'\nfi\n\n\n# Make Redis happier:\n# Redis doesn't want Transparent Huge Pages (THP) enabled, because that creates\n# latency and memory usage issues with Redis. Disable THP now directly, and also\n# after restart: (as recommended by Redis)\nif ! grep -q '\\[always\\]' /sys/kernel/mm/transparent_hugepage/enabled ; then\n echo \"Transparent Huge Pages is [madvise] or [never], fine, Redis happy.\"\nelse\n echo \"Setting Transparent Huge Pages to [madvise], Redis wants this ...\"\n echo madvise > /sys/kernel/mm/transparent_hugepage/enabled\n # We can use rc.local — also with Systemd, see: https://askubuntu.com/a/919598.\n rc_local_f=\"/etc/rc.local\"\n if [ ! -f $rc_local_f ]; then\n echo \"exit 0\" >> $rc_local_f\n fi\n if ! grep -q 'transparent_hugepage/enabled' $rc_local_f ; then\n echo \"Setting Transparent Huge Pages to [madvise] after reboot, in $rc_local_f...\"\n # Insert ('i') before the last line ('$') in rc.local, which always? is\n # 'exit 0' in a new Ubuntu installation ... no, Debian now.\n sed -i -e '$i # For Talkyard and the Redis Docker container:\\necho madvise > /sys/kernel/mm/transparent_hugepage/enabled\\n' $rc_local_f\n fi\n did_what=\"$did_what Set Transparent Huge Pages to [madvise].\"\nfi\n\n\n# Simplify troubleshooting:\nif ! grep -q 'HISTTIMEFORMAT' ~/.bashrc; then\n log_message 'Adding history settings to .bashrc...'\n cat <<-EOF >> ~/.bashrc\n\t\t\n\t\t###################################################################\n\t\texport HISTCONTROL=ignoredups\n\t\texport HISTCONTROL=ignoreboth\n\t\texport HISTSIZE=10100\n\t\texport HISTFILESIZE=10100\n\t\texport HISTTIMEFORMAT='%F %T %z '\n\t\tEOF\n did_what=\"$did_what Added HIST* settings to .bashrc.\"\nelse\n log_message 'Probably sensible settings found in .bashrc, leaving as is.'\nfi\n\n\n# [ty_v1] Auto upgr: Ask if, and recommend that, auto reboot if needed after security\n# upgrades, and if Yes, then, add Automatic-Reboot also if there's already\n# a 20auto-upgrades file. Seems such a file exists by default, nowadays, Debian 12.\n\n# Automatically apply OS security patches.\n# The --force-confdef/old tells Apt to not overwrite any existing configuration, and to ask no questions.\n# See e.g.: https://askubuntu.com/a/104912/48382.\n# APT::Periodic::AutoremoveInterval \"14\"; = remove auto-installed dependencies that are no longer needed.\n# APT::Periodic::AutocleanInterval \"14\"; = remove downloaded installation archives that are nowadays out-of-date.\n# APT::Periodic::MinAge \"8\" = packages won't be deleted until they're these many days old (default is 2).\n# more docs: less /usr/lib/apt/apt.systemd.daily\nauto_upgr_f=\"/etc/apt/apt.conf.d/20auto-upgrades\"\nif [ -f $auto_upgr_f ]; then\n log_message \"There's already an auto upgrades config file: $auto_upgr_f.\"\n log_message \"I'll leave it as is — I won't (re)configure automatic upgrades.\"\n log_message \"---- It's contents: ------\"\n cat $auto_upgr_f\n log_message \"--------------------------\"\n log_message \"Consider adding the below line, if it's missing,\"\n log_message \"so your server will reboot if needed, for upgrades to take effect:\"\n echo\n echo 'Unattended-Upgrade::Automatic-Reboot \"true\";'\n echo\nelse\n log_message 'Enabling automatic security updates and reboots...'\n did_what=\"$did_what Enabled automatic security updates and reboots.\"\n # About the packages we install:\n # apt-config-auto-update: Makes APT automatically update its package cache.\n # unattended-upgrades: Downloads and installs security upgrades automatically and unattended.\n DEBIAN_FRONTEND=noninteractive \\\n apt-get install -y \\\n -o Dpkg::Options::=\"--force-confdef\" \\\n -o Dpkg::Options::=\"--force-confold\" \\\n unattended-upgrades \\\n apt-config-auto-update\n cat <