From 09d4cc9b837b42d4ed6d8e6706b90df10c31742e Mon Sep 17 00:00:00 2001 From: Sovran Contributor Date: Mon, 21 Sep 2026 18:57:00 +0000 Subject: [PATCH] nextcloud, postgresql: fix Nextcloud 35 DB warnings on 32 GB hosts Nextcloud 35's Database checks flag three Performance issues out of the box: buffer cache hit ratio ~96% (wants 99%+), 100k+ dead tuples, and million-plus sequential scans on oc_mail_tags / oc_guests_users. Root causes in Sovran: stock 128MB shared_buffers, stock 60s autovacuum naptime, APCu file locking, and db:add-missing-indices running exactly once at install time (never on upgrades or app installs). Size Postgres for the README's Server + Desktop recommendation (32 GB RAM, NVMe): 2GB shared_buffers, 12GB effective_cache_size, 512MB maintenance_work_mem, 32MB work_mem, 4GB max_wal_size, 30s autovacuum naptime with 4 workers. shared_buffers stays below the 25% rule because Postgres shares the box with bitcoind, Electrs, LND, MariaDB and PHP-FPM. Scope the aggressive autovacuum to nextclouddb via ALTER DATABASE so the shared matrix-synapse DB keeps the milder cluster defaults. Add a local Redis (127.0.0.1:6379, Nextcloud only) and move memcache.distributed/locking to Redis; migrate existing installs with a one-shot since nextcloud-init never re-runs. Add a weekly nextcloud-db-maintenance timer (VACUUM ANALYZE + db:add-missing-*) so upgrades and later app installs can't regress the checks again. Note: shared_buffers needs one 'systemctl restart postgresql', which briefly takes down both Nextcloud and Matrix. Everything else is reload-only or scoped to nextclouddb. --- configuration.nix | 36 +++++++++++++ modules/nextcloud.nix | 115 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/configuration.nix b/configuration.nix index 7f50f43..aed8325 100644 --- a/configuration.nix +++ b/configuration.nix @@ -165,6 +165,14 @@ programs.fish = { enable = true; promptInit = "fastfetch"; }; # ── PostgreSQL base ──────────────────────────────────────── + # Shared cluster for Nextcloud (nextclouddb) + Matrix Synapse. + # Sized for the README's Server + Desktop recommendation (32 GB RAM, + # 500 GB NVMe OS + 2 TB NVMe timechain). Postgres shares the box with + # Bitcoin Core, Electrs, LND, MariaDB, PHP-FPM and GNOME, so + # shared_buffers stays below the 25%-of-RAM dedicated-server rule. + # Fixes Nextcloud 35 Database checks (pg.cache_hit_ratio, + # pg.dead_tuples). Override in custom.nix for other hosts, e.g.: + # services.postgresql.settings.shared_buffers = lib.mkForce "512MB"; services.postgresql = { enable = true; authentication = lib.mkForce '' @@ -172,6 +180,34 @@ host all all 127.0.0.1/32 trust host all all ::1/128 trust ''; + settings = { + # Memory — fixes low buffer cache hit ratio (stock default is + # 128MB shared_buffers). effective_cache_size is only a planner + # hint, not an allocation, so it can be generous. + # NOTE: changing shared_buffers requires a Postgres restart. + shared_buffers = "2GB"; + effective_cache_size = "12GB"; + maintenance_work_mem = "512MB"; + work_mem = "32MB"; + wal_buffers = "64MB"; + + # Checkpoints — spread write bursts out on NVMe. Reload-only. + min_wal_size = "1GB"; + max_wal_size = "4GB"; + checkpoint_completion_target = 0.9; + + # Autovacuum — the stock 60s naptime can't keep up with + # Nextcloud's and Synapse's write-heavy tables (filecache, + # activity, jobs, state). Reload-only. + autovacuum_naptime = "30s"; + autovacuum_vacuum_scale_factor = 0.05; + autovacuum_analyze_scale_factor = 0.025; + autovacuum_max_workers = 4; + + # NVMe planner assumptions (README: NVMe OS + data disks). + random_page_cost = "1.1"; + effective_io_concurrency = 200; + }; }; # ── Backups ──────────────────────────────────────────────── diff --git a/modules/nextcloud.nix b/modules/nextcloud.nix index 0b7d146..6b5b599 100755 --- a/modules/nextcloud.nix +++ b/modules/nextcloud.nix @@ -3,10 +3,22 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud { # ── PostgreSQL database ─────────────────────────────────── + # Cluster-wide tuning (shared_buffers, autovacuum) lives in + # configuration.nix so it is shared with Matrix Synapse. services.postgresql = { enable = true; }; + # ── Redis for Nextcloud distributed cache + file locking ─── + # Nextcloud does not recommend APCu for memcache.locking in production. + # TCP on localhost avoids unix-socket permission juggling with the caddy user. + # Scoped to Nextcloud only — Synapse / MariaDB / Bitcoin are unaffected. + services.redis.servers.nextcloud = { + enable = true; + bind = "127.0.0.1"; + port = 6379; + }; + # ── Auto-generate DB password and initialize ────────────── systemd.services.nextcloud-db-init = { description = "Initialize Nextcloud PostgreSQL database with auto-generated password"; @@ -47,14 +59,20 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud { if ! psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "nextclouddb"; then psql -U postgres -c "CREATE DATABASE nextclouddb WITH OWNER ncusr TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C';" fi + + # Per-database autovacuum, scoped to nextclouddb only. + # The shared matrix-synapse DB keeps the milder cluster defaults. + # Fixes Nextcloud 35 pg.dead_tuples warning. Idempotent. + psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_vacuum_scale_factor = '0.05';" + psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_analyze_scale_factor = '0.025';" ''; }; # ── Fully automated Nextcloud setup ─────────────────────── systemd.services.nextcloud-init = { description = "Download, extract, and fully configure Nextcloud"; - after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" ]; - wants = [ "network-online.target" ]; + after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" "redis-nextcloud.service" ]; + wants = [ "network-online.target" "redis-nextcloud.service" ]; requires = [ "postgresql.service" "nextcloud-db-init.service" ]; wantedBy = [ "multi-user.target" ]; @@ -150,7 +168,11 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud { php $INSTALL_DIR/occ config:system:set default_phone_region --value='US' php $INSTALL_DIR/occ config:system:set maintenance_window_start --type=integer --value=1 php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu' - php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\APCu' + php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis' + php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis' + php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1' + php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379 + php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5' php $INSTALL_DIR/occ config:system:set server_id --value='$SERVER_ID' php $INSTALL_DIR/occ background:cron " @@ -247,6 +269,93 @@ CREDS ''; }; + # ── Migrate existing installs to Redis locking ──────────── + # nextcloud-init only runs on fresh installs (ConditionPathExists + # !config.php), so pre-existing / pre-Sovran installs would keep + # APCu locking forever. This one-shot is idempotent and safe to + # re-run on every boot — occ just overwrites the same values. + systemd.services.nextcloud-redis-migrate = { + description = "Point existing Nextcloud installs at Redis locking"; + after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ]; + wants = [ "redis-nextcloud.service" ]; + wantedBy = [ "multi-user.target" ]; + unitConfig = { + ConditionPathExists = [ + "/var/lib/www/nextcloud/occ" + "/var/lib/www/nextcloud/config/config.php" + ]; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + path = with pkgs; [ coreutils shadow ]; + script = '' + set -euo pipefail + INSTALL_DIR="/var/lib/www/nextcloud" + # Wait briefly for Redis (TCP localhost:6379). + for i in $(seq 1 15); do + if (echo > /dev/tcp/127.0.0.1/6379) >/dev/null 2>&1; then + break + fi + sleep 2 + done + /run/wrappers/bin/su -s /bin/sh caddy -c " + php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu' + php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis' + php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis' + php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1' + php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379 + php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5' + " + ''; + }; + + # ── Recurring DB maintenance (Nextcloud 35 checks) ─────────── + # nextcloud-init runs db:add-missing-indices exactly once. Upgrades + # (e.g. to NC35) and later app installs (Mail, Guests) add tables + # like oc_mail_tags / oc_guests_users that then seq-scan forever. + # Weekly: VACUUM ANALYZE (dead tuples) + backfill missing indices. + # Scoped to nextclouddb only — matrix-synapse is untouched. + systemd.services.nextcloud-db-maintenance = { + description = "Nextcloud DB maintenance: VACUUM + missing indices"; + after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ]; + wants = [ "postgresql.service" ]; + unitConfig = { + ConditionPathExists = [ + "/var/lib/www/nextcloud/occ" + "/var/lib/www/nextcloud/config/config.php" + ]; + }; + serviceConfig = { + Type = "oneshot"; + }; + path = [ config.services.postgresql.package pkgs.coreutils pkgs.shadow ]; + script = '' + set -euo pipefail + INSTALL_DIR="/var/lib/www/nextcloud" + echo "Vacuuming nextclouddb..." + psql -U postgres -d nextclouddb -c "VACUUM (ANALYZE);" + echo "Backfilling Nextcloud indices..." + /run/wrappers/bin/su -s /bin/sh caddy -c " + php $INSTALL_DIR/occ db:add-missing-indices + php $INSTALL_DIR/occ db:add-missing-columns + php $INSTALL_DIR/occ db:add-missing-primary-keys + " + echo "Nextcloud DB maintenance complete." + ''; + }; + + systemd.timers.nextcloud-db-maintenance = { + description = "Weekly Nextcloud DB maintenance"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "Sun 03:30"; + Persistent = true; + RandomizedDelaySec = "30m"; + }; + }; + services.cron.systemCronJobs = [ "*/5 * * * * caddy /run/current-system/sw/bin/php -f /var/lib/www/nextcloud/cron.php" ];