nextcloud, postgresql: fix Nextcloud 35 DB warnings on 32 GB hosts

Nextcloud 35's Database checks flag three Performance issues out of the
box: buffer cache hit ratio ~96% (wants 99%+), 100k+ dead tuples, and
million-plus sequential scans on oc_mail_tags / oc_guests_users.

Root causes in Sovran: stock 128MB shared_buffers, stock 60s autovacuum
naptime, APCu file locking, and db:add-missing-indices running exactly
once at install time (never on upgrades or app installs).

Size Postgres for the README's Server + Desktop recommendation (32 GB
RAM, NVMe): 2GB shared_buffers, 12GB effective_cache_size, 512MB
maintenance_work_mem, 32MB work_mem, 4GB max_wal_size, 30s autovacuum
naptime with 4 workers. shared_buffers stays below the 25% rule because
Postgres shares the box with bitcoind, Electrs, LND, MariaDB and PHP-FPM.

Scope the aggressive autovacuum to nextclouddb via ALTER DATABASE so the
shared matrix-synapse DB keeps the milder cluster defaults.

Add a local Redis (127.0.0.1:6379, Nextcloud only) and move
memcache.distributed/locking to Redis; migrate existing installs with a
one-shot since nextcloud-init never re-runs.

Add a weekly nextcloud-db-maintenance timer (VACUUM ANALYZE +
db:add-missing-*) so upgrades and later app installs can't regress the
checks again.

Note: shared_buffers needs one 'systemctl restart postgresql', which
briefly takes down both Nextcloud and Matrix. Everything else is
reload-only or scoped to nextclouddb.

configuration.nix

@@ -165,6 +165,14 @@
   programs.fish = { enable = true; promptInit = "fastfetch"; };
 
   # ── PostgreSQL base ────────────────────────────────────────
+  # Shared cluster for Nextcloud (nextclouddb) + Matrix Synapse.
+  # Sized for the README's Server + Desktop recommendation (32 GB RAM,
+  # 500 GB NVMe OS + 2 TB NVMe timechain). Postgres shares the box with
+  # Bitcoin Core, Electrs, LND, MariaDB, PHP-FPM and GNOME, so
+  # shared_buffers stays below the 25%-of-RAM dedicated-server rule.
+  # Fixes Nextcloud 35 Database checks (pg.cache_hit_ratio,
+  # pg.dead_tuples). Override in custom.nix for other hosts, e.g.:
+  #   services.postgresql.settings.shared_buffers = lib.mkForce "512MB";
   services.postgresql = {
     enable = true;
     authentication = lib.mkForce ''
@@ -172,6 +180,34 @@
       host    all   all   127.0.0.1/32  trust
       host    all   all   ::1/128       trust
     '';
+    settings = {
+      # Memory — fixes low buffer cache hit ratio (stock default is
+      # 128MB shared_buffers). effective_cache_size is only a planner
+      # hint, not an allocation, so it can be generous.
+      # NOTE: changing shared_buffers requires a Postgres restart.
+      shared_buffers = "2GB";
+      effective_cache_size = "12GB";
+      maintenance_work_mem = "512MB";
+      work_mem = "32MB";
+      wal_buffers = "64MB";
+
+      # Checkpoints — spread write bursts out on NVMe. Reload-only.
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      checkpoint_completion_target = 0.9;
+
+      # Autovacuum — the stock 60s naptime can't keep up with
+      # Nextcloud's and Synapse's write-heavy tables (filecache,
+      # activity, jobs, state). Reload-only.
+      autovacuum_naptime = "30s";
+      autovacuum_vacuum_scale_factor = 0.05;
+      autovacuum_analyze_scale_factor = 0.025;
+      autovacuum_max_workers = 4;
+
+      # NVMe planner assumptions (README: NVMe OS + data disks).
+      random_page_cost = "1.1";
+      effective_io_concurrency = 200;
+    };
   };
 
   # ── Backups ────────────────────────────────────────────────

modules/nextcloud.nix

@@ -3,10 +3,22 @@
 lib.mkIf config.sovran_systemsOS.services.nextcloud {
 
   # ── PostgreSQL database ───────────────────────────────────
+  # Cluster-wide tuning (shared_buffers, autovacuum) lives in
+  # configuration.nix so it is shared with Matrix Synapse.
   services.postgresql = {
     enable = true;
   };
 
+  # ── Redis for Nextcloud distributed cache + file locking ───
+  # Nextcloud does not recommend APCu for memcache.locking in production.
+  # TCP on localhost avoids unix-socket permission juggling with the caddy user.
+  # Scoped to Nextcloud only — Synapse / MariaDB / Bitcoin are unaffected.
+  services.redis.servers.nextcloud = {
+    enable = true;
+    bind = "127.0.0.1";
+    port = 6379;
+  };
+
   # ── Auto-generate DB password and initialize ──────────────
   systemd.services.nextcloud-db-init = {
     description = "Initialize Nextcloud PostgreSQL database with auto-generated password";
@@ -47,14 +59,20 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
       if ! psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "nextclouddb"; then
         psql -U postgres -c "CREATE DATABASE nextclouddb WITH OWNER ncusr TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C';"
       fi
+
+      # Per-database autovacuum, scoped to nextclouddb only.
+      # The shared matrix-synapse DB keeps the milder cluster defaults.
+      # Fixes Nextcloud 35 pg.dead_tuples warning. Idempotent.
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_vacuum_scale_factor = '0.05';"
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_analyze_scale_factor = '0.025';"
     '';
   };
 
   # ── Fully automated Nextcloud setup ───────────────────────
   systemd.services.nextcloud-init = {
     description = "Download, extract, and fully configure Nextcloud";
-    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" ];
+    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" "redis-nextcloud.service" ];
-    wants = [ "network-online.target" ];
+    wants = [ "network-online.target" "redis-nextcloud.service" ];
     requires = [ "postgresql.service" "nextcloud-db-init.service" ];
     wantedBy = [ "multi-user.target" ];
 
@@ -150,7 +168,11 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
         php $INSTALL_DIR/occ config:system:set default_phone_region --value='US'
         php $INSTALL_DIR/occ config:system:set maintenance_window_start --type=integer --value=1
         php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
-        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
         php $INSTALL_DIR/occ config:system:set server_id --value='$SERVER_ID'
         php $INSTALL_DIR/occ background:cron
       "
@@ -247,6 +269,93 @@ CREDS
     '';
   };
 
+  # ── Migrate existing installs to Redis locking ────────────
+  # nextcloud-init only runs on fresh installs (ConditionPathExists
+  # !config.php), so pre-existing / pre-Sovran installs would keep
+  # APCu locking forever. This one-shot is idempotent and safe to
+  # re-run on every boot — occ just overwrites the same values.
+  systemd.services.nextcloud-redis-migrate = {
+    description = "Point existing Nextcloud installs at Redis locking";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "redis-nextcloud.service" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    path = with pkgs; [ coreutils shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      # Wait briefly for Redis (TCP localhost:6379).
+      for i in $(seq 1 15); do
+        if (echo > /dev/tcp/127.0.0.1/6379) >/dev/null 2>&1; then
+          break
+        fi
+        sleep 2
+      done
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
+      "
+    '';
+  };
+
+  # ── Recurring DB maintenance (Nextcloud 35 checks) ───────────
+  # nextcloud-init runs db:add-missing-indices exactly once. Upgrades
+  # (e.g. to NC35) and later app installs (Mail, Guests) add tables
+  # like oc_mail_tags / oc_guests_users that then seq-scan forever.
+  # Weekly: VACUUM ANALYZE (dead tuples) + backfill missing indices.
+  # Scoped to nextclouddb only — matrix-synapse is untouched.
+  systemd.services.nextcloud-db-maintenance = {
+    description = "Nextcloud DB maintenance: VACUUM + missing indices";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "postgresql.service" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+    };
+    path = [ config.services.postgresql.package pkgs.coreutils pkgs.shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      echo "Vacuuming nextclouddb..."
+      psql -U postgres -d nextclouddb -c "VACUUM (ANALYZE);"
+      echo "Backfilling Nextcloud indices..."
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ db:add-missing-indices
+        php $INSTALL_DIR/occ db:add-missing-columns
+        php $INSTALL_DIR/occ db:add-missing-primary-keys
+      "
+      echo "Nextcloud DB maintenance complete."
+    '';
+  };
+
+  systemd.timers.nextcloud-db-maintenance = {
+    description = "Weekly Nextcloud DB maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "Sun 03:30";
+      Persistent = true;
+      RandomizedDelaySec = "30m";
+    };
+  };
+
   services.cron.systemCronJobs = [
     "*/5 * * * * caddy /run/current-system/sw/bin/php -f /var/lib/www/nextcloud/cron.php"
   ];