synapse: performance tuning for 32 GB Server+Desktop hosts

Monolith Synapse spends most of its RAM on caches to avoid Postgres
round-trips, but Sovran ships stock cache settings (global_factor 0.5,
10K event cache, no autotuning) and a default 5-connection DB pool.

- caches.global_factor 4.0 + 100K event cache + autotuning capped at
  2G (target 1G), with boosts for the /sync and room-join hot paths.
- DB pool cp_min 5 / cp_max 15, txn_limit 10000 (fewer reconnects).
- gc_thresholds raised to cut GC pauses on a 32 GB box.
- cache-memory extra for cache-size statistics.
- Per-database autovacuum (ALTER DATABASE, scoped to matrix-synapse)
  matching the nextclouddb tuning.

Deliberately unchanged: presence and URL previews stay enabled
(disabling them is faster but user-visible), and no workers — monolith
is the right call under ~100 users. Workers would need Redis
replication, the redis extra, and Caddy reverse-proxy rework; revisit
if federation load ever justifies it.

configuration.nix

@@ -165,6 +165,14 @@
   programs.fish = { enable = true; promptInit = "fastfetch"; };
 
   # ── PostgreSQL base ────────────────────────────────────────
+  # Shared cluster for Nextcloud (nextclouddb) + Matrix Synapse.
+  # Sized for the README's Server + Desktop recommendation (32 GB RAM,
+  # 500 GB NVMe OS + 2 TB NVMe timechain). Postgres shares the box with
+  # Bitcoin Core, Electrs, LND, MariaDB, PHP-FPM and GNOME, so
+  # shared_buffers stays below the 25%-of-RAM dedicated-server rule.
+  # Fixes Nextcloud 35 Database checks (pg.cache_hit_ratio,
+  # pg.dead_tuples). Override in custom.nix for other hosts, e.g.:
+  #   services.postgresql.settings.shared_buffers = lib.mkForce "512MB";
   services.postgresql = {
     enable = true;
     authentication = lib.mkForce ''
@@ -172,6 +180,34 @@
       host    all   all   127.0.0.1/32  trust
       host    all   all   ::1/128       trust
     '';
+    settings = {
+      # Memory — fixes low buffer cache hit ratio (stock default is
+      # 128MB shared_buffers). effective_cache_size is only a planner
+      # hint, not an allocation, so it can be generous.
+      # NOTE: changing shared_buffers requires a Postgres restart.
+      shared_buffers = "2GB";
+      effective_cache_size = "12GB";
+      maintenance_work_mem = "512MB";
+      work_mem = "32MB";
+      wal_buffers = "64MB";
+
+      # Checkpoints — spread write bursts out on NVMe. Reload-only.
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      checkpoint_completion_target = 0.9;
+
+      # Autovacuum — the stock 60s naptime can't keep up with
+      # Nextcloud's and Synapse's write-heavy tables (filecache,
+      # activity, jobs, state). Reload-only.
+      autovacuum_naptime = "30s";
+      autovacuum_vacuum_scale_factor = 0.05;
+      autovacuum_analyze_scale_factor = 0.025;
+      autovacuum_max_workers = 4;
+
+      # NVMe planner assumptions (README: NVMe OS + data disks).
+      random_page_cost = "1.1";
+      effective_io_concurrency = 200;
+    };
   };
 
   # ── Backups ────────────────────────────────────────────────

modules/nextcloud.nix

@@ -3,10 +3,22 @@
 lib.mkIf config.sovran_systemsOS.services.nextcloud {
 
   # ── PostgreSQL database ───────────────────────────────────
+  # Cluster-wide tuning (shared_buffers, autovacuum) lives in
+  # configuration.nix so it is shared with Matrix Synapse.
   services.postgresql = {
     enable = true;
   };
 
+  # ── Redis for Nextcloud distributed cache + file locking ───
+  # Nextcloud does not recommend APCu for memcache.locking in production.
+  # TCP on localhost avoids unix-socket permission juggling with the caddy user.
+  # Scoped to Nextcloud only — Synapse / MariaDB / Bitcoin are unaffected.
+  services.redis.servers.nextcloud = {
+    enable = true;
+    bind = "127.0.0.1";
+    port = 6379;
+  };
+
   # ── Auto-generate DB password and initialize ──────────────
   systemd.services.nextcloud-db-init = {
     description = "Initialize Nextcloud PostgreSQL database with auto-generated password";
@@ -47,14 +59,20 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
       if ! psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "nextclouddb"; then
         psql -U postgres -c "CREATE DATABASE nextclouddb WITH OWNER ncusr TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C';"
       fi
+
+      # Per-database autovacuum, scoped to nextclouddb only.
+      # The shared matrix-synapse DB keeps the milder cluster defaults.
+      # Fixes Nextcloud 35 pg.dead_tuples warning. Idempotent.
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_vacuum_scale_factor = '0.05';"
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_analyze_scale_factor = '0.025';"
     '';
   };
 
   # ── Fully automated Nextcloud setup ───────────────────────
   systemd.services.nextcloud-init = {
     description = "Download, extract, and fully configure Nextcloud";
-    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" ];
+    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" "redis-nextcloud.service" ];
-    wants = [ "network-online.target" ];
+    wants = [ "network-online.target" "redis-nextcloud.service" ];
     requires = [ "postgresql.service" "nextcloud-db-init.service" ];
     wantedBy = [ "multi-user.target" ];
 
@@ -150,7 +168,11 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
         php $INSTALL_DIR/occ config:system:set default_phone_region --value='US'
         php $INSTALL_DIR/occ config:system:set maintenance_window_start --type=integer --value=1
         php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
-        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
         php $INSTALL_DIR/occ config:system:set server_id --value='$SERVER_ID'
         php $INSTALL_DIR/occ background:cron
       "
@@ -247,6 +269,93 @@ CREDS
     '';
   };
 
+  # ── Migrate existing installs to Redis locking ────────────
+  # nextcloud-init only runs on fresh installs (ConditionPathExists
+  # !config.php), so pre-existing / pre-Sovran installs would keep
+  # APCu locking forever. This one-shot is idempotent and safe to
+  # re-run on every boot — occ just overwrites the same values.
+  systemd.services.nextcloud-redis-migrate = {
+    description = "Point existing Nextcloud installs at Redis locking";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "redis-nextcloud.service" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    path = with pkgs; [ coreutils shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      # Wait briefly for Redis (TCP localhost:6379).
+      for i in $(seq 1 15); do
+        if (echo > /dev/tcp/127.0.0.1/6379) >/dev/null 2>&1; then
+          break
+        fi
+        sleep 2
+      done
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
+      "
+    '';
+  };
+
+  # ── Recurring DB maintenance (Nextcloud 35 checks) ───────────
+  # nextcloud-init runs db:add-missing-indices exactly once. Upgrades
+  # (e.g. to NC35) and later app installs (Mail, Guests) add tables
+  # like oc_mail_tags / oc_guests_users that then seq-scan forever.
+  # Weekly: VACUUM ANALYZE (dead tuples) + backfill missing indices.
+  # Scoped to nextclouddb only — matrix-synapse is untouched.
+  systemd.services.nextcloud-db-maintenance = {
+    description = "Nextcloud DB maintenance: VACUUM + missing indices";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "postgresql.service" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+    };
+    path = [ config.services.postgresql.package pkgs.coreutils pkgs.shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      echo "Vacuuming nextclouddb..."
+      psql -U postgres -d nextclouddb -c "VACUUM (ANALYZE);"
+      echo "Backfilling Nextcloud indices..."
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ db:add-missing-indices
+        php $INSTALL_DIR/occ db:add-missing-columns
+        php $INSTALL_DIR/occ db:add-missing-primary-keys
+      "
+      echo "Nextcloud DB maintenance complete."
+    '';
+  };
+
+  systemd.timers.nextcloud-db-maintenance = {
+    description = "Weekly Nextcloud DB maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "Sun 03:30";
+      Persistent = true;
+      RandomizedDelaySec = "30m";
+    };
+  };
+
   services.cron.systemCronJobs = [
     "*/5 * * * * caddy /run/current-system/sw/bin/php -f /var/lib/www/nextcloud/cron.php"
   ];

modules/synapse.nix

@@ -60,6 +60,35 @@ lib.mkIf config.sovran_systemsOS.services.synapse {
     '';
   };
 
+  # ── Per-database Postgres tuning (matrix-synapse only) ─────
+  # Mirrors the nextclouddb tuning: Synapse's state and event tables are
+  # write-heavy and bloat fast under stock autovacuum. Scoped via ALTER
+  # DATABASE so each DB gets what suits it. Idempotent.
+  systemd.services.matrix-synapse-db-tune = {
+    description = "Apply per-database Postgres tuning for Matrix Synapse";
+    after = [ "postgresql.service" ];
+    requires = [ "postgresql.service" ];
+    before = [ "matrix-synapse.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    path = [ config.services.postgresql.package pkgs.coreutils ];
+    script = ''
+      set -euo pipefail
+      # Wait for ensureDatabases to have created the DB on first boot.
+      for i in $(seq 1 30); do
+        if psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "matrix-synapse"; then
+          break
+        fi
+        sleep 2
+      done
+      psql -U postgres -d matrix-synapse -c "ALTER DATABASE \"matrix-synapse\" SET autovacuum_vacuum_scale_factor = '0.05';"
+      psql -U postgres -d matrix-synapse -c "ALTER DATABASE \"matrix-synapse\" SET autovacuum_analyze_scale_factor = '0.025';"
+    '';
+  };
+
   # ── Generate runtime config from domain files ───────────────
   systemd.services.matrix-synapse-runtime-config = {
     description = "Generate Synapse runtime config from domain files";
@@ -94,16 +123,23 @@ EOF
   # ── Synapse service ─────────────────────────────────────────
   services.matrix-synapse = {
     enable = true;
+    # cache-memory provides cache-size statistics for the autotuning below
+    # (in addition to the NixOS defaults).
+    extras = [ "systemd" "postgres" "url-preview" "cache-memory" ];
     extraConfigFiles = [
       "/run/matrix-synapse/runtime-config.yaml"
     ];
     settings = {
       database = {
         name = "psycopg2";
+        # Recycle pooled connections less often (fewer reconnects).
+        txn_limit = 10000;
         args = {
           host = "localhost";
           database = "matrix-synapse";
           user = "matrix-synapse";
+          cp_min = 5;
+          cp_max = 15;
         };
       };
       push.include_content = false;
@@ -120,6 +156,32 @@ EOF
       ];
       presence.enabled = true;
       enable_registration = false;
+      # ── Performance (32 GB Server + Desktop) ─────────────────
+      # Synapse trades RAM for fewer Postgres round-trips; most RAM goes
+      # to caches. Stock is global_factor 0.5 + 10K event cache, which
+      # leaves syncs hitting the database on every request.
+      # Deliberately unchanged: presence and URL previews stay enabled —
+      # disabling them is faster but changes user-visible behavior.
+      event_cache_size = "100K";
+      caches = {
+        global_factor = 4.0;
+        expire_caches = true;
+        cache_entry_ttl = "30m";
+        sync_response_cache_duration = "2m";
+        cache_autotuning = {
+          max_cache_memory_usage = "2G";
+          target_cache_memory_usage = "1G";
+          min_cache_ttl = "30s";
+        };
+        per_cache_factors = {
+          # Hot paths for /sync and room joins.
+          get_users_in_room = 3.0;
+          get_current_state_ids = 3.0;
+          get_unread_event_push_actions_by_room_for_user = 5.0;
+        };
+      };
+      # Fewer GC pauses at the cost of a little more memory.
+      gc_thresholds = [ 1500 20 10 ];
       listeners = [
         {
           port = 8008;