synapse: performance tuning for 32 GB Server+Desktop hosts

Monolith Synapse spends most of its RAM on caches to avoid Postgres
round-trips, but Sovran ships stock cache settings (global_factor 0.5,
10K event cache, no autotuning) and a default 5-connection DB pool.

- caches.global_factor 4.0 + 100K event cache + autotuning capped at
  2G (target 1G), with boosts for the /sync and room-join hot paths.
- DB pool cp_min 5 / cp_max 15, txn_limit 10000 (fewer reconnects).
- gc_thresholds raised to cut GC pauses on a 32 GB box.
- cache-memory extra for cache-size statistics.
- Per-database autovacuum (ALTER DATABASE, scoped to matrix-synapse)
  matching the nextclouddb tuning.

Deliberately unchanged: presence and URL previews stay enabled
(disabling them is faster but user-visible), and no workers — monolith
is the right call under ~100 users. Workers would need Redis
replication, the redis extra, and Caddy reverse-proxy rework; revisit
if federation load ever justifies it.

configuration.nix

@@ -165,6 +165,14 @@
   programs.fish = { enable = true; promptInit = "fastfetch"; };
 
   # ── PostgreSQL base ────────────────────────────────────────
+  # Shared cluster for Nextcloud (nextclouddb) + Matrix Synapse.
+  # Sized for the README's Server + Desktop recommendation (32 GB RAM,
+  # 500 GB NVMe OS + 2 TB NVMe timechain). Postgres shares the box with
+  # Bitcoin Core, Electrs, LND, MariaDB, PHP-FPM and GNOME, so
+  # shared_buffers stays below the 25%-of-RAM dedicated-server rule.
+  # Fixes Nextcloud 35 Database checks (pg.cache_hit_ratio,
+  # pg.dead_tuples). Override in custom.nix for other hosts, e.g.:
+  #   services.postgresql.settings.shared_buffers = lib.mkForce "512MB";
   services.postgresql = {
     enable = true;
     authentication = lib.mkForce ''
@@ -172,6 +180,34 @@
       host    all   all   127.0.0.1/32  trust
       host    all   all   ::1/128       trust
     '';
+    settings = {
+      # Memory — fixes low buffer cache hit ratio (stock default is
+      # 128MB shared_buffers). effective_cache_size is only a planner
+      # hint, not an allocation, so it can be generous.
+      # NOTE: changing shared_buffers requires a Postgres restart.
+      shared_buffers = "2GB";
+      effective_cache_size = "12GB";
+      maintenance_work_mem = "512MB";
+      work_mem = "32MB";
+      wal_buffers = "64MB";
+
+      # Checkpoints — spread write bursts out on NVMe. Reload-only.
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      checkpoint_completion_target = 0.9;
+
+      # Autovacuum — the stock 60s naptime can't keep up with
+      # Nextcloud's and Synapse's write-heavy tables (filecache,
+      # activity, jobs, state). Reload-only.
+      autovacuum_naptime = "30s";
+      autovacuum_vacuum_scale_factor = 0.05;
+      autovacuum_analyze_scale_factor = 0.025;
+      autovacuum_max_workers = 4;
+
+      # NVMe planner assumptions (README: NVMe OS + data disks).
+      random_page_cost = "1.1";
+      effective_io_concurrency = 200;
+    };
   };
 
   # ── Backups ────────────────────────────────────────────────

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1788871704,
+        "lastModified": 1789651044,
-        "narHash": "sha256-Vtc2SqCB7NOO028+K5JHbpWQ3XVpLyoC75MZQ12l13o=",
+        "narHash": "sha256-4VAbiqMJXW/fgdMqDN4+03PkCa3xRnWi4zEX9J6/QRk=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "e14502cba22806341f8c54c6f0c349094830c58b",
+        "rev": "bc71039666a17100c49e67a93e1d7a15f31dee47",
         "type": "github"
       },
       "original": {
@@ -41,11 +41,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1788179970,
+        "lastModified": 1789563672,
-        "narHash": "sha256-r5LmxzIhsu5+oDybatN/HJ8roYOKjb2Apa5xI6v46VU=",
+        "narHash": "sha256-05W7P4/pqgpliOA/HyZM6IkKd6ojI8L5X/x9tgthckI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1db62ab7d2ccf1916bbf7deb61fc9d16f1c4ab49",
+        "rev": "88142f2dc8f2727eec89047694ec8000f92ce542",
         "type": "github"
       },
       "original": {
@@ -56,11 +56,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1789344334,
+        "lastModified": 1789542786,
-        "narHash": "sha256-vdhpDJ3Lr24lkZ+fDCjmBRRtw9/vcSzkqGJOJyF5h2U=",
+        "narHash": "sha256-ajQuqcxnj6RYnwCjA/9FTCnrdR2+BDzstE8UembXf60=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "c3eea5b2156db11c7eeeada3dc737711255b253e",
+        "rev": "4c7870105e7f1fdf9c48688c8d7efc21abf0688a",
         "type": "github"
       },
       "original": {
@@ -72,11 +72,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1789286504,
+        "lastModified": 1789546076,
-        "narHash": "sha256-eiEK7cKZORNEvX0GeF3RtNEF/JXhgf2RqSp3230q13E=",
+        "narHash": "sha256-zVxLZiSnmaaPLwnhj7pwmqe3axBg/C6nG5JZsJMh2g4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ef34387ddd751e1ab8857adf4676492d32eb24ec",
+        "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3",
         "type": "github"
       },
       "original": {

flake.nix

@@ -6,8 +6,6 @@
 		nixvim.url = "github:nix-community/nixvim";
 		btc-clients.url = "github:emmanuelrosa/btc-clients-nix";
 		nixpkgs-stable.url = "github:nixos/nixpkgs/nixos-26.05";
-
-		# Bitcoin / Lightning stack — standalone flake, consumed as a module.
 		sovran-bitcoin.url = "github:naturallaw777/Sovran_Bitcoin";
 	};
 
@@ -19,24 +17,6 @@
 				system = prev.stdenv.hostPlatform.system;
 				config.allowUnfree = true;
 			};
-
-			# Pin LiveKit to 1.13.6: element-calling.nix sets
-			# rtc.advertise_internal_ip, which gives LAN callers a host candidate
-			# so calls work on Wi-Fi without the router needing NAT-hairpin. That
-			# flag is only honoured when node_ip is set manually from LiveKit
-			# v1.13.6 (mediatransportutil f234b53); nixpkgs-unstable currently
-			# ships 1.13.5. Remove this override once nixpkgs-unstable reaches
-			# >= 1.13.6.
-			livekit = prev.livekit.overrideAttrs (old: {
-				version = "1.13.6";
-				src = prev.fetchFromGitHub {
-					owner = "livekit";
-					repo = "livekit";
-					rev = "v1.13.6";
-					hash = "sha256-sUAx6ooeEUUqot5xuZv7xiQa3DdRFVULteTwYgFUzCI=";
-				};
-				vendorHash = "sha256-nOGSmoNuQQm/sIVI1HojsiS4GkbhA68uYMQ6X7d4a5Q=";
-			});
 		};
 	in
 	{
@@ -77,14 +57,5 @@
 				];
 			};
 		};
-
-		checks.x86_64-linux = let
-			pkgs = import nixpkgs {
-				system = "x86_64-linux";
-			};
-		in {
-			# Bitcoin hardening and package checks now live in the Sovran_Bitcoin flake.
-			# Run them with: nix build github:naturallaw777/Sovran_Bitcoin#checks.x86_64-linux
-		};
 	};
 }

modules/nextcloud.nix

@@ -3,10 +3,22 @@
 lib.mkIf config.sovran_systemsOS.services.nextcloud {
 
   # ── PostgreSQL database ───────────────────────────────────
+  # Cluster-wide tuning (shared_buffers, autovacuum) lives in
+  # configuration.nix so it is shared with Matrix Synapse.
   services.postgresql = {
     enable = true;
   };
 
+  # ── Redis for Nextcloud distributed cache + file locking ───
+  # Nextcloud does not recommend APCu for memcache.locking in production.
+  # TCP on localhost avoids unix-socket permission juggling with the caddy user.
+  # Scoped to Nextcloud only — Synapse / MariaDB / Bitcoin are unaffected.
+  services.redis.servers.nextcloud = {
+    enable = true;
+    bind = "127.0.0.1";
+    port = 6379;
+  };
+
   # ── Auto-generate DB password and initialize ──────────────
   systemd.services.nextcloud-db-init = {
     description = "Initialize Nextcloud PostgreSQL database with auto-generated password";
@@ -47,14 +59,20 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
       if ! psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "nextclouddb"; then
         psql -U postgres -c "CREATE DATABASE nextclouddb WITH OWNER ncusr TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C';"
       fi
+
+      # Per-database autovacuum, scoped to nextclouddb only.
+      # The shared matrix-synapse DB keeps the milder cluster defaults.
+      # Fixes Nextcloud 35 pg.dead_tuples warning. Idempotent.
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_vacuum_scale_factor = '0.05';"
+      psql -U postgres -d nextclouddb -c "ALTER DATABASE nextclouddb SET autovacuum_analyze_scale_factor = '0.025';"
     '';
   };
 
   # ── Fully automated Nextcloud setup ───────────────────────
   systemd.services.nextcloud-init = {
     description = "Download, extract, and fully configure Nextcloud";
-    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" ];
+    after = [ "network-online.target" "postgresql.service" "phpfpm-nextcloud.service" "nextcloud-db-init.service" "redis-nextcloud.service" ];
-    wants = [ "network-online.target" ];
+    wants = [ "network-online.target" "redis-nextcloud.service" ];
     requires = [ "postgresql.service" "nextcloud-db-init.service" ];
     wantedBy = [ "multi-user.target" ];
 
@@ -150,7 +168,11 @@ lib.mkIf config.sovran_systemsOS.services.nextcloud {
         php $INSTALL_DIR/occ config:system:set default_phone_region --value='US'
         php $INSTALL_DIR/occ config:system:set maintenance_window_start --type=integer --value=1
         php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
-        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
         php $INSTALL_DIR/occ config:system:set server_id --value='$SERVER_ID'
         php $INSTALL_DIR/occ background:cron
       "
@@ -247,6 +269,93 @@ CREDS
     '';
   };
 
+  # ── Migrate existing installs to Redis locking ────────────
+  # nextcloud-init only runs on fresh installs (ConditionPathExists
+  # !config.php), so pre-existing / pre-Sovran installs would keep
+  # APCu locking forever. This one-shot is idempotent and safe to
+  # re-run on every boot — occ just overwrites the same values.
+  systemd.services.nextcloud-redis-migrate = {
+    description = "Point existing Nextcloud installs at Redis locking";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "redis-nextcloud.service" ];
+    wantedBy = [ "multi-user.target" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    path = with pkgs; [ coreutils shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      # Wait briefly for Redis (TCP localhost:6379).
+      for i in $(seq 1 15); do
+        if (echo > /dev/tcp/127.0.0.1/6379) >/dev/null 2>&1; then
+          break
+        fi
+        sleep 2
+      done
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ config:system:set memcache.local --value='\OC\Memcache\APCu'
+        php $INSTALL_DIR/occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set memcache.locking --value='\OC\Memcache\Redis'
+        php $INSTALL_DIR/occ config:system:set redis host --value='127.0.0.1'
+        php $INSTALL_DIR/occ config:system:set redis port --type=integer --value=6379
+        php $INSTALL_DIR/occ config:system:set redis timeout --value='1.5'
+      "
+    '';
+  };
+
+  # ── Recurring DB maintenance (Nextcloud 35 checks) ───────────
+  # nextcloud-init runs db:add-missing-indices exactly once. Upgrades
+  # (e.g. to NC35) and later app installs (Mail, Guests) add tables
+  # like oc_mail_tags / oc_guests_users that then seq-scan forever.
+  # Weekly: VACUUM ANALYZE (dead tuples) + backfill missing indices.
+  # Scoped to nextclouddb only — matrix-synapse is untouched.
+  systemd.services.nextcloud-db-maintenance = {
+    description = "Nextcloud DB maintenance: VACUUM + missing indices";
+    after = [ "postgresql.service" "redis-nextcloud.service" "phpfpm-nextcloud.service" ];
+    wants = [ "postgresql.service" ];
+    unitConfig = {
+      ConditionPathExists = [
+        "/var/lib/www/nextcloud/occ"
+        "/var/lib/www/nextcloud/config/config.php"
+      ];
+    };
+    serviceConfig = {
+      Type = "oneshot";
+    };
+    path = [ config.services.postgresql.package pkgs.coreutils pkgs.shadow ];
+    script = ''
+      set -euo pipefail
+      INSTALL_DIR="/var/lib/www/nextcloud"
+      echo "Vacuuming nextclouddb..."
+      psql -U postgres -d nextclouddb -c "VACUUM (ANALYZE);"
+      echo "Backfilling Nextcloud indices..."
+      /run/wrappers/bin/su -s /bin/sh caddy -c "
+        php $INSTALL_DIR/occ db:add-missing-indices
+        php $INSTALL_DIR/occ db:add-missing-columns
+        php $INSTALL_DIR/occ db:add-missing-primary-keys
+      "
+      echo "Nextcloud DB maintenance complete."
+    '';
+  };
+
+  systemd.timers.nextcloud-db-maintenance = {
+    description = "Weekly Nextcloud DB maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "Sun 03:30";
+      Persistent = true;
+      RandomizedDelaySec = "30m";
+    };
+  };
+
   services.cron.systemCronJobs = [
     "*/5 * * * * caddy /run/current-system/sw/bin/php -f /var/lib/www/nextcloud/cron.php"
   ];

modules/php.nix

@@ -1,29 +1,70 @@
 { config, pkgs, lib, ... }:
 
+# ── Shared PHP for Nextcloud + WordPress ──────────────────────────────────────
+#
+# One interpreter (with one extension set and one php.ini) is shared by the
+# phpfpm-nextcloud and phpfpm-wordpress pools, the Nextcloud cron job and the
+# occ / wp-cli helper scripts.  Every consumer must reference
+# config.sovran_systemsOS.phpPackage (or /run/current-system/sw/bin/php) so
+# that the CLI and the FPM pools always run the *same* PHP.
+#
+# Version policy (September 2026):
+#   • Nextcloud 35 supports PHP 8.3 / 8.4 / 8.5 and recommends 8.5.  Its setup
+#     check flags 8.3 as "deprecated since Nextcloud 35" and warns that
+#     Nextcloud 36 may require at least 8.4.
+#   • WordPress 6.9 / 7.0 fully support PHP 8.4 and 8.5.
+#   • PHP 8.3 has been security-only since 2025-12-31; PHP 8.4 leaves active
+#     support on 2026-12-31; PHP 8.5 is actively supported until 2027-12-31.
+#
+# To fall back to PHP 8.4 (nixpkgs' current default `pkgs.php`) change only
+# the `phpBase` line below.
 
 let
+	phpBase = pkgs.php85;
+
+	custom-php = phpBase.buildEnv {
+		# `enabled` is nixpkgs' default extension set.  It already contains every
+		# module Nextcloud lists as required or recommended (ctype, curl, dom,
+		# fileinfo, gd, intl, mbstring, openssl, posix, session, simplexml,
+		# xmlreader, xmlwriter, zip, zlib, pdo_pgsql, pdo_mysql, bcmath, gmp,
+		# exif, sodium, sysvsem, pcntl, ...).  OPcache is compiled into PHP >= 8.5
+		# and no longer appears as a separate extension.
+		extensions = { enabled, all }: enabled ++ (with all; [
+			bz2        # Nextcloud: bz2 archive support
+			apcu       # Nextcloud: memcache.local  (apc.enable_cli=1 below is mandatory for occ + cron)
+			redis      # Nextcloud: memcache.distributed / file locking once a Redis server is configured
+			imagick    # Nextcloud: previews + theming (nixpkgs ImageMagick is built with SVG support)
+			memcached  # WordPress object-cache plugins (legacy option for Nextcloud)
+		]);
 
-	custom-php = pkgs.php83.buildEnv {
-		extensions = { enabled, all }: enabled ++ (with all; [ bz2 apcu redis imagick memcached ]);
 		extraConfig = ''
+			; ── Error handling (production) ─────────────────────────────────
+			display_errors = Off
+			display_startup_errors = Off
+			log_errors = On
 
-			display_errors = On
+			; ── Limits ──────────────────────────────────────────────────────
-			display_startup_errors = On
 			max_execution_time = 10000
 			max_input_time = 3000
-			memory_limit = 1G;
+			memory_limit = 1G
-			opcache.enable=1;
-			opcache.memory_consumption=512;
-			opcache_revalidate_freq = 240;
-			opcache.max_accelerated_files=20000;
 			post_max_size = 3G
 			upload_max_filesize = 3G
-			apc.enable_cli=1
+
+			; ── OPcache (Nextcloud "Server tuning" recommendations) ─────────
+			opcache.enable = 1
+			opcache.memory_consumption = 512
 			opcache.interned_strings_buffer = 192
+			opcache.max_accelerated_files = 20000
+			opcache.revalidate_freq = 240
+			opcache.save_comments = 1
+
+			; ── APCu ────────────────────────────────────────────────────────
+			apc.enable_cli = 1
+
+			; ── phpredis session locking (only used with session.save_handler = redis)
 			redis.session.locking_enabled = 1
 			redis.session.lock_retries = -1
 			redis.session.lock_wait_time = 10000
-			
 		'';
 	};
 in
@@ -55,3 +96,4 @@ in
 		];
 	};
 }
+

modules/synapse.nix

@@ -60,6 +60,35 @@ lib.mkIf config.sovran_systemsOS.services.synapse {
     '';
   };
 
+  # ── Per-database Postgres tuning (matrix-synapse only) ─────
+  # Mirrors the nextclouddb tuning: Synapse's state and event tables are
+  # write-heavy and bloat fast under stock autovacuum. Scoped via ALTER
+  # DATABASE so each DB gets what suits it. Idempotent.
+  systemd.services.matrix-synapse-db-tune = {
+    description = "Apply per-database Postgres tuning for Matrix Synapse";
+    after = [ "postgresql.service" ];
+    requires = [ "postgresql.service" ];
+    before = [ "matrix-synapse.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    path = [ config.services.postgresql.package pkgs.coreutils ];
+    script = ''
+      set -euo pipefail
+      # Wait for ensureDatabases to have created the DB on first boot.
+      for i in $(seq 1 30); do
+        if psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "matrix-synapse"; then
+          break
+        fi
+        sleep 2
+      done
+      psql -U postgres -d matrix-synapse -c "ALTER DATABASE \"matrix-synapse\" SET autovacuum_vacuum_scale_factor = '0.05';"
+      psql -U postgres -d matrix-synapse -c "ALTER DATABASE \"matrix-synapse\" SET autovacuum_analyze_scale_factor = '0.025';"
+    '';
+  };
+
   # ── Generate runtime config from domain files ───────────────
   systemd.services.matrix-synapse-runtime-config = {
     description = "Generate Synapse runtime config from domain files";
@@ -94,16 +123,23 @@ EOF
   # ── Synapse service ─────────────────────────────────────────
   services.matrix-synapse = {
     enable = true;
+    # cache-memory provides cache-size statistics for the autotuning below
+    # (in addition to the NixOS defaults).
+    extras = [ "systemd" "postgres" "url-preview" "cache-memory" ];
     extraConfigFiles = [
       "/run/matrix-synapse/runtime-config.yaml"
     ];
     settings = {
       database = {
         name = "psycopg2";
+        # Recycle pooled connections less often (fewer reconnects).
+        txn_limit = 10000;
         args = {
           host = "localhost";
           database = "matrix-synapse";
           user = "matrix-synapse";
+          cp_min = 5;
+          cp_max = 15;
         };
       };
       push.include_content = false;
@@ -120,6 +156,32 @@ EOF
       ];
       presence.enabled = true;
       enable_registration = false;
+      # ── Performance (32 GB Server + Desktop) ─────────────────
+      # Synapse trades RAM for fewer Postgres round-trips; most RAM goes
+      # to caches. Stock is global_factor 0.5 + 10K event cache, which
+      # leaves syncs hitting the database on every request.
+      # Deliberately unchanged: presence and URL previews stay enabled —
+      # disabling them is faster but changes user-visible behavior.
+      event_cache_size = "100K";
+      caches = {
+        global_factor = 4.0;
+        expire_caches = true;
+        cache_entry_ttl = "30m";
+        sync_response_cache_duration = "2m";
+        cache_autotuning = {
+          max_cache_memory_usage = "2G";
+          target_cache_memory_usage = "1G";
+          min_cache_ttl = "30s";
+        };
+        per_cache_factors = {
+          # Hot paths for /sync and room joins.
+          get_users_in_room = 3.0;
+          get_current_state_ids = 3.0;
+          get_unread_event_push_actions_by_room_for_user = 5.0;
+        };
+      };
+      # Fewer GC pauses at the cost of a little more memory.
+      gc_thresholds = [ 1500 20 10 ];
       listeners = [
         {
           port = 8008;