Merge pull request #408 from naturallaw777/copilot/vendor-mempool-packages

Vendor mempool packages and make `services.mempool` self-contained

modules/bitcoin/mempool.nix

@@ -66,8 +66,8 @@ let
         };
         staticContentRoot = mkOption {
           type = types.path;
-          default = pkgs.mempool-frontend.withConfig cfg.frontend.settings;
+          default = (pkgs.callPackage ../../packages/mempool {}).mempool-frontend.withConfig cfg.frontend.settings;
-          defaultText = "pkgs.mempool-frontend";
+          defaultText = "mempoolPkgs.mempool-frontend";
           description = "
             Path of the static frontend content root.
           ";
@@ -95,7 +95,7 @@ let
         description = "Mempool backend port.";
       };
       electrumServer = mkOption {
-        type = types.enum [ "electrs" "fulcrum" ];
+        type = types.enum [ "electrs" ];
         default = "electrs";
         description = ''
           The Electrum server to use for fetching address information.
@@ -103,8 +103,6 @@ let
           Possible options:
           - electrs:
             Small database size, slow when querying new addresses.
-          - fulcrum:
-            Large database size, quickly serves arbitrary address queries.
         '';
       };
       settings = mkOption {
@@ -133,8 +131,8 @@ let
       };
       package = mkOption {
         type = types.package;
-        default = pkgs.mempool-backend;
+        default = (pkgs.callPackage ../../packages/mempool {}).mempool-backend;
-        defaultText = "pkgs.mempool-backend";
+        defaultText = "mempoolPkgs.mempool-backend";
         description = "The package providing mempool binaries.";
       };
       user = mkOption {
@@ -173,14 +171,16 @@ let
     electrs;
 
   torSocket = config.services.tor.client.socksListenAddress;
+  # Vendored mempool package
+  mempoolPkgs = pkgs.callPackage ../../packages/mempool {};
 
-  # See the `services.nginx` definition further below below
+  # See the `services.nginx` definition further below
   # on how to use these snippets.
   frontend.nginxConfig = {
     # This must be added to `services.nginx.commonHttpConfig` when
     # `mempool/location-static.conf` is used
     httpConfig = ''
-      include ${if pkgs ? mempool-nginx-conf then "${pkgs.mempool-nginx-conf}/http-language.conf" else "/dev/null"};
+      include ${mempoolPkgs.mempool-nginx-conf}/http-language.conf;
     '';
 
     # Config for static website content.
@@ -193,7 +193,7 @@ let
       add_header Vary Accept-Language;
       add_header Vary Cookie;
 
-      include ${if pkgs ? mempool-nginx-conf then "${pkgs.mempool-nginx-conf}/location-static.conf" else "/dev/null"};
+      include ${mempoolPkgs.mempool-nginx-conf}/location-static.conf;
 
       # Redirect /api to /docs/api
       location = /api {
@@ -239,8 +239,7 @@ in {
 
   config = mkIf cfg.enable {
     services.bitcoind.txindex = true;
-    services.electrs.enable = mkIf (cfg.electrumServer == "electrs" ) true;
+    services.electrs.enable = true;
-    # vendored fix: fulcrum may not exist    # fulcrum removed - Sovran uses electrs only
     services.mysql = {
       enable = true;
       package = pkgs.mariadb;

packages/mempool/0001-allow-disabling-mining-pool-fetching.patch

@@ -0,0 +1,41 @@
+From e4b3ebaf0451c1bddbd7dcf8527c296938ebb607 Mon Sep 17 00:00:00 2001
+From: Erik Arvstedt <erik.arvstedt@gmail.com>
+Date: Sun, 1 Jun 2025 11:17:22 +0200
+Subject: [PATCH] allow disabling mining pool fetching in offline environments
+
+Previously, Mempool strictly required fetching mining pool data from
+Github and failed when this was not possible, e.g. in offline
+environments.
+
+This patch allows disabling pool fetching.
+When disabled, empty pool data is inserted into the DB, which
+effectively turns off block pool classification.
+---
+ backend/src/tasks/pools-updater.ts | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts
+index 6b0520dfc..a74259b95 100644
+--- a/backend/src/tasks/pools-updater.ts
++++ b/backend/src/tasks/pools-updater.ts
+@@ -75,7 +75,7 @@ class PoolsUpdater {
+       } else {
+         logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag);
+       }
+-      const poolsJson = await this.query(this.poolsUrl);
++      const poolsJson = (githubSha == "disable-pool-fetching") ? [] : await this.query(this.poolsUrl);
+       if (poolsJson === undefined) {
+         return;
+       }
+@@ -136,6 +136,9 @@ class PoolsUpdater {
+    * Fetch our latest pools-v2.json sha from github
+    */
+   private async fetchPoolsSha(): Promise<string | null> {
++    if (this.poolsUrl == "disable-pool-fetching") {
++      return "disable-pool-fetching";
++    }
+     const response = await this.query(this.treeUrl);
+ 
+     if (response !== undefined) {
+-- 
+2.47.2

packages/mempool/default.nix

@@ -0,0 +1,201 @@
+{ lib
+, stdenvNoCC
+, nodejs_22
+, nodejs-slim_22
+, fetchFromGitHub
+, fetchNodeModules
+, runCommand
+, makeWrapper
+, curl
+, cacert
+, rsync
+# for rust-gbt (backend module)
+, cargo
+, rustc
+, rustPlatform
+, napi-rs-cli
+}:
+rec {
+  nodejs = nodejs_22;
+  nodejsRuntime = nodejs-slim_22;
+
+  version = "3.2.1";
+
+  src = fetchFromGitHub {
+    owner = "mempool";
+    repo = "mempool";
+    tag = "v${version}";
+    hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU=";
+  };
+
+  nodeModules = {
+    frontend = fetchNodeModules {
+      inherit src nodejs;
+      sourceRoot = "source/frontend";
+      hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0=";
+    };
+    backend = fetchNodeModules {
+      inherit src nodejs;
+      sourceRoot = "source/backend";
+      hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4=";
+    };
+  };
+
+  frontendAssets = fetchFiles {
+    name = "mempool-frontend-assets";
+    hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ=";
+    fetcher = ./frontend-assets-fetch.sh;
+  };
+
+  mempool-backend = mkDerivationMempool {
+    pname = "mempool-backend";
+
+    patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ];
+
+    buildPhase = ''
+      cd backend
+      ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
+      patchShebangs node_modules
+
+      ${sync} ${mempool-rust-gbt}/ rust-gbt
+      npm run package
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mkdir -p $out/lib/mempool-backend
+      ${sync} package/ $out/lib/mempool-backend
+
+      makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \
+        --add-flags $out/lib/mempool-backend/index.js
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit nodejs nodejsRuntime;
+      nodeModules = nodeModules.backend;
+    };
+  };
+
+  mempool-frontend = mkFrontend {};
+
+  # Argument `config` (type: attrset) defines the mempool frontend config.
+  # If `{}`, the default config is used.
+  # See here for available options:
+  # https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts
+  # (`interface Env` and `defaultEnv`)
+  mkFrontend = config: mkDerivationMempool {
+    pname = "mempool-frontend";
+
+    buildPhase = ''
+      cd frontend
+
+      ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules .
+      patchShebangs node_modules
+
+      # sync-assets.js is called during `npm run build` and downloads assets from the
+      # internet. Disable this script and instead add the assets manually after building.
+      : > sync-assets.js
+
+      ${lib.optionalString (config != {}) ''
+        ln -s ${builtins.toFile "mempool-frontend-config" (builtins.toJSON config)} mempool-frontend-config.json
+      ''}
+
+      npm run build
+
+      # Add assets that would otherwise be downloaded by sync-assets.js
+      ${sync} ${frontendAssets}/ dist/mempool/browser/resources
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      ${sync} dist/mempool/browser/ $out
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      withConfig = mkFrontend;
+      assets = frontendAssets;
+      nodeModules = nodeModules.frontend;
+    };
+  };
+
+  mempool-rust-gbt = stdenvNoCC.mkDerivation rec {
+    pname = "mempool-rust-gbt";
+    inherit version src meta;
+
+    sourceRoot = "source/rust/gbt";
+
+    nativeBuildInputs = [
+      rustPlatform.cargoSetupHook
+      cargo
+      rustc
+      napi-rs-cli
+    ];
+
+    cargoDeps = rustPlatform.fetchCargoVendor {
+      inherit src;
+      name = "${pname}-${version}";
+      inherit sourceRoot;
+      hash = "sha256-eox/K3ipjAqNyFt87lZnxaU/okQLF/KIhqXrX86n+qw=";
+    };
+
+    buildPhase = ''
+      runHook preBuild
+      # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out
+      napi build --platform --release --strip out
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mv out $out
+      cp package.json $out
+    '';
+
+    passthru = { inherit cargoDeps; };
+  };
+
+  mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
+    ${sync} --chmod=u+w ${./nginx-conf}/ $out
+    ${sync} ${src}/production/nginx/http-language.conf $out
+  '';
+
+  sync = "${rsync}/bin/rsync -a --inplace";
+
+  mkDerivationMempool = args: stdenvNoCC.mkDerivation ({
+    inherit version src meta;
+
+    nativeBuildInputs = [
+      makeWrapper
+      nodejs
+      rsync
+    ];
+
+    phases = "unpackPhase patchPhase buildPhase installPhase";
+  } // args);
+
+  fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation {
+    inherit name;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = hash;
+    nativeBuildInputs = [ curl cacert ];
+    buildCommand = ''
+      mkdir $out
+      cd $out
+      ${builtins.readFile fetcher}
+    '';
+  };
+
+  meta = with lib; {
+    description = "Bitcoin blockchain and mempool explorer";
+    homepage = "https://github.com/mempool/mempool/";
+    license = licenses.agpl3Plus;
+    maintainers = with maintainers; [ erikarvstedt ];
+    platforms = platforms.unix;
+  };
+}

packages/mempool/frontend-assets-fetch.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Fetch hash-locked versions of assets that are dynamically fetched via
+# https://github.com/mempool/mempool/blob/master/frontend/sync-assets.js
+# when running `npm run build` in the frontend.
+#
+# This file is updated by ./frontend-assets-update.sh
+
+declare -A revs=(
+    ["mempool/mining-pool-logos"]=53972ebbd08373cf4910cbb3e6421a1f3bba4563
+)
+
+fetchFile() {
+    repo=$1
+    file=$2
+    rev=${revs["$repo"]}
+    curl -fsS "https://raw.githubusercontent.com/$repo/$rev/$file"
+}
+
+fetchRepo() {
+    repo=$1
+    rev=${revs["$repo"]}
+    curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz"
+}
+
+mkdir mining-pools
+fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools

packages/mempool/nginx-conf/location-static.conf

@@ -0,0 +1,47 @@
+# Settings adapted from
+# https://github.com/mempool/mempool/blob/v3.2.1/production/nginx/server-common.conf
+
+# see order of nginx location rules
+# https://stackoverflow.com/questions/5238377/nginx-location-priority
+
+# for exact / requests, redirect based on $lang
+# cache redirect for 5 minutes
+location = / {
+	if ($lang != '') {
+		return 302 $scheme://$host/$lang/;
+	}
+	try_files /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /<lang>/main.f40e91d908a068a2.js forever since they never change
+location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
+	try_files $uri =404;
+	expires 1y;
+}
+# cache everything else for 5 minutes
+location ~ ^/([a-z][a-z])$ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+location ~ ^/([a-z][a-z])/ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /resources/** for 1 week since they don't change often
+location /resources {
+	try_files $uri /en-US/index.html;
+	expires 1w;
+}
+# cache /main.f40e91d908a068a2.js forever since they never change
+location ~* ^/.+\..+\.(js|css)$ {
+	try_files /$lang/$uri /en-US/$uri =404;
+	expires 1y;
+}
+# catch-all for all URLs i.e. /address/foo /tx/foo /block/000
+# cache 5 minutes since they change frequently
+location / {
+	try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
+	expires 5m;
+}