Restore upstream nix-bitcoin fetchNodeModules for mempool and RTL packages

Co-authored-by: naturallaw777 <99053422+naturallaw777@users.noreply.github.com>

modules/bitcoin/mempool.nix

@@ -66,7 +66,7 @@ let
         };
         staticContentRoot = mkOption {
           type = types.path;
-          default = (pkgs.callPackage ../../packages/mempool {}).mempool-frontend.withConfig cfg.frontend.settings;
+          default = (pkgs.callPackage ../../packages/mempool { fetchNodeModules = pkgs.callPackage ../../packages/build-support/fetch-node-modules.nix {}; }).mempool-frontend.withConfig cfg.frontend.settings;
           defaultText = "mempoolPkgs.mempool-frontend";
           description = "
             Path of the static frontend content root.
@@ -131,7 +131,7 @@ let
       };
       package = mkOption {
         type = types.package;
-        default = (pkgs.callPackage ../../packages/mempool {}).mempool-backend;
+        default = (pkgs.callPackage ../../packages/mempool { fetchNodeModules = pkgs.callPackage ../../packages/build-support/fetch-node-modules.nix {}; }).mempool-backend;
         defaultText = "mempoolPkgs.mempool-backend";
         description = "The package providing mempool binaries.";
       };
@@ -172,7 +172,8 @@ let
 
   torSocket = config.services.tor.client.socksListenAddress;
   # Vendored mempool package
-  mempoolPkgs = pkgs.callPackage ../../packages/mempool {};
+  fetchNodeModules = pkgs.callPackage ../../packages/build-support/fetch-node-modules.nix {};
+  mempoolPkgs = pkgs.callPackage ../../packages/mempool { inherit fetchNodeModules; };
 
   # See the `services.nginx` definition further below
   # on how to use these snippets.

modules/bitcoin/rtl.nix

@@ -81,7 +81,8 @@ let
   secretsDir = config.nix-bitcoin.secretsDir;
 
   # Vendored RTL package
-  rtlPackage = pkgs.callPackage ../../packages/rtl {};
+  fetchNodeModules = pkgs.callPackage ../../packages/build-support/fetch-node-modules.nix {};
+  rtlPackage = pkgs.callPackage ../../packages/rtl { inherit fetchNodeModules; };
 
   runePath = "${cfg.dataDir}/CLN-Rune.env";
 

packages/build-support/fetch-node-modules.nix

@@ -0,0 +1,77 @@
+# This is a modified version of
+# https://github.com/NixOS/nixpkgs/pull/128749
+
+{ lib, stdenvNoCC, makeWrapper, nodejs, cacert }:
+
+{ src
+, hash ? ""
+, runScripts ? false
+, preferLocalBuild ? true
+, npmFlags ? ""
+, ...
+} @ args:
+stdenvNoCC.mkDerivation ({
+  inherit src preferLocalBuild;
+
+  name = "${src.name}-node_modules";
+  nativeBuildInputs = [
+    makeWrapper
+    (if args ? nodejs then args.nodejs else nodejs)
+  ];
+
+  outputHashMode =  "recursive";
+
+  impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+
+  phases = "unpackPhase patchPhase buildPhase installPhase";
+
+  # npm doesn't support var `SSL_CERT_FILE`.
+  NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+  buildPhase = ''
+    runHook preBuild
+
+    if [[ ! -f package.json ]]; then
+        echo "Error: file `package.json` doesn't exist"
+        exit 1
+    fi
+    if [[ ! -f package-lock.json ]]; then
+        echo "Error: file `package-lock.json` doesn't exist"
+        exit 1
+    fi
+
+    export SOURCE_DATE_EPOCH=1
+    export npm_config_cache=/tmp
+    NPM_FLAGS="--omit=dev --omit=optional --no-update-notifier $npmFlags"
+    # Scripts may result in non-deterministic behavior.
+    # Some packages (e.g., Puppeteer) use postinstall scripts to download extra data.
+    if [[ ! $runScripts ]]; then
+        NPM_FLAGS+=" --ignore-scripts"
+    fi
+
+    echo "Running npm ci $NPM_FLAGS"
+    npm ci $NPM_FLAGS
+
+    cp package.json \
+       package-lock.json node_modules/
+    rm -f node_modules/.package-lock.json
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out/lib
+    cp -r node_modules $out/lib
+
+    runHook postInstall
+  '';
+} // (
+  if hash == "" then {
+    outputHashAlgo = "sha256";
+    outputHash = "";
+  } else {
+    outputHash = hash;
+  }
+) // (builtins.removeAttrs args [ "hash" ]))

packages/mempool/default.nix

@@ -2,20 +2,23 @@
 , stdenvNoCC
 , nodejs_22
 , nodejs-slim_22
-, buildNpmPackage
 , fetchFromGitHub
+, fetchNodeModules
 , runCommand
 , makeWrapper
 , curl
 , cacert
 , rsync
+# for rust-gbt (backend module)
 , cargo
 , rustc
 , rustPlatform
 , napi-rs-cli
 }:
+rec {
+  nodejs = nodejs_22;
+  nodejsRuntime = nodejs-slim_22;
 
-let
   version = "3.2.1";
 
   src = fetchFromGitHub {
@@ -25,22 +28,105 @@ let
     hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU=";
   };
 
-  frontendAssets = stdenvNoCC.mkDerivation {
+  nodeModules = {
+    frontend = fetchNodeModules {
+      inherit src nodejs;
+      sourceRoot = "source/frontend";
+      hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0=";
+    };
+    backend = fetchNodeModules {
+      inherit src nodejs;
+      sourceRoot = "source/backend";
+      hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4=";
+    };
+  };
+
+  frontendAssets = fetchFiles {
     name = "mempool-frontend-assets";
-    outputHashMode = "recursive";
+    hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ=";
-    outputHashAlgo = "sha256";
+    fetcher = ./frontend-assets-fetch.sh;
-    outputHash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ=";
+  };
-    nativeBuildInputs = [ curl cacert ];
+
-    buildCommand = ''
+  mempool-backend = mkDerivationMempool {
-      mkdir $out
+    pname = "mempool-backend";
-      cd $out
+
-      ${builtins.readFile ./frontend-assets-fetch.sh}
+    patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ];
+
+    buildPhase = ''
+      cd backend
+      ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
+      patchShebangs node_modules
+
+      ${sync} ${mempool-rust-gbt}/ rust-gbt
+      npm run package
+
+      runHook postBuild
     '';
+
+    installPhase = ''
+      mkdir -p $out/lib/mempool-backend
+      ${sync} package/ $out/lib/mempool-backend
+
+      makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \
+        --add-flags $out/lib/mempool-backend/index.js
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit nodejs nodejsRuntime;
+      nodeModules = nodeModules.backend;
+    };
+  };
+
+  mempool-frontend = mkFrontend {};
+
+  # Argument `config` (type: attrset) defines the mempool frontend config.
+  # If `{}`, the default config is used.
+  # See here for available options:
+  # https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts
+  # (`interface Env` and `defaultEnv`)
+  mkFrontend = config: mkDerivationMempool {
+    pname = "mempool-frontend";
+
+    buildPhase = ''
+      cd frontend
+
+      ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules .
+      patchShebangs node_modules
+
+      # sync-assets.js is called during `npm run build` and downloads assets from the
+      # internet. Disable this script and instead add the assets manually after building.
+      : > sync-assets.js
+
+      ${lib.optionalString (config != {}) ''
+        ln -s ${builtins.toFile "mempool-frontend-config" (builtins.toJSON config)} mempool-frontend-config.json
+      ''}
+
+      npm run build
+
+      # Add assets that would otherwise be downloaded by sync-assets.js
+      ${sync} ${frontendAssets}/ dist/mempool/browser/resources
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      ${sync} dist/mempool/browser/ $out
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      withConfig = mkFrontend;
+      assets = frontendAssets;
+      nodeModules = nodeModules.frontend;
+    };
   };
 
   mempool-rust-gbt = stdenvNoCC.mkDerivation rec {
     pname = "mempool-rust-gbt";
-    inherit version src;
+    inherit version src meta;
 
     sourceRoot = "source/rust/gbt";
 
@@ -60,6 +146,7 @@ let
 
     buildPhase = ''
       runHook preBuild
+      # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out
       napi build --platform --release --strip out
       runHook postBuild
     '';
@@ -72,121 +159,43 @@ let
     passthru = { inherit cargoDeps; };
   };
 
-  sync = "${rsync}/bin/rsync -a --inplace";
-
-in rec {
-  mempool-backend = buildNpmPackage {
-    pname = "mempool-backend";
-    inherit version src;
-    npmRoot = "backend";
-
-    # postPatch runs in the npmDeps fetcher too. Copy the lock file to
-    # the repo root so prefetch-npm-deps can find it, then apply patches.
-    # Also create a dummy rust-gbt stub so `npm ci --offline` can resolve
-    # the `file:./rust-gbt` dependency before the real native module is
-    # synced in buildPhase.
-    postPatch = ''
-      cp backend/package-lock.json ./package-lock.json
-      cp backend/package.json ./package.json
-      patch -p1 < ${./0001-allow-disabling-mining-pool-fetching.patch}
-      mkdir -p backend/rust-gbt
-      cat > backend/rust-gbt/package.json <<'EOF'
-      {"name":"rust-gbt","version":"0.0.1","main":"index.js"}
-      EOF
-      # also need stub at repo root for the fetcher's npm ci at root
-      mkdir -p rust-gbt
-      cat > rust-gbt/package.json <<'EOF'
-      {"name":"rust-gbt","version":"0.0.1","main":"index.js"}
-      EOF
-    '';
-
-    npmDepsHash = "sha256-tuMrdc9vw5CWzaL1xRxZnskgGwElWv8qz4LSNvSUdXI=";
-    npmDepsFetcherVersion = 2;
-    makeCacheWritable = true;
-    npmFlags = [ "--legacy-peer-deps" ];
-
-    nativeBuildInputs = [ makeWrapper rsync ];
-
-    dontNpmBuild = true;
-    dontNpmInstall = true;
-
-    buildPhase = ''
-      runHook preBuild
-      cd backend
-      patchShebangs node_modules
-      ${sync} ${mempool-rust-gbt}/ rust-gbt
-      npm run package
-      runHook postBuild
-    '';
-
-    installPhase = ''
-      mkdir -p $out/lib/mempool-backend
-      ${sync} package/ $out/lib/mempool-backend
-      makeWrapper ${nodejs-slim_22}/bin/node $out/bin/mempool-backend \
-        --add-flags $out/lib/mempool-backend/index.js
-      runHook postInstall
-    '';
-
-    passthru = { nodejs = nodejs_22; nodejsRuntime = nodejs-slim_22; };
-
-    meta = with lib; {
-      description = "Bitcoin blockchain and mempool explorer (backend)";
-      homepage = "https://github.com/mempool/mempool/";
-      license = licenses.agpl3Plus;
-      platforms = platforms.unix;
-    };
-  };
-
-  mempool-frontend = mkFrontend {};
-
-  mkFrontend = config: buildNpmPackage {
-    pname = "mempool-frontend";
-    inherit version src;
-    npmRoot = "frontend";
-
-    postPatch = ''
-      cp frontend/package-lock.json ./package-lock.json
-      cp frontend/package.json ./package.json
-    '';
-
-    npmDepsHash = "sha256-/UwK0X9knsqTSAmnh2+jk35SK/J7DjBUhsR7e6OOn8Y=";
-    npmDepsFetcherVersion = 2;
-
-    nativeBuildInputs = [ makeWrapper rsync ];
-
-    dontNpmBuild = true;
-    dontNpmInstall = true;
-
-    buildPhase = ''
-      runHook preBuild
-      cd frontend
-      patchShebangs node_modules
-      : > sync-assets.js
-      ${lib.optionalString (config != {}) ''
-        ln -s ${builtins.toFile "mempool-frontend-config" (builtins.toJSON config)} mempool-frontend-config.json
-      ''}
-      npm run build
-      ${sync} ${frontendAssets}/ dist/mempool/browser/resources
-      runHook postBuild
-    '';
-
-    installPhase = ''
-      ${sync} dist/mempool/browser/ $out
-      runHook postInstall
-    '';
-
-    passthru = { withConfig = mkFrontend; assets = frontendAssets; };
-
-    meta = with lib; {
-      description = "Bitcoin blockchain and mempool explorer (frontend)";
-      homepage = "https://github.com/mempool/mempool/";
-      license = licenses.agpl3Plus;
-      platforms = platforms.unix;
-    };
-  };
-
   mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
     ${sync} --chmod=u+w ${./nginx-conf}/ $out
     ${sync} ${src}/production/nginx/http-language.conf $out
   '';
+
+  sync = "${rsync}/bin/rsync -a --inplace";
+
+  mkDerivationMempool = args: stdenvNoCC.mkDerivation ({
+    inherit version src meta;
+
+    nativeBuildInputs = [
+      makeWrapper
+      nodejs
+      rsync
+    ];
+
+    phases = "unpackPhase patchPhase buildPhase installPhase";
+  } // args);
+
+  fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation {
+    inherit name;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = hash;
+    nativeBuildInputs = [ curl cacert ];
+    buildCommand = ''
+      mkdir $out
+      cd $out
+      ${builtins.readFile fetcher}
+    '';
+  };
+
+  meta = with lib; {
+    description = "Bitcoin blockchain and mempool explorer";
+    homepage = "https://github.com/mempool/mempool/";
+    license = licenses.agpl3Plus;
+    maintainers = with maintainers; [ erikarvstedt ];
+    platforms = platforms.unix;
+  };
 }

packages/rtl/default.nix

@@ -1,35 +1,39 @@
 { lib
-, buildNpmPackage
+, stdenvNoCC
 , nodejs_22
 , nodejs-slim_22
+, fetchNodeModules
+, fetchpatch
 , fetchurl
 , makeWrapper
 }:
-
+let self = stdenvNoCC.mkDerivation {
-let
+  pname = "rtl";
   version = "0.15.8";
 
   src = fetchurl {
-    url = "https://github.com/Ride-The-Lightning/RTL/archive/refs/tags/v${version}.tar.gz";
+    url = "https://github.com/Ride-The-Lightning/RTL/archive/refs/tags/v${self.version}.tar.gz";
     hash = "sha256-8XdGyORxB2dkZRB/Yl7zh+Quqo4L/Y0VmC6Brbr/hqU=";
   };
 
-in buildNpmPackage {
+  passthru = {
-  pname = "rtl";
+    nodejs = nodejs_22;
-  inherit version src;
+    nodejsRuntime = nodejs-slim_22;
 
-  # Placeholder: build once, then replace with the hash from the
+    nodeModules = fetchNodeModules {
-  # "hash mismatch" error output.
+      inherit (self) src nodejs;
-  npmDepsHash = "sha256-hNBPdIBTHkAQ0Kztj9xvBEqYIkFW8mLZ3W6LAwYPOqM=";
+      # TODO-EXTERNAL: Remove `npmFlags` when no longer required
-
+      # See: https://github.com/Ride-The-Lightning/RTL/issues/1182
-  npmFlags = [ "--legacy-peer-deps" ];
+      npmFlags = "--legacy-peer-deps";
+      hash = "sha256-oMqd6nLzS6iQ9w4z2yzpR2unA5qhOq5YdvfoS8IgYLY=";
+    };
+  };
 
   nativeBuildInputs = [
     makeWrapper
   ];
 
-  dontNpmBuild = true;
+  phases = "unpackPhase patchPhase installPhase";
-  dontNpmInstall = true;
 
   # `src` already contains the precompiled frontend and backend.
   # Copy all files required for packaging, like in
@@ -42,25 +46,20 @@ in buildNpmPackage {
       package.json \
       frontend \
       backend \
-      node_modules \
+      ${self.nodeModules}/lib/node_modules \
       $dest
 
-    makeWrapper ${nodejs-slim_22}/bin/node "$out/bin/rtl" \
+    makeWrapper ${self.nodejsRuntime}/bin/node "$out/bin/rtl" \
       --add-flags "$dest/rtl.js"
 
     runHook postInstall
   '';
 
-  passthru = {
-    nodejs = nodejs_22;
-    nodejsRuntime = nodejs-slim_22;
-  };
-
   meta = with lib; {
-    description = "A web interface for LND";
+    description = "A web interface for LND, c-lightning and Eclair";
     homepage = "https://github.com/Ride-The-Lightning/RTL";
     license = licenses.mit;
-    maintainers = with maintainers; [ ];
+    maintainers = with maintainers; [ nixbitcoin erikarvstedt ];
     platforms = platforms.unix;
   };
-}
+}; in self