mastodon: 4.2.13 -> 4.3.0

Changelog:
https://github.com/mastodon/mastodon/releases/tag/v4.3.0

nixos/modules/services/web-apps/mastodon.nix

@@ -12,9 +12,12 @@ let
     RAILS_ENV = "production";
     NODE_ENV = "production";
 
+    BOOTSNAP_CACHE_DIR="/var/cache/mastodon/precompile";
     LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
 
-    # mastodon-web concurrency.
+    MASTODON_USE_LIBVIPS = "true";
+
+    # Concurrency mastodon-web
     WEB_CONCURRENCY = toString cfg.webProcesses;
     MAX_THREADS = toString cfg.webThreads;
 
@@ -24,7 +27,7 @@ let
     DB_NAME = cfg.database.name;
     LOCAL_DOMAIN = cfg.localDomain;
     SMTP_SERVER = cfg.smtp.host;
-    SMTP_PORT = toString(cfg.smtp.port);
+    SMTP_PORT = toString cfg.smtp.port;
     SMTP_FROM_ADDRESS = cfg.smtp.fromAddress;
     PAPERCLIP_ROOT_PATH = "/var/lib/mastodon/public-system";
     PAPERCLIP_ROOT_URL = "/system";
@@ -33,12 +36,12 @@ let
     TRUSTED_PROXY_IP = cfg.trustedProxy;
   }
   // lib.optionalAttrs (cfg.redis.host != null) { REDIS_HOST = cfg.redis.host; }
-  // lib.optionalAttrs (cfg.redis.port != null) { REDIS_PORT = toString(cfg.redis.port); }
+  // lib.optionalAttrs (cfg.redis.port != null) { REDIS_PORT = toString cfg.redis.port; }
   // lib.optionalAttrs (cfg.redis.createLocally && cfg.redis.enableUnixSocket) { REDIS_URL = "unix://${config.services.redis.servers.mastodon.unixSocket}"; }
   // lib.optionalAttrs (cfg.database.host != "/run/postgresql" && cfg.database.port != null) { DB_PORT = toString cfg.database.port; }
   // lib.optionalAttrs cfg.smtp.authenticate { SMTP_LOGIN  = cfg.smtp.user; }
   // lib.optionalAttrs (cfg.elasticsearch.host != null) { ES_HOST = cfg.elasticsearch.host; }
-  // lib.optionalAttrs (cfg.elasticsearch.host != null) { ES_PORT = toString(cfg.elasticsearch.port); }
+  // lib.optionalAttrs (cfg.elasticsearch.host != null) { ES_PORT = toString cfg.elasticsearch.port; }
   // lib.optionalAttrs (cfg.elasticsearch.host != null) { ES_PRESET = cfg.elasticsearch.preset; }
   // lib.optionalAttrs (cfg.elasticsearch.user != null) { ES_USER = cfg.elasticsearch.user; }
   // cfg.extraConfig;
@@ -51,6 +54,9 @@ let
     Group = cfg.group;
     # Working directory
     WorkingDirectory = cfg.package;
+    # Cache directory and mode
+    CacheDirectory = "mastodon";
+    CacheDirectoryMode = "0750";
     # State directory and mode
     StateDirectory = "mastodon";
     StateDirectoryMode = "0750";
@@ -127,7 +133,7 @@ let
       description = "Mastodon sidekiq${jobClassLabel}";
       wantedBy = [ "mastodon.target" ];
       environment = env // {
-        PORT = toString(cfg.sidekiqPort);
+        PORT = toString cfg.sidekiqPort;
         DB_POOL = threads;
       };
       serviceConfig = {
@@ -309,7 +315,7 @@ in {
           Voluntary Application Server Identification.  A new keypair can
           be generated by running:
 
-          `nix build -f '<nixpkgs>' mastodon; cd result; bin/rake webpush:generate_keys`
+          `nix build -f '<nixpkgs>' mastodon; cd result; RAILS_ENV=production bin/rake webpush:generate_keys`
 
           If {option}`mastodon.vapidPrivateKeyFile`does not
           exist, it and this file will be created with a new keypair.
@@ -324,12 +330,57 @@ in {
         type = lib.types.str;
       };
 
+      activeRecordEncryptionDeterministicKeyFile = lib.mkOption {
+        description = ''
+          This key must be set to enable the Active Record Encryption feature within
+          Rails that Mastodon uses to encrypt and decrypt some database attributes.
+          A new Active Record keys can be generated by running:
+
+          `nix build -f '<nixpkgs>' mastodon; cd result; RAILS_ENV=production ./bin/rails db:encryption:init`
+
+          If this file does not exist, it will be created with a new Active Record
+          keys.
+        '';
+        default = "/var/lib/mastodon/secrets/active-record-encryption-deterministic-key";
+        type = lib.types.str;
+      };
+
+      activeRecordEncryptionKeyDerivationSaltFile = lib.mkOption {
+        description = ''
+          This key must be set to enable the Active Record Encryption feature within
+          Rails that Mastodon uses to encrypt and decrypt some database attributes.
+          A new Active Record keys can be generated by running:
+
+          `nix build -f '<nixpkgs>' mastodon; cd result; RAILS_ENV=production ./bin/rails db:encryption:init`
+
+          If this file does not exist, it will be created with a new Active Record
+          keys.
+        '';
+        default = "/var/lib/mastodon/secrets/active-record-encryption-key-derivation-salt";
+        type = lib.types.str;
+      };
+
+      activeRecordEncryptionPrimaryKeyFile = lib.mkOption {
+        description = ''
+          This key must be set to enable the Active Record Encryption feature within
+          Rails that Mastodon uses to encrypt and decrypt some database attributes.
+          A new Active Record keys can be generated by running:
+
+          `nix build -f '<nixpkgs>' mastodon; cd result; RAILS_ENV=production ./bin/rails db:encryption:init`
+
+          If this file does not exist, it will be created with a new Active Record
+          keys.
+        '';
+        default = "/var/lib/mastodon/secrets/active-record-encryption-primary-key";
+        type = lib.types.str;
+      };
+
       secretKeyBaseFile = lib.mkOption {
         description = ''
           Path to file containing the secret key base.
           A new secret key base can be generated by running:
 
-          `nix build -f '<nixpkgs>' mastodon; cd result; bin/rake secret`
+          `nix build -f '<nixpkgs>' mastodon; cd result; bin/bundle exec rails secret`
 
           If this file does not exist, it will be created with a new secret key base.
         '';
@@ -342,7 +393,7 @@ in {
           Path to file containing the OTP secret.
           A new OTP secret can be generated by running:
 
-          `nix build -f '<nixpkgs>' mastodon; cd result; bin/rake secret`
+          `nix build -f '<nixpkgs>' mastodon; cd result; bin/bundle exec rails secret`
 
           If this file does not exist, it will be created with a new OTP secret.
         '';
@@ -708,13 +759,28 @@ in {
       script = ''
         umask 077
 
+        if ! test -d /var/cache/mastodon/precompile; then
+          ${cfg.package}/bin/bundle exec bootsnap precompile --gemfile ${cfg.package}/app ${cfg.package}/lib
+        fi
+        if ! test -f ${cfg.activeRecordEncryptionDeterministicKeyFile}; then
+          mkdir -p $(dirname ${cfg.activeRecordEncryptionDeterministicKeyFile})
+          bin/rails db:encryption:init | grep --only-matching "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=[^ ]\+" | sed 's/^ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=//' > ${cfg.activeRecordEncryptionDeterministicKeyFile}
+        fi
+        if ! test -f ${cfg.activeRecordEncryptionKeyDerivationSaltFile}; then
+          mkdir -p $(dirname ${cfg.activeRecordEncryptionKeyDerivationSaltFile})
+          bin/rails db:encryption:init | grep --only-matching "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=[^ ]\+" | sed 's/^ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=//' > ${cfg.activeRecordEncryptionKeyDerivationSaltFile}
+        fi
+        if ! test -f ${cfg.activeRecordEncryptionPrimaryKeyFile}; then
+          mkdir -p $(dirname ${cfg.activeRecordEncryptionPrimaryKeyFile})
+          bin/rails db:encryption:init | grep --only-matching "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=[^ ]\+" | sed 's/^ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=//' > ${cfg.activeRecordEncryptionPrimaryKeyFile}
+        fi
         if ! test -f ${cfg.secretKeyBaseFile}; then
           mkdir -p $(dirname ${cfg.secretKeyBaseFile})
-          bin/rake secret > ${cfg.secretKeyBaseFile}
+          bin/bundle exec rails secret > ${cfg.secretKeyBaseFile}
         fi
         if ! test -f ${cfg.otpSecretFile}; then
           mkdir -p $(dirname ${cfg.otpSecretFile})
-          bin/rake secret > ${cfg.otpSecretFile}
+          bin/bundle exec rails secret > ${cfg.otpSecretFile}
         fi
         if ! test -f ${cfg.vapidPrivateKeyFile}; then
           mkdir -p $(dirname ${cfg.vapidPrivateKeyFile}) $(dirname ${cfg.vapidPublicKeyFile})
@@ -724,6 +790,9 @@ in {
         fi
 
         cat > /var/lib/mastodon/.secrets_env <<EOF
+        ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY="$(cat ${cfg.activeRecordEncryptionDeterministicKeyFile})"
+        ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT="$(cat ${cfg.activeRecordEncryptionKeyDerivationSaltFile})"
+        ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY="$(cat ${cfg.activeRecordEncryptionPrimaryKeyFile})"
         SECRET_KEY_BASE="$(cat ${cfg.secretKeyBaseFile})"
         OTP_SECRET="$(cat ${cfg.otpSecretFile})"
         VAPID_PRIVATE_KEY="$(cat ${cfg.vapidPrivateKeyFile})"
@@ -802,7 +871,7 @@ in {
       description = "Mastodon web";
       environment = env // (if cfg.enableUnixSocket
         then { SOCKET = "/run/mastodon-web/web.socket"; }
-        else { PORT = toString(cfg.webPort); }
+        else { PORT = toString cfg.webPort; }
       );
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/puma -C config/puma.rb";
@@ -816,7 +885,7 @@ in {
         # System Call Filtering
         SystemCallFilter = [ ("~" + lib.concatStringsSep " " systemCallsList) "@chown" "pipe" "pipe2" ];
       } // cfgService;
-      path = with pkgs; [ ffmpeg-headless file imagemagick ];
+      path = with pkgs; [ ffmpeg-headless file ];
     };
 
     systemd.services.mastodon-media-auto-remove = lib.mkIf cfg.mediaAutoRemove.enable {
@@ -851,7 +920,7 @@ in {
         };
 
         locations."@proxy" = {
-          proxyPass = (if cfg.enableUnixSocket then "http://unix:/run/mastodon-web/web.socket" else "http://127.0.0.1:${toString(cfg.webPort)}");
+          proxyPass = (if cfg.enableUnixSocket then "http://unix:/run/mastodon-web/web.socket" else "http://127.0.0.1:${toString cfg.webPort}");
           proxyWebsockets = true;
         };
 
@@ -903,7 +972,7 @@ in {
           inherit (cfg) group;
         };
       })
-      (lib.attrsets.setAttrByPath [ cfg.user "packages" ] [ cfg.package pkgs.imagemagick ])
+      (lib.attrsets.setAttrByPath [ cfg.user "packages" ] [ cfg.package ])
       (lib.mkIf (cfg.redis.createLocally && cfg.redis.enableUnixSocket) {${config.services.mastodon.user}.extraGroups = [ "redis-mastodon" ];})
     ];
 

pkgs/servers/mastodon/default.nix

@@ -1,7 +1,6 @@
 { lib, stdenv, nodejs-slim, bundlerEnv, nixosTests
-, yarn, callPackage, ruby, writeShellScript
+, yarn-berry, callPackage, ruby, writeShellScript
-, fetchYarnDeps, fixup-yarn-lock
+, brotli, python3
-, brotli
 
   # Allow building a fork or custom version of Mastodon:
 , pname ? "mastodon"
@@ -28,12 +27,12 @@ stdenv.mkDerivation rec {
     pname = "${pname}-modules";
     inherit src version;
 
-    yarnOfflineCache = fetchYarnDeps {
+    yarnOfflineCache = callPackage ./yarn.nix {
-      yarnLock = "${src}/yarn.lock";
+      inherit version src;
       hash = yarnHash;
     };
 
-    nativeBuildInputs = [ fixup-yarn-lock nodejs-slim yarn mastodonGems mastodonGems.wrappedRuby brotli ];
+    nativeBuildInputs = [ nodejs-slim yarn-berry mastodonGems mastodonGems.wrappedRuby brotli python3 ];
 
     RAILS_ENV = "production";
     NODE_ENV = "production";
@@ -42,29 +41,33 @@ stdenv.mkDerivation rec {
       runHook preBuild
 
       export HOME=$PWD
-      fixup-yarn-lock ~/yarn.lock
+      export YARN_ENABLE_TELEMETRY=0
-      yarn config --offline set yarn-offline-mirror $yarnOfflineCache
+      export npm_config_nodedir=${nodejs-slim}
-      yarn install --offline --frozen-lockfile --ignore-engines --ignore-scripts --no-progress
+      export SECRET_KEY_BASE_DUMMY=1
+
+      mkdir -p ~/.yarn/berry
+      ln -s $yarnOfflineCache ~/.yarn/berry/cache
+
+      yarn install --immutable --immutable-cache
 
       patchShebangs ~/bin
       patchShebangs ~/node_modules
 
-      # skip running yarn install
+      bundle exec rails assets:precompile
-      rm -rf ~/bin/yarn
 
-      OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder \
+      yarn cache clean --all
-        rails assets:precompile
-      yarn cache clean --offline
       rm -rf ~/node_modules/.cache
 
+      # Remove execute permissions
+      find ~/public/assets -type f ! -perm 0555 \
+        -exec chmod 0444 {} ';'
+
       # Create missing static gzip and brotli files
-      gzip --best --keep ~/public/assets/500.html
+      find ~/public/assets -type f -regextype posix-extended -iregex '.*\.(css|html|js|json|svg)' \
-      gzip --best --keep ~/public/packs/report.html
+        -exec gzip --best --keep --force {} ';' \
-      find ~/public/assets -maxdepth 1 -type f -name '.*.json' \
-        -exec gzip --best --keep --force {} ';'
-      brotli --best --keep ~/public/packs/report.html
-      find ~/public/assets -type f -regextype posix-extended -iregex '.*\.(css|js|json|html)' \
         -exec brotli --best --keep {} ';'
+      gzip --best --keep ~/public/packs/report.html
+      brotli --best --keep ~/public/packs/report.html
 
       runHook postBuild
     '';
@@ -101,13 +104,14 @@ stdenv.mkDerivation rec {
     done
 
     # Remove execute permissions
-    chmod 0444 public/emoji/*.svg
+    find public/emoji -type f ! -perm 0555 \
+      -exec chmod 0444 {} ';'
 
     # Create missing static gzip and brotli files
-    find public -maxdepth 1 -type f -regextype posix-extended -iregex '.*\.(css|js|svg|txt|xml)' \
+    find public -maxdepth 1 -type f -regextype posix-extended -iregex '.*\.(js|txt)' \
       -exec gzip --best --keep --force {} ';' \
       -exec brotli --best --keep {} ';'
-    find public/emoji -type f -name '.*.svg' \
+    find public/emoji -type f -name '*.svg' \
       -exec gzip --best --keep --force {} ';' \
       -exec brotli --best --keep {} ';'
     ln -s assets/500.html.gz public/500.html.gz
@@ -133,7 +137,8 @@ stdenv.mkDerivation rec {
     runHook preInstall
 
     mkdir -p $out
-    cp -r * $out/
+    mv .{env*,ruby*} $out/
+    mv * $out/
     ln -s ${run-streaming} $out/run-streaming.sh
 
     runHook postInstall

pkgs/servers/mastodon/source.nix

@@ -1,7 +1,7 @@
 # This file was generated by pkgs.mastodon.updateScript.
 { fetchFromGitHub, applyPatches, patches ? [] }:
 let
-  version = "4.2.13";
+  version = "4.3.0";
 in
 (
   applyPatches {
@@ -9,10 +9,10 @@ in
       owner = "mastodon";
       repo = "mastodon";
       rev = "v${version}";
-      hash = "sha256-+HGu02fjYJ1x6Tk9AdqmFN7JHk3UnlvCdiQ/5yMu69M=";
+      hash = "sha256-nZtxildQmT/7JMCTx89ZSWxb9I7xMLGHTJv7v4gfdd4=";
     };
     patches = patches ++ [];
   }) // {
   inherit version;
-  yarnHash = "sha256-qoLesubmSvRsXhKwMEWHHXcpcqRszqcdZgHQqnTpNPE=";
+  yarnHash = "sha256-V/kBkxv6akTyzlFzdR1F53b7RD0NYtap58Xt5yOAbYA=";
 }

pkgs/servers/mastodon/update.sh

@@ -106,7 +106,5 @@ echo "Creating gemset.nix"
 bundix --lockfile="$SOURCE_DIR/Gemfile.lock" --gemfile="$SOURCE_DIR/Gemfile"
 echo "" >> gemset.nix  # Create trailing newline to please EditorConfig checks
 
-echo "Creating yarn-hash.nix"
+echo "Required manual update of yarn-hash"
-YARN_HASH="$(prefetch-yarn-deps "$SOURCE_DIR/yarn.lock")"
+exit 1
-YARN_HASH="$(nix hash to-sri --type sha256 "$YARN_HASH")"
-sed -i "s/sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=/$YARN_HASH/g" source.nix

pkgs/servers/mastodon/yarn.nix

@@ -0,0 +1,34 @@
+{
+  stdenvNoCC,
+  yarn-berry,
+  cacert,
+  version,
+  src,
+  hash,
+}:
+stdenvNoCC.mkDerivation {
+  pname = "yarn-deps";
+  inherit version src;
+
+  nativeBuildInputs = [
+    yarn-berry
+  ];
+
+  dontInstall = true;
+
+  NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+  buildPhase = ''
+    export HOME=$(mktemp -d)
+    export YARN_ENABLE_TELEMETRY=0
+
+    cache="$(yarn config get cacheFolder)"
+    yarn install --immutable --mode skip-build
+
+    mkdir -p $out
+    cp -r $cache/* $out/
+  '';
+
+  outputHash = hash;
+  outputHashMode = "recursive";
+}

pkgs/top-level/all-packages.nix

@@ -24522,8 +24522,10 @@ with pkgs;
   maker-panel = callPackage ../tools/misc/maker-panel { };
 
   mastodon = callPackage ../servers/mastodon {
-    nodejs-slim = nodejs-slim_20;
+    nodejs-slim = nodejs-slim_22;
-    ruby = ruby_3_2;
+    python3 = python311;
+    ruby = ruby_3_3;
+    yarn-berry = yarn-berry.override { nodejs = nodejs-slim_22; };
   };
 
   gotosocial = callPackage ../servers/gotosocial { };