Unverified Commit 9e2866d2 authored by Luflosi's avatar Luflosi
Browse files

kubo-migrator: rewrite 

- Migrate to pkgs/by-name
- Format with nixfmt-rfc-style
- Make it possible to remove support for very old migrations in the future by increasing the `minRepoVersion` parameter
- Rename kubo-migrator-all-fs-repo-migrations to kubo-fs-repo-migrations since it may no longer include all migrations
- Add an alias for kubo-migrator-all-fs-repo-migrations to keep backwards compatibility
- Update descriptions to differentiate between kubo-migrator and kubo-migrator-unwrapped and better describe the purpose of the migrator
- Add a description to every individual migration
- Add a description to kubo-fs-repo-migrations
- Fetch the source code of the individual migrations from their specific Git tags, like upstream intends
- Enable tests for some migrations
- Check that the migrations don't crash on startup
- Mark two broken migrations as broken. They are not compatible with the latest Go versions and upstream is not interested in fixing this
- Change code to allow most updates to be done by only changing three lines (add new version and change git tag and hash)
- Add a stub for any disabled or broken migration to prevent downloading unsigned binaries from the internet, see https://github.com/ipfs/fs-repo-migrations/issues/148#issuecomment-2351355627 and https://github.com/ipfs/fs-repo-migrations/issues/188
- Use `lib.getExe` instead of hardcoding the binary name in the kubo NixOS module
- Use `substituteInPlace` with `--replace-fail` instead of `--replace`
parent 5624e133

nixos/modules/services/network-filesystems/kubo.nix

+1 −1
Original line number Diff line number Diff line
@@ -339,7 +339,7 @@ in 
          # After an unclean shutdown this file may exist which will cause the config command to attempt to talk to the daemon. This will hang forever if systemd is holding our sockets open.

          rm -vf "$IPFS_PATH/api"

      '' + lib.optionalString cfg.autoMigrate ''

        ${pkgs.kubo-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y

        '${lib.getExe pkgs.kubo-migrator}' -to '${cfg.package.repoVersion}' -y

      '' + ''

        fi

        ipfs --offline config show |

pkgs/applications/networking/kubo-migrator/all-migrations.nix

deleted100644 → 0
+0 −82
Original line number Diff line number Diff line 
{ lib

, stdenv

, symlinkJoin

, buildGoModule

, kubo-migrator-unwrapped

}:



# This package contains all the individual migrations in the bin directory.

# This is used by fs-repo-migrations and could also be used by Kubo itself

# when starting it like this: ipfs daemon --migrate



let

  fs-repo-common = pname: version: buildGoModule {

    inherit pname version;

    inherit (kubo-migrator-unwrapped) src;

    sourceRoot = "${kubo-migrator-unwrapped.src.name}/${pname}";

    vendorHash = null;

    # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match

    # See https://github.com/ipfs/fs-repo-migrations/pull/163

    postPatch = lib.optionalString (lib.elem pname [ "fs-repo-10-to-11" "fs-repo-11-to-12" ]) ''

      substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \

        --replace \

          '"container/list"' \

          '"container/list"

          "context"' \

        --replace \

          'config *Config' \

          'config *Config

          ctx context.Context'

    '';

    doCheck = false;

    meta = kubo-migrator-unwrapped.meta // {

      mainProgram = pname;

      description = "Individual migration for the filesystem repository of Kubo clients";

    };

  };



  # Concatenation of the latest repo version and the version of that migration

  version = "15.1.0.1";



  fs-repo-14-to-15 = fs-repo-common "fs-repo-14-to-15" "1.0.1";

  fs-repo-13-to-14 = fs-repo-common "fs-repo-13-to-14" "1.0.0";

  fs-repo-12-to-13 = fs-repo-common "fs-repo-12-to-13" "1.0.0";

  fs-repo-11-to-12 = fs-repo-common "fs-repo-11-to-12" "1.0.2";

  fs-repo-10-to-11 = fs-repo-common "fs-repo-10-to-11" "1.0.1";

  fs-repo-9-to-10  = fs-repo-common "fs-repo-9-to-10"  "1.0.1";

  fs-repo-8-to-9   = fs-repo-common "fs-repo-8-to-9"   "1.0.1";

  fs-repo-7-to-8   = fs-repo-common "fs-repo-7-to-8"   "1.0.1";

  fs-repo-6-to-7   = fs-repo-common "fs-repo-6-to-7"   "1.0.1";

  fs-repo-5-to-6   = fs-repo-common "fs-repo-5-to-6"   "1.0.1";

  fs-repo-4-to-5   = fs-repo-common "fs-repo-4-to-5"   "1.0.1";

  fs-repo-3-to-4   = fs-repo-common "fs-repo-3-to-4"   "1.0.1";

  fs-repo-2-to-3   = fs-repo-common "fs-repo-2-to-3"   "1.0.1";

  fs-repo-1-to-2   = fs-repo-common "fs-repo-1-to-2"   "1.0.1";

  fs-repo-0-to-1   = fs-repo-common "fs-repo-0-to-1"   "1.0.1";



  all-migrations = [

    fs-repo-14-to-15

    fs-repo-13-to-14

    fs-repo-12-to-13

    fs-repo-11-to-12

    fs-repo-10-to-11

    fs-repo-9-to-10

    fs-repo-8-to-9

    fs-repo-7-to-8

  ] ++ lib.optional (!stdenv.hostPlatform.isDarwin) # I didn't manage to fix this on macOS:

    fs-repo-6-to-7                     # gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2

  ++ [

    fs-repo-5-to-6

    fs-repo-4-to-5

    fs-repo-3-to-4

    fs-repo-2-to-3

    fs-repo-1-to-2

    fs-repo-0-to-1

  ];



in



symlinkJoin {

  name = "kubo-migrator-all-fs-repo-migrations-${version}";

  paths = all-migrations;

}

pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix

0 → 100644
+253 −0
Original line number Diff line number Diff line 
{

  lib,

  stdenv,

  symlinkJoin,

  buildGoModule,

  fetchFromGitHub,

  kubo-migrator-unwrapped,

  writeShellApplication,

  minRepoVersion ? 0, # The minimum supported Kubo repo version from which the migrations can start. Increasing this reduces the closure size

  stubBrokenMigrations ? true, # This prevents the fs-repo-migrations program from downloading binaries off the internet without even checking any signatures

}:



let

  mkMigration =

    from: to: version: hash:

    let

      pname = "fs-repo-${toString from}-to-${toString to}";

      src = fetchFromGitHub {

        owner = "ipfs";

        repo = "fs-repo-migrations";

        rev = "${pname}/v${version}";

        inherit hash;

        sparseCheckout = [ pname ];

      };

    in

    buildGoModule {

      inherit pname version src;

      sourceRoot = "${src.name}/${pname}";

      vendorHash = null;



      # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match

      # See https://github.com/ipfs/fs-repo-migrations/pull/163

      postPatch =

        lib.optionalString

          (lib.elem to [

            11

            12

          ])

          ''

            substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \

              --replace-fail \

                '"container/list"' \

                '"container/list"

                "context"' \

              --replace-fail \

                'config *Config' \

                'config *Config

                ctx context.Context'

          '';



      checkPhase = ''

        runHook preCheck

        ${

          if to <= 11 then

            "" # Migrations fs-repo-10-to-11 and earlier require too much effort to test, making it not worth it

          else if to == 12 then

            ''

              cd migration

              go test -mod=vendor

            ''

          else

            ''

              cd not-sharness

              ./test.sh

            ''

        }

        runHook postCheck

      '';



      # Check that it does not crash

      doInstallCheck = true;

      installCheckPhase = ''

        runHook preInstallCheck

        "$out/bin/${pname}" -help

        runHook postInstallCheck

      '';



      meta = {

        inherit (kubo-migrator-unwrapped.meta)

          homepage

          license

          platforms

          maintainers

          ;

        mainProgram = pname;

        description = "Migrate the filesystem repository of Kubo from repo version ${toString from} to ${toString to}";



        broken =

          to == 7 && stdenv.hostPlatform.isDarwin # fs-repo-6-to-7 is broken on macOS: gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2

          || (lib.elem to [

            11 # fs-repo-10-to-11 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match

            12 # fs-repo-11-to-12 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match

          ]);

      };

    };



  stubBecauseDisabled =

    from: to: release:

    let

      pname = "fs-repo-${toString from}-to-${toString to}";

    in

    writeShellApplication {

      name = pname;

      text = ''

        echo 'The kubo-fs-repo-migrations package was not buit with support for ${pname}.'

        echo 'To enable support, set the minRepoVersion argument of this package to a lower value.'

        echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.'

      '';

    };



  stubBecauseBroken =

    pname:

    writeShellApplication {

      name = pname;

      text = ''

        echo '${pname} is broken with the latest Go version.'

        echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.'

      '';

    };



  releases = [

    {

      from = 0;

      to = 1;

      release = "1.0.1";

      hash = "sha256-2mKtr6ZXZdOOY+9GNaC85HKjOMsfeM91oxVuxHIWDO4=";

    }

    {

      from = 1;

      to = 2;

      release = "1.0.1";

      hash = "sha256-6/BewNcZc/fIBa8G1luNO2wqTdeHi8vL7ojJDjBfWYI=";

    }

    {

      from = 2;

      to = 3;

      release = "1.0.1";

      hash = "sha256-kESX/R25nb7G/uggwa7GB7I2IrdgeKe0chRzjr70Kuw=";

    }

    {

      from = 3;

      to = 4;

      release = "1.0.1";

      hash = "sha256-Mv3/7eUS8j7ZzbNR52baekDcXPwcaNpUfqkt0eRpP20=";

    }

    {

      from = 4;

      to = 5;

      release = "1.0.1";

      hash = "sha256-aEqXFhZGOBU5ql2RRqzwD5IXGySVGroaHxjrkpIGAeU=";

    }

    {

      from = 5;

      to = 6;

      release = "1.0.1";

      hash = "sha256-EhMe/3gIl3VjSh6KzBPGH4s6B3AWRnbJ+eHSc8GOHMw=";

    }

    {

      from = 6;

      to = 7;

      release = "1.0.1";

      hash = "sha256-+5kIPQZckloPujLS0QQT+ojIIndfCQaH6grftZdYQ88=";

    }

    {

      from = 7;

      to = 8;

      release = "1.0.1";

      hash = "sha256-82oSU7qhldPVTdbbol3xSnl8Ko7NUPvGpAnmFxvAceQ=";

    }

    {

      from = 8;

      to = 9;

      release = "1.0.1";

      hash = "sha256-9knC2CfiTUNJRlrOLRpKy70Hl9p9DQf6rfXnU2a0fig=";

    }

    {

      from = 9;

      to = 10;

      release = "1.0.1";

      hash = "sha256-732k76Kijs5izu404ES/YSnYfC9V88d9Qq5oHv5Qon0=";

    }

    {

      from = 10;

      to = 11;

      release = "1.0.1";

      hash = "sha256-WieBZpD8dpFDif7QxTGjRoZtNBbkI3KU4w4av7b+d4Q=";

    }

    {

      from = 11;

      to = 12;

      release = "1.0.2";

      hash = "sha256-x/4ps705Hnf+/875/tn3AsEHgaHHCc+cGXymXpRt0xA=";

    }

    {

      from = 12;

      to = 13;

      release = "1.0.0";

      hash = "sha256-HjtZ2izoZ+0BrhzXG/QJHcnwsxi0oY4Q3CHjTi29W9o=";

    }

    {

      from = 13;

      to = 14;

      release = "1.0.0";

      hash = "sha256-zvNq+AFp7HDHHZCJOh9OW/lalk3bXOl1Pi+rvJtjuSA=";

    }

    {

      from = 14;

      to = 15;

      release = "1.0.1";

      hash = "sha256-u7PM6kFCQUn07NGpeRYpBDEwc2pP+r5mf44LZU4DV5Y=";

    }

  ];



  maxRepoVersion = builtins.length releases;



  minRepoVersionValidated =

    if minRepoVersion >= 0 then

      minRepoVersion

    else

      throw "The minimum supported repo version is 0. Set `minRepoVersion` to a non-zero value.";



  latestMigration = builtins.foldl' (x: y: if y.to == maxRepoVersion then y else x) {

    release = throw "Could not get the latest Kubo migration";

  } releases;

  version = "${toString maxRepoVersion}.${latestMigration.release}";



  mkMigrationOrStub =

    x:

    let

      builder = if x.from >= minRepoVersionValidated then mkMigration else stubBecauseDisabled;

    in

    builder x.from x.to x.release x.hash;

  migrations = builtins.map mkMigrationOrStub releases;



  packageNotBroken = package: !package.meta.broken;

  migrationsBrokenRemoved = builtins.filter packageNotBroken migrations;

  migrationsBrokenStubbed = builtins.map (

    x: if packageNotBroken x then x else (stubBecauseBroken x.pname)

  ) migrations;

in



symlinkJoin {

  name = "kubo-fs-repo-migrations-${version}";

  paths = if stubBrokenMigrations then migrationsBrokenStubbed else migrationsBrokenRemoved;

  meta = (builtins.removeAttrs kubo-migrator-unwrapped.meta [ "mainProgram" ]) // {

    description = "Several individual migrations for migrating the filesystem repository of Kubo one version at a time";

    longDescription = ''

      This package contains all the individual migrations in the bin directory.

      This is used by fs-repo-migrations and could also be used by Kubo itself

      when starting it like this: ipfs daemon --migrate

    '';

  };

}

pkgs/applications/networking/kubo-migrator/unwrapped.nix→pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix

+36 −0
Original line number Diff line number Diff line 
{ lib

, buildGoModule

, fetchFromGitHub

{

  lib,

  buildGoModule,

  fetchFromGitHub,

}:



buildGoModule rec {

  pname = "kubo-migrator";

  version = "2.0.2";

  version = "2.0.2-unstable-2024-08-02";



  src = fetchFromGitHub {

    owner = "ipfs";

    repo = "fs-repo-migrations";

    # Use the latest git tag here, since v2.0.2 does not

    # contain the latest migration fs-repo-14-to-15/v1.0.1

    # The fs-repo-migrations code itself is the same between

    # the two versions but the migration code, which is built

    # into separate binaries, is not.

    rev = "fs-repo-14-to-15/v1.0.1";

    hash = "sha256-oIGDZr0cv+TIl5glHr3U+eIqAlPAOWyFzgfQGGM+xNM=";

    rev = "cbc31a03fb2f6aba80d577224c09472101427771";

    hash = "sha256-wgWwDuL5Yv7dSYFrBiC4OS7SuTHh1D8RSabBnOTUiZ0=";

    sparseCheckout = [ "fs-repo-migrations" ];

  };



  sourceRoot = "${src.name}/fs-repo-migrations";
@@ -25,11 +22,15 @@ buildGoModule rec { 


  doCheck = false;



  meta = with lib; {

    description = "Migrations for the filesystem repository of Kubo clients";

  meta = {

    description = "Run the appripriate migrations for migrating the filesystem repository of Kubo (migrations not included)";

    homepage = "https://github.com/ipfs/fs-repo-migrations";

    license = licenses.mit;

    maintainers = with maintainers; [ Luflosi elitak ];

    license = lib.licenses.mit;

    platforms = lib.platforms.unix;

    maintainers = with lib.maintainers; [

      Luflosi

      elitak

    ];

    mainProgram = "fs-repo-migrations";

  };

}

pkgs/applications/networking/kubo-migrator/default.nix→pkgs/by-name/ku/kubo-migrator/package.nix

+26 −0
Original line number Diff line number Diff line 
{ lib

, buildEnv

, makeWrapper

, kubo-migrator-unwrapped

, kubo-migrator-all-fs-repo-migrations

{

  lib,

  buildEnv,

  makeWrapper,

  kubo-migrator-unwrapped,

  kubo-fs-repo-migrations,

}:



buildEnv {
@@ -16,8 +17,10 @@ buildEnv { 


  postBuild = ''

    wrapProgram "$out/bin/fs-repo-migrations" \

      --prefix PATH ':' '${lib.makeBinPath [ kubo-migrator-all-fs-repo-migrations ]}'

      --prefix PATH ':' '${lib.makeBinPath [ kubo-fs-repo-migrations ]}'

  '';



  inherit (kubo-migrator-unwrapped) meta;

  meta = kubo-migrator-unwrapped.meta // {

    description = "Run the appripriate migrations for migrating the filesystem repository of Kubo";

  };

}