Merge pull request #249708 from NixOS/haskell-updates

{-# OPTIONS_GHC -Wall #-}

{-# LANGUAGE DataKinds #-}

import Control.Monad (forM_, (<=<))

import Control.Monad (forM_, forM, (<=<))

import Control.Monad.Trans (MonadIO (liftIO))

import Data.Aeson (

   FromJSON,

data Eval = Eval

   { id :: Int

   , jobsetevalinputs :: JobsetEvalInputs

   , builds :: Seq Int

   }

   deriving (Generic, ToJSON, FromJSON, Show)

   }

   deriving (Generic, ToJSON, FromJSON, Show)

data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround

data RequestLogsFlag = RequestLogs | NoRequestLogs

main :: IO ()

main = do

   args <- getArgs

   case args of

      ["get-report"] -> getBuildReports

      ["get-report", "--slow"] -> getBuildReports HydraSlownessWorkaround

      ["get-report"] -> getBuildReports NoHydraSlownessWorkaround

      ["ping-maintainers"] -> printMaintainerPing

      ["mark-broken-list"] -> printMarkBrokenList

      ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs

      ["mark-broken-list"] -> printMarkBrokenList RequestLogs

      ["eval-info"] -> printEvalInfo

      _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list | eval-info"

      _ -> putStrLn "Usage: get-report [--slow] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info"

reportFileName :: IO FilePath

reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"

showT :: Show a => a -> Text

showT = Text.pack . show

getBuildReports :: IO ()

getBuildReports = runReq defaultHttpConfig do

getBuildReports :: HydraSlownessWorkaroundFlag -> IO ()

getBuildReports opt = runReq defaultHttpConfig do

   evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"]

   eval@Eval{id} <- maybe (liftIO $ fail "No Evalution found") pure evalMay

   eval@Eval{id} <- maybe (liftIO $ fail "No Evaluation found") pure evalMay

   liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..."

   buildReports :: Seq Build <- hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"]

   buildReports <- getEvalBuilds opt id

   liftIO do

      fileName <- reportFileName

      putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName

      now <- getCurrentTime

      encodeFile fileName (eval, now, buildReports)

getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build)

getEvalBuilds NoHydraSlownessWorkaround id =

  hydraJSONQuery (responseTimeout 900000000) ["eval", showT id, "builds"]

getEvalBuilds HydraSlownessWorkaround id = do

  Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ]

  forM builds $ \buildId -> do

    liftIO $ putStrLn $ "Querying build " <> show buildId

    hydraJSONQuery mempty [ "build", showT buildId ]

hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a)

hydraQuery responseType option query =

   responseBody

         (foldl' (/:) (https "hydra.nixos.org") query)

         NoReqBody

         responseType

         (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)" <> option)

         (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" <> option)

hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a

hydraJSONQuery = hydraQuery jsonResponse

       textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps

   Text.putStrLn textBuildSummary

printMarkBrokenList :: IO ()

printMarkBrokenList = do

printMarkBrokenList :: RequestLogsFlag -> IO ()

printMarkBrokenList reqLogs = do

   (_, fetchTime, buildReport) <- readBuildReports

   runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} ->

      case (getBuildState build, Text.splitOn "." $ unJobName job) of

         (Failed, ["haskellPackages", name, "x86_64-linux"]) -> do

            -- We use the last probable error cause found in the build log file.

            error_message <- fromMaybe "failure" <$>

              case reqLogs of

                NoRequestLogs -> pure Nothing

                RequestLogs -> do

                  -- Fetch build log from hydra to figure out the cause of the error.

                  build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]

            -- We use the last probable error cause found in the build log file.

            let error_message = fromMaybe " failure " $ safeLast $ mapMaybe probableErrorCause build_log

                  pure $ safeLast $ mapMaybe probableErrorCause build_log

            liftIO $ putStrLn $ "  - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime

         _ -> pure ()

set -euo pipefail

do_commit=false

mark_broken_list_flags=""

for arg in "$@"; do

    case "$arg" in

        --do-commit)

            do_commit=true

            ;;

        --no-request-logs)

            mark_broken_list_flags="$mark_broken_list_flags $arg"

            ;;

        *)

            echo "$0: unknown flag: $arg"

            exit 100

            ;;

    esac

done

broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"

tmpfile=$(mktemp)

echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script."

echo "Generating a list of broken builds and displaying for manual confirmation ..."

maintainers/scripts/haskell/hydra-report.hs mark-broken-list | sort -i > "$tmpfile"

maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | sort -i > "$tmpfile"

$EDITOR "$tmpfile"

$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh

evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info)

if [[ "${1:-}" == "--do-commit" ]]; then

if $do_commit; then

git add $broken_config

git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml

git add pkgs/development/haskell-modules/hackage-packages.nix

{

  "commit": "4cdb9878496fdb36b8b9c5f2ab0ef8a44a0f859f",

  "url": "https://github.com/commercialhaskell/all-cabal-hashes/archive/4cdb9878496fdb36b8b9c5f2ab0ef8a44a0f859f.tar.gz",

  "sha256": "0yhymzcsls48hf44ncd79xn786rfh4k70h78w7b0ihn7lrjgsynv",

  "msg": "Update from Hackage at 2023-07-24T19:28:29Z"

  "commit": "69066b0daf2bbb4ca6f2b6de0bc9b8f27fffe4bc",

  "url": "https://github.com/commercialhaskell/all-cabal-hashes/archive/69066b0daf2bbb4ca6f2b6de0bc9b8f27fffe4bc.tar.gz",

  "sha256": "16ij50f7cx8gl3ypzwy50f5dr68y6m6n732sa1hwsng5db4vqzv7",

  "msg": "Update from Hackage at 2023-08-17T07:12:25Z"

}

  # There will probably be a new revision soon.

  hls-brittany-plugin = assert super.hls-brittany-plugin.version == "1.1.0.0"; doJailbreak super.hls-brittany-plugin;

  hls-hlint-plugin = super.hls-hlint-plugin.override {

    apply-refact = self.apply-refact_0_11_0_0;

  };

  # For -f-auto see cabal.project in haskell-language-server.

  ghc-lib-parser-ex = addBuildDepend self.ghc-lib-parser (disableCabalFlag "auto" super.ghc-lib-parser-ex);

    })

  ] super.aeson);

  # aeson 2.2.0.0 requires th-abstraction >= 0.5 & < 0.6

  aeson_2_2_0_0 = super.aeson_2_2_0_0.overrideScope (hfinal: hprev: {

    th-abstraction = hfinal.th-abstraction_0_5_0_0;

  });

  # 2023-06-28: Test error: https://hydra.nixos.org/build/225565149

  orbits = dontCheck super.orbits;

  # Arion's test suite needs a Nixpkgs, which is cumbersome to do from Nixpkgs

  # itself. For instance, pkgs.path has dirty sources and puts a huge .git in the

  # store. Testing is done upstream.

  # 2023-07-27: Allow base-4.17

  arion-compose = dontCheck (assert super.arion-compose.version == "0.2.0.0"; doJailbreak super.arion-compose);

  arion-compose = dontCheck super.arion-compose;

  # 2023-07-17: Outdated base bound https://github.com/srid/lvar/issues/5

  lvar = doJailbreak super.lvar;

  # https://github.com/glguy/config-value/commit/c5558c8258598fab686c259bff510cc1b19a0c50#commitcomment-119514821

  config-value = doJailbreak super.config-value;

  # path-io bound is adjusted in 0.6.1 release

  # https://github.com/tek/hix/commit/019426f6a3db256e4c96558ffe6fa2114e2f19a0

  hix = doJailbreak super.hix;

  # waiting for release: https://github.com/jwiegley/c2hsc/issues/41

  c2hsc = appendPatch (fetchpatch {

    url = "https://github.com/jwiegley/c2hsc/commit/490ecab202e0de7fc995eedf744ad3cb408b53cc.patch";

  # Overriding the version pandoc dependency uses as the latest release has version bounds

  # defined as >= 3.1  && < 3.2, can be removed once pandoc gets bumped by Stackage.

  patat = super.patat.override { pandoc = self.pandoc_3_1_6; };

  patat = super.patat.override { pandoc = self.pandoc_3_1_6_1; };

  # http2 also overridden in all-packages.nix for mailctl.

  # twain is currently only used by mailctl, so the .overrideScope shouldn't

  # negatively affect any other packages, at least currently...

  twain = super.twain.overrideScope (self: _: { http2 = self.http2_3_0_3; });

  # The latest release on hackage has an upper bound on containers which

  # breaks the build, though it works with the version of containers present

      name = "git-annex-${super.git-annex.version}-src";

      url = "git://git-annex.branchable.com/";

      rev = "refs/tags/" + super.git-annex.version;

      sha256 = "1i14mv8z9sr5sckckwiba4cypgs3iwk19pyrl9xzcrzz426dxrba";

      sha256 = "0fg3q7apdijnlgyb0yps1znjjd2nv3016r9cyxyw209sqn3whnx5";

      # delete android and Android directories which cause issues on

      # darwin (case insensitive directory). Since we don't need them

      # during the build process, we can delete it to prevent a hash

  elm-server = markBroken super.elm-server;

  elm-yesod = markBroken super.elm-yesod;

  # Tests failure with GHC >= 9.0.1, fixed in 1.6.24.4

  yesod-core = assert super.yesod-core.version == "1.6.24.3"; dontCheck super.yesod-core;

  # https://github.com/Euterpea/Euterpea2/issues/40

  Euterpea = doJailbreak super.Euterpea;

  # It does not support aeson 2.0

  descriptive = super.descriptive.override { aeson = self.aeson_1_5_6_0; };

  # Apply compatibility patches until a new release arrives

  # https://github.com/phadej/spdx/issues/33

  spdx = appendPatches [

    (fetchpatch {

      name = "spdx-ghc-9.4.patch";

      url = "https://github.com/phadej/spdx/pull/30/commits/545dc69f433225c837375fba4cbbdb7f9cc7b09b.patch";

      sha256 = "0p2h8dxkjy2v0dx7h6v62clmx5n5j3c4zh4myh926fijympi1glz";

    })

    (fetchpatch {

      name = "spdx-ghc-9.6.patch";

      url = "https://github.com/phadej/spdx/pull/32/commits/b51f665e9960614274ff6a9ac658802c1a785687.patch";

      sha256 = "01vf1h0djr84yxsjfhym715ncx0w5q4l02k3dkbmg40pnc62ql4h";

      excludes = [ ".github/**" ];

    })

  ] super.spdx;

  # 2022-03-19: Testsuite is failing: https://github.com/puffnfresh/haskell-jwt/issues/2

  jwt = dontCheck super.jwt;

  # https://github.com/basvandijk/concurrent-extra/issues/12

  concurrent-extra = dontCheck super.concurrent-extra;

  bloomfilter = appendPatches [

    # https://github.com/bos/bloomfilter/issues/7

    ./patches/bloomfilter-fix-on-32bit.patch

    # Fix build with GHC >= 9.2 by using stock unsafeShift* functions

    # https://github.com/bos/bloomfilter/pull/20

    (pkgs.fetchpatch {

      name = "bloomfilter-ghc-9.2-shift.patch";

      url = "https://github.com/bos/bloomfilter/pull/20/commits/fb79b39c44404fd791a3bed973e9d844fb084f1e.patch";

      sha256 = "0clmr5iar4mhp8nbgh1c1rh4fl7dy0g2kbqqh0af8aqmhjpqzrq3";

    })

  ] (overrideCabal (drv: {

    # Make sure GHC 9.2 patch applies correctly

    revision = null;

    editedCabalFile = null;

    prePatch = drv.prePatch or "" + ''

      "${pkgs.buildPackages.dos2unix}/bin/dos2unix" *.cabal

    '';

  }) super.bloomfilter);

  # https://github.com/pxqr/base32-bytestring/issues/4

  base32-bytestring = dontCheck super.base32-bytestring;

  github-backup = doJailbreak super.github-backup;

  # dontCheck: https://github.com/haskell-servant/servant-auth/issues/113

  # doJailbreak: waiting on revision 1 to hit hackage

  servant-auth-client = doJailbreak (dontCheck super.servant-auth-client);

  servant-auth-client = dontCheck super.servant-auth-client;

  # Allow lens-aeson >= 1.2 https://github.com/haskell-servant/servant/issues/1703

  servant-auth-server = doJailbreak super.servant-auth-server;

  # Allow hspec >= 2.10 https://github.com/haskell-servant/servant/issues/1704

  servant-foreign = doJailbreak super.servant-foreign;

  # Generate cli completions for dhall.

  dhall = self.generateOptparseApplicativeCompletions [ "dhall" ] super.dhall;

  inherit (let

    pandoc-cli-overlay = self: super: {

      # pandoc-cli requires pandoc >= 3.1

      pandoc = self.pandoc_3_1_6;

      pandoc = self.pandoc_3_1_6_1;

      # pandoc depends on crypton-connection, which requires tls >= 1.7

      tls = self.tls_1_7_0;

      tls = self.tls_1_7_1;

      crypton-connection = unmarkBroken super.crypton-connection;

      # pandoc depends on http-client-tls, which only starts depending

      # on crypton-connection in http-client-tls-0.3.6.2.

      http-client-tls = self.http-client-tls_0_3_6_2;

      # pandoc and skylighting are developed in tandem

      skylighting-core = self.skylighting-core_0_13_4_1;

      skylighting = self.skylighting_0_13_4_1;

      http-client-tls = self.http-client-tls_0_3_6_3;

    };

  in {

    pandoc-cli = super.pandoc-cli.overrideScope pandoc-cli-overlay;

    pandoc_3_1_6 = doDistribute (super.pandoc_3_1_6.overrideScope pandoc-cli-overlay);

    pandoc_3_1_6_1 = doDistribute (super.pandoc_3_1_6_1.overrideScope pandoc-cli-overlay);

    pandoc-lua-engine = super.pandoc-lua-engine.overrideScope pandoc-cli-overlay;

  })

    pandoc-cli

    pandoc_3_1_6

    pandoc_3_1_6_1

    pandoc-lua-engine

    ;

  })

  super.polynomial);

  # Unreleased bound relaxing patch allowing scotty 0.12

  taffybar = appendPatch (pkgs.fetchpatch {

    name = "taffybar-allow-scotty-0.12.patch";

    url = "https://github.com/taffybar/taffybar/commit/2e428ba550fc51067526a0350b91185acef72d19.patch";

    sha256 = "1lpcz671mk5cwqffjfi9ncc0d67bmwgzypy3i37a2fhfmxd0y3nl";

  }) ((p: assert p.version == "4.0.0"; p) super.taffybar);

  # Tests likely broke because of https://github.com/nick8325/quickcheck/issues/359,

  # but fft is not on GitHub, so no issue reported.

  fft = dontCheck super.fft;

  # has been resolved.

  lucid-htmx = doJailbreak super.lucid-htmx;

  # Needs lsp >= 2.1

  futhark = super.futhark.overrideScope (fself: _: {

    lsp = fself.lsp_2_1_0_0;

    lsp-types = fself.lsp-types_2_0_1_0;

  });

  # Too strict bounds on hspec

  # https://github.com/klapaucius/vector-hashtables/issues/11

  vector-hashtables = doJailbreak super.vector-hashtables;

  # Tests fail due to the newly-build fourmolu not being in PATH

  # https://github.com/fourmolu/fourmolu/issues/231

  fourmolu_0_13_1_0 = dontCheck (super.fourmolu_0_13_1_0.overrideScope (lself: lsuper: {

    Cabal-syntax = lself.Cabal-syntax_3_10_1_0;

    ghc-lib-parser = lself.ghc-lib-parser_9_6_2_20230523;

    parsec = lself.parsec_3_1_16_1;

    text = lself.text_2_0_2;

  }));

  fourmolu_0_13_1_0 = dontCheck super.fourmolu_0_13_1_0;

  # Merged upstream, but never released. Allows both intel and aarch64 darwin to build.

  # https://github.com/vincenthz/hs-gauge/pull/106

    # These aren't included in hackage-packages.nix because hackage2nix is configured for GHC 9.2, under which these plugins aren't supported.

    # See https://github.com/NixOS/nixpkgs/pull/205902 for why we use `self.<package>.scope`

    additionalDeps = with self.haskell-language-server.scope; [

      hls-brittany-plugin

      hls-haddock-comments-plugin

      (unmarkBroken hls-splice-plugin)

      hls-tactics-plugin

    Cabal = lself.Cabal_3_6_3_0;

    aeson = lself.aeson_1_5_6_0;

    lens-aeson = doJailbreak lself.lens-aeson_1_1_3;

    lsp-types = doJailbreak lsuper.lsp-types; # Checks require aeson >= 2.0

    lsp-types = dontCheck (doJailbreak lsuper.lsp-types); # Checks require aeson >= 2.0

    hls-overloaded-record-dot-plugin = null;

  }));

  ghc-lib-parser = doDistribute self.ghc-lib-parser_9_2_7_20230228;

  ghc-lib-parser = doDistribute self.ghc-lib-parser_9_2_8_20230729;

  ghc-lib-parser-ex = doDistribute self.ghc-lib-parser-ex_9_2_1_1;

  ghc-lib = doDistribute self.ghc-lib_9_2_7_20230228;

  ghc-lib = doDistribute self.ghc-lib_9_2_8_20230729;

  mod = super.mod_0_1_2_2;

  path-io = doJailbreak super.path-io;

  hls-cabal-plugin = super.hls-cabal-plugin.override {

    Cabal-syntax = self.Cabal-syntax_3_8_1_0;

  };

  ormolu = self.ormolu_0_5_0_1;

  fourmolu = dontCheck self.fourmolu_0_9_0_0;

  hlint = self.hlint_3_4_1;

    parser-combinators prettyprinter refinery retrie syb unagi-chan unordered-containers

  ]) super.hls-tactics-plugin);

  hls-brittany-plugin =  unmarkBroken (addBuildDepends (with self.hls-brittany-plugin.scope; [

    brittany czipwith extra ghc-exactprint ghcide hls-plugin-api hls-test-utils lens lsp-types

    ]) (super.hls-brittany-plugin.overrideScope (lself: lsuper: {

    brittany = doJailbreak (unmarkBroken lself.brittany_0_13_1_2);

    aeson = lself.aeson_1_5_6_0;

    multistate = unmarkBroken (dontCheck lsuper.multistate);

    lsp-types = doJailbreak lsuper.lsp-types; # Checks require aeson >= 2.0

  })));

  # This package is marked as unbuildable on GHC 9.2, so hackage2nix doesn't include any dependencies.

  # See https://github.com/NixOS/nixpkgs/pull/205902 for why we use `self.<package>.scope`

  hls-haddock-comments-plugin =  unmarkBroken (addBuildDepends (with self.hls-haddock-comments-plugin.scope; [