Unverified Commit 7f8a16b9 authored by Janik's avatar Janik Committed by GitHub
Browse files

Merge pull request #235466 from 06kellyjac/semgrep

parents ea12f277 f3dde5bf

pkgs/tools/security/semgrep/common.nix

+19 −36
Original line number Diff line number Diff line 
{ lib, fetchFromGitHub, fetchzip, stdenv }:

{ lib }:



rec {

  version = "1.15.0";

  version = "1.27.0";



  src = fetchFromGitHub {

    owner = "returntocorp";

    repo = "semgrep";

    rev = "v${version}";

    sha256 = "sha256-x+AOt6nn2hN4MODFZCvlq0kZ3VLoS7rVcFGGCEssIu0=";

  };

  srcHash = "sha256-F6n3LQY4a5sO6c8SMQF9YjjgOS+v2SH+UQPwhg2EX7Q=";



  # submodule dependencies

  # these are fetched so we:

  #   1. don't fetch the many submodules we don't need

  #   2. avoid fetchSubmodules since it's prone to impurities

  submodules = {

    "cli/src/semgrep/lang" = fetchFromGitHub {

      owner = "returntocorp";

      repo = "semgrep-langs";

      rev = "08656cdefc9e6818c64e168cf51ee1e76ea8829e";

      sha256 = "sha256-vYf33JhfvEDmt/VW0hBOmqailIERS0GdUgrPuCxWt9I=";

    };

    "cli/src/semgrep/semgrep_interfaces" = fetchFromGitHub {

    "cli/src/semgrep/semgrep_interfaces" = {

      owner = "returntocorp";

      repo = "semgrep-interfaces";

      rev = "ba9241ca8f13dea72a4ca5c5eae99f45c071c8b4";

      sha256 = "sha256-2rcMmN42445AivcyYLPeE+HBYOyxJijQME1UUr9HISA=";

      rev = "213f67abea73546ca6111e1bbf0ef96aa917c940";

      hash = "sha256-HeNHJkTje9j16+dwsfyMhoqQn/J18q/7XvQPRwgTw/Y=";

    };

  };



  # fetch pre-built semgrep-core since the ocaml build is complex and relies on

  # the opam package manager at some point

  core = rec {

    data = {

  # pulling it out of the python wheel as r2c no longer release a built binary

  # on github releases

  core = {

    x86_64-linux = {

        suffix = "-ubuntu-16.04.tgz";

        sha256 = "sha256-vLtV1WAnOD6HhgrWYIP0NfXHKfvXORksdNp5UTG1QWc=";

      platform = "any";

      hash = "sha256-cRj81dXpAE6S0EXajsRikOIAPzlUf42FhiDCWjv+wZQ=";

    };

    x86_64-darwin = {

        suffix = "-osx.zip";

        sha256 = "sha256-6+ENjOOIJ5TSjpnJ5pDudblrWj/FLUe66UGr6V9c0HQ=";

      };

      platform = "macosx_10_14_x86_64";

      hash = "sha256-jqfGVZGF/DFgXkr7kQg6QyqEELSr8AKE3Ga8kTftnIY=";

    };

    src = let

      inherit (stdenv.hostPlatform) system;

      selectSystemData = data: data.${system} or (throw "Unsupported system: ${system}");

      inherit (selectSystemData data) suffix sha256;

    in fetchzip {

      url = "https://github.com/returntocorp/semgrep/releases/download/v${version}/semgrep-v${version}${suffix}";

      inherit sha256;

    aarch64-darwin = {

      platform = "macosx_11_0_arm64";

      hash = "sha256-e/uCSRMdbVD0lvc0hukbiUzheqRNIIh1LgMq6Ae7JYI=";

    };

  };
@@ -66,7 +51,5 @@ rec { 
    '';

    license = licenses.lgpl21Plus;

    maintainers = with maintainers; [ jk ambroisie ];

    # limited by semgrep-core

    platforms = [ "x86_64-linux" "x86_64-darwin" ];

  };

}

pkgs/tools/security/semgrep/default.nix

+13 −4
Original line number Diff line number Diff line 
{ lib

, fetchFromGitHub

, callPackage

, semgrep-core

, buildPythonApplication

, pythonPackages
@@ -11,12 +10,20 @@ 
}:



let

  common = callPackage ./common.nix { };

  common = import ./common.nix { inherit lib; };

in

buildPythonApplication rec {

  pname = "semgrep";

  inherit (common) src version;

  inherit (common) version;

  src = fetchFromGitHub {

    owner = "returntocorp";

    repo = "semgrep";

    rev = "v${version}";

    hash = common.srcHash;

  };



  # prepare a subset of the submodules as we only need a handful

  # and there are many many submodules total

  postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList

    (

      path: submodule: ''
@@ -27,7 +34,7 @@ buildPythonApplication rec { 
        ln -s ${submodule}/ ${path}

      ''

    )

    common.submodules)) + ''

    passthru.submodulesSubset)) + ''

    cd cli

  '';
@@ -97,10 +104,12 @@ buildPythonApplication rec { 


  passthru = {

    inherit common;

    submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules;

    updateScript = ./update.sh;

  };



  meta = common.meta // {

    description = common.meta.description + " - cli";

    inherit (semgrep-core.meta) platforms;

  };

}

pkgs/tools/security/semgrep/semgrep-core.nix

+35 −4
Original line number Diff line number Diff line 
{ lib, stdenvNoCC, callPackage }:

{ lib, stdenvNoCC, fetchPypi, unzip }:



let

  common = callPackage ./common.nix { };

  common = import ./common.nix { inherit lib; };

in

stdenvNoCC.mkDerivation rec {

  pname = "semgrep-core";

  inherit (common) version;

  inherit (common.core) src;

  # fetch pre-built semgrep-core since the ocaml build is complex and relies on

  # the opam package manager at some point

  # pulling it out of the python wheel as r2c no longer release a built binary

  # on github releases

  src =

    let

      inherit (stdenvNoCC.hostPlatform) system;

      data = common.core.${system} or (throw "Unsupported system: ${system}");

    in

    fetchPypi rec {

      pname = "semgrep";

      inherit version;

      format = "wheel";

      dist = python;

      python = "cp37.cp38.cp39.py37.py38.py39";

      inherit (data) platform hash;

    };



  nativeBuildInputs = [ unzip ];



  # _tryUnzip from unzip's setup-hook doesn't recognise .whl

  # "do not know how to unpack source archive"

  # perform unpack by hand

  unpackPhase = ''

    runHook preUnpack

    LANG=en_US.UTF-8 unzip -qq "$src"

    runHook postUnpack

  '';



  dontConfigure = true;

  dontBuild = true;



  installPhase = ''

    runHook preInstall

    install -Dm 755 -t $out/bin semgrep-core

    install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core

    runHook postInstall

  '';



  meta = common.meta // {

    description = common.meta.description + " - core binary";

    sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];

    platforms = lib.attrNames common.core;

  };

}

pkgs/tools/security/semgrep/update.sh

+35 −25
Original line number Diff line number Diff line 
#!/usr/bin/env nix-shell

#!nix-shell -i bash -p curl gnused jq

#!nix-shell -i bash -p curl gnused jq nix-prefetch



set -euxo pipefail
@@ -33,7 +33,7 @@ NEW_VERSION=$( 
)

# trim v prefix

NEW_VERSION="${NEW_VERSION:1}"

OLD_VERSION="$(instantiateClean semgrep.common.version)"

OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)"



if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then

    echo "Already up to date"
@@ -50,43 +50,54 @@ fetchgithub() { 
    set -eo pipefail

}



fetchzip() {

    set +eo pipefail

    nix-build -E "with import $NIXPKGS_ROOT {}; fetchzip {url = \"$1\"; sha256 = lib.fakeSha256; }" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g'

    set -eo pipefail

fetch_arch() {

  VERSION=$1

  PLATFORM=$2

  nix-prefetch "{ fetchPypi }:

fetchPypi rec {

  pname = \"semgrep\";

  version = \"$VERSION\";

  format = \"wheel\";

  dist = python;

  python = \"cp37.cp38.cp39.py37.py38.py39\";

  platform = \"$PLATFORM\";

}

"

}



replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE"



echo "Updating src"



OLD_HASH="$(instantiateClean semgrep.common.src.outputHash)"

OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)"

echo "Old hash $OLD_HASH"

TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="

replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"

NEW_HASH="$(fetchgithub semgrep.common.src)"

NEW_HASH="$(fetchgithub semgrep.src)"

echo "New hash $NEW_HASH"

replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"



echo "Updated src"



# loop through platforms for core

nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.common.core.data" --eval --strict --json \

| jq '.[]' -r \

| while read -r PLATFORM; do

    echo "Updating core for $PLATFORM"

    SUFFIX=$(instantiateClean semgrep.common.core.data."$PLATFORM".suffix)

    OLD_HASH=$(instantiateClean semgrep.common.core.data."$PLATFORM".sha256)

    echo "Old hash $OLD_HASH"



    NEW_URL="https://github.com/returntocorp/semgrep/releases/download/v$NEW_VERSION/semgrep-v$NEW_VERSION$SUFFIX"

    NEW_HASH="$(fetchzip "$NEW_URL")"

    echo "New hash $NEW_HASH"

update_core_platform() {

    SYSTEM=$1

    echo "Updating core src $SYSTEM"



    PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")"



    OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")"

    echo "Old core hash $OLD_HASH"

    NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")"

    echo "New core hash $NEW_HASH"

    replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE"



    echo "Updated core for $PLATFORM"

done

    echo "Updated core src $SYSTEM"

}



update_core_platform "x86_64-linux"

update_core_platform "x86_64-darwin"

update_core_platform "aarch64-darwin"



OLD_PWD=$PWD

TMPDIR="$(mktemp -d)"
@@ -109,7 +120,7 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas 
    echo "Updating $SUBMODULE"

    OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev)

    echo "Old commit $OLD_REV"

    OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".outputHash)

    OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash)

    echo "Old hash $OLD_HASH"



    NEW_REV=$(get_submodule_commit "$SUBMODULE")
@@ -120,13 +131,12 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas 
      continue

    fi



    NEW_URL=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")

    NEW_HASH=$(nix --experimental-features nix-command hash to-sri "sha256:$(nix-prefetch-url "$NEW_URL")")

    NEW_URL=$(instantiateClean semgrep.passthru.submodulesSubset."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")



    TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="

    replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE"

    replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"

    NEW_HASH="$(fetchgithub semgrep.passthru.common.submodules."$SUBMODULE")"

    NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")"

    echo "New hash $NEW_HASH"

    replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"