semgrep{,-core}: 1.15.0 -> 1.27.0 (f3dde5bf) · Commits · nix / nixpkgs

pkgs/tools/security/semgrep/common.nix

+19 −36

Original line number	Diff line number	Diff line
		{ lib, fetchFromGitHub, fetchzip, stdenv }:
		{ lib }:

		rec {
		version = "1.15.0";
		version = "1.27.0";

		src = fetchFromGitHub {
		owner = "returntocorp";
		repo = "semgrep";
		rev = "v${version}";
		sha256 = "sha256-x+AOt6nn2hN4MODFZCvlq0kZ3VLoS7rVcFGGCEssIu0=";
		};
		srcHash = "sha256-F6n3LQY4a5sO6c8SMQF9YjjgOS+v2SH+UQPwhg2EX7Q=";

		# submodule dependencies
		# these are fetched so we:
		# 1. don't fetch the many submodules we don't need
		# 2. avoid fetchSubmodules since it's prone to impurities
		submodules = {
		"cli/src/semgrep/lang" = fetchFromGitHub {
		owner = "returntocorp";
		repo = "semgrep-langs";
		rev = "08656cdefc9e6818c64e168cf51ee1e76ea8829e";
		sha256 = "sha256-vYf33JhfvEDmt/VW0hBOmqailIERS0GdUgrPuCxWt9I=";
		};
		"cli/src/semgrep/semgrep_interfaces" = fetchFromGitHub {
		"cli/src/semgrep/semgrep_interfaces" = {
		owner = "returntocorp";
		repo = "semgrep-interfaces";
		rev = "ba9241ca8f13dea72a4ca5c5eae99f45c071c8b4";
		sha256 = "sha256-2rcMmN42445AivcyYLPeE+HBYOyxJijQME1UUr9HISA=";
		rev = "213f67abea73546ca6111e1bbf0ef96aa917c940";
		hash = "sha256-HeNHJkTje9j16+dwsfyMhoqQn/J18q/7XvQPRwgTw/Y=";
		};
		};

		# fetch pre-built semgrep-core since the ocaml build is complex and relies on
		# the opam package manager at some point
		core = rec {
		data = {
		# pulling it out of the python wheel as r2c no longer release a built binary
		# on github releases
		core = {
		x86_64-linux = {
		suffix = "-ubuntu-16.04.tgz";
		sha256 = "sha256-vLtV1WAnOD6HhgrWYIP0NfXHKfvXORksdNp5UTG1QWc=";
		platform = "any";
		hash = "sha256-cRj81dXpAE6S0EXajsRikOIAPzlUf42FhiDCWjv+wZQ=";
		};
		x86_64-darwin = {
		suffix = "-osx.zip";
		sha256 = "sha256-6+ENjOOIJ5TSjpnJ5pDudblrWj/FLUe66UGr6V9c0HQ=";
		};
		platform = "macosx_10_14_x86_64";
		hash = "sha256-jqfGVZGF/DFgXkr7kQg6QyqEELSr8AKE3Ga8kTftnIY=";
		};
		src = let
		inherit (stdenv.hostPlatform) system;
		selectSystemData = data: data.${system} or (throw "Unsupported system: ${system}");
		inherit (selectSystemData data) suffix sha256;
		in fetchzip {
		url = "https://github.com/returntocorp/semgrep/releases/download/v${version}/semgrep-v${version}${suffix}";
		inherit sha256;
		aarch64-darwin = {
		platform = "macosx_11_0_arm64";
		hash = "sha256-e/uCSRMdbVD0lvc0hukbiUzheqRNIIh1LgMq6Ae7JYI=";
		};
		};

		@@ -66,7 +51,5 @@ rec {
		'';
		license = licenses.lgpl21Plus;
		maintainers = with maintainers; [ jk ambroisie ];
		# limited by semgrep-core
		platforms = [ "x86_64-linux" "x86_64-darwin" ];
		};
		}

pkgs/tools/security/semgrep/default.nix

+13 −4

Original line number	Diff line number	Diff line
		{ lib
		, fetchFromGitHub
		, callPackage
		, semgrep-core
		, buildPythonApplication
		, pythonPackages
		@@ -11,12 +10,20 @@
		}:

		let
		common = callPackage ./common.nix { };
		common = import ./common.nix { inherit lib; };
		in
		buildPythonApplication rec {
		pname = "semgrep";
		inherit (common) src version;
		inherit (common) version;
		src = fetchFromGitHub {
		owner = "returntocorp";
		repo = "semgrep";
		rev = "v${version}";
		hash = common.srcHash;
		};

		# prepare a subset of the submodules as we only need a handful
		# and there are many many submodules total
		postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList
		(
		path: submodule: ''
		@@ -27,7 +34,7 @@ buildPythonApplication rec {
		ln -s ${submodule}/ ${path}
		''
		)
		common.submodules)) + ''
		passthru.submodulesSubset)) + ''
		cd cli
		'';

		@@ -97,10 +104,12 @@ buildPythonApplication rec {

		passthru = {
		inherit common;
		submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules;
		updateScript = ./update.sh;
		};

		meta = common.meta // {
		description = common.meta.description + " - cli";
		inherit (semgrep-core.meta) platforms;
		};
		}

pkgs/tools/security/semgrep/semgrep-core.nix

+35 −4

Original line number	Diff line number	Diff line
		{ lib, stdenvNoCC, callPackage }:
		{ lib, stdenvNoCC, fetchPypi, unzip }:

		let
		common = callPackage ./common.nix { };
		common = import ./common.nix { inherit lib; };
		in
		stdenvNoCC.mkDerivation rec {
		pname = "semgrep-core";
		inherit (common) version;
		inherit (common.core) src;
		# fetch pre-built semgrep-core since the ocaml build is complex and relies on
		# the opam package manager at some point
		# pulling it out of the python wheel as r2c no longer release a built binary
		# on github releases
		src =
		let
		inherit (stdenvNoCC.hostPlatform) system;
		data = common.core.${system} or (throw "Unsupported system: ${system}");
		in
		fetchPypi rec {
		pname = "semgrep";
		inherit version;
		format = "wheel";
		dist = python;
		python = "cp37.cp38.cp39.py37.py38.py39";
		inherit (data) platform hash;
		};

		nativeBuildInputs = [ unzip ];

		# _tryUnzip from unzip's setup-hook doesn't recognise .whl
		# "do not know how to unpack source archive"
		# perform unpack by hand
		unpackPhase = ''
		runHook preUnpack
		LANG=en_US.UTF-8 unzip -qq "$src"
		runHook postUnpack
		'';

		dontConfigure = true;
		dontBuild = true;

		installPhase = ''
		runHook preInstall
		install -Dm 755 -t $out/bin semgrep-core
		install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core
		runHook postInstall
		'';

		meta = common.meta // {
		description = common.meta.description + " - core binary";
		sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];
		platforms = lib.attrNames common.core;
		};
		}

pkgs/tools/security/semgrep/update.sh

+35 −25

Original line number	Diff line number	Diff line
		#!/usr/bin/env nix-shell
		#!nix-shell -i bash -p curl gnused jq
		#!nix-shell -i bash -p curl gnused jq nix-prefetch

		set -euxo pipefail

		@@ -33,7 +33,7 @@ NEW_VERSION=$(
		)
		# trim v prefix
		NEW_VERSION="${NEW_VERSION:1}"
		OLD_VERSION="$(instantiateClean semgrep.common.version)"
		OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)"

		if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
		echo "Already up to date"
		@@ -50,43 +50,54 @@ fetchgithub() {
		set -eo pipefail
		}

		fetchzip() {
		set +eo pipefail
		nix-build -E "with import $NIXPKGS_ROOT {}; fetchzip {url = \"$1\"; sha256 = lib.fakeSha256; }" 2>&1 >/dev/null \| grep "got:" \| cut -d':' -f2 \| sed 's\| \|\|g'
		set -eo pipefail
		fetch_arch() {
		VERSION=$1
		PLATFORM=$2
		nix-prefetch "{ fetchPypi }:
		fetchPypi rec {
		pname = \"semgrep\";
		version = \"$VERSION\";
		format = \"wheel\";
		dist = python;
		python = \"cp37.cp38.cp39.py37.py38.py39\";
		platform = \"$PLATFORM\";
		}
		"
		}

		replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE"

		echo "Updating src"

		OLD_HASH="$(instantiateClean semgrep.common.src.outputHash)"
		OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)"
		echo "Old hash $OLD_HASH"
		TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
		replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
		NEW_HASH="$(fetchgithub semgrep.common.src)"
		NEW_HASH="$(fetchgithub semgrep.src)"
		echo "New hash $NEW_HASH"
		replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"

		echo "Updated src"

		# loop through platforms for core
		nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.common.core.data" --eval --strict --json \
		\| jq '.[]' -r \
		\| while read -r PLATFORM; do
		echo "Updating core for $PLATFORM"
		SUFFIX=$(instantiateClean semgrep.common.core.data."$PLATFORM".suffix)
		OLD_HASH=$(instantiateClean semgrep.common.core.data."$PLATFORM".sha256)
		echo "Old hash $OLD_HASH"

		NEW_URL="https://github.com/returntocorp/semgrep/releases/download/v$NEW_VERSION/semgrep-v$NEW_VERSION$SUFFIX"
		NEW_HASH="$(fetchzip "$NEW_URL")"
		echo "New hash $NEW_HASH"
		update_core_platform() {
		SYSTEM=$1
		echo "Updating core src $SYSTEM"

		PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")"

		OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")"
		echo "Old core hash $OLD_HASH"
		NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")"
		echo "New core hash $NEW_HASH"
		replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE"

		echo "Updated core for $PLATFORM"
		done
		echo "Updated core src $SYSTEM"
		}

		update_core_platform "x86_64-linux"
		update_core_platform "x86_64-darwin"
		update_core_platform "aarch64-darwin"

		OLD_PWD=$PWD
		TMPDIR="$(mktemp -d)"
		@@ -109,7 +120,7 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
		echo "Updating $SUBMODULE"
		OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev)
		echo "Old commit $OLD_REV"
		OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".outputHash)
		OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash)
		echo "Old hash $OLD_HASH"

		NEW_REV=$(get_submodule_commit "$SUBMODULE")
		@@ -120,13 +131,12 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
		continue
		fi

		NEW_URL=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".url \| sed "s@$OLD_REV@$NEW_REV@g")
		NEW_HASH=$(nix --experimental-features nix-command hash to-sri "sha256:$(nix-prefetch-url "$NEW_URL")")
		NEW_URL=$(instantiateClean semgrep.passthru.submodulesSubset."$SUBMODULE".url \| sed "s@$OLD_REV@$NEW_REV@g")

		TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
		replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE"
		replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
		NEW_HASH="$(fetchgithub semgrep.passthru.common.submodules."$SUBMODULE")"
		NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")"
		echo "New hash $NEW_HASH"
		replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"

Admin message