Unverified Commit 7541ec60 authored by annalee's avatar annalee
Browse files

Merge remote-tracking branch 'upstream/master' into staging-next

parents 90e2c2cd 88e7ad7c

lib/systems/default.nix

+54 −20
Original line number Diff line number Diff line 
{ lib }:

  let inherit (lib.attrsets) mapAttrs; in



rec {

let

  inherit (lib)

    any

    filterAttrs

    foldl

    hasInfix

    isFunction

    isList

    isString

    mapAttrs

    optional

    optionalAttrs

    optionalString

    removeSuffix

    replaceStrings

    toUpper

    ;



  inherit (lib.strings) toJSON;



  doubles = import ./doubles.nix { inherit lib; };

  parse = import ./parse.nix { inherit lib; };

  inspect = import ./inspect.nix { inherit lib; };
@@ -24,7 +42,7 @@ rec { 
    both arguments have been `elaborate`-d.

  */

  equals =

    let removeFunctions = a: lib.filterAttrs (_: v: !builtins.isFunction v) a;

    let removeFunctions = a: filterAttrs (_: v: !isFunction v) a;

    in a: b: removeFunctions a == removeFunctions b;



  /* List of all Nix system doubles the nixpkgs flake will expose the package set
@@ -41,7 +59,7 @@ rec { 
  # clearly preferred, and to prevent cycles. A simpler fixed point where the RHS

  # always just used `final.*` would fail on both counts.

  elaborate = args': let

    args = if lib.isString args' then { system = args'; }

    args = if isString args' then { system = args'; }

           else args';



    # TODO: deprecate args.rustc in favour of args.rust after 23.05 is EOL.
@@ -96,7 +114,7 @@ rec { 
        then "lib64"

        else "lib"

      else null;

      extensions = lib.optionalAttrs final.hasSharedLibraries {

      extensions = optionalAttrs final.hasSharedLibraries {

        sharedLibrary =

          if      final.isDarwin  then ".dylib"

          else if final.isWindows then ".dll"
@@ -134,9 +152,9 @@ rec { 
         # uname -m

         processor =

           if final.isPower64

           then "ppc64${lib.optionalString final.isLittleEndian "le"}"

           then "ppc64${optionalString final.isLittleEndian "le"}"

           else if final.isPower

           then "ppc${lib.optionalString final.isLittleEndian "le"}"

           then "ppc${optionalString final.isLittleEndian "le"}"

           else if final.isMips64

           then "mips64"  # endianness is *not* included on mips64

           else final.parsed.cpu.name;
@@ -202,8 +220,8 @@ rec { 
        else if final.isS390 && !final.isS390x then null

        else if final.isx86_64 then "x86_64"

        else if final.isx86 then "i386"

        else if final.isMips64n32 then "mipsn32${lib.optionalString final.isLittleEndian "el"}"

        else if final.isMips64 then "mips64${lib.optionalString final.isLittleEndian "el"}"

        else if final.isMips64n32 then "mipsn32${optionalString final.isLittleEndian "el"}"

        else if final.isMips64 then "mips64${optionalString final.isLittleEndian "el"}"

        else final.uname.processor;



      # Name used by UEFI for architectures.
@@ -259,7 +277,7 @@ rec { 
          if pkgs.stdenv.hostPlatform.canExecute final

          then "${pkgs.runtimeShell} -c '\"$@\"' --"

          else if final.isWindows

          then "${wine}/bin/wine${lib.optionalString (final.parsed.cpu.bits == 64) "64"}"

          then "${wine}/bin/wine${optionalString (final.parsed.cpu.bits == 64) "64"}"

          else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux && final.qemuArch != null

          then "${qemu-user}/bin/qemu-${final.qemuArch}"

          else if final.isWasi
@@ -310,10 +328,10 @@ rec { 
                  let

                    f = args.rustc.platform.target-family;

                  in

                    if builtins.isList f then f else [ f ]

                    if isList f then f else [ f ]

                )

              else lib.optional final.isUnix "unix"

                   ++ lib.optional final.isWindows "windows";

              else optional final.isUnix "unix"

                   ++ optional final.isWindows "windows";



            # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor

            vendor = let
@@ -337,13 +355,13 @@ rec { 
            vendor_ = final.rust.platform.vendor;

          # TODO: deprecate args.rustc in favour of args.rust after 23.05 is EOL.

          in args.rust.rustcTarget or args.rustc.config

            or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}";

            or "${cpu_}-${vendor_}-${kernel.name}${optionalString (abi.name != "unknown") "-${abi.name}"}";



          # The name of the rust target if it is standard, or the json file

          # containing the custom target spec.

          rustcTargetSpec = rust.rustcTargetSpec or (

            /**/ if rust ? platform

            then builtins.toFile (final.rust.rustcTarget + ".json") (builtins.toJSON rust.platform)

            then builtins.toFile (final.rust.rustcTarget + ".json") (toJSON rust.platform)

            else final.rust.rustcTarget);



          # The name of the rust target if it is standard, or the
@@ -352,7 +370,7 @@ rec { 
          #

          # This is the name used by Cargo for target subdirectories.

          cargoShortTarget =

            lib.removeSuffix ".json" (baseNameOf "${final.rust.rustcTargetSpec}");

            removeSuffix ".json" (baseNameOf "${final.rust.rustcTargetSpec}");



          # When used as part of an environment variable name, triples are

          # uppercased and have all hyphens replaced by underscores:
@@ -360,17 +378,17 @@ rec { 
          # https://github.com/rust-lang/cargo/pull/9169

          # https://github.com/rust-lang/cargo/issues/8285#issuecomment-634202431

          cargoEnvVarTarget =

            lib.strings.replaceStrings ["-"] ["_"]

              (lib.strings.toUpper final.rust.cargoShortTarget);

            replaceStrings ["-"] ["_"]

              (toUpper final.rust.cargoShortTarget);



          # True if the target is no_std

          # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421

          isNoStdTarget =

            builtins.any (t: lib.hasInfix t final.rust.rustcTarget) ["-none" "nvptx" "switch" "-uefi"];

            any (t: hasInfix t final.rust.rustcTarget) ["-none" "nvptx" "switch" "-uefi"];

        };

      };

  in assert final.useAndroidPrebuilt -> final.isAndroid;

     assert lib.foldl

     assert foldl

       (pass: { assertion, message }:

         if assertion final

         then pass
@@ -378,4 +396,20 @@ rec { 
       true

       (final.parsed.abi.assertions or []);

    final;



in



# Everything in this attrset is the public interface of the file.

{

  inherit

    architectures

    doubles

    elaborate

    equals

    examples

    flakeExposed

    inspect

    parse

    platforms

    ;

}

lib/systems/inspect.nix

+35 −14
Original line number Diff line number Diff line 
{ lib }:

with import ./parse.nix { inherit lib; };

with lib.attrsets;

with lib.lists;



let abis_ = abis; in

let abis = lib.mapAttrs (_: abi: builtins.removeAttrs abi [ "assertions" ]) abis_; in

let

  inherit (lib)

    any

    attrValues

    concatMap

    filter

    hasPrefix

    isList

    mapAttrs

    matchAttrs

    recursiveUpdateUntil

    toList

    ;



  inherit (lib.strings) toJSON;



  inherit (lib.systems.parse)

    kernels

    kernelFamilies

    significantBytes

    cpuTypes

    execFormats

    ;



  abis = mapAttrs (_: abi: removeAttrs abi [ "assertions" ]) lib.systems.parse.abis;

in



rec {

  # these patterns are to be matched against {host,build,target}Platform.parsed
@@ -32,8 +53,8 @@ rec { 
    isx86          = { cpu = { family = "x86"; }; };

    isAarch32      = { cpu = { family = "arm"; bits = 32; }; };

    isArmv7        = map ({ arch, ... }: { cpu = { inherit arch; }; })

                       (lib.filter (cpu: lib.hasPrefix "armv7" cpu.arch or "")

                         (lib.attrValues cpuTypes));

                       (filter (cpu: hasPrefix "armv7" cpu.arch or "")

                         (attrValues cpuTypes));

    isAarch64      = { cpu = { family = "arm"; bits = 64; }; };

    isAarch        = { cpu = { family = "arm"; }; };

    isMicroBlaze   = { cpu = { family = "microblaze"; }; };
@@ -111,19 +132,19 @@ rec { 
    let

      # patterns can be either a list or a (bare) singleton; turn

      # them into singletons for uniform handling

      pat1 = lib.toList pat1_;

      pat2 = lib.toList pat2_;

      pat1 = toList pat1_;

      pat2 = toList pat2_;

    in

      lib.concatMap (attr1:

      concatMap (attr1:

        map (attr2:

          lib.recursiveUpdateUntil

          recursiveUpdateUntil

            (path: subattr1: subattr2:

              if (builtins.intersectAttrs subattr1 subattr2) == {} || subattr1 == subattr2

              then true

              else throw ''

                pattern conflict at path ${toString path}:

                  ${builtins.toJSON subattr1}

                  ${builtins.toJSON subattr2}

                  ${toJSON subattr1}

                  ${toJSON subattr2}

                '')

            attr1

            attr2
@@ -132,7 +153,7 @@ rec { 
        pat1;



  matchAnyAttrs = patterns:

    if builtins.isList patterns then attrs: any (pattern: matchAttrs pattern attrs) patterns

    if isList patterns then attrs: any (pattern: matchAttrs pattern attrs) patterns

    else matchAttrs patterns;



  predicates = mapAttrs (_: matchAnyAttrs) patterns;

lib/systems/parse.nix

+51 −17
Original line number Diff line number Diff line
@@ -15,14 +15,45 @@ 
# systems that overlap with existing ones and won't notice something amiss.

#

{ lib }:

with lib.lists;

with lib.types;

with lib.attrsets;

with lib.strings;

with (import ./inspect.nix { inherit lib; }).predicates;



let

  inherit (lib.options) mergeOneOption;

  inherit (lib)

    all

    any

    attrValues

    elem

    elemAt

    hasPrefix

    id

    length

    mapAttrs

    mergeOneOption

    optionalString

    splitString

    versionAtLeast

    ;



  inherit (lib.strings) match;



  inherit (lib.systems.inspect.predicates)

    isAarch32

    isBigEndian

    isDarwin

    isLinux

    isPower64

    isWindows

    ;



  inherit (lib.types)

    enum

    float

    isType

    mkOptionType

    number

    setType

    string

    types

    ;



  setTypes = type:

    mapAttrs (name: value:
@@ -33,10 +64,10 @@ let 
  # regex `e?abi.*$` when determining the validity of a triple.  In

  # other words, `i386-linuxabichickenlips` is a valid triple.

  removeAbiSuffix = x:

    let match = builtins.match "(.*)e?abi.*" x;

    in if match==null

    let found = match "(.*)e?abi.*" x;

    in if found == null

       then x

       else lib.elemAt match 0;

       else elemAt found 0;



in
@@ -76,7 +107,7 @@ rec { 


  types.cpuType = enum (attrValues cpuTypes);



  cpuTypes = with significantBytes; setTypes types.openCpuType {

  cpuTypes = let inherit (significantBytes) bigEndian littleEndian; in setTypes types.openCpuType {

    arm      = { bits = 32; significantByte = littleEndian; family = "arm"; };

    armv5tel = { bits = 32; significantByte = littleEndian; family = "arm"; version = "5"; arch = "armv5t"; };

    armv6m   = { bits = 32; significantByte = littleEndian; family = "arm"; version = "6"; arch = "armv6-m"; };
@@ -166,7 +197,7 @@ rec { 
  # Note: Since 22.11 the archs of a mode switching CPU are no longer considered

  # pairwise compatible. Mode switching implies that binaries built for A

  # and B respectively can't be executed at the same time.

  isCompatible = a: b: with cpuTypes; lib.any lib.id [

  isCompatible = with cpuTypes; a: b: any id [

    # x86

    (b == i386 && isCompatible a i486)

    (b == i486 && isCompatible a i586)
@@ -287,7 +318,10 @@ rec { 


  types.kernel = enum (attrValues kernels);



  kernels = with execFormats; with kernelFamilies; setTypes types.openKernel {

  kernels = let

    inherit (execFormats) elf pe wasm unknown macho;

    inherit (kernelFamilies) bsd darwin;

  in setTypes types.openKernel {

    # TODO(@Ericson2314): Don't want to mass-rebuild yet to keeping 'darwin' as

    # the normalized name for macOS.

    macos    = { execFormat = macho;   families = { inherit darwin; }; name = "darwin"; };
@@ -359,7 +393,7 @@ rec { 
            The "gnu" ABI is ambiguous on 32-bit ARM. Use "gnueabi" or "gnueabihf" instead.

          '';

        }

        { assertion = platform: with platform; !(isPower64 && isBigEndian);

        { assertion = platform: !(platform.isPower64 && platform.isBigEndian);

          message = ''

            The "gnu" ABI is ambiguous on big-endian 64-bit PowerPC. Use "gnuabielfv2" or "gnuabielfv1" instead.

          '';
@@ -480,7 +514,7 @@ rec { 
        /**/ if args ? abi       then getAbi args.abi

        else if isLinux parsed || isWindows parsed then

          if isAarch32 parsed then

            if lib.versionAtLeast (parsed.cpu.version or "0") "6"

            if versionAtLeast (parsed.cpu.version or "0") "6"

            then abis.gnueabihf

            else abis.gnueabi

          # Default ppc64 BE to ELFv2
@@ -491,7 +525,7 @@ rec { 


  in mkSystem parsed;



  mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s));

  mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (splitString "-" s));



  kernelName = kernel:

    kernel.name + toString (kernel.version or "");
@@ -503,10 +537,10 @@ rec { 


  tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let

    optExecFormat =

      lib.optionalString (kernel.name == "netbsd" &&

      optionalString (kernel.name == "netbsd" &&

                          gnuNetBSDDefaultExecFormat cpu != kernel.execFormat)

        kernel.execFormat.name;

    optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}";

    optAbi = optionalString (abi != abis.unknown) "-${abi.name}";

  in "${cpu.name}-${vendor.name}-${kernelName kernel}${optExecFormat}${optAbi}";



  ################################################################################

nixos/tests/nix-ld.nix

+37 −15
Original line number Diff line number Diff line 
import ./make-test-python.nix ({ lib, pkgs, ...} :

{ system ? builtins.currentSystem,

  config ? {},

  pkgs ? import ../.. { inherit system config; }

}:

let

  inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;

  shared =

    { config, pkgs, ... }:

    {

  name = "nix-ld";

  nodes.machine = { pkgs, ... }: {

      programs.nix-ld.enable = true;

      environment.systemPackages = [

        (pkgs.runCommand "patched-hello" { } ''

          install -D -m755 ${pkgs.hello}/bin/hello $out/bin/hello

        patchelf $out/bin/hello --set-interpreter $(cat ${pkgs.nix-ld}/nix-support/ldpath)

          patchelf $out/bin/hello --set-interpreter $(cat ${config.programs.nix-ld.package}/nix-support/ldpath)

        '')

      ];

    };

in

{

  nix-ld = makeTest {

    name = "nix-ld";

    nodes.machine = shared;

    testScript = ''

      start_all()

      machine.succeed("hello")

    '';

  };

  nix-ld-rs = makeTest {

    name = "nix-ld-rs";

    nodes.machine = {

      imports = [ shared ];

      programs.nix-ld.package = pkgs.nix-ld-rs;

    };

    testScript = ''

      start_all()

      machine.succeed("hello")

    '';

})
 
  };

}

nixos/tests/tracee.nix

+45 −48
Original line number Diff line number Diff line 
import ./make-test-python.nix ({ pkgs, ... }: {

import ./make-test-python.nix ({ pkgs, ... }: rec {

  name = "tracee-integration";

  meta.maintainers = pkgs.tracee.meta.maintainers;



  passthru.hello-world-builder = pkgs: pkgs.dockerTools.buildImage {

    name = "hello-world";

    tag = "latest";

    config.Cmd = [ "${pkgs.hello}/bin/hello" ];

  };



  nodes = {

    machine = { config, pkgs, ... }: {

      # EventFilters/trace_only_events_from_new_containers and
@@ -12,57 +18,48 @@ import ./make-test-python.nix ({ pkgs, ... }: { 
      environment.systemPackages = with pkgs; [

        # required by Test_EventFilters/trace_events_from_ls_and_which_binary_in_separate_scopes

        which

        # build the go integration tests as a binary

        (tracee.overrideAttrs (oa: {

          pname = oa.pname + "-integration";

          postPatch = oa.postPatch or "" + ''

            # prepare tester.sh (which will be embedded in the test binary)

            patchShebangs tests/integration/tester.sh



            # fix the test to look at nixos paths for running programs

            substituteInPlace tests/integration/integration_test.go \

              --replace "bin=/usr/bin/" "comm=" \

              --replace "binary=/usr/bin/" "comm=" \

              --replace "/usr/bin/dockerd" "dockerd" \

              --replace "/usr/bin" "/run/current-system/sw/bin"

          '';

          nativeBuildInputs = oa.nativeBuildInputs or [ ] ++ [ makeWrapper ];

          buildPhase = ''

            runHook preBuild

            # just build the static lib we need for the go test binary

            make $makeFlags ''${enableParallelBuilding:+-j$NIX_BUILD_CORES} bpf-core ./dist/btfhub



            # then compile the tests to be ran later

            CGO_LDFLAGS="$(pkg-config --libs libbpf)" go test -tags core,ebpf,integration -p 1 -c -o $GOPATH/tracee-integration ./tests/integration/...

            runHook postBuild

          '';

          doCheck = false;

          outputs = [ "out" ];

          installPhase = ''

            mkdir -p $out/bin

            mv $GOPATH/tracee-integration $out/bin/

          '';

          doInstallCheck = false;



          meta = oa.meta // {

            outputsToInstall = [];

          };

        }))

        # the go integration tests as a binary

        tracee.passthru.tests.integration-test-cli

      ];

    };

  };



  testScript = ''

  testScript =

    let

      skippedTests = [

        # these comm tests for some reason do not resolve.

        # something about the test is different as it works fine if I replicate

        # the policies and run tracee myself but doesn't work in the integration

        # test either with the automatic run or running the commands by hand

        # while it's searching.

        "Test_EventFilters/comm:_event:_args:_trace_event_set_in_a_specific_policy_with_args_from_ls_command"

        "Test_EventFilters/comm:_event:_trace_events_set_in_two_specific_policies_from_ls_and_uname_commands"



        # worked at some point, seems to be flakey

        "Test_EventFilters/pid:_event:_args:_trace_event_sched_switch_with_args_from_pid_0"

      ];

    in

    ''

      with subtest("prepare for integration tests"):

        machine.wait_for_unit("docker.service")

        machine.succeed('which bash')



    with subtest("run integration tests"):

      # EventFilters/trace_only_events_from_new_containers also requires a container called "alpine"

      machine.succeed('tar c -C ${pkgs.pkgsStatic.busybox} . | docker import - alpine --change "ENTRYPOINT [\"sleep\"]"')

        # EventFilters/trace_only_events_from_new_containers also requires a container called "hello-world"

        machine.succeed('docker load < ${passthru.hello-world-builder pkgs}')



        # exec= needs fully resolved paths

        machine.succeed(

          'mkdir /tmp/testdir',

          'cp $(which who) /tmp/testdir/who',

          'cp $(which uname) /tmp/testdir/uname',

        )



      with subtest("run integration tests"):

        # Test_EventFilters/trace_event_set_in_a_specific_scope expects to be in a dir that includes "integration"

        # tests must be ran with 1 process

        print(machine.succeed(

          'mkdir /tmp/integration',

        'cd /tmp/integration && tracee-integration -test.v'

          'cd /tmp/integration && export PATH="/tmp/testdir:$PATH" && integration.test -test.v -test.parallel 1 -test.skip="^${builtins.concatStringsSep "$|^" skippedTests}$"'

        ))

    '';

})