Commit 49e6e3b3 authored by Joachim Jenke's avatar Joachim Jenke
Browse files

[TSan] Add instrumentation of AVX2 and AVX512 instructions 

Currently, ThreadSanitizer only instruments memory accesses up to a width of
128 bit and explicitly skips instrumentation of wider memory accesses. This
means that TSan is blind for AVX2 and AVX512 memory instructions.

This patch adds instrumentation and runtime support for 256bit and 512bit memory
loads/stores. Additionally, vector gather/scatter instructions are considered for
instrumentation. These instructions allow to gather individual data elements from
memory into a single vector register and scatter the elements from a vector
register into individual memory locations.
Since the vector of addresses is passed as a 256bit / 512bit vector, the new
interface functions are compiled separately with the specific compiler flags.
This avoids that AVX instructions are introduced into other parts of the runtime.
Since the new interface is only called on architectures that actually support AVX
instructions, this separation maintains the portability of the runtime.

Some of the tests use #pragma omp simd as a portable way to generate vector
instructions across architectures. The construct is independent of the OpenMP
runtime. Therefore the tests used base-language threading.
Some of the tests directly call into the new runtime functions, since we found
no way to actually generate scatter/gather instructions with masks different
from 0xFF.

Under review as #74636
parent 4a11bb40

compiler-rt/cmake/config-ix.cmake

+2 −0
Original line number Diff line number Diff line
@@ -89,6 +89,8 @@ check_cxx_compiler_flag(-fno-profile-instr-use COMPILER_RT_HAS_FNO_PROFILE_INSTR 
check_cxx_compiler_flag(-fno-coverage-mapping COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)

check_cxx_compiler_flag("-Werror -mcrc32"    COMPILER_RT_HAS_MCRC32_FLAG)

check_cxx_compiler_flag("-Werror -msse4.2"   COMPILER_RT_HAS_MSSE4_2_FLAG)

check_cxx_compiler_flag("-Werror -mavx2"   COMPILER_RT_HAS_MAVX2_FLAG)

check_cxx_compiler_flag("-Werror -mavx512f"   COMPILER_RT_HAS_MAVX512F_FLAG)

check_cxx_compiler_flag(--sysroot=.          COMPILER_RT_HAS_SYSROOT_FLAG)

check_cxx_compiler_flag("-Werror -mcrc"      COMPILER_RT_HAS_MCRC_FLAG)

check_cxx_compiler_flag(-fno-partial-inlining COMPILER_RT_HAS_FNO_PARTIAL_INLINING_FLAG)

compiler-rt/lib/tsan/rtl/CMakeLists.txt

+15 −0
Original line number Diff line number Diff line
@@ -237,6 +237,17 @@ else() 
    else()

      set(TSAN_ASM_SOURCES)

    endif()

    add_compiler_rt_object_libraries(RTTSanAVX2

        ARCHS ${arch}

        SOURCES tsan_interface_avx2.cpp

        ADDITIONAL_HEADERS tsan_interface_avx2.h 

        #CFLAGS ${TSAN_RTL_CFLAGS} $<IF:"$COMPILER_RT_HAS_MAVX2_FLAG","-mavx2","">)

        CFLAGS ${TSAN_RTL_CFLAGS} $<IF:$<BOOL:${COMPILER_RT_HAS_MAVX2_FLAG}>,-mavx2,"">)

    add_compiler_rt_object_libraries(RTTSanAVX512

        ARCHS ${arch}

        SOURCES tsan_interface_avx512.cpp

        ADDITIONAL_HEADERS tsan_interface_avx512.h 

        CFLAGS ${TSAN_RTL_CFLAGS} $<IF:$<BOOL:${COMPILER_RT_HAS_MAVX512F_FLAG}>,-mavx512f,"">)

    add_compiler_rt_runtime(clang_rt.tsan

      STATIC

      ARCHS ${arch}
@@ -247,6 +258,8 @@ else() 
              $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>

              $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>

              $<TARGET_OBJECTS:RTUbsan.${arch}>

              $<TARGET_OBJECTS:RTTSanAVX2.${arch}>

              $<TARGET_OBJECTS:RTTSanAVX512.${arch}>

      ADDITIONAL_HEADERS ${TSAN_HEADERS}

      CFLAGS ${TSAN_RTL_CFLAGS}

      PARENT_TARGET tsan)
@@ -270,6 +283,8 @@ else() 
              $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>

              $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>

              $<TARGET_OBJECTS:RTUbsan.${arch}>

              $<TARGET_OBJECTS:RTTSanAVX2.${arch}>

              $<TARGET_OBJECTS:RTTSanAVX512.${arch}>

      ADDITIONAL_HEADERS ${TSAN_HEADERS}

      CFLAGS ${TSAN_RTL_DYNAMIC_CFLAGS}

      DEFS SANITIZER_SHARED

compiler-rt/lib/tsan/rtl/tsan_interface.cpp

+31 −6
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include "tsan_interface.h"

#include "tsan_interface_ann.h"

#include "tsan_rtl.h"



#include "sanitizer_common/sanitizer_internal_defs.h"

#include "sanitizer_common/sanitizer_ptrauth.h"
@@ -42,18 +43,42 @@ void __tsan_write16_pc(void *addr, void *pc) { 


// __tsan_unaligned_read/write calls are emitted by compiler.



void __tsan_unaligned_read16(const void *addr) {

template <unsigned int N>

void __tsan_unaligned_readx(const void *addr) {

  uptr pc = CALLERPC;

  ThreadState *thr = cur_thread();

  UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);

  UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);

  for (unsigned int i = 0; i < N / 8; i++)

    UnalignedMemoryAccess(thr, pc, (uptr)addr + (i * 8), 8, kAccessRead);

}



void __tsan_unaligned_write16(void *addr) {

template <unsigned int N>

void __tsan_unaligned_writex(void *addr) {

  uptr pc = CALLERPC;

  ThreadState *thr = cur_thread();

  UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);

  UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);

  for (unsigned int i = 0; i < N / 8; i++)

    UnalignedMemoryAccess(thr, pc, (uptr)addr + (i * 8), 8, kAccessWrite);

}



void __tsan_unaligned_read16(const void *addr) {

  __tsan_unaligned_readx<16>(addr);

}



void __tsan_unaligned_write16(void *addr) { __tsan_unaligned_writex<16>(addr); }



extern "C" void __tsan_unaligned_read32(const void *addr) {

  __tsan_unaligned_readx<32>(addr);

}



extern "C" void __tsan_unaligned_write32(void *addr) {

  __tsan_unaligned_writex<32>(addr);

}



extern "C" void __tsan_unaligned_read64(const void *addr) {

  __tsan_unaligned_readx<64>(addr);

}



extern "C" void __tsan_unaligned_write64(void *addr) {

  __tsan_unaligned_writex<64>(addr);

}



extern "C" {

compiler-rt/lib/tsan/rtl/tsan_interface.h

+4 −0
Original line number Diff line number Diff line
@@ -53,11 +53,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr); 
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read32(const void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read64(const void *addr);



SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write32(void *addr);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write64(void *addr);



SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1_pc(void *addr, void *pc);

SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2_pc(void *addr, void *pc);

compiler-rt/lib/tsan/rtl/tsan_interface.inc

+27 −0
Original line number Diff line number Diff line
@@ -38,6 +38,18 @@ void __tsan_read16(void *addr) { 
  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);

}



extern "C" void __tsan_read32(void *addr) {

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessRead);

}



extern "C" void __tsan_read64(void *addr) {

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessRead);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 32, kAccessRead);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 48, kAccessRead);

}



void __tsan_write1(void *addr) {

  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);

}
@@ -58,6 +70,21 @@ void __tsan_write16(void *addr) { 
  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);

}



extern "C" void __tsan_write32(void *addr) {

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessWrite);

}



extern "C" void __tsan_write64(void *addr) {

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessWrite);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 32, kAccessWrite);

  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 48, kAccessWrite);

}



// Our vector instructions

// TODO



void __tsan_read1_pc(void *addr, void *pc) {

  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);

}