Commit 993e3c92 authored by Kostya Kortchinsky's avatar Kostya Kortchinsky
Browse files

[scudo][standalone] Secondary & general other improvements 

Summary:
This CL changes multiple things to improve performance (notably on
Android).We introduce a cache class for the Secondary that is taking
care of this mechanism now.

The changes:
- change the Secondary "freelist" to an array. By keeping free secondary
  blocks linked together through their headers, we were keeping a page
  per block, which isn't great. Also we know touch less pages when
  walking the new "freelist".
- fix an issue with the freelist getting full: if the pattern is an ever
  increasing size malloc then free, the freelist would fill up and
  entries would not be used. So now we empty the list if we get to many
  "full" events;
- use the global release to os interval option for the secondary: it
  was too costly to release all the time, particularly for pattern that
  are malloc(X)/free(X)/malloc(X). Now the release will only occur
  after the selected interval, when going through the deallocate path;
- allow release of the `BatchClassId` class: it is releasable, we just
  have to make sure we don't mark the batches containing batches
  pointers as free.
- change the default release interval to 1s for Android to match the
  current Bionic allocator configuration. A patch is coming up to allow
  changing it through `mallopt`.
- lower the smallest class that can be released to `PageSize/64`.

Reviewers: cferris, pcc, eugenis, morehouse, hctim

Subscribers: phosek, #sanitizers, llvm-commits

Tags: #sanitizers, #llvm

Differential Revision: https://reviews.llvm.org/D73507
parent 731b140a

compiler-rt/lib/scudo/standalone/allocator_config.h

+4 −4
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ struct DefaultConfig { 
  // 512KB regions

  typedef SizeClassAllocator32<SizeClassMap, 19U> Primary;

#endif

  typedef MapAllocator<> Secondary;

  typedef MapAllocator<MapAllocatorCache<>> Secondary;

  template <class A> using TSDRegistryT = TSDRegistryExT<A>; // Exclusive

};
@@ -47,7 +47,7 @@ struct AndroidConfig { 
  // 512KB regions

  typedef SizeClassAllocator32<SizeClassMap, 19U> Primary;

#endif

  typedef MapAllocator<> Secondary;

  typedef MapAllocator<MapAllocatorCache<>> Secondary;

  template <class A>

  using TSDRegistryT = TSDRegistrySharedT<A, 2U>; // Shared, max 2 TSDs.

};
@@ -61,7 +61,7 @@ struct AndroidSvelteConfig { 
  // 64KB regions

  typedef SizeClassAllocator32<SizeClassMap, 16U> Primary;

#endif

  typedef MapAllocator<0U> Secondary;

  typedef MapAllocator<MapAllocatorCache<4U, 1UL << 18>> Secondary;

  template <class A>

  using TSDRegistryT = TSDRegistrySharedT<A, 1U>; // Shared, only 1 TSD.

};
@@ -70,7 +70,7 @@ struct AndroidSvelteConfig { 
struct FuchsiaConfig {

  // 1GB Regions

  typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary;

  typedef MapAllocator<0U> Secondary;

  typedef MapAllocator<MapAllocatorNoCache> Secondary;

  template <class A>

  using TSDRegistryT = TSDRegistrySharedT<A, 8U>; // Shared, max 8 TSDs.

};

compiler-rt/lib/scudo/standalone/combined.h

+3 −2
Original line number Diff line number Diff line
@@ -141,8 +141,9 @@ public: 
        static_cast<u32>(getFlags()->quarantine_max_chunk_size);



    Stats.initLinkerInitialized();

    Primary.initLinkerInitialized(getFlags()->release_to_os_interval_ms);

    Secondary.initLinkerInitialized(&Stats);

    const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;

    Primary.initLinkerInitialized(ReleaseToOsIntervalMs);

    Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs);



    Quarantine.init(

        static_cast<uptr>(getFlags()->quarantine_size_kb << 10),

compiler-rt/lib/scudo/standalone/flags.inc

+1 −1
Original line number Diff line number Diff line
@@ -45,6 +45,6 @@ SCUDO_FLAG(bool, may_return_null, true, 
           "returning NULL in otherwise non-fatal error scenarios, eg: OOM, "

           "invalid allocation alignments, etc.")



SCUDO_FLAG(int, release_to_os_interval_ms, 5000,

SCUDO_FLAG(int, release_to_os_interval_ms, SCUDO_ANDROID ? 1000 : 5000,

           "Interval (in milliseconds) at which to attempt release of unused "

           "memory to the OS. Negative values disable the feature.")

compiler-rt/lib/scudo/standalone/primary32.h

+2 −3
Original line number Diff line number Diff line
@@ -74,8 +74,7 @@ public: 
      Sci->RandState = getRandomU32(&Seed);

      // See comment in the 64-bit primary about releasing smaller size classes.

      Sci->CanRelease = (ReleaseToOsInterval >= 0) &&

                        (I != SizeClassMap::BatchClassId) &&

                        (getSizeByClassId(I) >= (PageSize / 32));

                        (getSizeByClassId(I) >= (PageSize / 64));

    }

    ReleaseToOsIntervalMs = ReleaseToOsInterval;

  }
@@ -385,7 +384,7 @@ private: 
      if (IntervalMs < 0)

        return 0;

      if (Sci->ReleaseInfo.LastReleaseAtNs +

              static_cast<uptr>(IntervalMs) * 1000000ULL >

              static_cast<u64>(IntervalMs) * 1000000 >

          getMonotonicTime()) {

        return 0; // Memory was returned recently.

      }

compiler-rt/lib/scudo/standalone/primary64.h

+2 −3
Original line number Diff line number Diff line
@@ -87,8 +87,7 @@ public: 
      // limit is mostly arbitrary and based on empirical observations.

      // TODO(kostyak): make the lower limit a runtime option

      Region->CanRelease = (ReleaseToOsInterval >= 0) &&

                           (I != SizeClassMap::BatchClassId) &&

                           (getSizeByClassId(I) >= (PageSize / 32));

                           (getSizeByClassId(I) >= (PageSize / 64));

      Region->RandState = getRandomU32(&Seed);

    }

    ReleaseToOsIntervalMs = ReleaseToOsInterval;
@@ -401,7 +400,7 @@ private: 
      if (IntervalMs < 0)

        return 0;

      if (Region->ReleaseInfo.LastReleaseAtNs +

              static_cast<uptr>(IntervalMs) * 1000000ULL >

              static_cast<u64>(IntervalMs) * 1000000 >

          getMonotonicTime()) {

        return 0; // Memory was returned recently.

      }