Commit d38d6ca1 authored by William Huang's avatar William Huang
Browse files

[llvm-profdata] Deprecate Compact Binary Sample Profile Format 

Remove support for compact binary sample profile format

Reviewed By: davidxl, wenlei

Differential Revision: https://reviews.llvm.org/D149400
parent 9b4faa11

llvm/include/llvm/ProfileData/SampleProf.h

+1 −1
Original line number Diff line number Diff line
@@ -90,7 +90,7 @@ namespace sampleprof { 
enum SampleProfileFormat {

  SPF_None = 0,

  SPF_Text = 0x1,

  SPF_Compact_Binary = 0x2,

  SPF_Compact_Binary = 0x2, // Deprecated

  SPF_GCC = 0x3,

  SPF_Ext_Binary = 0x4,

  SPF_Binary = 0xff

llvm/include/llvm/ProfileData/SampleProfReader.h

+0 −35
Original line number Diff line number Diff line
@@ -811,41 +811,6 @@ public: 
  static bool hasFormat(const MemoryBuffer &Buffer);

};



class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary {

private:

  /// Function name table.

  std::vector<std::string> NameTable;

  /// The table mapping from function name to the offset of its FunctionSample

  /// towards file start.

  DenseMap<StringRef, uint64_t> FuncOffsetTable;

  /// The set containing the functions to use when compiling a module.

  DenseSet<StringRef> FuncsToUse;

  std::error_code verifySPMagic(uint64_t Magic) override;

  std::error_code readNameTable() override;

  /// Read a string indirectly via the name table.

  ErrorOr<StringRef> readStringFromTable() override;

  std::error_code readHeader() override;

  std::error_code readFuncOffsetTable();



public:

  SampleProfileReaderCompactBinary(std::unique_ptr<MemoryBuffer> B,

                                   LLVMContext &C)

      : SampleProfileReaderBinary(std::move(B), C, SPF_Compact_Binary) {}



  /// \brief Return true if \p Buffer is in the format supported by this class.

  static bool hasFormat(const MemoryBuffer &Buffer);



  /// Read samples only for functions to use.

  std::error_code readImpl() override;



  /// Collect functions with definitions in Module M. Return true if

  /// the reader has been given a module.

  bool collectFuncsFromModule() override;



  /// Return whether names in the profile are all MD5 numbers.

  bool useMD5() override { return true; }

};



using InlineCallStack = SmallVector<FunctionSamples *, 10>;



// Supported histogram types in GCC.  Currently, we only need support for

llvm/include/llvm/ProfileData/SampleProfWriter.h

+0 −49
Original line number Diff line number Diff line
@@ -414,55 +414,6 @@ private: 
  }

};



// CompactBinary is a compact format of binary profile which both reduces

// the profile size and the load time needed when compiling. It has two

// major difference with Binary format.

// 1. It represents all the strings in name table using md5 hash.

// 2. It saves a function offset table which maps function name index to

// the offset of its function profile to the start of the binary profile,

// so by using the function offset table, for those function profiles which

// will not be needed when compiling a module, the profile reader does't

// have to read them and it saves compile time if the profile size is huge.

// The layout of the compact format is shown as follows:

//

//    Part1: Profile header, the same as binary format, containing magic

//           number, version, summary, name table...

//    Part2: Function Offset Table Offset, which saves the position of

//           Part4.

//    Part3: Function profile collection

//             function1 profile start

//                 ....

//             function2 profile start

//                 ....

//             function3 profile start

//                 ....

//                ......

//    Part4: Function Offset Table

//             function1 name index --> function1 profile start

//             function2 name index --> function2 profile start

//             function3 name index --> function3 profile start

//

// We need Part2 because profile reader can use it to find out and read

// function offset table without reading Part3 first.

class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary {

  using SampleProfileWriterBinary::SampleProfileWriterBinary;



public:

  std::error_code writeSample(const FunctionSamples &S) override;

  std::error_code write(const SampleProfileMap &ProfileMap) override;



protected:

  /// The table mapping from function name to the offset of its FunctionSample

  /// towards profile start.

  MapVector<StringRef, uint64_t> FuncOffsetTable;

  /// The offset of the slot to be filled with the offset of FuncOffsetTable

  /// towards profile start.

  uint64_t TableOffset;

  std::error_code writeNameTable() override;

  std::error_code writeHeader(const SampleProfileMap &ProfileMap) override;

  std::error_code writeFuncOffsetTable();

};



} // end namespace sampleprof

} // end namespace llvm

llvm/lib/ProfileData/SampleProfReader.cpp

+2 −122
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ using namespace sampleprof; 
#define DEBUG_TYPE "samplepgo-reader"



// This internal option specifies if the profile uses FS discriminators.

// It only applies to text, binary and compact binary format profiles.

// It only applies to text, and binary format profiles.

// For ext-binary format profiles, the flag is set in the summary.

static cl::opt<bool> ProfileIsFSDisciminator(

    "profile-isfs", cl::Hidden, cl::init(false),
@@ -568,14 +568,6 @@ ErrorOr<StringRef> SampleProfileReaderExtBinaryBase::readStringFromTable() { 
  return SR;

}



ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() {

  auto Idx = readStringIndex(NameTable);

  if (std::error_code EC = Idx.getError())

    return EC;



  return StringRef(NameTable[*Idx]);

}



std::error_code

SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {

  auto NumSamples = readNumber<uint64_t>();
@@ -1012,40 +1004,6 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() { 
  return sampleprof_error::success;

}



std::error_code SampleProfileReaderCompactBinary::readImpl() {

  // Collect functions used by current module if the Reader has been

  // given a module.

  bool LoadFuncsToBeUsed = collectFuncsFromModule();

  ProfileIsFS = ProfileIsFSDisciminator;

  FunctionSamples::ProfileIsFS = ProfileIsFS;

  std::vector<uint64_t> OffsetsToUse;

  if (!LoadFuncsToBeUsed) {

    // load all the function profiles.

    for (auto FuncEntry : FuncOffsetTable) {

      OffsetsToUse.push_back(FuncEntry.second);

    }

  } else {

    // load function profiles on demand.

    for (auto Name : FuncsToUse) {

      auto GUID = std::to_string(MD5Hash(Name));

      auto iter = FuncOffsetTable.find(StringRef(GUID));

      if (iter == FuncOffsetTable.end())

        continue;

      OffsetsToUse.push_back(iter->second);

    }

  }



  for (auto Offset : OffsetsToUse) {

    const uint8_t *SavedData = Data;

    if (std::error_code EC = readFuncProfile(

            reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +

            Offset))

      return EC;

    Data = SavedData;

  }

  return sampleprof_error::success;

}



std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {

  if (Magic == SPMagic())

    return sampleprof_error::success;
@@ -1058,13 +1016,6 @@ std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { 
  return sampleprof_error::bad_magic;

}



std::error_code

SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {

  if (Magic == SPMagic(SPF_Compact_Binary))

    return sampleprof_error::success;

  return sampleprof_error::bad_magic;

}



std::error_code SampleProfileReaderBinary::readNameTable() {

  auto Size = readNumber<size_t>();

  if (std::error_code EC = Size.getError())
@@ -1234,20 +1185,6 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { 
  return sampleprof_error::success;

}



std::error_code SampleProfileReaderCompactBinary::readNameTable() {

  auto Size = readNumber<uint64_t>();

  if (std::error_code EC = Size.getError())

    return EC;

  NameTable.reserve(*Size);

  for (uint64_t I = 0; I < *Size; ++I) {

    auto FID = readNumber<uint64_t>();

    if (std::error_code EC = FID.getError())

      return EC;

    NameTable.push_back(std::to_string(*FID));

  }

  return sampleprof_error::success;

}



std::error_code

SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {

  SecHdrTableEntry Entry;
@@ -1427,54 +1364,6 @@ std::error_code SampleProfileReaderBinary::readHeader() { 
  return sampleprof_error::success;

}



std::error_code SampleProfileReaderCompactBinary::readHeader() {

  SampleProfileReaderBinary::readHeader();

  if (std::error_code EC = readFuncOffsetTable())

    return EC;

  return sampleprof_error::success;

}



std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {

  auto TableOffset = readUnencodedNumber<uint64_t>();

  if (std::error_code EC = TableOffset.getError())

    return EC;



  const uint8_t *SavedData = Data;

  const uint8_t *TableStart =

      reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +

      *TableOffset;

  Data = TableStart;



  auto Size = readNumber<uint64_t>();

  if (std::error_code EC = Size.getError())

    return EC;



  FuncOffsetTable.reserve(*Size);

  for (uint64_t I = 0; I < *Size; ++I) {

    auto FName(readStringFromTable());

    if (std::error_code EC = FName.getError())

      return EC;



    auto Offset = readNumber<uint64_t>();

    if (std::error_code EC = Offset.getError())

      return EC;



    FuncOffsetTable[*FName] = *Offset;

  }

  End = TableStart;

  Data = SavedData;

  return sampleprof_error::success;

}



bool SampleProfileReaderCompactBinary::collectFuncsFromModule() {

  if (!M)

    return false;

  FuncsToUse.clear();

  for (auto &F : *M)

    FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));

  return true;

}



std::error_code SampleProfileReaderBinary::readSummaryEntry(

    std::vector<ProfileSummaryEntry> &Entries) {

  auto Cutoff = readNumber<uint64_t>();
@@ -1545,13 +1434,6 @@ bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) { 
  return Magic == SPMagic(SPF_Ext_Binary);

}



bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) {

  const uint8_t *Data =

      reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());

  uint64_t Magic = decodeULEB128(Data);

  return Magic == SPMagic(SPF_Compact_Binary);

}



std::error_code SampleProfileReaderGCC::skipNextWord() {

  uint32_t dummy;

  if (!GcovBuffer.readInt(dummy))
@@ -1803,7 +1685,7 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { 
    Ctx.diagnose(DiagnosticInfoSampleProfile(

        Reader.getBuffer()->getBufferIdentifier(),

        "Profile data remapping cannot be applied to profile data "

        "in compact format (original mangled names are not available).",

        "using MD5 names (original mangled names are not available).",

        DS_Warning));

    return;

  }
@@ -1934,8 +1816,6 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, 
    Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));

  else if (SampleProfileReaderExtBinary::hasFormat(*B))

    Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));

  else if (SampleProfileReaderCompactBinary::hasFormat(*B))

    Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C));

  else if (SampleProfileReaderGCC::hasFormat(*B))

    Reader.reset(new SampleProfileReaderGCC(std::move(B), C));

  else if (SampleProfileReaderText::hasFormat(*B))

llvm/lib/ProfileData/SampleProfWriter.cpp

+2 −71
Original line number Diff line number Diff line
@@ -557,15 +557,6 @@ std::error_code SampleProfileWriterExtBinary::writeSections( 
  return EC;

}



std::error_code

SampleProfileWriterCompactBinary::write(const SampleProfileMap &ProfileMap) {

  if (std::error_code EC = SampleProfileWriter::write(ProfileMap))

    return EC;

  if (std::error_code EC = writeFuncOffsetTable())

    return EC;

  return sampleprof_error::success;

}



/// Write samples to a text file.

///

/// Note: it may be tempting to implement this in terms of
@@ -712,41 +703,6 @@ std::error_code SampleProfileWriterBinary::writeNameTable() { 
  return sampleprof_error::success;

}



std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {

  auto &OS = *OutputStream;



  // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.

  uint64_t FuncOffsetTableStart = OS.tell();

  support::endian::SeekableWriter Writer(static_cast<raw_pwrite_stream &>(OS),

                                         support::little);

  Writer.pwrite(FuncOffsetTableStart, TableOffset);



  // Write out the table size.

  encodeULEB128(FuncOffsetTable.size(), OS);



  // Write out FuncOffsetTable.

  for (auto Entry : FuncOffsetTable) {

    if (std::error_code EC = writeNameIdx(Entry.first))

      return EC;

    encodeULEB128(Entry.second, OS);

  }

  FuncOffsetTable.clear();

  return sampleprof_error::success;

}



std::error_code SampleProfileWriterCompactBinary::writeNameTable() {

  auto &OS = *OutputStream;

  std::set<StringRef> V;

  stablizeNameTable(NameTable, V);



  // Write out the name table.

  encodeULEB128(NameTable.size(), OS);

  for (auto N : V) {

    encodeULEB128(MD5Hash(N), OS);

  }

  return sampleprof_error::success;

}



std::error_code

SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {

  auto &OS = *OutputStream;
@@ -848,19 +804,6 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader( 
  return sampleprof_error::success;

}



std::error_code SampleProfileWriterCompactBinary::writeHeader(

    const SampleProfileMap &ProfileMap) {

  support::endian::Writer Writer(*OutputStream, support::little);

  if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))

    return EC;



  // Reserve a slot for the offset of function offset table. The slot will

  // be populated with the offset of FuncOffsetTable later.

  TableOffset = OutputStream->tell();

  Writer.write(static_cast<uint64_t>(-2));

  return sampleprof_error::success;

}



std::error_code SampleProfileWriterBinary::writeSummary() {

  auto &OS = *OutputStream;

  encodeULEB128(Summary->getTotalCount(), OS);
@@ -930,15 +873,6 @@ SampleProfileWriterBinary::writeSample(const FunctionSamples &S) { 
  return writeBody(S);

}



std::error_code

SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {

  uint64_t Offset = OutputStream->tell();

  StringRef Name = S.getName();

  FuncOffsetTable[Name] = Offset;

  encodeULEB128(S.getHeadSamples(), *OutputStream);

  return writeBody(S);

}



/// Create a sample profile file writer based on the specified format.

///

/// \param Filename The file to create.
@@ -950,8 +884,7 @@ ErrorOr<std::unique_ptr<SampleProfileWriter>> 
SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {

  std::error_code EC;

  std::unique_ptr<raw_ostream> OS;

  if (Format == SPF_Binary || Format == SPF_Ext_Binary ||

      Format == SPF_Compact_Binary)

  if (Format == SPF_Binary || Format == SPF_Ext_Binary)

    OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));

  else

    OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
@@ -976,15 +909,13 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, 


  // Currently only Text and Extended Binary format are supported for CSSPGO.

  if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&

      (Format == SPF_Binary || Format == SPF_Compact_Binary))

      Format == SPF_Binary)

    return sampleprof_error::unsupported_writing_format;



  if (Format == SPF_Binary)

    Writer.reset(new SampleProfileWriterRawBinary(OS));

  else if (Format == SPF_Ext_Binary)

    Writer.reset(new SampleProfileWriterExtBinary(OS));

  else if (Format == SPF_Compact_Binary)

    Writer.reset(new SampleProfileWriterCompactBinary(OS));

  else if (Format == SPF_Text)

    Writer.reset(new SampleProfileWriterText(OS));

  else if (Format == SPF_GCC)