[SampleFDO] Add flag for partial profile.

The common profile usage is to collect profile from a target and then use the profile to guide the optimized build for the same target. There are some cases that no profile can be collected for a target. In those cases, although no full profile is available, it is possible to have some partial profile collected from other targets to optimize common libraries and utilities. A flag is needed to tell the partial profile from the full profile apart, so compiler can use different strategy for them.

Differential Revision: https://reviews.llvm.org/D77426

Author: wmi-11

llvm/include/llvm/IR/ProfileSummary.h

@@ -49,6 +49,12 @@ class ProfileSummary {
   SummaryEntryVector DetailedSummary;
   uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount;
   uint32_t NumCounts, NumFunctions;
+  /// If 'Partial' is false, it means the profile being used to optimize
+  /// a target is collected from the same target.
+  /// If 'Partial' is true, it means the profile is for common/shared
+  /// code. The common profile is usually merged from profiles collected
+  /// from running other targets.
+  bool Partial = false;
   /// Return detailed summary as metadata.
   Metadata *getDetailedSummaryMD(LLVMContext &Context);
 
@@ -76,6 +82,8 @@ class ProfileSummary {
   uint64_t getTotalCount() { return TotalCount; }
   uint64_t getMaxCount() { return MaxCount; }
   uint64_t getMaxInternalCount() { return MaxInternalCount; }
+  void setPartialProfile(bool PP) { Partial = PP; }
+  bool isPartialProfile() { return Partial; }
 };
 
 } // end namespace llvm

llvm/include/llvm/ProfileData/SampleProf.h

@@ -169,6 +169,13 @@ enum class SecNameTableFlags : uint32_t {
   SecFlagInValid = 0,
   SecFlagMD5Name = (1 << 0)
 };
+enum class SecProfSummaryFlags : uint32_t {
+  SecFlagInValid = 0,
+  /// SecFlagPartial means the profile is for common/shared code.
+  /// The common profile is usually merged from profiles collected
+  /// from running other targets.
+  SecFlagPartial = (1 << 0)
+};
 
 // Verify section specific flag is used for the correct section.
 template <class SecFlagType>
@@ -183,6 +190,9 @@ static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
   case SecNameTable:
     IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
     break;
+  case SecProfSummary:
+    IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
+    break;
   default:
     break;
   }

llvm/include/llvm/ProfileData/SampleProfWriter.h

@@ -59,6 +59,7 @@ class SampleProfileWriter {
   virtual void setProfileSymbolList(ProfileSymbolList *PSL) {}
   virtual void setToCompressAllSections() {}
   virtual void setUseMD5() {}
+  virtual void setPartialProfile() {}
 
 protected:
   SampleProfileWriter(std::unique_ptr<raw_ostream> &OS)
@@ -217,6 +218,13 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
     addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name);
   }
 
+  // Set the profile to be partial. It means the profile is for
+  // common/shared code. The common profile is usually merged from
+  // profiles collected from running other targets.
+  virtual void setPartialProfile() override {
+    addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial);
+  }
+
 private:
   virtual void initSectionHdrLayout() override {
     // Note that SecFuncOffsetTable section is written after SecLBRProfile

llvm/lib/ProfileData/SampleProfReader.cpp

@@ -478,6 +478,8 @@ std::error_code SampleProfileReaderExtBinary::readOneSection(
   case SecProfSummary:
     if (std::error_code EC = readSummary())
       return EC;
+    if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
+      Summary->setPartialProfile(true);
     break;
   case SecNameTable:
     if (std::error_code EC = readNameTableSec(
@@ -831,11 +833,40 @@ uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
   return FileSize;
 }
 
+static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
+  std::string Flags;
+  if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
+    Flags.append("{compressed,");
+  else
+    Flags.append("{");
+
+  switch (Entry.Type) {
+  case SecNameTable:
+    if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))
+      Flags.append("md5,");
+    break;
+  case SecProfSummary:
+    if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
+      Flags.append("partial,");
+    break;
+  default:
+    break;
+  }
+  char &last = Flags.back();
+  if (last == ',')
+    last = '}';
+  else
+    Flags.append("}");
+  return Flags;
+}
+
 bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
   uint64_t TotalSecsSize = 0;
   for (auto &Entry : SecHdrTable) {
     OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
-       << ", Size: " << Entry.Size << "\n";
+       << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
+       << "\n";
+    ;
     TotalSecsSize += getSectionSize(Entry.Type);
   }
   uint64_t HeaderSize = SecHdrTable.front().Offset;

llvm/test/tools/llvm-profdata/show-prof-info.test

@@ -0,0 +1,11 @@
+; RUN: llvm-profdata merge -sample -extbinary -use-md5 -compress-all-sections -partial-profile -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output
+; RUN: wc -c < %t.1.output > %t.txt
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt
+; RUN: FileCheck %s --input-file=%t.txt
+; CHECK: [[FILESIZE:.*]]
+; To check llvm-profdata shows the correct flags for ProfileSummarySection.
+; CHECK: ProfileSummarySection {{.*}} Flags: {compressed,partial}
+; To check llvm-profdata shows the correct flags for NameTableSection.
+; CHECK: NameTableSection {{.*}} Flags: {compressed,md5}
+; To check llvm-profdata shows the correct file size.
+; CHECK: [[FILESIZE]]

llvm/test/tools/llvm-profdata/show-prof-size.test

@@ -448,7 +448,8 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
                                   ProfileFormat OutputFormat,
                                   MemoryBuffer *Buffer,
                                   sampleprof::ProfileSymbolList &WriterList,
-                                  bool CompressAllSections, bool UseMD5) {
+                                  bool CompressAllSections, bool UseMD5,
+                                  bool PartialProfile) {
   populateProfileSymbolList(Buffer, WriterList);
   if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
     warn("Profile Symbol list is not empty but the output format is not "
@@ -468,13 +469,19 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
     else
       Writer.setUseMD5();
   }
+  if (PartialProfile) {
+    if (OutputFormat != PF_Ext_Binary)
+      warn("-partial-profile is ignored. Specify -extbinary to enable it");
+    else
+      Writer.setPartialProfile();
+  }
 }
 
 static void
 mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
                    StringRef OutputFilename, ProfileFormat OutputFormat,
                    StringRef ProfileSymbolListFile, bool CompressAllSections,
-                   bool UseMD5, FailureMode FailMode) {
+                   bool UseMD5, bool PartialProfile, FailureMode FailMode) {
   using namespace sampleprof;
   StringMap<FunctionSamples> ProfileMap;
   SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
@@ -531,7 +538,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
   // Make sure Buffer lives as long as WriterList.
   auto Buffer = getInputFileBuf(ProfileSymbolListFile);
   handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList,
-                        CompressAllSections, UseMD5);
+                        CompressAllSections, UseMD5, PartialProfile);
   Writer->write(ProfileMap);
 }
 
@@ -663,6 +670,10 @@ static int merge_main(int argc, const char *argv[]) {
       "use-md5", cl::init(false), cl::Hidden,
       cl::desc("Choose to use MD5 to represent string in name table (only "
                "meaningful for -extbinary)"));
+  cl::opt<bool> PartialProfile(
+      "partial-profile", cl::init(false), cl::Hidden,
+      cl::desc("Set the profile to be a partial profile (only meaningful "
+               "for -extbinary)"));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
 
@@ -697,7 +708,7 @@ static int merge_main(int argc, const char *argv[]) {
   else
     mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename,
                        OutputFormat, ProfileSymbolListFile, CompressAllSections,
-                       UseMD5, FailureMode);
+                       UseMD5, PartialProfile, FailureMode);
 
   return 0;
 }