RewriteInstance.cpp

//===- bolt/Rewrite/RewriteInstance.cpp - ELF rewriter --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include"bolt/Rewrite/RewriteInstance.h"
#include"bolt/Core/AddressMap.h"
#include"bolt/Core/BinaryContext.h"
#include"bolt/Core/BinaryEmitter.h"
#include"bolt/Core/BinaryFunction.h"
#include"bolt/Core/DebugData.h"
#include"bolt/Core/Exceptions.h"
#include"bolt/Core/FunctionLayout.h"
#include"bolt/Core/MCPlusBuilder.h"
#include"bolt/Core/ParallelUtilities.h"
#include"bolt/Core/Relocation.h"
#include"bolt/Passes/BinaryPasses.h"
#include"bolt/Passes/CacheMetrics.h"
#include"bolt/Passes/IdenticalCodeFolding.h"
#include"bolt/Passes/PAuthGadgetScanner.h"
#include"bolt/Passes/ReorderFunctions.h"
#include"bolt/Profile/BoltAddressTranslation.h"
#include"bolt/Profile/DataAggregator.h"
#include"bolt/Profile/DataReader.h"
#include"bolt/Profile/YAMLProfileReader.h"
#include"bolt/Profile/YAMLProfileWriter.h"
#include"bolt/Rewrite/BinaryPassManager.h"
#include"bolt/Rewrite/DWARFRewriter.h"
#include"bolt/Rewrite/ExecutableFileMemoryManager.h"
#include"bolt/Rewrite/JITLinkLinker.h"
#include"bolt/Rewrite/MetadataRewriters.h"
#include"bolt/RuntimeLibs/HugifyRuntimeLibrary.h"
#include"bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
#include"bolt/Utils/CommandLineOpts.h"
#include"bolt/Utils/Utils.h"
#include"llvm/ADT/AddressRanges.h"
#include"llvm/ADT/STLExtras.h"
#include"llvm/DebugInfo/DWARF/DWARFContext.h"
#include"llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
#include"llvm/MC/MCAsmBackend.h"
#include"llvm/MC/MCAsmInfo.h"
#include"llvm/MC/MCDisassembler/MCDisassembler.h"
#include"llvm/MC/MCObjectStreamer.h"
#include"llvm/MC/MCStreamer.h"
#include"llvm/MC/MCSymbol.h"
#include"llvm/MC/TargetRegistry.h"
#include"llvm/Object/ObjectFile.h"
#include"llvm/Support/Alignment.h"
#include"llvm/Support/Casting.h"
#include"llvm/Support/CommandLine.h"
#include"llvm/Support/DataExtractor.h"
#include"llvm/Support/Errc.h"
#include"llvm/Support/Error.h"
#include"llvm/Support/FileSystem.h"
#include"llvm/Support/ManagedStatic.h"
#include"llvm/Support/Timer.h"
#include"llvm/Support/ToolOutputFile.h"
#include"llvm/Support/raw_ostream.h"
#include<algorithm>
#include<fstream>
#include<memory>
#include<optional>
#include<system_error>

#undef DEBUG_TYPE
#defineDEBUG_TYPE"bolt"

usingnamespacellvm;
usingnamespaceobject;
usingnamespacebolt;

extern cl::opt<uint32_t> X86AlignBranchBoundary;
extern cl::opt<bool> X86AlignBranchWithin32BBoundaries;

namespaceopts {

extern cl::list<std::string> HotTextMoveSections;
extern cl::opt<bool> Hugify;
extern cl::opt<bool> Instrument;
extern cl::opt<bool> KeepNops;
extern cl::opt<bool> Lite;
extern cl::list<std::string> ReorderData;
extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
extern cl::opt<bool> TerminalTrap;
extern cl::opt<bool> TimeBuild;
extern cl::opt<bool> TimeRewrite;
extern cl::opt<bolt::IdenticalCodeFolding::ICFLevel, false,
 llvm::bolt::DeprecatedICFNumericOptionParser>
 ICF;

static cl::opt<bool>
AllowStripped("allow-stripped",
cl::desc("allow processing of stripped binaries"), cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<bool> ForceToDataRelocations(
"force-data-relocations",
cl::desc("force relocations to data sections to always be processed"),

 cl::Hidden, cl::cat(BoltCategory));

static cl::opt<std::string>
BoltID("bolt-id",
cl::desc("add any string to tag this execution in the "
"output binary via bolt info section"),
 cl::cat(BoltCategory));

cl::opt<bool> DumpDotAll(
"dump-dot-all",
cl::desc("dump function CFGs to graphviz format after each stage;"
"enable '-print-loops' for color-coded blocks"),
 cl::Hidden, cl::cat(BoltCategory));

static cl::list<std::string>
ForceFunctionNames("funcs",
 cl::CommaSeparated,
cl::desc("limit optimizations to functions from the list"),
 cl::value_desc("func1,func2,func3,..."),
 cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<std::string>
FunctionNamesFile("funcs-file",
cl::desc("file with list of functions to optimize"),
 cl::Hidden,
 cl::cat(BoltCategory));

static cl::list<std::string> ForceFunctionNamesNR(
"funcs-no-regex", cl::CommaSeparated,
cl::desc("limit optimizations to functions from the list (non-regex)"),
 cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));

static cl::opt<std::string> FunctionNamesFileNR(
"funcs-file-no-regex",
cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,
 cl::cat(BoltCategory));

cl::opt<bool>
KeepTmp("keep-tmp",
cl::desc("preserve intermediate .o file"),
 cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<unsigned>
LiteThresholdPct("lite-threshold-pct",
cl::desc("threshold (in percent) for selecting functions to process in lite "
"mode. Higher threshold means fewer functions to process. E.g "
"threshold of 90 means only top 10 percent of functions with "
"profile will be processed."),
 cl::init(0),
 cl::ZeroOrMore,
 cl::Hidden,
 cl::cat(BoltOptCategory));

static cl::opt<unsigned> LiteThresholdCount(
"lite-threshold-count",
cl::desc("similar to '-lite-threshold-pct' but specify threshold using "
"absolute function call count. I.e. limit processing to functions "
"executed at least the specified number of times."),
 cl::init(0), cl::Hidden, cl::cat(BoltOptCategory));

static cl::opt<unsigned>
MaxFunctions("max-funcs",
cl::desc("maximum number of functions to process"), cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<unsigned> MaxDataRelocations(
"max-data-relocations",
cl::desc("maximum number of data relocations to process"), cl::Hidden,
 cl::cat(BoltCategory));

cl::opt<bool> PrintAll("print-all",
cl::desc("print functions after each stage"), cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<bool>
PrintProfile("print-profile",
cl::desc("print functions after attaching profile"),
 cl::Hidden, cl::cat(BoltCategory));

cl::opt<bool> PrintCFG("print-cfg",
cl::desc("print functions after CFG construction"),
 cl::Hidden, cl::cat(BoltCategory));

cl::opt<bool> PrintDisasm("print-disasm",
cl::desc("print function after disassembly"),
 cl::Hidden, cl::cat(BoltCategory));

static cl::opt<bool>
PrintGlobals("print-globals",
cl::desc("print global symbols after disassembly"), cl::Hidden,
 cl::cat(BoltCategory));

extern cl::opt<bool> PrintSections;

static cl::opt<bool> PrintLoopInfo("print-loops",
cl::desc("print loop related information"),
 cl::Hidden, cl::cat(BoltCategory));

static cl::opt<cl::boolOrDefault> RelocationMode(
"relocs", cl::desc("use relocations in the binary (default=autodetect)"),
 cl::cat(BoltCategory));

extern cl::opt<std::string> SaveProfile;

static cl::list<std::string>
SkipFunctionNames("skip-funcs",
 cl::CommaSeparated,
cl::desc("list of functions to skip"),
 cl::value_desc("func1,func2,func3,..."),
 cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<std::string>
SkipFunctionNamesFile("skip-funcs-file",
cl::desc("file with list of functions to skip"),
 cl::Hidden,
 cl::cat(BoltCategory));

static cl::opt<bool> TrapOldCode(
"trap-old-code",
cl::desc("insert traps in old function bodies (relocation mode)"),
 cl::Hidden, cl::cat(BoltCategory));

static cl::opt<std::string> DWPPathName("dwp",
cl::desc("Path and name to DWP file."),
 cl::Hidden, cl::init(""),
 cl::cat(BoltCategory));

static cl::opt<bool>
UseGnuStack("use-gnu-stack",
cl::desc("use GNU_STACK program header for new segment (workaround for "
"issues with strip/objcopy)"),
 cl::ZeroOrMore,
 cl::cat(BoltCategory));

static cl::opt<uint64_t> CustomAllocationVMA(
"custom-allocation-vma",
cl::desc("use a custom address at which new code will be put, "
"bypassing BOLT's logic to detect where to put code"),
 cl::Hidden, cl::cat(BoltCategory));

static cl::opt<bool>
SequentialDisassembly("sequential-disassembly",
cl::desc("performs disassembly sequentially"),
 cl::init(false),
 cl::cat(BoltOptCategory));

static cl::opt<bool> WriteBoltInfoSection(
"bolt-info", cl::desc("write bolt info section in the output binary"),
 cl::init(true), cl::Hidden, cl::cat(BoltOutputCategory));

cl::bits<GadgetScannerKind> GadgetScannersToRun(
"scanners", cl::desc("which gadget scanners to run"),
 cl::values(
clEnumValN(GS_PACRET, "pacret",
"pac-ret: return address protection (subset of \"pauth\")"),
 clEnumValN(GS_PAUTH, "pauth", "All Pointer Authentication scanners"),
 clEnumValN(GS_ALL, "all", "All implemented scanners")),
 cl::ZeroOrMore, cl::CommaSeparated, cl::cat(BinaryAnalysisCategory));

} // namespace opts

// FIXME: implement a better way to mark sections for replacement.
constexprconstchar *RewriteInstance::SectionsToOverwrite[];
std::vector<std::string> RewriteInstance::DebugSectionsToOverwrite = {
".debug_abbrev", ".debug_aranges", ".debug_line", ".debug_line_str",
".debug_loc", ".debug_loclists", ".debug_ranges", ".debug_rnglists",
".gdb_index", ".debug_addr", ".debug_abbrev", ".debug_info",
".debug_types", ".pseudo_probe"};

constchar RewriteInstance::TimerGroupName[] = "rewrite";
constchar RewriteInstance::TimerGroupDesc[] = "Rewrite passes";

namespacellvm {
namespacebolt {

externconstchar *BoltRevision;

// Weird location for createMCPlusBuilder, but this is here to avoid a
// cyclic dependency of libCore (its natural place) and libTarget. libRewrite
// can depend on libTarget, but not libCore. Since libRewrite is the only
// user of this function, we define it here.
MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
const MCInstrAnalysis *Analysis,
const MCInstrInfo *Info,
const MCRegisterInfo *RegInfo,
const MCSubtargetInfo *STI) {
#ifdef X86_AVAILABLE
if (Arch == Triple::x86_64)
returncreateX86MCPlusBuilder(Analysis, Info, RegInfo, STI);
#endif

#ifdef AARCH64_AVAILABLE
if (Arch == Triple::aarch64)
returncreateAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
#endif

#ifdef RISCV_AVAILABLE
if (Arch == Triple::riscv64)
returncreateRISCVMCPlusBuilder(Analysis, Info, RegInfo, STI);
#endif

llvm_unreachable("architecture unsupported by MCPlusBuilder");
}

} // namespace bolt
} // namespace llvm

using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;

namespace {

boolrefersToReorderedSection(ErrorOr<BinarySection &> Section) {
returnllvm::any_of(opts::ReorderData, [&](const std::string &SectionName) {
return Section && Section->getName() == SectionName;
 });
}

} // anonymous namespace

Expected<std::unique_ptr<RewriteInstance>>
RewriteInstance::create(ELFObjectFileBase *File, constint Argc,
constchar *const *Argv, StringRef ToolPath,
 raw_ostream &Stdout, raw_ostream &Stderr) {
 Error Err = Error::success();
auto RI = std::make_unique<RewriteInstance>(File, Argc, Argv, ToolPath,
 Stdout, Stderr, Err);
if (Err)
returnstd::move(Err);
returnstd::move(RI);
}

RewriteInstance::RewriteInstance(ELFObjectFileBase *File, constint Argc,
constchar *const *Argv, StringRef ToolPath,
 raw_ostream &Stdout, raw_ostream &Stderr,
 Error &Err)
 : InputFile(File), Argc(Argc), Argv(Argv), ToolPath(ToolPath),
 SHStrTab(StringTableBuilder::ELF) {
 ErrorAsOutParameter EAO(&Err);
auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(InputFile);
if (!ELF64LEFile) {
 Err = createStringError(errc::not_supported,
"Only 64-bit LE ELF binaries are supported");
return;
 }

bool IsPIC = false;
const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
if (Obj.getHeader().e_type != ELF::ET_EXEC) {
 Stdout << "BOLT-INFO: shared object or position-independent executable "
"detected\n";
 IsPIC = true;
 }

// Make sure we don't miss any output on core dumps.
 Stdout.SetUnbuffered();
 Stderr.SetUnbuffered();
LLVM_DEBUG(dbgs().SetUnbuffered());

// Read RISCV subtarget features from input file
 std::unique_ptr<SubtargetFeatures> Features;
 Triple TheTriple = File->makeTriple();
if (TheTriple.getArch() == llvm::Triple::riscv64) {
 Expected<SubtargetFeatures> FeaturesOrErr = File->getFeatures();
if (auto E = FeaturesOrErr.takeError()) {
 Err = std::move(E);
return;
 } else {
 Features.reset(newSubtargetFeatures(*FeaturesOrErr));
 }
 }

 Relocation::Arch = TheTriple.getArch();
auto BCOrErr = BinaryContext::createBinaryContext(
 TheTriple, std::make_shared<orc::SymbolStringPool>(), File->getFileName(),
 Features.get(), IsPIC,
DWARFContext::create(*File, DWARFContext::ProcessDebugRelocations::Ignore,
nullptr, opts::DWPPathName,
 WithColor::defaultErrorHandler,
 WithColor::defaultWarningHandler),
 JournalingStreams{Stdout, Stderr});
if (Error E = BCOrErr.takeError()) {
 Err = std::move(E);
return;
 }
 BC = std::move(BCOrErr.get());
 BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(
createMCPlusBuilder(BC->TheTriple->getArch(), BC->MIA.get(),
 BC->MII.get(), BC->MRI.get(), BC->STI.get())));

 BAT = std::make_unique<BoltAddressTranslation>();

if (opts::UpdateDebugSections)
 DebugInfoRewriter = std::make_unique<DWARFRewriter>(*BC);

if (opts::Instrument)
 BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
elseif (opts::Hugify)
 BC->setRuntimeLibrary(std::make_unique<HugifyRuntimeLibrary>());
}

RewriteInstance::~RewriteInstance() {}

Error RewriteInstance::setProfile(StringRef Filename) {
if (!sys::fs::exists(Filename))
returnerrorCodeToError(make_error_code(errc::no_such_file_or_directory));

if (ProfileReader) {
// Already exists
return make_error<StringError>(Twine("multiple profiles specified: ") +
 ProfileReader->getFilename() + " and " +
 Filename,
inconvertibleErrorCode());
 }

// Spawn a profile reader based on file contents.
if (DataAggregator::checkPerfDataMagic(Filename))
 ProfileReader = std::make_unique<DataAggregator>(Filename);
elseif (YAMLProfileReader::isYAML(Filename))
 ProfileReader = std::make_unique<YAMLProfileReader>(Filename);
else
 ProfileReader = std::make_unique<DataReader>(Filename);

returnError::success();
}

/// Return true if the function \p BF should be disassembled.
staticboolshouldDisassemble(const BinaryFunction &BF) {
if (BF.isPseudo())
returnfalse;

if (opts::processAllFunctions())
returntrue;

return !BF.isIgnored();
}

// Return if a section stored in the image falls into a segment address space.
// If not, Set \p Overlap to true if there's a partial overlap.
template <classELFT>
staticboolcheckOffsets(consttypename ELFT::Phdr &Phdr,
consttypename ELFT::Shdr &Sec, bool &Overlap) {
// SHT_NOBITS sections don't need to have an offset inside the segment.
if (Sec.sh_type == ELF::SHT_NOBITS)
returntrue;

// Only non-empty sections can be at the end of a segment.
uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
 AddressRange SectionAddressRange((uint64_t)Sec.sh_offset,
 Sec.sh_offset + SectionSize);
 AddressRange SegmentAddressRange(Phdr.p_offset,
 Phdr.p_offset + Phdr.p_filesz);
if (SegmentAddressRange.contains(SectionAddressRange))
returntrue;

 Overlap = SegmentAddressRange.intersects(SectionAddressRange);
returnfalse;
}

// Check that an allocatable section belongs to a virtual address
// space of a segment.
template <classELFT>
staticboolcheckVMA(consttypename ELFT::Phdr &Phdr,
consttypename ELFT::Shdr &Sec, bool &Overlap) {
// Only non-empty sections can be at the end of a segment.
uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
 AddressRange SectionAddressRange((uint64_t)Sec.sh_addr,
 Sec.sh_addr + SectionSize);
 AddressRange SegmentAddressRange(Phdr.p_vaddr, Phdr.p_vaddr + Phdr.p_memsz);

if (SegmentAddressRange.contains(SectionAddressRange))
returntrue;
 Overlap = SegmentAddressRange.intersects(SectionAddressRange);
returnfalse;
}

voidRewriteInstance::markGnuRelroSections() {
using ELFT = ELF64LE;
using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);
const ELFFile<ELFT> &Obj = ELF64LEFile->getELFFile();

auto handleSection = [&](const ELFT::Phdr &Phdr, SectionRef SecRef) {
 BinarySection *BinarySection = BC->getSectionForSectionRef(SecRef);
// If the section is non-allocatable, ignore it for GNU_RELRO purposes:
// it can't be made read-only after runtime relocations processing.
if (!BinarySection || !BinarySection->isAllocatable())
return;
const ELFShdrTy *Sec = cantFail(Obj.getSection(SecRef.getIndex()));
bool ImageOverlap{false}, VMAOverlap{false};
bool ImageContains = checkOffsets<ELFT>(Phdr, *Sec, ImageOverlap);
bool VMAContains = checkVMA<ELFT>(Phdr, *Sec, VMAOverlap);
if (ImageOverlap) {
if (opts::Verbosity >= 1)
 BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial file offset "
 << "overlap with section " << BinarySection->getName()
 << '\n';
return;
 }
if (VMAOverlap) {
if (opts::Verbosity >= 1)
 BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial VMA overlap "
 << "with section " << BinarySection->getName() << '\n';
return;
 }
if (!ImageContains || !VMAContains)
return;
 BinarySection->setRelro();
if (opts::Verbosity >= 1)
 BC->outs() << "BOLT-INFO: marking " << BinarySection->getName()
 << " as GNU_RELRO\n";
 };

for (const ELFT::Phdr &Phdr : cantFail(Obj.program_headers()))
if (Phdr.p_type == ELF::PT_GNU_RELRO)
for (SectionRef SecRef : InputFile->sections())
handleSection(Phdr, SecRef);
}

Error RewriteInstance::discoverStorage() {
 NamedRegionTimer T("discoverStorage", "discover storage", TimerGroupName,
 TimerGroupDesc, opts::TimeRewrite);

auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);
const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();

 BC->StartFunctionAddress = Obj.getHeader().e_entry;

 NextAvailableAddress = 0;
uint64_t NextAvailableOffset = 0;
 Expected<ELF64LE::PhdrRange> PHsOrErr = Obj.program_headers();
if (Error E = PHsOrErr.takeError())
return E;

 ELF64LE::PhdrRange PHs = PHsOrErr.get();
for (const ELF64LE::Phdr &Phdr : PHs) {
switch (Phdr.p_type) {
case ELF::PT_LOAD:
 BC->FirstAllocAddress = std::min(BC->FirstAllocAddress,
static_cast<uint64_t>(Phdr.p_vaddr));
 NextAvailableAddress = std::max(NextAvailableAddress,
 Phdr.p_vaddr + Phdr.p_memsz);
 NextAvailableOffset = std::max(NextAvailableOffset,
 Phdr.p_offset + Phdr.p_filesz);

 BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
 Phdr.p_vaddr, Phdr.p_memsz, Phdr.p_offset,
 Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
 Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
 BC->IsLinuxKernel = true;
break;
case ELF::PT_INTERP:
 BC->HasInterpHeader = true;
break;
 }
 }

if (BC->IsLinuxKernel)
 BC->outs() << "BOLT-INFO: Linux kernel binary detected\n";

for (const SectionRef &Section : InputFile->sections()) {
 Expected<StringRef> SectionNameOrErr = Section.getName();
if (Error E = SectionNameOrErr.takeError())
return E;
 StringRef SectionName = SectionNameOrErr.get();
if (SectionName == BC->getMainCodeSectionName()) {
 BC->OldTextSectionAddress = Section.getAddress();
 BC->OldTextSectionSize = Section.getSize();

 Expected<StringRef> SectionContentsOrErr = Section.getContents();
if (Error E = SectionContentsOrErr.takeError())
return E;
 StringRef SectionContents = SectionContentsOrErr.get();
 BC->OldTextSectionOffset =
 SectionContents.data() - InputFile->getData().data();
 }

if (!opts::HeatmapMode &&
 !(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&
 (SectionName.starts_with(getOrgSecPrefix()) ||
 SectionName == getBOLTTextSectionName()))
returncreateStringError(
 errc::function_not_supported,
"BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize");
 }

if (!NextAvailableAddress || !NextAvailableOffset)
returncreateStringError(errc::executable_format_error,
"no PT_LOAD pheader seen");

 BC->outs() << "BOLT-INFO: first alloc address is 0x"
 << Twine::utohexstr(BC->FirstAllocAddress) << '\n';

 FirstNonAllocatableOffset = NextAvailableOffset;

if (opts::CustomAllocationVMA) {
// If user specified a custom address where we should start writing new
// data, honor that.
 NextAvailableAddress = opts::CustomAllocationVMA;
// Sanity check the user-supplied address and emit warnings if something
// seems off.
for (const ELF64LE::Phdr &Phdr : PHs) {
switch (Phdr.p_type) {
case ELF::PT_LOAD:
if (NextAvailableAddress >= Phdr.p_vaddr &&
 NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) {
 BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x"
 << Twine::utohexstr(NextAvailableAddress)
 << " conflicts with ELF segment at 0x"
 << Twine::utohexstr(Phdr.p_vaddr) << "\n";
 }
 }
 }
 }
 NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);
 NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);

// Hugify: Additional huge page from left side due to
// weird ASLR mapping addresses (4KB aligned)
if (opts::Hugify && !BC->HasFixedLoadAddress) {
 NextAvailableAddress += BC->PageAlign;
 }

if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
// This is where the black magic happens. Creating PHDR table in a segment
// other than that containing ELF header is tricky. Some loaders and/or
// parts of loaders will apply e_phoff from ELF header assuming both are in
// the same segment, while others will do the proper calculation.
// We create the new PHDR table in such a way that both of the methods
// of loading and locating the table work. There's a slight file size
// overhead because of that.
//
// NB: bfd's strip command cannot do the above and will corrupt the
// binary during the process of stripping non-allocatable sections.
if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress)
 NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress;
else
 NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress;

assert(NextAvailableOffset ==
 NextAvailableAddress - BC->FirstAllocAddress &&
"PHDR table address calculation error");

 BC->outs() << "BOLT-INFO: creating new program header table at address 0x"
 << Twine::utohexstr(NextAvailableAddress) << ", offset 0x"
 << Twine::utohexstr(NextAvailableOffset) << '\n';

 PHDRTableAddress = NextAvailableAddress;
 PHDRTableOffset = NextAvailableOffset;

// Reserve space for 3 extra pheaders.
unsigned Phnum = Obj.getHeader().e_phnum;
 Phnum += 3;

// Reserve two more pheaders to avoid having writeable and executable
// segment in instrumented binary.
if (opts::Instrument)
 Phnum += 2;

 NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
 NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
 }

// Align at cache line.
 NextAvailableAddress = alignTo(NextAvailableAddress, 64);
 NextAvailableOffset = alignTo(NextAvailableOffset, 64);

 NewTextSegmentAddress = NextAvailableAddress;
 NewTextSegmentOffset = NextAvailableOffset;
 BC->LayoutStartAddress = NextAvailableAddress;

// Tools such as objcopy can strip section contents but leave header
// entries. Check that at least .text is mapped in the file.
if (!getFileOffsetForAddress(BC->OldTextSectionAddress))
returncreateStringError(errc::executable_format_error,
"BOLT-ERROR: input binary is not a valid ELF "
"executable as its text section is not "
"mapped to a valid segment");
returnError::success();
}

Error RewriteInstance::run() {
assert(BC && "failed to create a binary context");

 BC->outs() << "BOLT-INFO: Target architecture: "
 << Triple::getArchTypeName(
 (llvm::Triple::ArchType)InputFile->getArch())
 << "\n";
 BC->outs() << "BOLT-INFO: BOLT version: " << BoltRevision << "\n";

if (Error E = discoverStorage())
return E;
if (Error E = readSpecialSections())
return E;
adjustCommandLineOptions();
discoverFileObjects();

if (opts::Instrument && !BC->IsStaticExecutable)
if (Error E = discoverRtFiniAddress())
return E;

preprocessProfileData();

// Skip disassembling if we have a translation table and we are running an
// aggregation job.
if (opts::AggregateOnly && BAT->enabledFor(InputFile)) {
// YAML profile in BAT mode requires CFG for .bolt.org.text functions
if (!opts::SaveProfile.empty() ||
 opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) {
selectFunctionsToProcess();
disassembleFunctions();
processMetadataPreCFG();
buildFunctionsCFG();
 }
processProfileData();
returnError::success();
 }

selectFunctionsToProcess();

readDebugInfo();

disassembleFunctions();

processMetadataPreCFG();

buildFunctionsCFG();

processProfileData();

// Save input binary metadata if BAT section needs to be emitted
if (opts::EnableBAT)
 BAT->saveMetadata(*BC);

postProcessFunctions();

processMetadataPostCFG();

if (opts::DiffOnly)
returnError::success();

if (opts::BinaryAnalysisMode) {
runBinaryAnalyses();
returnError::success();
 }

preregisterSections();

runOptimizationPasses();

finalizeMetadataPreEmit();

emitAndLink();

updateMetadata();

if (opts::Instrument && !BC->IsStaticExecutable)
updateRtFiniReloc();

if (opts::OutputFilename == "/dev/null") {
 BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n";
returnError::success();
 } elseif (BC->IsLinuxKernel) {
 BC->errs() << "BOLT-WARNING: Linux kernel support is experimental\n";
 }

// Rewrite allocatable contents and copy non-allocatable parts with mods.
rewriteFile();
returnError::success();
}

voidRewriteInstance::discoverFileObjects() {
 NamedRegionTimer T("discoverFileObjects", "discover file objects",
 TimerGroupName, TimerGroupDesc, opts::TimeRewrite);

// For local symbols we want to keep track of associated FILE symbol name for
// disambiguation by combined name.
 StringRef FileSymbolName;
bool SeenFileName = false;
structSymbolRefHash {
size_toperator()(SymbolRef const &S) const {
return std::hash<decltype(DataRefImpl::p)>{}(S.getRawDataRefImpl().p);
 }
 };
 std::unordered_map<SymbolRef, StringRef, SymbolRefHash> SymbolToFileName;
for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
 Expected<StringRef> NameOrError = Symbol.getName();
if (NameOrError && NameOrError->starts_with("__asan_init")) {
 BC->errs()
 << "BOLT-ERROR: input file was compiled or linked with sanitizer "
"support. Cannot optimize.\n";
exit(1);
 }
if (NameOrError && NameOrError->starts_with("__llvm_coverage_mapping")) {
 BC->errs()
 << "BOLT-ERROR: input file was compiled or linked with coverage "
"support. Cannot optimize.\n";
exit(1);
 }

if (cantFail(Symbol.getFlags()) & SymbolRef::SF_Undefined)
continue;

if (cantFail(Symbol.getType()) == SymbolRef::ST_File) {
 FileSymbols.emplace_back(Symbol);
 StringRef Name =
cantFail(std::move(NameOrError), "cannot get symbol name for file");
// Ignore Clang LTO artificial FILE symbol as it is not always generated,
// and this uncertainty is causing havoc in function name matching.
if (Name == "ld-temp.o")
continue;
 FileSymbolName = Name;
 SeenFileName = true;
continue;
 }
if (!FileSymbolName.empty() &&
 !(cantFail(Symbol.getFlags()) & SymbolRef::SF_Global))
 SymbolToFileName[Symbol] = FileSymbolName;
 }

// Sort symbols in the file by value. Ignore symbols from non-allocatable
// sections. We memoize getAddress(), as it has rather high overhead.
structSymbolInfo {
uint64_t Address;
 SymbolRef Symbol;
 };
 std::vector<SymbolInfo> SortedSymbols;
auto isSymbolInMemory = [this](const SymbolRef &Sym) {
if (cantFail(Sym.getType()) == SymbolRef::ST_File)
returnfalse;
if (cantFail(Sym.getFlags()) & SymbolRef::SF_Absolute)
returntrue;
if (cantFail(Sym.getFlags()) & SymbolRef::SF_Undefined)
returnfalse;
 BinarySection Section(*BC, *cantFail(Sym.getSection()));
return Section.isAllocatable();
 };
auto checkSymbolInSection = [this](const SymbolInfo &S) {
// Sometimes, we encounter symbols with addresses outside their section. If
// such symbols happen to fall into another section, they can interfere with
// disassembly. Notably, this occurs with AArch64 marker symbols ($d and $t)
// that belong to .eh_frame, but end up pointing into .text.
// As a workaround, we ignore all symbols that lie outside their sections.
auto Section = cantFail(S.Symbol.getSection());

// Accept all absolute symbols.
if (Section == InputFile->section_end())
returntrue;

uint64_t SecStart = Section->getAddress();
uint64_t SecEnd = SecStart + Section->getSize();
uint64_t SymEnd = S.Address + ELFSymbolRef(S.Symbol).getSize();
if (S.Address >= SecStart && SymEnd <= SecEnd)
returntrue;

auto SymType = cantFail(S.Symbol.getType());
// Skip warnings for common benign cases.
if (opts::Verbosity < 1 && SymType == SymbolRef::ST_Other)
returnfalse; // E.g. ELF::STT_TLS.

auto SymName = S.Symbol.getName();
auto SecName = cantFail(S.Symbol.getSection())->getName();
 BC->errs() << "BOLT-WARNING: ignoring symbol "
 << (SymName ? *SymName : "[unnamed]") << " at 0x"
 << Twine::utohexstr(S.Address) << ", which lies outside "
 << (SecName ? *SecName : "[unnamed]") << "\n";

returnfalse;
 };
for (const SymbolRef &Symbol : InputFile->symbols())
if (isSymbolInMemory(Symbol)) {
 SymbolInfo SymInfo{cantFail(Symbol.getAddress()), Symbol};
if (checkSymbolInSection(SymInfo))
 SortedSymbols.push_back(SymInfo);
 }

auto CompareSymbols = [this](const SymbolInfo &A, const SymbolInfo &B) {
if (A.Address != B.Address)
return A.Address < B.Address;

constbool AMarker = BC->isMarker(A.Symbol);
constbool BMarker = BC->isMarker(B.Symbol);
if (AMarker || BMarker) {
return AMarker && !BMarker;
 }

constauto AType = cantFail(A.Symbol.getType());
constauto BType = cantFail(B.Symbol.getType());
if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function)
returntrue;
if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug)
returntrue;

returnfalse;
 };
llvm::stable_sort(SortedSymbols, CompareSymbols);

auto LastSymbol = SortedSymbols.end();
if (!SortedSymbols.empty())
 --LastSymbol;

// For aarch64, the ABI defines mapping symbols so we identify data in the
// code section (see IHI0056B). $d identifies data contents.
// Compilers usually merge multiple data objects in a single $d-$x interval,
// but we need every data object to be marked with $d. Because of that we
// create a vector of MarkerSyms with all locations of data objects.

structMarkerSym {
uint64_t Address;
 MarkerSymType Type;
 };

 std::vector<MarkerSym> SortedMarkerSymbols;
auto addExtraDataMarkerPerSymbol = [&]() {
bool IsData = false;
uint64_t LastAddr = 0;
for (constauto &SymInfo : SortedSymbols) {
if (LastAddr == SymInfo.Address) // don't repeat markers
continue;

 MarkerSymType MarkerType = BC->getMarkerType(SymInfo.Symbol);
if (MarkerType != MarkerSymType::NONE) {
 SortedMarkerSymbols.push_back(MarkerSym{SymInfo.Address, MarkerType});
 LastAddr = SymInfo.Address;
 IsData = MarkerType == MarkerSymType::DATA;
continue;
 }

if (IsData) {
 SortedMarkerSymbols.push_back({SymInfo.Address, MarkerSymType::DATA});
 LastAddr = SymInfo.Address;
 }
 }
 };

if (BC->isAArch64() || BC->isRISCV()) {
addExtraDataMarkerPerSymbol();
 LastSymbol = std::stable_partition(
 SortedSymbols.begin(), SortedSymbols.end(),
 [this](const SymbolInfo &S) { return !BC->isMarker(S.Symbol); });
if (!SortedSymbols.empty())
 --LastSymbol;
 }

 BinaryFunction *PreviousFunction = nullptr;
unsigned AnonymousId = 0;

constauto SortedSymbolsEnd =
 LastSymbol == SortedSymbols.end() ? LastSymbol : std::next(LastSymbol);
for (auto Iter = SortedSymbols.begin(); Iter != SortedSymbolsEnd; ++Iter) {
const SymbolRef &Symbol = Iter->Symbol;
constuint64_t SymbolAddress = Iter->Address;
constauto SymbolFlags = cantFail(Symbol.getFlags());
const SymbolRef::Type SymbolType = cantFail(Symbol.getType());

if (SymbolType == SymbolRef::ST_File)
continue;

 StringRef SymName = cantFail(Symbol.getName(), "cannot get symbol name");
if (SymbolAddress == 0) {
if (opts::Verbosity >= 1 && SymbolType == SymbolRef::ST_Function)
 BC->errs() << "BOLT-WARNING: function with 0 address seen\n";
continue;
 }

// Ignore input hot markers
if (SymName == "__hot_start" || SymName == "__hot_end")
continue;

 FileSymRefs.emplace(SymbolAddress, Symbol);

// Skip section symbols that will be registered by disassemblePLT().
if (SymbolType == SymbolRef::ST_Debug) {
 ErrorOr<BinarySection &> BSection =
 BC->getSectionForAddress(SymbolAddress);
if (BSection && getPLTSectionInfo(BSection->getName()))
continue;
 }

/// It is possible we are seeing a globalized local. LLVM might treat it as
/// a local if it has a "private global" prefix, e.g. ".L". Thus we have to
/// change the prefix to enforce global scope of the symbol.
 std::string Name =
 SymName.starts_with(BC->AsmInfo->getPrivateGlobalPrefix())
 ? "PG" + std::string(SymName)
 : std::string(SymName);

// Disambiguate all local symbols before adding to symbol table.
// Since we don't know if we will see a global with the same name,
// always modify the local name.
//
// NOTE: the naming convention for local symbols should match
// the one we use for profile data.
 std::string UniqueName;
 std::string AlternativeName;