From 421999a7c27f3c141cbafaaedd02dc73f0eed3af Mon Sep 17 00:00:00 2001 From: Shanzhi Date: Sat, 6 Dec 2025 16:22:01 +0800 Subject: [PATCH 1/2] Add Profile-Guilded Prefetch Support --- .../llvm/Transforms/Scalar/PGOPrefetch.h | 32 ++++ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassBuilderPipelines.cpp | 6 + llvm/lib/Passes/PassRegistry.def | 1 + llvm/lib/Transforms/Scalar/CMakeLists.txt | 1 + llvm/lib/Transforms/Scalar/PGOPrefetch.cpp | 149 ++++++++++++++++++ 6 files changed, 190 insertions(+) create mode 100644 llvm/include/llvm/Transforms/Scalar/PGOPrefetch.h create mode 100644 llvm/lib/Transforms/Scalar/PGOPrefetch.cpp diff --git a/llvm/include/llvm/Transforms/Scalar/PGOPrefetch.h b/llvm/include/llvm/Transforms/Scalar/PGOPrefetch.h new file mode 100644 index 000000000000..0f79b0c684e0 --- /dev/null +++ b/llvm/include/llvm/Transforms/Scalar/PGOPrefetch.h @@ -0,0 +1,32 @@ +//===------- PGOPrefetch.h - Profile-Guilded Prefetching Pass ---*- C++ -*-===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file provides the interface for Profile-Guilded Prefetching Pass. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_PGOPREFETCH_H +#define LLVM_TRANSFORMS_SCALAR_PGOPREFETCH_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Support/CommandLine.h" + +namespace llvm { + +extern cl::opt EnablePGOPrefetch; + +class PGOPrefetchPass : public PassInfoMixin { +public: + PGOPrefetchPass() = default; + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_PGOPREFETCH_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 2ae9bb8f8d65..80cb688b1f21 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -208,6 +208,7 @@ #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/Transforms/Scalar/NaryReassociate.h" #include "llvm/Transforms/Scalar/NewGVN.h" +#include "llvm/Transforms/Scalar/PGOPrefetch.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/PlaceSafepoints.h" #include "llvm/Transforms/Scalar/Reassociate.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index f48c7ac57248..c000bf0e0c73 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -112,6 +112,7 @@ #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" #include "llvm/Transforms/Scalar/NewGVN.h" +#include "llvm/Transforms/Scalar/PGOPrefetch.h" #include "llvm/Transforms/Scalar/Reassociate.h" #include "llvm/Transforms/Scalar/SCCP.h" #include "llvm/Transforms/Scalar/SROA.h" @@ -1281,6 +1282,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, MPM.addPass(GlobalOptPass()); MPM.addPass(GlobalDCEPass()); + if (EnablePGOPrefetch) + MPM.addPass(createModuleToFunctionPassAdaptor(PGOPrefetchPass())); return MPM; } @@ -1930,6 +1933,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM), PTO.EagerlyInvalidateAnalyses)); + if (EnablePGOPrefetch) + MPM.addPass(createModuleToFunctionPassAdaptor(PGOPrefetchPass())); + // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index f6df0fcb7f80..94b2af6e281b 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -394,6 +394,7 @@ FUNCTION_PASS("objc-arc-contract", ObjCARCContractPass()) FUNCTION_PASS("objc-arc-expand", ObjCARCExpandPass()) FUNCTION_PASS("pa-eval", PAEvalPass()) FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt()) +FUNCTION_PASS("pgo-prefetch", PGOPrefetchPass()) FUNCTION_PASS("place-safepoints", PlaceSafepointsPass()) FUNCTION_PASS("print", PrintFunctionPass(dbgs())) FUNCTION_PASS("print", AssumptionPrinterPass(dbgs())) diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt index e5a82ea8f923..fb31faa1c185 100644 --- a/llvm/lib/Transforms/Scalar/CMakeLists.txt +++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt @@ -59,6 +59,7 @@ add_llvm_component_library(LLVMScalarOpts MergedLoadStoreMotion.cpp NaryReassociate.cpp NewGVN.cpp + PGOPrefetch.cpp PartiallyInlineLibCalls.cpp PlaceSafepoints.cpp Reassociate.cpp diff --git a/llvm/lib/Transforms/Scalar/PGOPrefetch.cpp b/llvm/lib/Transforms/Scalar/PGOPrefetch.cpp new file mode 100644 index 000000000000..368f30dfb8e8 --- /dev/null +++ b/llvm/lib/Transforms/Scalar/PGOPrefetch.cpp @@ -0,0 +1,149 @@ +//===-------- PGOPrefetch.cpp - Profile-Guided Prefetching Pass -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a Profile-Guilded Prefetching Pass. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Scalar/PGOPrefetch.h" + +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/ProfileData/SampleProfReader.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/VirtualFileSystem.h" + +#define DEBUG_TYPE "pgo-prefetch" + +using namespace llvm; + +namespace llvm { + +cl::opt EnablePGOPrefetch("enable-pgo-prefetch", cl::init(false), + cl::ReallyHidden, cl::desc("Enable PGO Prefetch Pass")); + +static cl::opt PrefetchHintsFile("pgo-prefetch-hints-file", + cl::desc("Path to the prefetch hints profile"), cl::Hidden, + cl::callback([](const std::string &Path) { + if (!Path.empty()) + EnablePGOPrefetch = true; + })); + +class PGOPrefetch { +public: + PGOPrefetch(LLVMContext &Ctx, const std::string &FilePath); + bool run(Function *M); + +private: + std::unique_ptr Reader; + + void readAfdoFile(LLVMContext &Ctx, const std::string &FilePath); + bool doByteOffsetPrefetch(LoadInst *I, uint64_t PrefetchDistance); +}; + +void PGOPrefetch::readAfdoFile(LLVMContext &Ctx, const std::string &FilePath) { + auto FS = vfs::getRealFileSystem(); + auto ReaderOrErr = SampleProfileReader::create(FilePath, Ctx, *FS); + if (auto EC = ReaderOrErr.getError()) { + errs() << "Could not open sample profile " << FilePath << "\n"; + Reader = nullptr; + return; + } + + Reader = std::move(ReaderOrErr.get()); + Reader->read(); +} + +PGOPrefetch::PGOPrefetch(LLVMContext &Ctx, const std::string &FilePath) { + readAfdoFile(Ctx, FilePath); +} + +bool PGOPrefetch::run(Function *F) { + if (!Reader || F->isDeclaration()) + return false; + + FunctionSamples *Samples = Reader->getSamplesFor(F->getName()); + if (!Samples) + return false; + + LLVM_DEBUG(dbgs() << "Found samples for function " << F->getName() << '\n'); + + bool Changed = false; + for (auto It = inst_begin(F); It != inst_end(F); ++It) { + Instruction &I = *It; + + LoadInst *LI = dyn_cast(&I); + if (!LI) + continue; + + const DILocation *DIL = I.getDebugLoc().get(); + if (!DIL) + continue; + + uint32_t Off = FunctionSamples::getOffset(DIL); + uint32_t Dis = DIL->getBaseDiscriminator(); + auto *FunctionSamples = Samples->findFunctionSamples(DIL); + if (!FunctionSamples) + continue; + + LLVM_DEBUG(dbgs() << "Instruction: " << *LI << '\n'); + LLVM_DEBUG(dbgs() << "Off: " << Off << ", Dis: " << Dis << '\n'); + + auto PrefetcherType = FunctionSamples->findCallTargetMapAt(Off, Dis); + if (!PrefetcherType) + continue; + + for (auto &KV : PrefetcherType.get()) { + auto Type = KV.first(); + if (Type == "__prefetch_nta_0") + Changed |= doByteOffsetPrefetch(LI, KV.second); + else + errs() << "Unsupported prefetcher type: " << Type << '\n'; + } + } + return Changed; +} + +bool PGOPrefetch::doByteOffsetPrefetch(LoadInst *I, uint64_t PrefetchDistance) { + LLVM_DEBUG(dbgs() << "Prefetch distance = " << PrefetchDistance + << "for load instruction: " << *I << '\n'); + + Value *PtrValue = I->getPointerOperand(); + IRBuilder<> Builder(I); + + Type *I8 = Type::getInt8Ty(I->getContext()); + Type *I32 = Type::getInt32Ty(I->getContext()); + Type *I64 = Type::getInt64Ty(I->getContext()); + + PointerType *I8PtrTy = PointerType::get(I8, + PtrValue->getType()->getPointerAddressSpace()); + Value *LoadPtrI8 = Builder.CreateBitCast(PtrValue, I8PtrTy, "load_ptr_i8"); + Value *PrefetchPtr = Builder.CreateGEP(I8, LoadPtrI8, + ConstantInt::get(I64, PrefetchDistance), "prefetch_ptr"); + + Function *PrefetchFunc = Intrinsic::getDeclaration( + I->getParent()->getModule(), Intrinsic::prefetch, PtrValue->getType()); + Builder.CreateCall(PrefetchFunc, {PrefetchPtr, ConstantInt::get(I32, 0), + ConstantInt::get(I32, 3), ConstantInt::get(I32, 1)}); + + return true; +} + +PreservedAnalyses PGOPrefetchPass::run(Function &F, + FunctionAnalysisManager &AM) { + PGOPrefetch Prefetcher(F.getContext(), PrefetchHintsFile); + bool Changed = Prefetcher.run(&F); + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + +} // namespace llvm + -- Gitee From cacf003bea8a3fa2c274366826e36983af53a382 Mon Sep 17 00:00:00 2001 From: Shanzhi Date: Sat, 6 Dec 2025 21:37:45 +0800 Subject: [PATCH 2/2] Add perf2afdo.sh --- llvm/utils/perf2afdo.sh | 203 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100755 llvm/utils/perf2afdo.sh diff --git a/llvm/utils/perf2afdo.sh b/llvm/utils/perf2afdo.sh new file mode 100755 index 000000000000..b36703d8e615 --- /dev/null +++ b/llvm/utils/perf2afdo.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +die() { + printf '%s\n' "$1" >&2 + exit 1 +} + +# External Dependency +create_llvm_prof=`which create_llvm_prof` + +check_extern_deps() { + if [ -z "${create_llvm_prof}" ]; then + die '[ERROR] "create_llvm_prof" is required but not found.' + fi +} + + +# Argument processing +prefetcher="" +binary_file="" +spe_sampling_file="" +prefetch_distance="" +top_N="" + + +parse_args() { + while :; do + case $1 in + --prefetcher) + if [ "$2" ]; then + prefetcher=$2 + shift + else + die '[ERROR] "--prefetcher" requires a non-empty option argument.' + fi + ;; + --prefetcher=?*) + prefetcher=${1#*=} + ;; + --prefetcher=) + die '[ERROR] "--prefetcher" requires a non-empty option argument.' + ;; + + --binary-file) + if [ "$2" ]; then + binary_file=$2 + shift + else + die '[ERROR] "--binary-file" requires a non-empty option argument.' + fi + ;; + --binary-file=?*) + binary_file=${1#*=} + ;; + --binary-file=) + die '[ERROR] "--binary-file" requires a non-empty option argument.' + ;; + + --spe-sampling-file) + if [ "$2" ]; then + spe_sampling_file=$2 + shift + else + die '[ERROR] "--spe-sampling-file" requires a non-empty option argument.' + fi + ;; + --spe-sampling-file=?*) + spe_sampling_file=${1#*=} + ;; + --spe-sampling-file=) + die '[ERROR] "--spe-sampling-file" requires a non-empty option argument.' + ;; + + + --prefetch-distance) + if [ "$2" ]; then + prefetch_distance=$2 + shift + else + die '[ERROR] --prefetch-distance" requires a non-empty option argument.' + fi + ;; + --prefetch-distance=?*) + prefetch_distance=${1#*=} + ;; + --prefetch-distance=) + die '[ERROR] "--prefetch-distance" requires a non-empty option argument.' + ;; + + --top-N) + if [ "$2" ]; then + top_N=$2 + shift + else + die '[ERROR] --top-N" requires a non-empty option argument.' + fi + ;; + --top-N=?*) + top_N=${1#*=} + ;; + --top-N=) + die '[ERROR] "--top-N" requires a non-empty option argument.' + ;; + + --) + shift + break + ;; + -?*) + printf '[WARNING] Unknown option (ignored): %s\n' "$1" >&2 + ;; + *) + break + ;; + esac + + shift + done +} + +verify_binary_file() { + if [ -z "${binary_file}" ]; then + die '[ERROR] "--binary-file=" should be specified.' + fi + if [ ! -f "${binary_file}" ]; then + die '[ERROR] No file exists in "${binary_file}".' + fi +} + +verify_spe_sampling_file() { + if [ -z "${spe_sampling_file}" ]; then + die '[ERROR] "--spe-sampling-file=" should be specified.' + fi + if [ ! -f "${spe_sampling_file}" ]; then + die '[ERROR] No file exists in "${spe_sampling_file}".' + fi +} + +verfiy_prefetch_distance() { + if [ -z "${prefetch_distance}" ]; then + die '[ERROR] "--prefetch-distance=" should be specified.' + fi + + local number_regex='^[0-9]+$' + if ! [[ ${prefetch_distance} =~ ${number_regex} ]]; then + die '[ERROR] Invalid number specified in "--prefetch-distance=${prefetch_distance}".' + fi +} + +verfiy_top_N() { + if [ -z "${top_N}" ]; then + die '[ERROR] "--top-N=" should be specified.' + fi + + local number_regex='^[0-9]+$' + if ! [[ ${top_N} =~ ${number_regex} ]]; then + die '[ERROR] Invalid number specified in "--top-N=${top_N}".' + fi +} + +run_byte_offset_prefetcher() { + printf '[INFO] Running byte-offset prefetcher...\n' + + verify_binary_file + verify_spe_sampling_file + verfiy_prefetch_distance + verfiy_top_N + + perf script -i "${spe_sampling_file}" --no-demangle -F event,ip,sym,symoff,dso --itrace=m \ + | awk -v event="llc-miss:" -v dso="(${binary_file})" '$1 == event && $NF == dso' \ + | awk 'NF--' | sort | uniq -c | sort -rn | tee llc-miss.txt + + head -n ${top_N} llc-miss.txt 2>&1 | tee llc-miss-top-N.txt + + printf '' >prefetch-hints.csv + while IFS= read -r line; do + sym_name_plus_offset=${line##* } + sym_name=${sym_name_plus_offset%+*} + sym_off=${sym_name_plus_offset#*+} + sym_regex=" ${sym_name}\$" + nm_result=$(nm ${binary_file} | grep ${sym_regex}) + sym_addr="0x${nm_result%% *}" + + ip_value=$((${sym_addr}+${sym_off})) + printf "0x%x,%d,nta\n" ${ip_value} ${prefetch_distance} >> prefetch-hints.csv + done < llc-miss-top-N.txt + + ${create_llvm_prof} --binary ${binary_file} --profile prefetch-hints.csv --profiler prefetch --format text \ + --out hints.afdo +} + + +check_extern_deps +parse_args $@ +case ${prefetcher} in + byte-offset) + run_byte_offset_prefetcher + ;; + *) + die '[ERROR] Unknown prefetcher' + ;; +esac + -- Gitee