From 050a449de46aadef9a73b181413657a758987bf6 Mon Sep 17 00:00:00 2001 From: Han-Chung Wang Date: Thu, 8 Aug 2024 09:05:55 -0700 Subject: [PATCH] [CPU] Switch to new pass generation tablegen definitions (#18132) This is mostly an NFC change. It does more cleanups on ConvertToLLVM pass which moves private options to tablegen definition. Also, the revision removes a duplicated option from the ConvertToLLVM pass. The additional change is that it adds dependent dialects to ConvertToLLVM pass. This is generally good because it removes the duplication of pass definition, and remove the magic numbers from the pass declarations. Before the change, we duplicate all the option default values to `Passes.h`. Some passes are still defined in `Passes.h` because passing tablegen option struct is a visual noise. It makes the pipeline configuration much cleaner. --------- Signed-off-by: hanhanW --- .../compiler/Codegen/Common/CPU/BUILD.bazel | 1 - .../Codegen/Common/CPU/CMakeLists.txt | 1 - .../Codegen/Common/CPU/CPULowerToUKernels.cpp | 30 ++--- .../Common/CPU/CPUMaterializeEncodings.cpp | 22 ++-- .../Codegen/Common/CPU/CPUPrepareUkernels.cpp | 11 +- .../compiler/Codegen/Common/CPU/PassDetail.h | 21 ---- .../iree/compiler/Codegen/Common/CPU/Passes.h | 24 ++-- .../compiler/Codegen/Common/CPU/Passes.td | 30 +++-- .../iree/compiler/Codegen/LLVMCPU/BUILD.bazel | 1 - .../compiler/Codegen/LLVMCPU/CMakeLists.txt | 1 - .../Codegen/LLVMCPU/ConvertToLLVM.cpp | 44 +++---- .../Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp | 11 +- .../LLVMCPU/LLVMCPU2DScalableTo1DScalable.cpp | 16 +-- .../LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp | 14 +-- .../LLVMCPU/LLVMCPUAssignImportOrdinals.cpp | 15 +-- .../LLVMCPUCheckIRBeforeLLVMConversion.cpp | 21 ++-- .../LLVMCPU/LLVMCPUDropVectorUnitDims.cpp | 17 ++- .../LLVMCPUEmitVectorizationRemarks.cpp | 13 +- .../LLVMCPU/LLVMCPULinkExecutables.cpp | 13 +- .../LLVMCPU/LLVMCPULowerExecutableTarget.cpp | 12 +- .../LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp | 20 ++- .../compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp | 13 +- .../LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp | 15 +-- .../Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp | 14 +-- .../LLVMCPUSynchronizeSymbolVisibility.cpp | 15 +-- .../compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp | 11 +- .../Codegen/LLVMCPU/LLVMCPUTileAndFuse.cpp | 13 +- ...LLVMCPUTileRootAndFuseProducerConsumer.cpp | 13 +- .../Codegen/LLVMCPU/LLVMCPUUnfuseFMAOps.cpp | 12 +- .../LLVMCPUVectorShapeCastLowering.cpp | 17 ++- .../LLVMCPU/LLVMCPUVectorTransferLowering.cpp | 16 ++- .../LLVMCPUVectorTransposeLowering.cpp | 22 ++-- .../LLVMCPUVerifyVectorSizeLegality.cpp | 13 +- .../LLVMCPU/LLVMCPUVirtualVectorLowering.cpp | 24 ++-- .../compiler/Codegen/LLVMCPU/PassDetail.h | 23 ---- .../iree/compiler/Codegen/LLVMCPU/Passes.cpp | 12 +- .../iree/compiler/Codegen/LLVMCPU/Passes.h | 115 ++---------------- .../iree/compiler/Codegen/LLVMCPU/Passes.td | 114 +++++++---------- .../LLVMCPU/VectorContractCustomKernels.cpp | 16 +-- .../LLVMCPU/VerifyLinalgTransformLegality.cpp | 13 +- 40 files changed, 286 insertions(+), 543 deletions(-) delete mode 100644 compiler/src/iree/compiler/Codegen/Common/CPU/PassDetail.h delete mode 100644 compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/CPU/BUILD.bazel index fb3b309c6d8d..acac444d2780 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/BUILD.bazel +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/BUILD.bazel @@ -28,7 +28,6 @@ iree_gentbl_cc_library( iree_compiler_cc_library( name = "PassHeaders", hdrs = [ - "PassDetail.h", "Passes.h", "Passes.h.inc", ], diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/CPU/CMakeLists.txt index 0b79e7d710c5..8b20fdfc321e 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/CMakeLists.txt @@ -23,7 +23,6 @@ iree_cc_library( NAME PassHeaders HDRS - "PassDetail.h" "Passes.h" "Passes.h.inc" DEPS diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/CPULowerToUKernels.cpp b/compiler/src/iree/compiler/Codegen/Common/CPU/CPULowerToUKernels.cpp index 7e09db44560a..8c1c76045a8e 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/CPULowerToUKernels.cpp +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/CPULowerToUKernels.cpp @@ -5,7 +5,6 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "iree/builtins/ukernel/exported_bits.h" -#include "iree/compiler/Codegen/Common/CPU/PassDetail.h" #include "iree/compiler/Codegen/Common/CPU/Passes.h" #include "iree/compiler/Codegen/Common/EncodingUtils.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" @@ -27,6 +26,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_CPULOWERTOUKERNELSPASS +#include "iree/compiler/Codegen/Common/CPU/Passes.h.inc" + // Returns the CastOpInterface op of the body, if // - the `genericOp` is element-wise with identity maps, and // - it has only a CastOpInterface op. @@ -54,32 +56,18 @@ getCastOpOfElementWiseCast(linalg::GenericOp genericOp) { namespace { class CPULowerToUKernelsPass - : public CPULowerToUKernelsBase { + : public impl::CPULowerToUKernelsPassBase { public: - CPULowerToUKernelsPass(bool skipIntermediateRoundings) - : skipIntermediateRoundings(skipIntermediateRoundings) {} - + using impl::CPULowerToUKernelsPassBase< + CPULowerToUKernelsPass>::CPULowerToUKernelsPassBase; + explicit CPULowerToUKernelsPass(bool skipIntermediateRoundings) { + this->skipIntermediateRoundings = skipIntermediateRoundings; + } void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } void runOnOperation() override; - - LogicalResult initializeOptions( - StringRef options, - function_ref errorHandler) override { - if (failed(Pass::initializeOptions(options, errorHandler))) { - return failure(); - } - // This option defaults to `true` both in Passes.td and in C++ code. - // If either side has `false`, that's a non-default choice, so we let that - // override a `true` on the other side. - skipIntermediateRoundings &= optionSkipIntermediateRoundings; - return success(); - } - -private: - bool skipIntermediateRoundings; }; } // namespace diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/CPUMaterializeEncodings.cpp b/compiler/src/iree/compiler/Codegen/Common/CPU/CPUMaterializeEncodings.cpp index cb46bd5c185a..bae4378eb371 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/CPUMaterializeEncodings.cpp +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/CPUMaterializeEncodings.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/Common/CPU/PassDetail.h" #include "iree/compiler/Codegen/Common/CPU/Passes.h" #include "iree/compiler/Codegen/Common/EncodingUtils.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" @@ -31,6 +30,10 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_CPUMATERIALIZEDEVICEENCODINGPASS +#define GEN_PASS_DEF_CPUMATERIALIZEHOSTENCODINGPASS +#include "iree/compiler/Codegen/Common/CPU/Passes.h.inc" + // Enumerate tile sizes to choose from when no specific architecture is // targeted. For narrow-{M,N} cases, this only enumerates on narrow M. The // narrow-N cases are handled by transposition in chooseMatmulTile. @@ -529,9 +532,8 @@ getFuncExecutableTargetAttrs(FunctionOpInterface funcOp, } struct CPUMaterializeHostEncodingPass - : public CPUMaterializeHostEncodingBase { - CPUMaterializeHostEncodingPass() = default; - + : public impl::CPUMaterializeHostEncodingPassBase< + CPUMaterializeHostEncodingPass> { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -585,19 +587,13 @@ struct CPUMaterializeHostEncodingPass } }; -std::unique_ptr createCPUMaterializeHostEncodingPass() { - return std::make_unique(); -} - // NOTE: this runs on host modules and executables and has two paths to handle // that. It should _not_ be running on both - target-specific codegen passes // are not allowed on host programs and it's a big violation of layering that // this exists. struct CPUMaterializeDeviceEncodingPass - : public CPUMaterializeDeviceEncodingBase< + : public impl::CPUMaterializeDeviceEncodingPassBase< CPUMaterializeDeviceEncodingPass> { - CPUMaterializeDeviceEncodingPass() = default; - void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -612,8 +608,4 @@ struct CPUMaterializeDeviceEncodingPass } }; -std::unique_ptr createCPUMaterializeDeviceEncodingPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/CPUPrepareUkernels.cpp b/compiler/src/iree/compiler/Codegen/Common/CPU/CPUPrepareUkernels.cpp index ac51f211234f..ca91e34af134 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/CPUPrepareUkernels.cpp +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/CPUPrepareUkernels.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/Common/CPU/PassDetail.h" #include "iree/compiler/Codegen/Common/CPU/Passes.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" #include "iree/compiler/Codegen/Utils/Utils.h" @@ -15,6 +14,8 @@ #include "mlir/Transforms/GreedyPatternRewriteDriver.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_CPUPREPAREUKERNELSPASS +#include "iree/compiler/Codegen/Common/CPU/Passes.h.inc" namespace { @@ -392,7 +393,7 @@ struct Convert5DUnPackto4DUnPackPattern }; struct CPUPrepareUkernelsPass - : public CPUPrepareUkernelsBase { + : public impl::CPUPrepareUkernelsPassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -437,10 +438,4 @@ void CPUPrepareUkernelsPass::runOnOperation() { return signalPassFailure(); } } - -std::unique_ptr> -createCPUPrepareUkernelsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/PassDetail.h b/compiler/src/iree/compiler/Codegen/Common/CPU/PassDetail.h deleted file mode 100644 index 25360a73c7e6..000000000000 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/PassDetail.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2023 The IREE Authors -// -// Licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#ifndef IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ -#define IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ - -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Pass/Pass.h" - -namespace mlir::iree_compiler { - -#define GEN_PASS_CLASSES -#include "iree/compiler/Codegen/Common/CPU/Passes.h.inc" - -} // namespace mlir::iree_compiler - -#endif // IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.h b/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.h index e6e39a032d48..df1b46562492 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.h +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.h @@ -19,24 +19,22 @@ namespace mlir::iree_compiler { -/// Convert encoding-specific operations based on target attributes. Examples: -/// encoding.set_encoding -> tensor.pack -/// encoding.unset_encoding -> tensor.unpack -/// linalg.matmul -> linalg.mmt4d -std::unique_ptr createCPUMaterializeHostEncodingPass(); -std::unique_ptr createCPUMaterializeDeviceEncodingPass(); +//------------------------------------------------------------------------------ +// Wrappers that not use tablegen options. +//------------------------------------------------------------------------------ + +std::unique_ptr> +createCPULowerToUKernelsPass(bool skipIntermediateRoundings); /// Adds CPU bufferization passes to the pipeline. void addCPUBufferizePasses(OpPassManager &funcPassManager); -/// Pass to lower a sequence of operations to a iree_codegen.ukernel.* -/// operation. -std::unique_ptr> -createCPULowerToUKernelsPass(bool skipIntermediateRoundings = true); +//----------------------------------------------------------------------------// +// Register Common CPU Passes +//----------------------------------------------------------------------------// -/// Pass to decompose batch_mmt4d/pack/etc to fit ukernel requirements. -std::unique_ptr> -createCPUPrepareUkernelsPass(); +#define GEN_PASS_DECL +#include "iree/compiler/Codegen/Common/CPU/Passes.h.inc" // IWYU pragma: keep void registerCodegenCommonCPUPasses(); diff --git a/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.td b/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.td index dd120bbe580a..8c73c5bca4a9 100644 --- a/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.td +++ b/compiler/src/iree/compiler/Codegen/Common/CPU/Passes.td @@ -13,37 +13,41 @@ include "mlir/Pass/PassBase.td" // Common Passes used for CPU-like backends (keep alphabetical) //===---------------------------------------------------------------------===// -def CPUMaterializeHostEncoding : +def CPUMaterializeHostEncodingPass : Pass<"iree-codegen-cpu-materialize-host-encoding", "mlir::ModuleOp"> { - let summary = "Materialize the encoding for tensor as specified by the backend."; - let constructor = "mlir::iree_compiler::createCPUMaterializeHostEncodingPass()"; + let summary = "Convert encoding-specific operations based on target attributes."; + let description = [{ + Examples: + encoding.set_encoding -> tensor.pack + encoding.unset_encoding -> tensor.unpack + linalg.matmul -> linalg.mmt4d "}]; } -def CPUMaterializeDeviceEncoding : +def CPUMaterializeDeviceEncodingPass : InterfacePass<"iree-codegen-cpu-materialize-device-encoding", "mlir::FunctionOpInterface"> { - let summary = "Materialize the encoding for tensor as specified by the backend."; - let constructor = "mlir::iree_compiler::createCPUMaterializeDeviceEncodingPass()"; + let summary = "Convert encoding-specific operations based on target attributes."; + let description = [{ + Examples: + encoding.set_encoding -> tensor.pack + encoding.unset_encoding -> tensor.unpack + linalg.matmul -> linalg.mmt4d "}]; } -def CPULowerToUKernels : +def CPULowerToUKernelsPass : Pass<"iree-codegen-cpu-lower-to-ukernels", ""> { let summary = "Separate out parts of the IR that lower to a micro-kernel"; - let constructor = - "mlir::iree_compiler::createCPULowerToUKernelsPass()"; let options = [ - Option<"optionSkipIntermediateRoundings", "skip-intermediate-roundings", + Option<"skipIntermediateRoundings", "skip-intermediate-roundings", "bool", /*default=*/"true", "Allow skipping intermediate roundings, e.g. in f16 ukernels internally doing f32 arithmetic.">, ]; } -def CPUPrepareUkernels : +def CPUPrepareUkernelsPass : InterfacePass<"iree-codegen-cpu-prepare-ukernels", "mlir::FunctionOpInterface"> { let summary = "Rank reduce operations to fit existing ukernels requirements." "For example, batch_mmt4d ops are decomposed to mmt4d ops"; - let constructor = - "mlir::iree_compiler::createCPUPrepareUkernelsPass()"; } #endif // IREE_CODEGEN_COMMON_CPU_PASSES diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel index 35c5e81e4284..b616bf0e426f 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel @@ -28,7 +28,6 @@ iree_gentbl_cc_library( iree_compiler_cc_library( name = "PassHeaders", hdrs = [ - "PassDetail.h", "Passes.h", "Passes.h.inc", ], diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt index a25c8973bf23..cd3f00904121 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt @@ -23,7 +23,6 @@ iree_cc_library( NAME PassHeaders HDRS - "PassDetail.h" "Passes.h" "Passes.h.inc" DEPS diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp index 107cae64a0f7..21847b418536 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp @@ -6,7 +6,6 @@ #include "iree/compiler/Codegen/Common/PassUtils.h" #include "iree/compiler/Codegen/LLVMCPU/DispatchABI.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "iree/compiler/Codegen/Utils/Utils.h" @@ -50,6 +49,7 @@ #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/Dialect/Math/IR/Math.h" #include "mlir/Dialect/Math/Transforms/Passes.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/MemRef/Transforms/Transforms.h" #include "mlir/Dialect/Tosa/IR/TosaOps.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" @@ -66,6 +66,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_CONVERTTOLLVMPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { template @@ -914,31 +917,21 @@ class ExpandMulSIExtended : public OpRewritePattern { } }; -class ConvertToLLVMPass : public ConvertToLLVMBase { +class ConvertToLLVMPass + : public impl::ConvertToLLVMPassBase { public: - ConvertToLLVMPass(bool reassociateFpReductions) { - targetReassociateFpReductions.setValue(reassociateFpReductions); + using impl::ConvertToLLVMPassBase::ConvertToLLVMPassBase; + explicit ConvertToLLVMPass(bool reassociateFpReductions) { + this->reassociateFpReductions = reassociateFpReductions; } - ConvertToLLVMPass(const ConvertToLLVMPass &pass) {} void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); + registry.insert(); } - void runOnOperation() override; - -private: - Option targetTriple{ - *this, "target-triple", llvm::cl::desc("Code generation target triple."), - llvm::cl::init("")}; - Option targetDataLayout{ - *this, "target-data-layout", - llvm::cl::desc("Code generation target data layout."), - llvm::cl::init("")}; - Option targetReassociateFpReductions{ - *this, "target-reassociate-fp-reductions", - llvm::cl::desc("Code generation target reassociate FP reductions."), - llvm::cl::init("false")}; }; } // namespace @@ -952,15 +945,14 @@ static std::string getStringAttrFromTargetAttr(ModuleOp module, void ConvertToLLVMPass::runOnOperation() { auto module = getOperation(); - std::string dataLayoutStr = targetDataLayout.getValue(); + std::string dataLayoutStr = targetDataLayout; if (targetDataLayout.empty()) { dataLayoutStr = getStringAttrFromTargetAttr(module, "data_layout"); } - std::string targetTripleStr = targetTriple.getValue(); + std::string targetTripleStr = targetTriple; if (targetTripleStr.empty()) { targetTripleStr = getStringAttrFromTargetAttr(module, "target_triple"); } - // Add required attributes to the module so that the lowering knows how to // handle structs and data layouts. module->setAttr(LLVM::LLVMDialect::getTargetTripleAttrName(), @@ -1056,8 +1048,8 @@ void ConvertToLLVMPass::runOnOperation() { // unroll them to 1-D before converting to the LLVM dialect. vector::populateVectorBitCastLoweringPatterns(patterns); populateVectorToLLVMMatrixConversionPatterns(typeConverter, patterns); - populateVectorToLLVMConversionPatterns( - typeConverter, patterns, targetReassociateFpReductions.getValue()); + populateVectorToLLVMConversionPatterns(typeConverter, patterns, + reassociateFpReductions); if (isAArch64(targetAttr) && (hasAnySVEFeature(targetAttr) || hasSMEFeature(targetAttr))) { diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp index 2c2eae55efee..378bd7bc3216 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" @@ -14,6 +13,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_EXPANDF16OPTOF32PASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// A pattern that expands floating-point arithmetic/math operations with f16 @@ -52,7 +54,7 @@ struct ExpandF16OpToF32Pattern : public OpRewritePattern { }; struct ExpandF16OpToF32Pass - : public ExpandF16OpToF32Base { + : public impl::ExpandF16OpToF32PassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -71,9 +73,4 @@ struct ExpandF16OpToF32Pass } }; } // namespace - -std::unique_ptr createExpandF16OpToF32Pass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPU2DScalableTo1DScalable.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPU2DScalableTo1DScalable.cpp index 144bd8cff9e7..917fe500513f 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPU2DScalableTo1DScalable.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPU2DScalableTo1DScalable.cpp @@ -6,7 +6,6 @@ #include "iree/compiler/Codegen/Common/TileSizeSelection.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "iree/compiler/Codegen/Utils/LinalgOpInfo.h" @@ -16,6 +15,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPU2DSCALABLETO1DSCALABLEPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// Currently, IREE requires `lowering_config`s to be propagated to all compute @@ -74,11 +76,11 @@ namespace { /// This can now be vectorized and lowered successfully, which produces a /// dispatch that mixes SME and SVE. class LLVMCPU2DScalableTo1DScalablePass - : public LLVMCPU2DScalableTo1DScalableBase< + : public impl::LLVMCPU2DScalableTo1DScalablePassBase< LLVMCPU2DScalableTo1DScalablePass> { public: - using LLVMCPU2DScalableTo1DScalableBase::LLVMCPU2DScalableTo1DScalableBase; - + using impl::LLVMCPU2DScalableTo1DScalablePassBase< + LLVMCPU2DScalableTo1DScalablePass>::LLVMCPU2DScalableTo1DScalablePassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry .insert(); @@ -171,10 +173,4 @@ void LLVMCPU2DScalableTo1DScalablePass::runOnOperation() { } } // namespace - -std::unique_ptr> -createLLVMCPU2DScalableTo1DScalablePass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp index 27f50274e331..3c1d9b79d01d 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp @@ -4,19 +4,20 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Pass/Pass.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUASSIGNCONSTANTORDINALSPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPUAssignConstantOrdinalsPass - : public LLVMCPUAssignConstantOrdinalsBase< + : public impl::LLVMCPUAssignConstantOrdinalsPassBase< LLVMCPUAssignConstantOrdinalsPass> { - LLVMCPUAssignConstantOrdinalsPass() = default; void runOnOperation() override { auto variantOp = getOperation(); @@ -48,12 +49,5 @@ struct LLVMCPUAssignConstantOrdinalsPass } } }; - } // namespace - -std::unique_ptr> -createLLVMCPUAssignConstantOrdinalsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp index d9665af3167f..c43fc8be75c6 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "llvm/ADT/MapVector.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -12,11 +11,14 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUASSIGNIMPORTORDINALSPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPUAssignImportOrdinalsPass - : public LLVMCPUAssignImportOrdinalsBase { - LLVMCPUAssignImportOrdinalsPass() = default; + : public impl::LLVMCPUAssignImportOrdinalsPassBase< + LLVMCPUAssignImportOrdinalsPass> { void runOnOperation() override { auto variantOp = getOperation(); @@ -71,12 +73,5 @@ struct LLVMCPUAssignImportOrdinalsPass } } }; - } // namespace - -std::unique_ptr> -createLLVMCPUAssignImportOrdinalsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp index 885681dbb117..674fd72a23e2 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp @@ -4,15 +4,18 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "llvm/Support/CommandLine.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" #include "mlir/Interfaces/ValueBoundsOpInterface.h" #include "mlir/Pass/Pass.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUCHECKIRBEFORELLVMCONVERSIONPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + static llvm::cl::opt clMaxAllocationSizeInBytes( "iree-llvmcpu-stack-allocation-limit", llvm::cl::desc("maximum allowed stack allocation size in bytes"), @@ -26,12 +29,11 @@ static llvm::cl::opt clAssumedVscaleValue( namespace { struct LLVMCPUCheckIRBeforeLLVMConversionPass - : LLVMCPUCheckIRBeforeLLVMConversionBase< + : impl::LLVMCPUCheckIRBeforeLLVMConversionPassBase< LLVMCPUCheckIRBeforeLLVMConversionPass> { - LLVMCPUCheckIRBeforeLLVMConversionPass(bool failOnOutOfBounds) { - this->failOnOutOfBounds = failOnOutOfBounds; - } - + using impl::LLVMCPUCheckIRBeforeLLVMConversionPassBase< + LLVMCPUCheckIRBeforeLLVMConversionPass>:: + LLVMCPUCheckIRBeforeLLVMConversionPassBase; void runOnOperation() override; }; } // namespace @@ -107,11 +109,4 @@ void LLVMCPUCheckIRBeforeLLVMConversionPass::runOnOperation() { return signalPassFailure(); } } - -std::unique_ptr> -createLLVMCPUCheckIRBeforeLLVMConversionPass(bool failOnOutOfBounds) { - return std::make_unique( - failOnOutOfBounds); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUDropVectorUnitDims.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUDropVectorUnitDims.cpp index 1b6770dde5dd..b27d0f0d7afd 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUDropVectorUnitDims.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUDropVectorUnitDims.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" @@ -14,11 +13,17 @@ #define DEBUG_TYPE "iree-llvmcpu-drop-vector-unit-dims" namespace mlir::iree_compiler { + +#define GEN_PASS_DEF_LLVMCPUDROPVECTORUNITDIMSPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { class LLVMCPUDropVectorUnitDimsPass - : public LLVMCPUDropVectorUnitDimsBase { + : public impl::LLVMCPUDropVectorUnitDimsPassBase< + LLVMCPUDropVectorUnitDimsPass> { public: - using LLVMCPUDropVectorUnitDimsBase::LLVMCPUDropVectorUnitDimsBase; + using impl::LLVMCPUDropVectorUnitDimsPassBase< + LLVMCPUDropVectorUnitDimsPass>::LLVMCPUDropVectorUnitDimsPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -46,10 +51,4 @@ void LLVMCPUDropVectorUnitDimsPass::runOnOperation() { (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns)); } } // namespace - -std::unique_ptr> -createLLVMCPUDropVectorUnitDimsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp index f3bcba90be69..09ad05deaf25 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp @@ -4,16 +4,19 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Linalg/Utils/Utils.h" #include "mlir/Pass/Pass.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUEMITVECTORIZATIONREMARKSPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPUEmitVectorizationRemarksPass - : LLVMCPUEmitVectorizationRemarksBase { + : impl::LLVMCPUEmitVectorizationRemarksPassBase< + LLVMCPUEmitVectorizationRemarksPass> { void runOnOperation() override; }; } // namespace @@ -29,10 +32,4 @@ void LLVMCPUEmitVectorizationRemarksPass::runOnOperation() { funcOp.emitWarning("found one or more ops not vectorized"); } } - -std::unique_ptr> -createLLVMCPUEmitVectorizationRemarksPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp index 6c7c45645d3f..8a2e91c6a646 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/Utils/LinkingUtils.h" #include "iree/compiler/Utils/ModuleUtils.h" @@ -13,10 +12,13 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPULINKEXECUTABLESPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPULinkExecutablesPass - : public LLVMCPULinkExecutablesBase { + : public impl::LLVMCPULinkExecutablesPassBase { LLVMCPULinkExecutablesPass() = default; void runOnOperation() override { auto moduleOp = getOperation(); @@ -69,12 +71,5 @@ struct LLVMCPULinkExecutablesPass } } }; - } // namespace - -std::unique_ptr> -createLLVMCPULinkExecutablesPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp index e0d94935e92f..766a7e206310 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp @@ -6,7 +6,6 @@ #include "iree/compiler/Codegen/Common/TileSizeSelection.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "iree/compiler/Codegen/Utils/CPUUtils.h" @@ -29,6 +28,9 @@ using mlir::iree_compiler::IREE::Codegen::LoweringConfigAttr; namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPULOWEREXECUTABLETARGETPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// Lowers an hal.executable.variant operation to scalar/native-vector /// code. Invokes different compilation pipeline to @@ -36,7 +38,7 @@ namespace { /// - then convert to LLVM dialect. /// In due course this could be used to generate code for all backends. class LLVMCPULowerExecutableTargetPass - : public LLVMCPULowerExecutableTargetBase< + : public impl::LLVMCPULowerExecutableTargetPassBase< LLVMCPULowerExecutableTargetPass> { public: LLVMCPULowerExecutableTargetPass() = default; @@ -167,10 +169,4 @@ void LLVMCPULowerExecutableTargetPass::runOnOperation() { return signalPassFailure(); } } - -std::unique_ptr> -createLLVMCPULowerExecutableTargetPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp index 7044f2c7c9c2..9134fa46c46c 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "llvm/Support/Debug.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -27,12 +26,16 @@ static llvm::cl::opt clMmt4dUseIntrinsics( namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUMMT4DVECTORLOWERINGPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPUMmt4dVectorLoweringPass - : public LLVMCPUMmt4dVectorLoweringBase { - LLVMCPUMmt4dVectorLoweringPass(bool enableVectorContractCustomKernels) { - this->enableVectorContractCustomKernels = enableVectorContractCustomKernels; - } + : public impl::LLVMCPUMmt4dVectorLoweringPassBase< + LLVMCPUMmt4dVectorLoweringPass> { + using impl::LLVMCPUMmt4dVectorLoweringPassBase< + LLVMCPUMmt4dVectorLoweringPass>::LLVMCPUMmt4dVectorLoweringPassBase; + void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -104,11 +107,4 @@ void LLVMCPUMmt4dVectorLoweringPass::runOnOperation() { } } } - -std::unique_ptr> -createLLVMCPUMmt4dVectorLoweringPass(bool enableVectorContractCustomKernels) { - return std::make_unique( - enableVectorContractCustomKernels); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp index e14807e14dd8..fcfd2c113732 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/Utils/Utils.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" @@ -21,6 +20,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUPEELPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { // Gathers tiled loops that aren't distribution loops from previous tiling @@ -49,7 +51,7 @@ void collectLoopsToPeel(Operation *op, } } -class LLVMCPUPeelPass : public LLVMCPUPeelBase { +class LLVMCPUPeelPass : public impl::LLVMCPUPeelPassBase { public: void getDependentDialects(DialectRegistry ®istry) const override { registry.insert> -createLLVMCPUPeelPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp index fdd3571a358d..5a680fce4213 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp @@ -8,7 +8,6 @@ #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" #include "iree/compiler/Codegen/LLVMCPU/KernelDispatch.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "iree/compiler/Dialect/HAL/IR/HALDialect.h" @@ -29,15 +28,15 @@ using mlir::iree_compiler::IREE::Codegen::LoweringConfigAttr; namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUSELECTLOWERINGSTRATEGYPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// Selects the lowering strategy for a hal.executable.variant operation. class LLVMCPUSelectLoweringStrategyPass - : public LLVMCPUSelectLoweringStrategyBase< + : public impl::LLVMCPUSelectLoweringStrategyPassBase< LLVMCPUSelectLoweringStrategyPass> { public: - LLVMCPUSelectLoweringStrategyPass() = default; - LLVMCPUSelectLoweringStrategyPass( - const LLVMCPUSelectLoweringStrategyPass &pass) {} void getDependentDialects(DialectRegistry ®istry) const override { // TODO(qedawkins): Once TransformStrategies is deprecated, drop the // unnecessary dialect registrations. @@ -113,10 +112,4 @@ void LLVMCPUSelectLoweringStrategyPass::runOnOperation() { } } } - -std::unique_ptr> -createLLVMCPUSelectLoweringStrategyPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp index a3ff80d91995..4ef391df6200 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp @@ -6,7 +6,6 @@ #include "iree/compiler/Codegen/Common/TileSizeSelection.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" @@ -24,6 +23,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUSPLITREDUCTIONPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// Make sure that @@ -163,12 +165,13 @@ LogicalResult splitReductionImpl(Operation *op, int64_t size, /// Pass to splitReduce linalg operations. class LLVMCPUSplitReductionPass - : public LLVMCPUSplitReductionBase { + : public impl::LLVMCPUSplitReductionPassBase { public: - LLVMCPUSplitReductionPass(bool fpReductionReordering) { + using impl::LLVMCPUSplitReductionPassBase< + LLVMCPUSplitReductionPass>::LLVMCPUSplitReductionPassBase; + explicit LLVMCPUSplitReductionPass(bool fpReductionReordering) { this->enableFpReductionReordering = fpReductionReordering; } - void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -215,13 +218,10 @@ void LLVMCPUSplitReductionPass::runOnOperation() { } } } - } // namespace - std::unique_ptr> createLLVMCPUSplitReductionPass(const bool enableFpReductionReordering) { return std::make_unique( enableFpReductionReordering); } - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp index 27ee115c2517..79cf17916dcd 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp @@ -4,13 +4,15 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Pass/Pass.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUSYNCHRONIZESYMBOLVISIBILITYPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { static void setVisibilityFromLinkage(SymbolOpInterface op, @@ -29,10 +31,8 @@ static void setVisibilityFromLinkage(SymbolOpInterface op, } struct LLVMCPUSynchronizeSymbolVisibilityPass - : public LLVMCPUSynchronizeSymbolVisibilityBase< + : public impl::LLVMCPUSynchronizeSymbolVisibilityPassBase< LLVMCPUSynchronizeSymbolVisibilityPass> { - LLVMCPUSynchronizeSymbolVisibilityPass() = default; - void runOnOperation() override { auto moduleOp = getOperation(); for (auto &op : moduleOp.getOps()) { @@ -44,12 +44,5 @@ struct LLVMCPUSynchronizeSymbolVisibilityPass } } }; - } // namespace - -std::unique_ptr> -createLLVMCPUSynchronizeSymbolVisibilityPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp index 47ad085c778b..aba35c0222ce 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp @@ -5,7 +5,6 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "llvm/Support/CommandLine.h" @@ -26,14 +25,18 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUTILEPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { /// This pass tiles all the TilingInterface operations. The `tilingLevel` must /// be specified. It picks the `tilingLevel`-th list as tiling sizes from /// lowering_config. -struct LLVMCPUTilePass : LLVMCPUTileBase { - LLVMCPUTilePass(int64_t tilingLevel = -1) { - this->tilingLevel.setValue(tilingLevel); +struct LLVMCPUTilePass : impl::LLVMCPUTilePassBase { + using impl::LLVMCPUTilePassBase::LLVMCPUTilePassBase; + explicit LLVMCPUTilePass(int64_t tilingLevel) { + this->tilingLevel = tilingLevel; } void getDependentDialects(DialectRegistry ®istry) const override { registry.insert { - LLVMCPUTileAndFusePass(int64_t tilingLevel = -1) { - this->tilingLevel.setValue(tilingLevel); +struct LLVMCPUTileAndFusePass + : impl::LLVMCPUTileAndFusePassBase { + using impl::LLVMCPUTileAndFusePassBase< + LLVMCPUTileAndFusePass>::LLVMCPUTileAndFusePassBase; + explicit LLVMCPUTileAndFusePass(int64_t tilingLevel) { + this->tilingLevel = tilingLevel; } void getDependentDialects(DialectRegistry ®istry) const override { registry.insert { - LLVMCPUTileRootAndFuseProducerConsumer(int64_t tilingLevel = -1) { - this->tilingLevel.setValue(tilingLevel); + using impl::LLVMCPUTileRootAndFuseProducerConsumerPassBase< + LLVMCPUTileRootAndFuseProducerConsumer>:: + LLVMCPUTileRootAndFuseProducerConsumerPassBase; + explicit LLVMCPUTileRootAndFuseProducerConsumer(int64_t tilingLevel) { + this->tilingLevel = tilingLevel; } void getDependentDialects(DialectRegistry ®istry) const override { registry.insert { namespace { struct LLVMCPUUnfuseFMAOpsPass - : LLVMCPUUnfuseFMAOpsBase { + : impl::LLVMCPUUnfuseFMAOpsPassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -58,10 +60,4 @@ void LLVMCPUUnfuseFMAOpsPass::runOnOperation() { return signalPassFailure(); } } - -std::unique_ptr> -createLLVMCPUUnfuseFMAOpsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorShapeCastLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorShapeCastLowering.cpp index 1114e1bafce4..3d4e37a511c2 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorShapeCastLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorShapeCastLowering.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" @@ -14,12 +13,18 @@ #define DEBUG_TYPE "iree-llvmcpu-vector-shape-cast-lowering" namespace mlir::iree_compiler { + +#define GEN_PASS_DEF_LLVMCPUVECTORSHAPECASTLOWERINGPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { class LLVMCPUVectorShapeCastLoweringPass - : public LLVMCPUVectorShapeCastLoweringBase< + : public impl::LLVMCPUVectorShapeCastLoweringPassBase< LLVMCPUVectorShapeCastLoweringPass> { public: - using LLVMCPUVectorShapeCastLoweringBase::LLVMCPUVectorShapeCastLoweringBase; + using impl::LLVMCPUVectorShapeCastLoweringPassBase< + LLVMCPUVectorShapeCastLoweringPass>:: + LLVMCPUVectorShapeCastLoweringPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -36,10 +41,4 @@ void LLVMCPUVectorShapeCastLoweringPass::runOnOperation() { (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns)); } } // namespace - -std::unique_ptr> -createLLVMCPUVectorShapeCastLoweringPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransferLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransferLowering.cpp index 017bc9554583..cd02cf39c66b 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransferLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransferLowering.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" @@ -17,12 +16,17 @@ #define DEBUG_TYPE "iree-llvmcpu-vector-transfer-lowering" namespace mlir::iree_compiler { + +#define GEN_PASS_DEF_LLVMCPUVECTORTRANSFERLOWERINGPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { class LLVMCPUVectorTransferLoweringPass - : public LLVMCPUVectorTransferLoweringBase< + : public impl::LLVMCPUVectorTransferLoweringPassBase< LLVMCPUVectorTransferLoweringPass> { public: - using LLVMCPUVectorTransferLoweringBase::LLVMCPUVectorTransferLoweringBase; + using impl::LLVMCPUVectorTransferLoweringPassBase< + LLVMCPUVectorTransferLoweringPass>::LLVMCPUVectorTransferLoweringPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert> -createLLVMCPUVectorTransferLoweringPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransposeLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransposeLowering.cpp index 7dd80e49e699..9c0081473f85 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransposeLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorTransposeLowering.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" @@ -15,6 +14,10 @@ #define DEBUG_TYPE "iree-llvmcpu-vector-transpose-lowering" namespace mlir::iree_compiler { + +#define GEN_PASS_DEF_LLVMCPUVECTORTRANSPOSELOWERINGPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { static bool has16x16Transpose(mlir::FunctionOpInterface funcOp) { @@ -36,14 +39,12 @@ static bool has16x16Transpose(mlir::FunctionOpInterface funcOp) { } class LLVMCPUVectorTransposeLoweringPass - : public LLVMCPUVectorTransposeLoweringBase< + : public impl::LLVMCPUVectorTransposeLoweringPassBase< LLVMCPUVectorTransposeLoweringPass> { public: - using LLVMCPUVectorTransposeLoweringBase::LLVMCPUVectorTransposeLoweringBase; - LLVMCPUVectorTransposeLoweringPass(bool lowerVectorTransposeToAVX2) { - this->lowerVectorTransposeToAVX2 = lowerVectorTransposeToAVX2; - } - + using impl::LLVMCPUVectorTransposeLoweringPassBase< + LLVMCPUVectorTransposeLoweringPass>:: + LLVMCPUVectorTransposeLoweringPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -84,11 +85,4 @@ void LLVMCPUVectorTransposeLoweringPass::runOnOperation() { (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns)); } } // namespace - -std::unique_ptr> -createLLVMCPUVectorTransposeLoweringPass(bool lowerVectorTransposeToAVX2) { - return std::make_unique( - lowerVectorTransposeToAVX2); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVerifyVectorSizeLegality.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVerifyVectorSizeLegality.cpp index 8c340840023d..a6e894032f97 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVerifyVectorSizeLegality.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVerifyVectorSizeLegality.cpp @@ -7,7 +7,6 @@ #include #include -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/Utils/Utils.h" #include "llvm/Support/Alignment.h" @@ -18,10 +17,18 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_LLVMCPUVERIFYVECTORSIZELEGALITYPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct LLVMCPUVerifyVectorSizeLegalityPass - : LLVMCPUVerifyVectorSizeLegalityBase { - LLVMCPUVerifyVectorSizeLegalityPass(int64_t maxAllowedNumberOfNativeVectors) { + : impl::LLVMCPUVerifyVectorSizeLegalityPassBase< + LLVMCPUVerifyVectorSizeLegalityPass> { + using impl::LLVMCPUVerifyVectorSizeLegalityPassBase< + LLVMCPUVerifyVectorSizeLegalityPass>:: + LLVMCPUVerifyVectorSizeLegalityPassBase; + explicit LLVMCPUVerifyVectorSizeLegalityPass( + int64_t maxAllowedNumberOfNativeVectors) { this->maxAllowedNumberOfNativeVectors = maxAllowedNumberOfNativeVectors; } diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVirtualVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVirtualVectorLowering.cpp index d836a70e6f90..ca293d15cfff 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVirtualVectorLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVirtualVectorLowering.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" #include "mlir/Dialect/ArmNeon/Transforms.h" @@ -17,18 +16,17 @@ #define DEBUG_TYPE "iree-llvmcpu-virtual-vector-lowering" namespace mlir::iree_compiler { + +#define GEN_PASS_DEF_LLVMCPUVIRTUALVECTORLOWERINGPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { class LLVMCPUVirtualVectorLoweringPass - : public LLVMCPUVirtualVectorLoweringBase< + : public impl::LLVMCPUVirtualVectorLoweringPassBase< LLVMCPUVirtualVectorLoweringPass> { public: - using LLVMCPUVirtualVectorLoweringBase::LLVMCPUVirtualVectorLoweringBase; - LLVMCPUVirtualVectorLoweringPass(std::string splitVectorTransfersTo, - bool enableArmI8mm) { - this->splitVectorTransfersTo = splitVectorTransfersTo; - this->enableArmI8mm = enableArmI8mm; - } - + using impl::LLVMCPUVirtualVectorLoweringPassBase< + LLVMCPUVirtualVectorLoweringPass>::LLVMCPUVirtualVectorLoweringPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -88,12 +86,4 @@ void LLVMCPUVirtualVectorLoweringPass::runOnOperation() { } } } // namespace - -std::unique_ptr> -createLLVMCPUVirtualVectorLoweringPass(std::string splitVectorTransfersTo, - bool enableArmI8mm) { - return std::make_unique( - splitVectorTransfersTo, enableArmI8mm); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h deleted file mode 100644 index f3801f73eb9e..000000000000 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 The IREE Authors -// -// Licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#ifndef IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ -#define IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ - -#include "iree/compiler/Dialect/HAL/IR/HALOps.h" -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Pass/Pass.h" - -namespace mlir::iree_compiler { - -#define GEN_PASS_CLASSES -#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" - -} // namespace mlir::iree_compiler - -#endif // IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_ diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp index 6a48eb7b7710..31accfce6a25 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp @@ -299,7 +299,8 @@ void buildLLVMCPUVectorLoweringPipeline( const LLVMCPUVectorLoweringPassOptions &options) { funcPassManager.addPass(createLLVMCPUDropVectorUnitDimsPass()); funcPassManager.addPass(createLLVMCPUVirtualVectorLoweringPass( - options.splitVectorTransfersTo, options.enableArmI8mm)); + LLVMCPUVirtualVectorLoweringPassOptions{options.splitVectorTransfersTo, + options.enableArmI8mm})); // Make sure we remove redundant vector ops (e.g., vector tranposes) before we // lower them and can't be optimized away anymore. @@ -307,7 +308,8 @@ void buildLLVMCPUVectorLoweringPipeline( funcPassManager.addPass(createLLVMCPUVectorTransferLoweringPass()); funcPassManager.addPass(createLLVMCPUVectorTransposeLoweringPass( - options.lowerVectorTransposeToAVX2)); + LLVMCPUVectorTransposeLoweringPassOptions{ + options.lowerVectorTransposeToAVX2})); // Potentially removes shape_cast and broadcast on unit dims before shape_cast // lowering. @@ -531,7 +533,8 @@ void addMmt4dTilingExpertPassPipeline(OpPassManager &funcPassManager, // Vector lowering of Mmt4d. funcPassManager.addPass(createLLVMCPUMmt4dVectorLoweringPass( - clEnableVectorContractCustomKernels)); + LLVMCPUMmt4dVectorLoweringPassOptions{ + clEnableVectorContractCustomKernels})); // Generic vector lowering. LLVMCPUVectorLoweringPassOptions options; @@ -702,7 +705,8 @@ static void addLowerToLLVMPasses(OpPassManager &modulePassManager, // Checking stack allocation before converting to CF dialect is easier. .addPass([&]() { return createLLVMCPUCheckIRBeforeLLVMConversionPass( - clFailOnOutOfBoundsStackAllocation); + LLVMCPUCheckIRBeforeLLVMConversionPassOptions{ + clFailOnOutOfBoundsStackAllocation}); }) // SCF -> CF .addPass(createConvertSCFToCFPass) diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h index 735e3561f036..bac30de708fa 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h @@ -19,77 +19,8 @@ namespace mlir::iree_compiler { class TilingConfig; -/// Performs the final conversion to LLVM dialect. -std::unique_ptr> -createConvertToLLVMPass(bool reassociateFpReordering = false); - -/// Checks CPU backend specific IR constraints (like no stack allocations) -std::unique_ptr> -createLLVMCPUCheckIRBeforeLLVMConversionPass(bool failOnOutOfBounds = true); - -std::unique_ptr> -createLLVMCPUEmitVectorizationRemarksPass(); - -/// Pass to select a lowering strategy for a hal.executable.variant operation. -/// The variant is annotated with the selected strategies, which are -/// subsequently ingested by LLVMCPULowerExecutableTargetPass. -std::unique_ptr> -createLLVMCPUSelectLoweringStrategyPass(); - -/// Pass to lower the module an hal.executable.variant operation to external -/// dialect. Currently this pass lowers to LLVM dialect, but could be -/// generalized to lower to any "final" dialect like SPIR-V/NVVM, etc. -std::unique_ptr> -createLLVMCPULowerExecutableTargetPass(); - -/// Pass to handel F16 bit operations, but converting f16 operands to F32. -/// Currently this pass is handeling fmaxf conversion from f16 to f32, -/// and then returing a f16 output back after preforming the operation. -/// Can handel more operations if required in future. -std::unique_ptr createExpandF16OpToF32Pass(); - -std::unique_ptr> -createLLVMCPUMmt4dVectorLoweringPass( - bool enableVectorContractCustomKernels = true); - -/// Pass to perform peeling on non-distributed loops. -std::unique_ptr> -createLLVMCPUPeelPass(); - -/// Pass to perform SplitReduction transformations of `LinalgOp`s. -std::unique_ptr> -createLLVMCPUSplitReductionPass(bool enableReassociateFpReductions = false); - -/// Synchronizes LLVM linkage with MLIR symbol visibility. -std::unique_ptr> -createLLVMCPUSynchronizeSymbolVisibilityPass(); - -/// Pass to tile and fuse TilingInterface ops with given tilingLevel. -std::unique_ptr> -createLLVMCPUTileAndFusePass(int64_t tilingLevel = -1); - -// Pass to Tile the Root Op and Fuse Producer and Consumer. -std::unique_ptr> -createLLVMCPUTileRootAndFuseProducerConsumer(int64_t tilingLevel = -1); - -std::unique_ptr> -createLLVMCPU2DScalableTo1DScalablePass(); - -/// Pass to tile TilingInterface ops with given tilingLevel. -std::unique_ptr> -createLLVMCPUTilePass(int64_t tilingLevel = -1); - -/// Replaces llvm.intr.fma with its unfused mul and add ops. -std::unique_ptr> -createLLVMCPUUnfuseFMAOpsPass(); - -/// Signals errors when there are large vectors in the IR. -std::unique_ptr> -createLLVMCPUVerifyVectorSizeLegalityPass( - int64_t maxAllowedNumberOfNativeVectors = 512); - //------------------------------------------------------------------------------ -// Passes to lower Vector ops before conversion to LLVM. +// Wrappers that not use tablegen options. //------------------------------------------------------------------------------ struct LLVMCPUVectorLoweringPassOptions { @@ -99,36 +30,23 @@ struct LLVMCPUVectorLoweringPassOptions { }; std::unique_ptr> -createLLVMCPUDropVectorUnitDimsPass(); +createLLVMCPUSplitReductionPass(bool enableReassociateFpReductions); std::unique_ptr> -createLLVMCPUVirtualVectorLoweringPass(std::string splitVectorTransfersTo = "", - bool enableArmI8mm = false); +createLLVMCPUTilePass(int64_t tilingLevel); std::unique_ptr> -createLLVMCPUVectorTransferLoweringPass(); +createLLVMCPUTileAndFusePass(int64_t tilingLevel); std::unique_ptr> -createLLVMCPUVectorTransposeLoweringPass( - bool lowerVectorTransposeToAVX2 = false); +createLLVMCPUTileRootAndFuseProducerConsumer(int64_t tilingLevel); std::unique_ptr> -createLLVMCPUVectorShapeCastLoweringPass(); - -std::unique_ptr> -createLLVMCPUVectorLoweringPass(); -std::unique_ptr> -createLLVMCPUVectorLoweringPass( - const LLVMCPUVectorLoweringPassOptions &options); - -/// A pass that converts certain vector.contract ops to custom kernels. -std::unique_ptr> -createVectorContractCustomKernelsPass(); +createLLVMCPUVerifyVectorSizeLegalityPass( + int64_t maxAllowedNumberOfNativeVectors); -// Verifies that only supported IR constructs are passed to the compiler (like -// no Linalg transform markers are set). -std::unique_ptr> -createVerifyLinalgTransformLegalityPass(); +std::unique_ptr> +createConvertToLLVMPass(bool reassociateFpReordering); //------------------------------------------------------------------------------ // LLVMCPU Codegen specific patterns. @@ -233,18 +151,6 @@ void buildLLVMCPUCodegenPassPipeline(OpPassManager &variantPassManager, // LLVMCPU Linking Passes and Pipelines //----------------------------------------------------------------------------// -/// Assigns executable constant ordinals across all LLVMCPU variants. -std::unique_ptr> -createLLVMCPUAssignConstantOrdinalsPass(); - -/// Assigns executable import ordinals across all LLVMCPU variants. -std::unique_ptr> -createLLVMCPUAssignImportOrdinalsPass(); - -/// Links LLVMCPU HAL executables within the top-level program module. -std::unique_ptr> -createLLVMCPULinkExecutablesPass(); - /// Populates passes needed to link HAL executables across LLVMCPU targets. void buildLLVMCPULinkingPassPipeline(OpPassManager &modulePassManager); @@ -252,6 +158,9 @@ void buildLLVMCPULinkingPassPipeline(OpPassManager &modulePassManager); // Register LLVMCPU Passes //----------------------------------------------------------------------------// +#define GEN_PASS_DECL +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" // IWYU pragma: keep + void registerCodegenLLVMCPUPasses(); } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td index 4a84d1a36cba..c85666bb1590 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td @@ -13,42 +13,47 @@ include "mlir/Pass/PassBase.td" // LLVMCPU Passes (keep alphabetical) //------------------------------------------------------------------------------ -def ConvertToLLVM : +def ConvertToLLVMPass : Pass<"iree-convert-to-llvm", "ModuleOp"> { let summary = "Perform final conversion from Linalg/HAL/Shape/Vector/Standard to LLVMIR dialect"; - let constructor = "mlir::iree_compiler::createConvertToLLVMPass()"; let options = [ Option<"reassociateFpReductions", "reassociateFpReductions", "bool", /*default=*/"false", "Specifies if FP add and mult reductions can be reordered">, + Option<"targetTriple", "target-triple", "std::string", /*default=*/"", + "Code generation target triple.">, + Option<"targetDataLayout", "target-data-layout", "std::string", + /*default=*/"", + "Code generation target data layout.">, ]; } -def ExpandF16OpToF32 : +def ExpandF16OpToF32Pass : Pass<"iree-llvmcpu-expand-f16-op-to-f32", ""> { let summary = "Preform f16 opertaions by expanding them to f32."; - let constructor = - "mlir::iree_compiler::createExpandF16OpToF32Pass()"; + let description = [{ + Pass to handel F16 bit operations, but converting f16 operands to F32. + Currently this pass is handeling fmaxf conversion from f16 to f32, + and then returing a f16 output back after preforming the operation. + Can handle more operations if required in future. + }]; } -def LLVMCPUAssignConstantOrdinals : +def LLVMCPUAssignConstantOrdinalsPass : Pass<"iree-llvmcpu-assign-constant-ordinals", "IREE::HAL::ExecutableVariantOp"> { let summary = "Assigns executable constant ordinals across all LLVMCPU variants."; - let constructor = "mlir::iree_compiler::createLLVMCPUAssignConstantOrdinalsPass()"; } -def LLVMCPUAssignImportOrdinals : +def LLVMCPUAssignImportOrdinalsPass : Pass<"iree-llvmcpu-assign-import-ordinals", "IREE::HAL::ExecutableVariantOp"> { let summary = "Assigns executable import ordinals across all LLVMCPU variants."; - let constructor = "mlir::iree_compiler::createLLVMCPUAssignImportOrdinalsPass()"; } -def LLVMCPUCheckIRBeforeLLVMConversion : +def LLVMCPUCheckIRBeforeLLVMConversionPass : InterfacePass<"iree-llvmcpu-check-ir-before-llvm-conversion", "mlir::FunctionOpInterface"> { let summary = "Checks CPU backend specific IR constraints (like no allocas)"; - let constructor = "mlir::iree_compiler::createLLVMCPUCheckIRBeforeLLVMConversionPass()"; let options = [ Option<"failOnOutOfBounds", "fail-on-out-of-bounds", "bool", "true", "Fails if the upper bound of dynamic stack allocation cannot be" @@ -56,28 +61,28 @@ def LLVMCPUCheckIRBeforeLLVMConversion : ]; } -def LLVMCPUEmitVectorizationRemarks : +def LLVMCPUEmitVectorizationRemarksPass : InterfacePass<"iree-llvmcpu-emit-vectorization-remarks", "mlir::FunctionOpInterface"> { let summary = "Emit vectorization remarks on Linalg ops"; - let constructor = - "mlir::iree_compiler::createLLVMCPUEmitVectorizationRemarksPass()"; } -def LLVMCPULinkExecutables : +def LLVMCPULinkExecutablesPass : Pass<"iree-llvmcpu-link-executables", "mlir::ModuleOp"> { let summary = "Links LLVMCPU HAL executables within the top-level program module."; - let constructor = "mlir::iree_compiler::createLLVMCPULinkExecutablesPass()"; } -def LLVMCPULowerExecutableTarget : +def LLVMCPULowerExecutableTargetPass : InterfacePass<"iree-llvmcpu-lower-executable-target", "mlir::FunctionOpInterface"> { let summary = "Lower executable target using an IREE::HAL::DispatchLoweringPassPipeline"; - let constructor = - "mlir::iree_compiler::createLLVMCPULowerExecutableTargetPass()"; + let description = [{ + Pass to lower the module an hal.executable.variant operation to external + dialect. Currently this pass lowers to LLVM dialect, but could be + generalized to lower to any "final" dialect like SPIR-V/NVVM, etc. + }]; } -def LLVMCPUMmt4dVectorLowering +def LLVMCPUMmt4dVectorLoweringPass : InterfacePass<"iree-llvmcpu-mmt4d-vector-lowering", "mlir::FunctionOpInterface"> { let summary = "Apply vector lowering logic to vector ops"; let options = [ @@ -85,28 +90,26 @@ def LLVMCPUMmt4dVectorLowering /*default=*/"true", "Flag to enable or disable vector contract custom kernels.">, ]; - let constructor = - "mlir::iree_compiler::createLLVMCPUMmt4dVectorLoweringPass()"; } -def LLVMCPUPeel : +def LLVMCPUPeelPass : InterfacePass<"iree-llvmcpu-peel", "mlir::FunctionOpInterface"> { let summary = "Pass to perform peeling on non-distributed loops."; - let constructor = - "mlir::iree_compiler::createLLVMCPUPeelPass()"; } -def LLVMCPUSelectLoweringStrategy : +def LLVMCPUSelectLoweringStrategyPass : Pass<"iree-llvmcpu-select-lowering-strategy", "ModuleOp"> { let summary = "Select a IREE::HAL::DispatchLoweringPassPipeline for lowering the variant"; - let constructor = - "mlir::iree_compiler::createLLVMCPUSelectLoweringStrategyPass()"; + let description = [{ + Pass to select a lowering strategy for a hal.executable.variant operation. + The variant is annotated with the selected strategies, which are + subsequently ingested by LLVMCPULowerExecutableTargetPass. + }]; } -def LLVMCPUSplitReduction : InterfacePass<"iree-llvmcpu-split-reduction", "mlir::FunctionOpInterface"> { +def LLVMCPUSplitReductionPass : InterfacePass<"iree-llvmcpu-split-reduction", "mlir::FunctionOpInterface"> { let summary = "Pass to splitReduce linalg operations."; - let constructor = "mlir::iree_compiler::createLLVMCPUSplitReductionPass()"; let options = [ Option<"enableFpReductionReordering", "enable-fp-reduction-reordering", "bool", /*default=*/"false", @@ -114,46 +117,39 @@ def LLVMCPUSplitReduction : InterfacePass<"iree-llvmcpu-split-reduction", "mlir: ]; } -def LLVMCPUSynchronizeSymbolVisibility : +def LLVMCPUSynchronizeSymbolVisibilityPass : Pass<"iree-llvmcpu-synchronize-symbol-visibility", "ModuleOp"> { let summary = "Synchronizes LLVM linkage with MLIR symbol visibility"; - let constructor = "mlir::iree_compiler::createLLVMCPUSynchronizeSymbolVisibilityPass()"; } -def LLVMCPUTile : +def LLVMCPUTilePass : InterfacePass<"iree-llvmcpu-tile", "mlir::FunctionOpInterface"> { let summary = "Pass to tile TilingInterface operations."; - let constructor = - "mlir::iree_compiler::createLLVMCPUTilePass()"; let options = [ Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1", "Use default tiling level used to retrieve the configuration from lowering_config"> ]; } -def LLVMCPUTileAndFuse : +def LLVMCPUTileAndFusePass : InterfacePass<"iree-llvmcpu-tile-and-fuse", "mlir::FunctionOpInterface"> { let summary = "Pass to tile and fuse TilingInterface operations."; - let constructor = - "mlir::iree_compiler::createLLVMCPUTileAndFusePass()"; let options = [ Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1", "Use default tiling level used to retrieve the configuration from lowering_config"> ]; } -def LLVMCPUTileRootAndFuseProducerConsumer : +def LLVMCPUTileRootAndFuseProducerConsumerPass : InterfacePass<"iree-llvmcpu-tile-root-and-fuse-producer-consumer", "mlir::FunctionOpInterface"> { let summary = "Pass to tile root op and fuse with producer and consumer TilingInterface ops."; - let constructor = - "mlir::iree_compiler::createLLVMCPUTileRootAndFuseProducerConsumer()"; let options = [ Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1", "Use default tiling level used to retrieve the configuration from lowering_config"> ]; } -def LLVMCPUVerifyVectorSizeLegality : +def LLVMCPUVerifyVectorSizeLegalityPass : InterfacePass<"iree-llvmcpu-verify-vector-size-legality", "mlir::FunctionOpInterface"> { let summary = "Signals errors when there are large vectors in the IR. I.e., one of" @@ -166,36 +162,30 @@ def LLVMCPUVerifyVectorSizeLegality : Option<"maxAllowedNumberOfNativeVectors", "max-allowed-number-of-native-vectors", "int64_t", /*default=*/"512", "The ratio used in the computation of max vector size."> ]; - let constructor = "mlir::iree_compiler::createLLVMCPUVerifyVectorSizeLegalityPass()"; } // Note: This pass is currently only required when targeting Arm SME (which is // the only target that currently has some concept of 2D scalability). -def LLVMCPU2DScalableTo1DScalable : +def LLVMCPU2DScalableTo1DScalablePass : InterfacePass<"iree-llvmcpu-2d-scalable-to-1d-scalable", "mlir::FunctionOpInterface"> { let summary = "Pass to replace unsupported scalable dimensions with loops."; - let constructor = - "mlir::iree_compiler::createLLVMCPU2DScalableTo1DScalablePass()"; let options = [ Option<"assumeArmSME", "assume-arm-sme", "bool", /*default=*/"false", "Assume the current target is ArmSME (used for testing)"> ]; } -def LLVMCPUUnfuseFMAOps : +def LLVMCPUUnfuseFMAOpsPass : InterfacePass<"iree-llvmcpu-unfuse-fma-pass", "mlir::FunctionOpInterface"> { let summary = "Convert llvm.fma into unfused mulf and addf ops"; - let constructor = "mlir::iree_compiler::createLLVMCPUUnfuseFMAOpsPass()"; } -def LLVMCPUDropVectorUnitDims : +def LLVMCPUDropVectorUnitDimsPass : InterfacePass<"iree-llvmcpu-drop-vector-unit-dims", "mlir::FunctionOpInterface"> { let summary = "Pass to drop vector unit dims."; - let constructor = - "mlir::iree_compiler::createLLVMCPUDropVectorUnitDimsPass()"; } -def LLVMCPUVirtualVectorLowering : +def LLVMCPUVirtualVectorLoweringPass : InterfacePass<"iree-llvmcpu-virtual-vector-lowering", "mlir::FunctionOpInterface"> { let summary = "Pass to lower high level vector operations like contract or multidim reduce ops to lower level vector ops."; let options = [ @@ -211,18 +201,14 @@ def LLVMCPUVirtualVectorLowering : /*default=*/ "false", "Enables arm i8mm lowering patterns">, ]; - let constructor = - "mlir::iree_compiler::createLLVMCPUVirtualVectorLoweringPass()"; } -def LLVMCPUVectorTransferLowering : +def LLVMCPUVectorTransferLoweringPass : InterfacePass<"iree-llvmcpu-vector-transfer-lowering", "mlir::FunctionOpInterface"> { let summary = "Pass to lower transfer ops to simpler ops like `vector.load`, `vector.store`, `vector.broadcast`, and a set of scf ops."; - let constructor = - "mlir::iree_compiler::createLLVMCPUVectorTransferLoweringPass()"; } -def LLVMCPUVectorTransposeLowering : +def LLVMCPUVectorTransposeLoweringPass : InterfacePass<"iree-llvmcpu-vector-transpose-lowering", "mlir::FunctionOpInterface"> { let summary = "Pass to lower vector.transpose ops."; let options = [ @@ -230,27 +216,21 @@ def LLVMCPUVectorTransposeLowering : /*default=*/"false", "Add specific transpose to avx2 lowering patterns.">, ]; - let constructor = - "mlir::iree_compiler::createLLVMCPUVectorTransposeLoweringPass()"; } -def LLVMCPUVectorShapeCastLowering : +def LLVMCPUVectorShapeCastLoweringPass : InterfacePass<"iree-llvmcpu-vector-shape-cast-lowering", "mlir::FunctionOpInterface"> { let summary = "Pass to lower vector.shape_cast ops."; - let constructor = - "mlir::iree_compiler::createLLVMCPUVectorShapeCastLoweringPass()"; } -def VectorContractCustomKernels : +def VectorContractCustomKernelsPass : InterfacePass<"iree-llvmcpu-vector-contract-custom-kernels", "mlir::FunctionOpInterface"> { let summary = "Enable custom kernels (inline assembly or intrinsics) for some vector.contract ops"; - let constructor = "mlir::iree_compiler::createVectorContractCustomKernelsPass()"; } -def VerifyLinalgTransformLegality : +def VerifyLinalgTransformLegalityPass : InterfacePass<"iree-llvmcpu-verify-linalg-transform-legality", "mlir::FunctionOpInterface"> { let summary = "Verify that only supported IR constructs are passed to the compiler."; - let constructor = "mlir::iree_compiler::createVerifyLinalgTransformLegalityPass()"; } #endif // IREE_CODEGEN_LLVMCPU_PASSES diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp index 695e8ec2b3d1..258fecb56052 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/LLVMCPU/Utils.h" #include "iree/compiler/Utils/StringUtils.h" @@ -26,6 +25,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_VECTORCONTRACTCUSTOMKERNELSPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { // Returns true if `contractionOp` is of the form @@ -1115,7 +1117,8 @@ struct MMT_8x4x8_i8i8i32_Aarch64Dotprod_Intrinsics }; class VectorContractCustomKernelsPass - : public VectorContractCustomKernelsBase { + : public impl::VectorContractCustomKernelsPassBase< + VectorContractCustomKernelsPass> { public: void getDependentDialects(DialectRegistry ®istry) const override { registry.insert> -createVectorContractCustomKernelsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp index 183b7898d0c5..0deb34b0778a 100644 --- a/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp +++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h" #include "iree/compiler/Codegen/LLVMCPU/Passes.h" #include "iree/compiler/Codegen/Utils/MarkerUtils.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" @@ -13,9 +12,13 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_VERIFYLINALGTRANSFORMLEGALITYPASS +#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc" + namespace { struct VerifyLinalgTransformLegalityPass - : VerifyLinalgTransformLegalityBase { + : impl::VerifyLinalgTransformLegalityPassBase< + VerifyLinalgTransformLegalityPass> { void runOnOperation() override; }; } // namespace @@ -33,10 +36,4 @@ void VerifyLinalgTransformLegalityPass::runOnOperation() { return signalPassFailure(); } } - -std::unique_ptr> -createVerifyLinalgTransformLegalityPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler