diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel index b6e7a7088373..ccd545988581 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel +++ b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel @@ -28,7 +28,6 @@ iree_gentbl_cc_library( iree_compiler_cc_library( name = "PassHeaders", hdrs = [ - "PassDetail.h", "Passes.h", "Passes.h.inc", ], diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt index a2ced1e0fc63..7f4ccd972613 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt @@ -23,7 +23,6 @@ iree_cc_library( NAME PassHeaders HDRS - "PassDetail.h" "Passes.h" "Passes.h.inc" DEPS diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp index 0aad84dcff4f..a5c7df978c95 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp @@ -17,7 +17,6 @@ #include #include -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Codegen/Utils/Utils.h" @@ -64,6 +63,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_CONVERTTOSPIRVPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { //===----------------------------------------------------------------------===// @@ -470,14 +472,17 @@ struct RemoveIdentityConversionCast final /// Converts remaining interface ops into SPIR-V global variables, GPU processor /// ID ops into SPIR-V global variables, loop/standard ops into corresponding /// SPIR-V ops. -class ConvertToSPIRVPass final : public ConvertToSPIRVBase { +class ConvertToSPIRVPass final + : public impl::ConvertToSPIRVPassBase { public: + using impl::ConvertToSPIRVPassBase< + ConvertToSPIRVPass>::ConvertToSPIRVPassBase; + explicit ConvertToSPIRVPass(unsigned indexBits) : indexBits(indexBits) {} + void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } - explicit ConvertToSPIRVPass(unsigned indexBits) : indexBits(indexBits) {} - LogicalResult initializeOptions( StringRef options, function_ref errorHandler) override { diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h b/compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h deleted file mode 100644 index bf460f532734..000000000000 --- a/compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 The IREE Authors -// -// Licensed under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#ifndef IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_ -#define IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_ - -#include "iree/compiler/Dialect/HAL/IR/HALOps.h" -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Pass/Pass.h" - -namespace mlir::iree_compiler { - -#define GEN_PASS_CLASSES -#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" - -} // namespace mlir::iree_compiler - -#endif // IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_ diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp index a9d50868ec3a..23e4fe5482f4 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp @@ -201,7 +201,7 @@ static void addMemRefLoweringPasses(OpPassManager &modulePassManager) { // Turn scalar load/store from memrefs into vectorized ones if possible. // This gives better memory access patterns, which is very important for // perf. - .addPass(createSPIRVVectorizeLoadStore) + .addPass(createSPIRVVectorizeLoadStorePass) // Perform optimizations that need to across the scf.for region boundary. .addPass(createForOpCanonicalizationPass) // Perform various vector-level cross-op optimizations like load-store @@ -385,8 +385,9 @@ void addSPIRVCooperativeMatrixVectorizePassPipeline( addBufferizePasses(funcPassManager, gpuAllocateWorkgroupMemoryFn); // Tile to GPU workgroups and promote. - funcPassManager.addPass(createSPIRVTileAndPromotePass( - /*promoteCMatrix=*/true, /*skipThreadLevel=*/true)); + funcPassManager.addPass( + createSPIRVTileAndPromotePass(SPIRVTileAndPromotePassOptions{ + /*promoteCMatrix=*/true, /*skipThreadLevel=*/true})); funcPassManager.addPass(createRemoveSingleIterationLoopPass()); // Run canonicalization patterns to propagate constant shape sizes after // removing trip-one loops. @@ -443,7 +444,7 @@ void addSPIRVCooperativeMatrixVectorizePassPipeline( funcPassManager.addPass(createForOpCanonicalizationPass()); funcPassManager.addPass(createCanonicalizerPass()); funcPassManager.addPass(createCSEPass()); - funcPassManager.addPass(createSPIRVVectorToGPUSubgroupMMAOpsPass()); + funcPassManager.addPass(createSPIRVVectorToGPUSubgroupMMAPass()); funcPassManager.addPass(createCanonicalizerPass()); funcPassManager.addPass(createCSEPass()); addSPIRVVectorLoweringPasses(funcPassManager); diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h index 31f9202d353c..fc391c7bff06 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h +++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h @@ -65,116 +65,6 @@ void buildSPIRVCodegenPassPipeline(OpPassManager &variantPassManager); /// Populates passes needed to link HAL executables across SPIRV targets. void buildSPIRVLinkingPassPipeline(OpPassManager &modulePassManager); -//===---------------------------------------------------------------------===// -// SPIR-V passes -//===---------------------------------------------------------------------===// - -/// Pass to perform the final conversion to SPIR-V dialect. -/// -/// This pass converts remaining interface ops into SPIR-V global variables, -/// GPU processor ID ops into SPIR-V global variables, loop/standard ops into -/// corresponding SPIR-V ops. -std::unique_ptr> -createConvertToSPIRVPass(unsigned indexWidth = 32); - -/// Annotates the innermost Winograd loops with the spirv distribute -/// attribute. -std::unique_ptr> -createSPIRVAnnotateWinogradLoopsPass(); - -/// Breaks down large vectors not natively supported by SPIR-V. -std::unique_ptr> -createSPIRVBreakDownLargeVectorPass(); - -// Converts #iree_gpu.target into #spirv.target_env. -std::unique_ptr> createSPIRVConvertGPUTargetPass(); - -/// Emulates bfloat 16 ops with 32-bit float ops. -std::unique_ptr> -createSPIRVEmulateBf16Pass(); - -/// Emulates 64-bit integer ops with 32-bit integer ops. -std::unique_ptr> createSPIRVEmulateI64Pass(); - -/// Turns static shaped storage buffer subspan ops into dynamic shaped ones. -std::unique_ptr> -createSPIRVEraseStorageBufferStaticShapePass(); - -/// Pass to perform final vector ops lowering to meet SPIR-V requirements. -std::unique_ptr> -createSPIRVFinalVectorLoweringPass(); - -/// Creates a pass to fold processor ID uses where possible. -std::unique_ptr> -createSPIRVFoldProcessorIDUsesPass(); - -/// Pass to perform initial vector ops lowering to meet SPIR-V requirements. -std::unique_ptr> -createSPIRVInitialVectorLoweringPass(); - -/// Links SPIR-V HAL executables within the top-level program module. -std::unique_ptr> createSPIRVLinkExecutablesPass(); - -/// Pass to set the lowering strategy for the target variant. -std::unique_ptr> -createSPIRVSelectLoweringStrategyPass(); - -/// Main pass to lower executables to scalar + vector code on SPIR-V path. -/// Invokes one of the pass pipelines that translate the executable to -/// scalar + vector code. -std::unique_ptr> -createSPIRVLowerExecutableTargetPass(); - -/// Pass to lower executables using Transform dialect on the SPIR-V backend. -/// This shouldnt be a separate pass, but it is since there are some -/// extra spir-v passes that need to be run as well. -std::unique_ptr> -createSPIRVLowerExecutableUsingTransformDialectPass(); - -/// Pass to map MemRef memory spaces to SPIR-V storage classes. -std::unique_ptr> -createSPIRVMapMemRefStorageClassPass(); - -/// Pass to materialize SPIR-V target requirements of hal.exectuable.variant -/// ops into hal.executable.condition regions. -std::unique_ptr> -createSPIRVMaterializeExecutableConditionsPass(); - -/// Pass to tile and distribute Linalg ops with buffer semantics to -/// invocations. -std::unique_ptr> -createSPIRVTileAndDistributePass(); - -/// Pass to promote Linalg ops with buffer semantics to use workgroup memory -/// and then tile to invocations. -std::unique_ptr> -createSPIRVTileAndPromotePass(bool promoteCMatrix = false, - bool skipThreadLevel = false); - -/// Pass to tile Linalg ops with buffer semantics suitable for lowering to -/// SPIR-V cooperative ops. -std::unique_ptr> -createSPIRVTileToCooperativeOpsPass(); - -// Trims the SPIR-V target environment of a HAL executable variant to the -// minimal requirement per the compiled spirv.module op needs. -std::unique_ptr> -createSPIRVTrimExecutableTargetEnvPass(); - -/// Converts vector ops to gpu subgroup MMA ops. -std::unique_ptr> -createSPIRVVectorToGPUSubgroupMMAOpsPass(); - -/// Converts memref of scalar to memref of vector of efficent size. This will -/// allow to convert memory accesses to vector load/store in SPIR-V without -/// having pointer bitcast. -std::unique_ptr> -createSPIRVVectorizeLoadStore(); - -/// Pass to do vectorization suitable for lowering to SPIR-V cooperative ops. -std::unique_ptr> -createSPIRVVectorizeToCooperativeOpsPass(); - /// Pass pipeline to lower IREE HAL executables by tiling and distributing to /// workgroups and invocations and vectorizing. Each invocation handles a /// vector. @@ -200,10 +90,25 @@ LogicalResult verifySPIRVMatmulPromoteVectorizePassPipeline( IREE::Codegen::TranslationInfoAttr translationInfo, ArrayRef workgroupSize); +//===---------------------------------------------------------------------===// +// Wrappers that not use tablegen options. +//===---------------------------------------------------------------------===// + +/// Pass to perform the final conversion to SPIR-V dialect. +/// +/// This pass converts remaining interface ops into SPIR-V global variables, +/// GPU processor ID ops into SPIR-V global variables, loop/standard ops into +/// corresponding SPIR-V ops. +std::unique_ptr> +createConvertToSPIRVPass(unsigned indexWidth); + //----------------------------------------------------------------------------// // Registration //----------------------------------------------------------------------------// +#define GEN_PASS_DECL +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" // IWYU pragma: keep + void registerCodegenSPIRVPasses(); } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td index 6436102b6e19..2420c3601074 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td +++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td @@ -13,157 +13,144 @@ include "mlir/Pass/PassBase.td" // SPIR-V passes (keep alphabetical) //===---------------------------------------------------------------------===// -def ConvertToSPIRV : Pass<"iree-convert-to-spirv", "ModuleOp"> { +def ConvertToSPIRVPass : Pass<"iree-convert-to-spirv", "ModuleOp"> { let summary = "Perform the final conversion to SPIR-V dialect"; - let constructor = "mlir::iree_compiler::createConvertToSPIRVPass()"; let options = [ Option<"indexBitsOption", "index-bits", "unsigned", /*default=*/"32", "Specify the bit widths for SPIR-V indices">, ]; } -def SPIRVAnnotateWinogradLoops : InterfacePass<"iree-spirv-annotate-winograd-loops", "mlir::FunctionOpInterface"> { +def SPIRVAnnotateWinogradLoopsPass : InterfacePass<"iree-spirv-annotate-winograd-loops", "mlir::FunctionOpInterface"> { let summary = "Annotate innermost Winograd loops with spirv distribute attribute"; - let constructor = "mlir::iree_compiler::createSPIRVAnnotateWinogradLoopsPass()"; } -def SPIRVBreakDownLargeVector : InterfacePass<"iree-spirv-breakdown-large-vector", - "mlir::FunctionOpInterface"> { +def SPIRVBreakDownLargeVectorPass : InterfacePass<"iree-spirv-breakdown-large-vector", + "mlir::FunctionOpInterface"> { let summary = "Break down vectors not natively supported by SPIR-V"; - let constructor = "mlir::iree_compiler::createSPIRVBreakDownLargeVectorPass()"; } -def SPIRVConvertGPUTarget : Pass<"iree-spirv-convert-gpu-target", "mlir::ModuleOp"> { +def SPIRVConvertGPUTargetPass : Pass<"iree-spirv-convert-gpu-target", "mlir::ModuleOp"> { let summary = "Convert #iree_gpu.target into #spirv.target_env"; - let constructor = "mlir::iree_compiler::createSPIRVConvertGPUTargetPass()"; } -def SPIRVEmulateI64 : +def SPIRVEmulateI64Pass : InterfacePass<"iree-spirv-emulate-i64", "mlir::FunctionOpInterface"> { let summary = "Emulate 64-bit integer ops with 32-bit integer ops"; - let constructor = "mlir::iree_compiler::createSPIRVEmulateI64Pass()"; } -def SPIRVEraseStorageBufferStaticShape : +def SPIRVEraseStorageBufferStaticShapePass : InterfacePass<"iree-spirv-erase-storage-buffer-static-shape", "mlir::FunctionOpInterface"> { let summary = "Turn static shaped storage buffer subspan ops into dynamic shaped ones"; - let constructor = "mlir::iree_compiler::createSPIRVEraseStorageBufferStaticShapePass()"; } -def SPIRVFinalVectorLowering : InterfacePass< +def SPIRVFinalVectorLoweringPass : InterfacePass< "iree-spirv-final-vector-lowering", "mlir::FunctionOpInterface"> { let summary = "Perform final lowering of vectors ops to fit SPIR-V"; - let constructor = "mlir::iree_compiler::createSPIRVFinalVectorLoweringPass()"; } -def SPIRVInitialVectorLowering : InterfacePass< +def SPIRVInitialVectorLoweringPass : InterfacePass< "iree-spirv-initial-vector-lowering", "mlir::FunctionOpInterface"> { let summary = "Perform initial lowering of vectors ops to fit SPIR-V"; - let constructor = "mlir::iree_compiler::createSPIRVInitialVectorLoweringPass()"; } -def SPIRVLinkExecutables : +def SPIRVLinkExecutablesPass : Pass<"iree-spirv-link-executables", "mlir::ModuleOp"> { let summary = "Links SPIR-V HAL executables within the top-level program module."; - let constructor = "mlir::iree_compiler::createSPIRVLinkExecutablesPass()"; } -def SPIRVLowerExecutableTarget : +def SPIRVLowerExecutableTargetPass : InterfacePass<"iree-spirv-lower-executable-target-pass", "mlir::FunctionOpInterface"> { let summary = "Lower the executable target to SPIR-V using one of the " "IREE::HAL::DispatchLoweringPassPipeline"; - let constructor = - "mlir::iree_compiler::createSPIRVLowerExecutableTargetPass()"; + let description = [{ + Main pass to lower executables to scalar + vector code on SPIR-V path. + Invokes one of the pass pipelines that translate the executable to + scalar + vector code. + }]; } -def SPIRVLowerExecutableUsingTransformDialect : +def SPIRVLowerExecutableUsingTransformDialectPass : Pass<"iree-spirv-lower-executable-using-transform-dialect", "ModuleOp"> { let summary = "Lower the executable target to SPIR-V using transform dialect" " followed by some passes to do SPIR-V specific vectorization"; - let constructor = "mlir::iree_compiler::createSPIRVLowerExecutableUsingTransformDialectPass()"; + let description = [{ + Pass to lower executables using Transform dialect on the SPIR-V backend. + This shouldnt be a separate pass, but it is since there are some + extra spir-v passes that need to be run as well. + }]; } -def SPIRVMapMemRefStorageClass : +def SPIRVMapMemRefStorageClassPass : InterfacePass<"iree-spirv-map-memref-storage-class", "mlir::FunctionOpInterface"> { let summary = "Map MemRef memory spaces to SPIR-V storage classes"; - let constructor = "mlir::iree_compiler::createSPIRVMapMemRefStorageClassPass()"; } -def SPIRVMaterializeExecutableConditions : +def SPIRVMaterializeExecutableConditionsPass : Pass<"iree-spirv-materialize-executable-conditions", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> { let summary = "Materialize SPIR-V target requirements of hal.exectuable.variant " "ops into hal.executable.condition regions"; - let constructor = - "mlir::iree_compiler::createSPIRVMaterializeExecutableConditionsPass()"; } -def SPIRVSelectLoweringStrategy : +def SPIRVSelectLoweringStrategyPass : Pass<"iree-spirv-select-lowering-strategy-pass", "ModuleOp"> { let summary = "Select the IREE::HAL::DispatchLoweringPassPipeline for lowering" "to SPIR-V"; - let constructor = - "mlir::iree_compiler::createSPIRVSelectLoweringStrategyPass()"; } -def SPIRVTileAndDistribute : InterfacePass<"iree-spirv-tile-and-distribute", "mlir::FunctionOpInterface"> { +def SPIRVTileAndDistributePass : InterfacePass<"iree-spirv-tile-and-distribute", "mlir::FunctionOpInterface"> { let summary = "Tile and distribute Linalg ops with buffer semantics to " "invocations"; - let constructor = "mlir::iree_compiler::createSPIRVTileAndDistributePass()"; } -def SPIRVTileAndPromote : InterfacePass<"iree-spirv-tile-and-promote", "mlir::FunctionOpInterface"> { +def SPIRVTileAndPromotePass : InterfacePass<"iree-spirv-tile-and-promote", "mlir::FunctionOpInterface"> { let summary = "Promote tiled Linalg ops with buffer semantics to use " "workgroup memory and then tile to invocations"; - let constructor = - "mlir::iree_compiler::createSPIRVTileAndPromotePass()"; let options = [ - Option<"promoteC", "promote-c", "bool", /*default=*/"false", + Option<"promoteCMatrix", "promote-c", "bool", /*default=*/"false", "Promote C matrix to use shared memory">, - Option<"skipThread", "skip-thread", "bool", /*default=*/"false", + Option<"skipThreadLevel", "skip-thread", "bool", /*default=*/"false", "Skip tiling and distributing to GPU threads">, ]; } -def SPIRVTileToCooperativeOps : InterfacePass< +def SPIRVTileToCooperativeOpsPass : InterfacePass< "iree-spirv-tile-to-cooperative-ops", "mlir::FunctionOpInterface"> { let summary = "Tile Linalg ops with buffer semantics to subgroups and " "vectorize to vector ops suitable for lowering to SPIR-V " "cooperative ops"; - let constructor = - "mlir::iree_compiler::createSPIRVTileToCooperativeOpsPass()"; } -def SPIRVTrimExecutableTargetEnv : +def SPIRVTrimExecutableTargetEnvPass : Pass<"iree-spirv-trim-executable-target-env", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> { let summary = "Trim the SPIR-V target environment of a HAL executable " "variant to the minimal requirement per the compiled " "spirv.module op needs"; - let constructor = - "mlir::iree_compiler::createSPIRVTrimExecutableTargetEnvPass()"; } -def SPIRVVectorizeLoadStore : +def SPIRVVectorizeLoadStorePass : InterfacePass<"iree-spirv-vectorize-load-store", "mlir::FunctionOpInterface"> { let summary = "Vectorize load/store of memrefs for better memory access"; - let constructor = "mlir::iree_compiler::createSPIRVVectorizeLoadStore()"; + let description = [{ + Converts memref of scalar to memref of vector of efficent size. This will + allow to convert memory accesses to vector load/store in SPIR-V without + having pointer bitcast. + }]; } -def SPIRVVectorizeToCooperativeOps : InterfacePass< +def SPIRVVectorizeToCooperativeOpsPass : InterfacePass< "iree-spirv-vectorize-to-cooperative-ops", "mlir::FunctionOpInterface"> { let summary = "Tile Linalg ops with buffer semantics to subgroups and " "vectorize to vector ops suitable for lowering to SPIR-V " "cooperative ops"; - let constructor = - "mlir::iree_compiler::createSPIRVVectorizeToCooperativeOpsPass()"; } -def SPIRVVectorToGPUSubgroupMMA : +def SPIRVVectorToGPUSubgroupMMAPass : InterfacePass<"iree-spirv-vector-to-gpu-subgroup-mma-ops", "mlir::FunctionOpInterface"> { let summary = "Pass to convert vector ops to GPU subgroup MMA ops."; - let constructor = "mlir::iree_compiler::createSPIRVVectorToGPUSubgroupMMAOpsPass()"; } #endif // IREE_CODEGEN_SPIRV_PASSES diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp index 00762649d5e8..474c334f6ca6 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp @@ -4,20 +4,23 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" #include "iree/compiler/Codegen/Utils/Utils.h" +#include "mlir/Pass/Pass.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVANNOTATEWINOGRADLOOPSPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { class SPIRVAnnotateWinogradLoopsPass final - : public SPIRVAnnotateWinogradLoopsBase { + : public impl::SPIRVAnnotateWinogradLoopsPassBase< + SPIRVAnnotateWinogradLoopsPass> { public: - SPIRVAnnotateWinogradLoopsPass() = default; - SPIRVAnnotateWinogradLoopsPass(const SPIRVAnnotateWinogradLoopsPass &pass) = - default; + using impl::SPIRVAnnotateWinogradLoopsPassBase< + SPIRVAnnotateWinogradLoopsPass>::SPIRVAnnotateWinogradLoopsPassBase; void runOnOperation() override { auto funcOp = getOperation(); @@ -38,10 +41,4 @@ class SPIRVAnnotateWinogradLoopsPass final } }; } // namespace - -std::unique_ptr> -createSPIRVAnnotateWinogradLoopsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp index f99aa307c070..3529e9281826 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "llvm/Support/MathExtras.h" #include "mlir/Dialect/Arith/IR/Arith.h" @@ -16,6 +15,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVBREAKDOWNLARGEVECTORPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { // Breaks down a chain of (bitcast -> extract -> extui) ops. @@ -136,7 +138,7 @@ struct BreakDownCastExtractExtend final : OpRewritePattern { }; struct SPIRVBreakDownLargeVectorPass final - : public SPIRVBreakDownLargeVectorBase { + : impl::SPIRVBreakDownLargeVectorPassBase { void runOnOperation() override { MLIRContext *context = &getContext(); RewritePatternSet patterns(context); @@ -163,10 +165,4 @@ struct SPIRVBreakDownLargeVectorPass final }; } // namespace - -std::unique_ptr> -createSPIRVBreakDownLargeVectorPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVConvertGPUTarget.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVConvertGPUTarget.cpp index d82ab9db9990..ee789500484e 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVConvertGPUTarget.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVConvertGPUTarget.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" #include "iree/compiler/Dialect/HAL/IR/HALOps.h" @@ -20,6 +19,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVCONVERTGPUTARGETPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { using IREE::GPU::ComputeBitwidths; @@ -252,7 +254,7 @@ convertGPUTarget(IREE::HAL::ExecutableVariantOp variant) { } struct SPIRVConvertGPUTargetPass final - : SPIRVConvertGPUTargetBase { + : impl::SPIRVConvertGPUTargetPassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -270,10 +272,4 @@ struct SPIRVConvertGPUTargetPass final }; } // namespace - -std::unique_ptr> -createSPIRVConvertGPUTargetPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp index 9fa07da472e5..6861e81781f8 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp @@ -12,7 +12,6 @@ //===----------------------------------------------------------------------===// #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUAttrs.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" #include "iree/compiler/Dialect/HAL/IR/HALDialect.h" @@ -38,6 +37,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVEMULATEI64PASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { //===----------------------------------------------------------------------===// @@ -165,7 +167,7 @@ static bool supportsI64(FunctionOpInterface op) { //===----------------------------------------------------------------------===// struct SPIRVEmulateI64Pass final - : public SPIRVEmulateI64Base { + : impl::SPIRVEmulateI64PassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -225,14 +227,4 @@ struct SPIRVEmulateI64Pass final }; } // namespace - -//===----------------------------------------------------------------------===// -// Public interface -//===----------------------------------------------------------------------===// - -std::unique_ptr> -createSPIRVEmulateI64Pass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp index f9c91e9e1499..817b121a6f2f 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Transforms/Transforms.h" #include "iree/compiler/Codegen/Utils/Utils.h" @@ -21,10 +20,13 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVERASESTORAGEBUFFERSTATICSHAPEPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { class EraseStorageBufferStaticShapePass final - : public SPIRVEraseStorageBufferStaticShapeBase< + : public impl::SPIRVEraseStorageBufferStaticShapePassBase< EraseStorageBufferStaticShapePass> { void runOnOperation() override; }; @@ -129,9 +131,4 @@ void EraseStorageBufferStaticShapePass::runOnOperation() { } } -std::unique_ptr> -createSPIRVEraseStorageBufferStaticShapePass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVFinalVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVFinalVectorLowering.cpp index f1dc537d0674..c6ccc2d94e25 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVFinalVectorLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVFinalVectorLowering.cpp @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "iree/compiler/Codegen/Common/Passes.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Transforms/Transforms.h" #include "llvm/Support/Debug.h" @@ -30,6 +29,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVFINALVECTORLOWERINGPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { void debugPrint(mlir::FunctionOpInterface funcOp, const char *message) { @@ -40,8 +42,9 @@ void debugPrint(mlir::FunctionOpInterface funcOp, const char *message) { }); } -class SPIRVFinalVectorLoweringPass - : public SPIRVFinalVectorLoweringBase { +class SPIRVFinalVectorLoweringPass final + : public impl::SPIRVFinalVectorLoweringPassBase< + SPIRVFinalVectorLoweringPass> { public: void getDependentDialects(DialectRegistry ®istry) const override { // vector.gather lowering patterns target scf ops. @@ -107,10 +110,4 @@ class SPIRVFinalVectorLoweringPass }; } // namespace - -std::unique_ptr> -createSPIRVFinalVectorLoweringPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVInitialVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVInitialVectorLowering.cpp index 500edacb99c2..493894675cd3 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVInitialVectorLowering.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVInitialVectorLowering.cpp @@ -14,7 +14,6 @@ #include "iree/compiler/Codegen/Common/Passes.h" #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUAttrs.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" #include "llvm/ADT/TypeSwitch.h" @@ -37,6 +36,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVINITIALVECTORLOWERINGPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { void debugPrint(Operation *op, const char *message) { @@ -276,8 +278,9 @@ bool supportsIntegerDotProductOps(mlir::FunctionOpInterface fn) { return true; } -class SPIRVInitialLoweringPass - : public SPIRVInitialVectorLoweringBase { +class SPIRVInitialLoweringPass final + : public impl::SPIRVInitialVectorLoweringPassBase< + SPIRVInitialLoweringPass> { public: void getDependentDialects(DialectRegistry ®istry) const override { // vector.gather lowering patterns target scf ops. @@ -501,10 +504,4 @@ class SPIRVInitialLoweringPass }; } // namespace - -std::unique_ptr> -createSPIRVInitialVectorLoweringPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLinkExecutables.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLinkExecutables.cpp index bd389783ebba..8716cd53d76e 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLinkExecutables.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLinkExecutables.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Codegen/Utils/LinkingUtils.h" @@ -20,6 +19,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVLINKEXECUTABLESPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace IREE::HAL { // Compares two ExecutableTargetAttr according to the alphabetical order of used // SPIR-V features. @@ -45,7 +47,7 @@ namespace { using IREE::HAL::ExecutableTargetAttr; struct SPIRVLinkExecutablesPass final - : SPIRVLinkExecutablesBase { + : impl::SPIRVLinkExecutablesPassBase { void runOnOperation() override { mlir::ModuleOp moduleOp = getOperation(); @@ -211,10 +213,4 @@ struct SPIRVLinkExecutablesPass final }; } // namespace - -std::unique_ptr> -createSPIRVLinkExecutablesPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp index 34e7a110321b..b706f82e7309 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp @@ -8,7 +8,6 @@ #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" #include "iree/compiler/Codegen/SPIRV/KernelConfig.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Utils/Utils.h" #include "iree/compiler/Dialect/HAL/IR/HALDialect.h" @@ -19,6 +18,7 @@ #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/MemRef/Transforms/Transforms.h" #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" @@ -32,6 +32,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVLOWEREXECUTABLETARGETPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + using CodeGenPipeline = IREE::Codegen::DispatchLoweringPassPipeline; namespace { @@ -39,11 +42,12 @@ namespace { /// code. Invokes different compilation pipeline to /// - first lower to scalar/native-vector code, /// - then convert to SPIRV dialect. -class SPIRVLowerExecutableTargetPass - : public SPIRVLowerExecutableTargetBase { +class SPIRVLowerExecutableTargetPass final + : public impl::SPIRVLowerExecutableTargetPassBase< + SPIRVLowerExecutableTargetPass> { public: - SPIRVLowerExecutableTargetPass() = default; - SPIRVLowerExecutableTargetPass(const SPIRVLowerExecutableTargetPass &pass) {} + using impl::SPIRVLowerExecutableTargetPassBase< + SPIRVLowerExecutableTargetPass>::SPIRVLowerExecutableTargetPassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry @@ -140,9 +144,4 @@ void SPIRVLowerExecutableTargetPass::runOnOperation() { } } -std::unique_ptr> -createSPIRVLowerExecutableTargetPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableUsingTransformDialect.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableUsingTransformDialect.cpp index efad52defef5..9c6f2a196fe6 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableUsingTransformDialect.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableUsingTransformDialect.cpp @@ -6,14 +6,16 @@ #include "iree-dialects/Dialect/LinalgTransform/Passes.h" #include "iree/compiler/Codegen/Common/Passes.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVLOWEREXECUTABLEUSINGTRANSFORMDIALECTPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { -class SPIRVLowerExecutableUsingTransformDialectPass - : public SPIRVLowerExecutableUsingTransformDialectBase< +class SPIRVLowerExecutableUsingTransformDialectPass final + : public impl::SPIRVLowerExecutableUsingTransformDialectPassBase< SPIRVLowerExecutableUsingTransformDialectPass> { public: void runOnOperation() override; @@ -69,9 +71,4 @@ void SPIRVLowerExecutableUsingTransformDialectPass::runOnOperation() { } } -std::unique_ptr> -createSPIRVLowerExecutableUsingTransformDialectPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp index e5d1f0602ffd..a94ad370fc8d 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp @@ -5,7 +5,6 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUAttrs.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" @@ -23,6 +22,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVMAPMEMREFSTORAGECLASSPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { template @@ -90,7 +92,7 @@ bool allowsKernelCapability(ArrayRef features) { } struct SPIRVMapMemRefStorageClassPass final - : public SPIRVMapMemRefStorageClassBase { + : impl::SPIRVMapMemRefStorageClassPassBase { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -138,10 +140,4 @@ struct SPIRVMapMemRefStorageClassPass final }; } // namespace - -std::unique_ptr> -createSPIRVMapMemRefStorageClassPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMaterializeExecutableConditions.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMaterializeExecutableConditions.cpp index aac9baded4a3..50086d04efa8 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMaterializeExecutableConditions.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMaterializeExecutableConditions.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Dialect/HAL/IR/HALOps.h" @@ -21,6 +20,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVMATERIALIZEEXECUTABLECONDITIONSPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { // The list of device features potentially required by a particular kernel. @@ -269,7 +271,7 @@ SmallVector getDeviceQueries(const KernelFeatures &features) { } struct SPIRVMaterializeExecutableConditionsPass final - : SPIRVMaterializeExecutableConditionsBase< + : impl::SPIRVMaterializeExecutableConditionsPassBase< SPIRVMaterializeExecutableConditionsPass> { void runOnOperation() override { IREE::HAL::ExecutableVariantOp variantOp = getOperation(); @@ -334,10 +336,4 @@ struct SPIRVMaterializeExecutableConditionsPass final }; } // namespace - -std::unique_ptr> -createSPIRVMaterializeExecutableConditionsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp index f1606cb7dad6..ef68e53ffe4f 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp @@ -8,13 +8,13 @@ #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUDialect.h" #include "iree/compiler/Codegen/SPIRV/KernelConfig.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Dialect/HAL/IR/HALDialect.h" #include "iree/compiler/Dialect/LinalgExt/IR/LinalgExtDialect.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" @@ -24,6 +24,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVSELECTLOWERINGSTRATEGYPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + using CodeGenPipeline = IREE::Codegen::DispatchLoweringPassPipeline; namespace { @@ -31,12 +34,12 @@ namespace { /// code. Invokes different compilation pipeline to /// - first lower to scalar/native-vector code, /// - then convert to SPIRV dialect. -class SPIRVSelectLoweringStrategyPass - : public SPIRVSelectLoweringStrategyBase { +class SPIRVSelectLoweringStrategyPass final + : public impl::SPIRVSelectLoweringStrategyPassBase< + SPIRVSelectLoweringStrategyPass> { public: - SPIRVSelectLoweringStrategyPass() = default; - SPIRVSelectLoweringStrategyPass(const SPIRVSelectLoweringStrategyPass &pass) { - } + using impl::SPIRVSelectLoweringStrategyPassBase< + SPIRVSelectLoweringStrategyPass>::SPIRVSelectLoweringStrategyPassBase; void getDependentDialects(DialectRegistry ®istry) const override { // TODO(qedawkins): Once TransformStrategies is deprecated, drop the @@ -120,9 +123,4 @@ void SPIRVSelectLoweringStrategyPass::runOnOperation() { } } -std::unique_ptr> -createSPIRVSelectLoweringStrategyPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp index 59d60714dc19..b8329ee73c99 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp @@ -13,7 +13,6 @@ #include "iree/compiler/Codegen/Common/Passes.h" #include "iree/compiler/Codegen/Common/Transforms.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Codegen/Transforms/Transforms.h" @@ -38,6 +37,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVTILEANDDISTRIBUTEPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + //===----------------------------------------------------------------------===// // Invocation tiling utils //===----------------------------------------------------------------------===// @@ -104,11 +106,11 @@ tileReduction(mlir::FunctionOpInterface funcOp, namespace { /// Function pass that implements tiling and distributing Linalg ops with /// buffer semantics. -class SPIRVTileAndDistributePass - : public SPIRVTileAndDistributeBase { +class SPIRVTileAndDistributePass final + : public impl::SPIRVTileAndDistributePassBase { public: - SPIRVTileAndDistributePass() = default; - SPIRVTileAndDistributePass(const SPIRVTileAndDistributePass &pass) = default; + using impl::SPIRVTileAndDistributePassBase< + SPIRVTileAndDistributePass>::SPIRVTileAndDistributePassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert> -createSPIRVTileAndDistributePass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp index 9ec9e775abc4..296bf8a031df 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp @@ -15,7 +15,6 @@ #include "iree/compiler/Codegen/Common/Passes.h" #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUDialect.h" #include "iree/compiler/Codegen/SPIRV/KernelConfig.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Codegen/Transforms/Transforms.h" @@ -37,6 +36,9 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVTILEANDPROMOTEPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + //====---------------------------------------------------------------------===// // Reduction tiling patterns //====---------------------------------------------------------------------===// @@ -107,37 +109,21 @@ static void populatePromotionPatterns(RewritePatternSet &patterns, namespace { class SPIRVTileAndPromotePass final - : public SPIRVTileAndPromoteBase { + : public impl::SPIRVTileAndPromotePassBase { public: - SPIRVTileAndPromotePass(bool promoteCMatrix, bool skipThreadLevel) - : promoteCMatrix(promoteCMatrix), skipThreadLevel(skipThreadLevel) {} + using impl::SPIRVTileAndPromotePassBase< + SPIRVTileAndPromotePass>::SPIRVTileAndPromotePassBase; void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } - LogicalResult initializeOptions( - StringRef options, - function_ref errorHandler) override { - if (failed(Pass::initializeOptions(options, errorHandler))) - return failure(); - // Consider pass option too - promoteCMatrix |= this->promoteC; - skipThreadLevel |= this->skipThread; - return success(); - } - void runOnOperation() override; private: /// Promotes C matrix to shared memory when necessary and returns success if /// no error happens. LogicalResult doPromoteCMatrix(mlir::FunctionOpInterface funcOp) const; - - // Whether to promote C matrix to use shared memory. - bool promoteCMatrix = false; - // Whether to skip thread level tiling and distribution. - bool skipThreadLevel = false; }; } // namespace @@ -352,10 +338,4 @@ LogicalResult SPIRVTileAndPromotePass::doPromoteCMatrix( return success(); } -std::unique_ptr> -createSPIRVTileAndPromotePass(bool promoteCMatrix, bool skipThreadLevel) { - return std::make_unique(promoteCMatrix, - skipThreadLevel); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp index 4a61471cd7f5..0161a4adc90b 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp @@ -18,7 +18,6 @@ #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenAttrs.h" #include "iree/compiler/Codegen/Dialect/GPU/IR/IREEGPUDialect.h" #include "iree/compiler/Codegen/SPIRV/KernelConfig.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Transforms/Transforms.h" #include "iree/compiler/Codegen/Utils/GPUUtils.h" @@ -44,6 +43,10 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVTILETOCOOPERATIVEOPSPASS +#define GEN_PASS_DEF_SPIRVVECTORIZETOCOOPERATIVEOPSPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { void debugPrint(Operation *op, const char *message) { @@ -324,7 +327,8 @@ class CombineContractTranspose final //===----------------------------------------------------------------------===// class SPIRVTileToCooperativeOpsPass final - : public SPIRVTileToCooperativeOpsBase { + : public impl::SPIRVTileToCooperativeOpsPassBase< + SPIRVTileToCooperativeOpsPass> { public: void getDependentDialects(DialectRegistry ®istry) const override { registry.insert { public: void getDependentDialects(DialectRegistry ®istry) const override { @@ -454,15 +458,4 @@ class SPIRVVectorizeToCooperativeOpsPass final }; } // namespace - -std::unique_ptr> -createSPIRVTileToCooperativeOpsPass() { - return std::make_unique(); -} - -std::unique_ptr> -createSPIRVVectorizeToCooperativeOpsPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTrimExecutableTargetEnv.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTrimExecutableTargetEnv.cpp index 9d07ebcd7d66..dbbb9c8e4542 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTrimExecutableTargetEnv.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTrimExecutableTargetEnv.cpp @@ -4,7 +4,6 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/SPIRV/Utils.h" #include "iree/compiler/Dialect/HAL/IR/HALOps.h" @@ -15,10 +14,14 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVTRIMEXECUTABLETARGETENVPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { struct SPIRVTrimExecutableTargetEnvPass final - : SPIRVTrimExecutableTargetEnvBase { + : impl::SPIRVTrimExecutableTargetEnvPassBase< + SPIRVTrimExecutableTargetEnvPass> { void runOnOperation() override { IREE::HAL::ExecutableVariantOp variant = getOperation(); if (!usesSPIRVCodeGen(variant)) { @@ -67,10 +70,4 @@ struct SPIRVTrimExecutableTargetEnvPass final }; } // namespace - -std::unique_ptr> -createSPIRVTrimExecutableTargetEnvPass() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp index 2f8f9a7e839a..15ab837197b5 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp @@ -5,7 +5,6 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h" -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "mlir/Conversion/VectorToGPU/VectorToGPU.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" @@ -16,9 +15,13 @@ namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVVECTORTOGPUSUBGROUPMMAPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + namespace { struct SPIRVVectorToGPUSubgroupMMAPass final - : public SPIRVVectorToGPUSubgroupMMABase { + : public impl::SPIRVVectorToGPUSubgroupMMAPassBase< + SPIRVVectorToGPUSubgroupMMAPass> { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp index 46343fb9291e..5934189e2805 100644 --- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp +++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#include "iree/compiler/Codegen/SPIRV/PassDetail.h" #include "iree/compiler/Codegen/SPIRV/Passes.h" #include "iree/compiler/Codegen/Utils/Utils.h" #include "iree/compiler/Dialect/Util/IR/UtilOps.h" @@ -37,6 +36,9 @@ constexpr int kMaxVectorNumElements = 4; namespace mlir::iree_compiler { +#define GEN_PASS_DEF_SPIRVVECTORIZELOADSTOREPASS +#include "iree/compiler/Codegen/SPIRV/Passes.h.inc" + //===----------------------------------------------------------------------===// // Utility Functions //===----------------------------------------------------------------------===// @@ -1019,7 +1021,8 @@ struct ReifyExtractOfCreateMask final //===----------------------------------------------------------------------===// class SPIRVVectorizeLoadStorePass final - : public SPIRVVectorizeLoadStoreBase { + : public impl::SPIRVVectorizeLoadStorePassBase< + SPIRVVectorizeLoadStorePass> { void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -1103,9 +1106,4 @@ void SPIRVVectorizeLoadStorePass::runOnOperation() { } } -std::unique_ptr> -createSPIRVVectorizeLoadStore() { - return std::make_unique(); -} - } // namespace mlir::iree_compiler