diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index d2c381d654bc..5a61b50db730 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -43,8 +43,10 @@ TORCHDYNAMO_CRASHING_SET, ONNX_CRASHING_SET, ONNX_XFAIL_SET, - FX_IMPORT_XFAIL_SET, + FX_IMPORTER_XFAIL_SET, FX_IMPORTER_CRASHING_SET, + FX_IMPORTER_STABLEHLO_XFAIL_SET, + FX_IMPORTER_STABLEHLO_CRASHING_SET, ) # Import tests to register them in the global registry. @@ -52,7 +54,8 @@ register_all_tests() def _get_argparse(): - config_choices = ["native_torch", "torchscript", "linalg", "stablehlo", "make_fx_tosa", "tosa", "lazy_tensor_core", "torchdynamo", "onnx", "fx_importer"] + config_choices = ["native_torch", "torchscript", "linalg", "stablehlo", "make_fx_tosa", "tosa", "lazy_tensor_core", + "torchdynamo", "onnx", "fx_importer", "fx_importer_stablehlo"] parser = argparse.ArgumentParser(description="Run torchscript e2e tests.") parser.add_argument("-c", "--config", choices=config_choices, @@ -67,6 +70,8 @@ def _get_argparse(): "lazy_tensor_core": run the model through the Lazy Tensor Core frontend and execute the traced graph. "torchdynamo": run the model through the TorchDynamo frontend and execute the graph using Linalg-on-Tensors. "onnx": export to the model via onnx and reimport using the torch-onnx-to-torch path. +"fx_importer": run the model through the fx importer frontend and execute the graph using Linalg-on-Tensors. +"fx_importer_stablehlo": run the model through the fx importer frontend and execute the graph using Stablehlo backend. """) parser.add_argument("-f", "--filter", default=".*", help=""" Regular expression specifying which tests to include in this run. @@ -127,8 +132,12 @@ def main(): crashing_set = LTC_CRASHING_SET elif args.config == "fx_importer": config = FxImporterTestConfig(RefBackendLinalgOnTensorsBackend()) - xfail_set = FX_IMPORT_XFAIL_SET + xfail_set = FX_IMPORTER_XFAIL_SET crashing_set = FX_IMPORTER_CRASHING_SET + elif args.config == "fx_importer_stablehlo": + config = FxImporterTestConfig(LinalgOnTensorsStablehloBackend(), "stablehlo") + xfail_set = FX_IMPORTER_STABLEHLO_XFAIL_SET + crashing_set = FX_IMPORTER_STABLEHLO_CRASHING_SET elif args.config == "torchdynamo": config = TorchDynamoTestConfig(RefBackendLinalgOnTensorsBackend()) xfail_set = TORCHDYNAMO_XFAIL_SET diff --git a/projects/pt1/e2e_testing/xfail_sets.py b/projects/pt1/e2e_testing/xfail_sets.py index 50c909936450..ed1f5eb8dd7a 100644 --- a/projects/pt1/e2e_testing/xfail_sets.py +++ b/projects/pt1/e2e_testing/xfail_sets.py @@ -391,7 +391,7 @@ "ElementwiseAddScalar_TensorLiteralInt32_Module_basic", } -FX_IMPORT_XFAIL_SET = { +FX_IMPORTER_XFAIL_SET = { 'AllBoolFalseModule_basic', 'AllBoolTrueModule_basic', 'AnyBoolFalseModule_basic', @@ -525,6 +525,408 @@ "HBC_basic", } +FX_IMPORTER_STABLEHLO_XFAIL_SET = { + "AdaptiveAvgPool3dDynamicNoBatch_basic", + "AdaptiveAvgPool3dDynamic_basic", + "AdaptiveMaxPool1dDynamicNoBatch_basic", + "AdaptiveMaxPool1dDynamic_basic", + "AdaptiveMaxPool1dStatic_basic", + "AdaptiveMaxPool2dDynamicNoBatch_basic", + "AdaptiveMaxPool2dDynamicWithIndices_basic", + "AdaptiveMaxPool2dDynamic_basic", + "AdaptiveMaxPool2dStaticWithIndices_basic", + "AdaptiveMaxPool2dStatic_basic", + "AdaptiveMaxPool3dDynamicNoBatch_basic", + "AdaptiveMaxPool3dDynamicWithIndices_basic", + "AdaptiveMaxPool3dDynamic_basic", + "AdaptiveMaxPool3dStaticWithIndices_basic", + "AdaptiveMaxPool3dStatic_basic", + "AllBoolFalseModule_basic", + "AllBoolTrueModule_basic", + "AnyBoolFalseModule_basic", + "AnyBoolTrueModule_basic", + "ArangeStartOutViewModule_basic", + "ArgminIntModule_basic", + "ArgminIntModule_multiple_mins", + "ArgminModule_basic", + "ArgminModule_keepDim", + "ArgminModule_with_dim", + "AtenComplexImagModule_basic", + "AtenComplexRealModule_basic", + "AtenComplexViewModule_basic", + "AtenDiagEmbedDefaultDiag_basic", + "AtenDiagEmbedDimDiag_basic", + "AtenDiagEmbedNegOffsetDiag_basic", + "AtenDiagEmbedNonDefault4DDiag_basic", + "AtenDiagEmbedOffsetDiag_basic", + "AtenDiagEmbedRevDimDiag_basic", + "AtenEmbeddingBagStaticModule_basic", + "AtenEmbeddingBagSumExample_basic", + "AtenFloatScalarModule_basic", + "AtenIntBoolOpConstFalseModule_basic", + "AtenIntBoolOpConstTrueModule_basic", + "AtenIntBoolOpModule_basic", + "AtenItemFpOpModule_basic", + "AtenMatmulQMixedSigni8Transpose_basic", + "AtenMatmulQMixedSigni8_basic", + "AtenMatmulQint8MV_basic", + "AtenMatmulQint8VM_basic", + "AtenMatmulQint8VV_basic", + "AtenMatmulQint8_basic", + "AtenMmQMixedSigni8_basic", + "AtenMmQint8_basic", + "AtenMmQuint8_basic", + "AtenRealView128Module_basic", + "AtenRealView64Module_basic", + "AtenSubFloatModule_basic", + "AtenTopKModule_basic", + "AtenTopKSmallestModule_basic", + "AtenTrilModule_basic", + "AtenTrilWithNegDiagonalModule_basic", + "AtenTrilWithPosDiagonalModule_basic", + "Aten_EmbeddingBagExample_basic", + "AvgPool2dDivisorOverrideModule_basic", + "BernoulliTensorModule_basic", + "BincountMinlengthModule_basic", + "BincountModule_basic", + "BincountStaticSizeModule_basic", + "BoolFloatConstantModule_basic", + "BoolFloatFalseModule_basic", + "BoolFloatTrueModule_basic", + "BoolIntConstantModule_basic", + "BoolIntFalseModule_basic", + "BoolIntTrueModule_basic", + "BroadcastDynamicDimModule_basic", + "CeilFloatModule_basic", + "ConstantBoolParameterModule_basic", + "ConstantPad2dStaticModule_basic", + "ConstantPadNdModule_basic", + "ConstantPadNdPartialStaticModule_basic", + "ConstantPadNdStaticModule_basic", + "ContainsIntList_False", + "ContainsIntList_True", + "Conv2dQInt8Module_basic", + "ConvTbcModule_basic", + "ConvolutionBackwardModule2DPadded_basic", + "ConvolutionBackwardModule2DStrided_basic", + "ConvolutionBackwardModule2D_basic", + "CumsumModule_basic", + "DiagonalModule_basic", + "DiagonalModule_nonsquare", + "DiagonalModule_transposed", + "DiagonalModule_with_dims", + "DiagonalModule_with_dims_and_offset", + "DiagonalModule_with_negative_dims", + "DiagonalModule_with_offset", + "DivFloatModule_basic", + "DivIntModule_basic", + "ElementwiseAcosIntModule_basic", + "ElementwiseAcosModule_basic", + "ElementwiseAcoshIntModule_basic", + "ElementwiseAcoshModule_basic", + "ElementwiseAddScalar_NumToTensorFloat_Module_basic", + "ElementwiseAsinIntModule_basic", + "ElementwiseAsinModule_basic", + "ElementwiseAsinhIntModule_basic", + "ElementwiseAsinhModule_basic", + "ElementwiseAtan2FloatIntModule_basic", + "ElementwiseAtan2TensorFloatModule_basic", + "ElementwiseAtan2TensorIntModule_basic", + "ElementwiseAtanTensorFloatModule_basic", + "ElementwiseAtanTensorIntModule_basic", + "ElementwiseAtanhIntModule_basic", + "ElementwiseAtanhModule_basic", + "ElementwiseBitwiseLeftShiftInt32Module_basic", + "ElementwiseBitwiseLeftShiftInt64Module_basic", + "ElementwiseBitwiseLeftShiftInt8Module_basic", + "ElementwiseBitwiseRightShiftInt32Module_basic", + "ElementwiseBitwiseRightShiftInt64Module_basic", + "ElementwiseBitwiseRightShiftInt8Module_basic", + "ElementwiseClampMinTensorFloatModule_basic", + "ElementwiseClampMinTensorIntModule_basic", + "ElementwiseClampTensorFloatModule_basic", + "ElementwiseClampTensorIntModule_basic", + "ElementwiseCosIntModule_basic", + "ElementwiseCoshIntModule_basic", + "ElementwiseCoshModule_basic", + "ElementwiseDequantizePerChannelModule_basic", + "ElementwiseDequantizePerTensorModule_basic", + "ElementwiseErfIntModule_basic", + "ElementwiseExpIntModule_basic", + "ElementwiseExpm1IntModule_basic", + "ElementwiseExpm1Module_basic", + "ElementwiseFmodTensor_Float_basic", + "ElementwiseFmodTensor_Int_Float_basic", + "ElementwiseFmodTensor_Int_basic", + "ElementwiseLog10IntModule_basic", + "ElementwiseLog10Module_basic", + "ElementwiseLog2IntModule_basic", + "ElementwiseLog2Module_basic", + "ElementwiseLogIntModule_basic", + "ElementwiseLogitModule_basic", + "ElementwiseMulTensorComplexModule_basic", + "ElementwisePowScalarModule_basic", + "ElementwiseQuantizePerTensorModule_basic", + "ElementwiseQuantizePerTensorUIntModule_basic", + "ElementwiseReciprocalIntModule_basic", + "ElementwiseRemainderTensorModule_Float_basic", + "ElementwiseRemainderTensorModule_Int_Float_basic", + "ElementwiseRemainderTensorModule_Int_basic", + "ElementwiseRsqrtIntModule_basic", + "ElementwiseSigmoidIntModule_basic", + "ElementwiseSinIntModule_basic", + "ElementwiseSqrtIntModule_basic", + "ElementwiseTanIntModule_basic", + "ElementwiseTanModule_basic", + "ElementwiseTernaryModule_basic", + "ElementwiseToDtypeI64ToUI8Module_basic", + "ElementwiseUnaryIntModule_basic", + "EmptyModule_uint8", + "EqIntModule_basic", + "ExponentialModule_basic", + "FakeQuantizePerTensorAffineDynamicShapeModule_basic", + "FakeQuantizePerTensorAffineModule_basic", + "FakeQuantizePerTensorAffineRoundToEvenModule_basic", + "Fill_TensorFloat32WithFloat32_basic", + "Fill_TensorFloat32WithFloat64_basic", + "Fill_TensorFloat32WithInt64_basic", + "FloatImplicitModule_basic", + "GeFloatIntModule_basic", + "GeFloatModule_basic", + "GeIntModule_basic", + "GtFloatIntModule_basic", + "GtIntModule_basic", + "HBC_basic", + "HardtanhBackward_basic", + "IndexPut1DFloatAccumulateModule_basic", + "IndexPut1DFloatNonAccumulateModule_basic", + "IndexPut1DIntAccumulateModule_basic", + "IndexPut1DIntNonAccumulateModule_basic", + "IndexPut2DFloatAccumulateModule_basic", + "IndexPut2DFloatNonAccumulateModule_basic", + "IndexPut2DIntAccumulateModule_basic", + "IndexPut2DIntNonAccumulateModule_basic", + "IndexPut3DFloatAccumulateModule_basic", + "IndexPut3DFloatNonAccumulateModule_basic", + "IndexPut3DIntAccumulateModule_basic", + "IndexPut3DIntNonAccumulateModule_basic", + "IndexPutHackedTwin1DFloatAccumulateModule_basic", + "IndexPutHackedTwin1DFloatNonAccumulateModule_basic", + "IndexPutHackedTwin1DIntAccumulateModule_basic", + "IndexPutHackedTwin1DIntNonAccumulateModule_basic", + "IndexPutHackedTwin2DFloatAccumulateModule_basic", + "IndexPutHackedTwin2DFloatNonAccumulateModule_basic", + "IndexPutHackedTwin2DIntAccumulateModule_basic", + "IndexPutHackedTwin2DIntNonAccumulateModule_basic", + "IndexPutHackedTwin3DFloatAccumulateModule_basic", + "IndexPutHackedTwin3DFloatNonAccumulateModule_basic", + "IndexPutHackedTwin3DIntAccumulateModule_basic", + "IndexPutHackedTwin3DIntNonAccumulateModule_basic", + "IndexPutImpl1DFloatAccumulateModule_basic", + "IndexPutImpl1DFloatNonAccumulateModule_basic", + "IndexPutImpl1DIntAccumulateModule_basic", + "IndexPutImpl1DIntNonAccumulateModule_basic", + "IndexPutImpl2DFloatAccumulateModule_basic", + "IndexPutImpl2DFloatNonAccumulateModule_basic", + "IndexPutImpl2DImplicitModule_basic", + "IndexPutImpl2DIndexModule_basic", + "IndexPutImpl2DNoneIndexStaticModule_basic", + "IndexPutImpl3DFloatAccumulateModule_basic", + "IndexPutImpl3DFloatNonAccumulateModule_basic", + "IndexPutImplIndexWithNoneModule_basic", + "IndexTensorNegativeIndexModule_basic", + "IntFloatModule_basic", + "IntImplicitModule_basic", + "IsFloatingPointFloat_True", + "IsFloatingPointInt_False", + "LenStrModule_basic", + "MaxPool2dCeilModeTrueModule_basic", + "MaxPool2dEmptyStrideStaticModule_basic", + "MaxPool2dStaticCeilModeTrueModule_basic", + "MaxPool2dWithIndicesBackwardDynamic3DModule_basic", + "MaxPool2dWithIndicesBackwardDynamic4DModule_basic", + "MaxPool2dWithIndicesBackwardStatic3DModule_basic", + "MaxPool2dWithIndicesBackwardStatic4DModule_basic", + "MaxPool3dCeilModeTrueModule_basic", + "MaxPool3dEmptyStrideStaticModule_basic", + "MaxPool3dLargeDatadModule_basic", + "MaxPool3dModuleRandomSimple_basic", + "MaxPool3dModule_basic", + "MaxPool3dStaticCeilModeTrueModule_basic", + "MaxPool3dStaticModule_basic", + "MeanDimNoneDimModule_basic", + "MseLossMeanReductionModule_basic", + "MseLossSumReductionWithDifferentElemTypeModule_basic", + "MulFloatModule_basic", + "NativeGroupNormBackwardModule_basic", + "NeFloatIntModule_basic", + "NeIntModule_basic", + "NllLossModuleBackward1DMeanWeight_basic", + "NllLossModuleBackward1DMean_basic", + "NllLossModuleBackward1DSumWeight_basic", + "NllLossModuleBackward1DSum_basic", + "NllLossModuleBackward1DWeight_basic", + "NllLossModuleBackward1D_basic", + "NllLossModuleBackwardMeanWeight_basic", + "NllLossModuleBackwardMean_basic", + "NllLossModuleBackwardSumWeight_basic", + "NllLossModuleBackwardSum_basic", + "NllLossModuleBackwardWeight_basic", + "NllLossModuleBackward_basic", + "NllLossModuleBackward_ignore_index", + "NormScalarComplexModule_basic", + "NormScalarModule_basic", + "NormalFunctionalModule_basic", + "NumToTensorFloatModule_basic", + "NumToTensorIntModule_basic", + "NumelModule_basic", + "NumelZeroRankModule_basic", + "PadModule_basic", + "PadWithNoneValModule_basic", + "PixelShuffleModuleFullDynamic_basic", + "PixelShuffleModuleSpatiallyDynamic_basic", + "PixelShuffleModuleSpatiallyStatic_basic", + "PixelShuffleModuleStaticRank3Int64_basic", + "PixelShuffleModuleStaticRank4Float32_basic", + "PowIntFloatModule_basic", + "PrimMaxIntModule_basic", + "PrimMinIntDynamicModule_basic", + "PrimMinIntModule_basic", + "PrimsSqueezeEmptyDimensionsModule_basic", + "PrimsSqueezeModule_basic", + "PrimsViewOfModule_basic", + "PrimsViewOfZeroRankModule_basic", + "QuantizedBatchedInputSingleLayer_basic", + "QuantizedMLP_basic", + "QuantizedNoLayer_basic", + "QuantizedSingleLayer_basic", + "RandnDtypeDeviceModule_basic", + "RandnGeneratorF64Module_basic", + "RandnGeneratorModule_basic", + "RandnLikeDtypeModule_basic", + "RandnLikeModule_basic", + "RandnModule_basic", + "ReduceAllDimBool_basic", + "ReduceAllDimEmpty_basic", + "ReduceAllDimFloat_basic", + "ReduceAllDimInt_basic", + "ReduceMaxAlongDimUnsignedInt_basic", + "ReduceMinAlongDimNegative_basic", + "ReduceMinAlongDimSignedInt_basic", + "ReduceMinAlongDimUnsignedInt_basic", + "ReduceMinAlongDim_basic", + "ReduceMinKeepDimReturnBoth_basic", + "ReduceMinKeepDim_basic", + "ReduceProdDimIntFloatModule_basic", + "ReflectionPad1dModule2dInput_Right", + "ReflectionPad1dModule2dInput_basic", + "ReflectionPad1dModule3dInput_Left", + "ReflectionPad1dModule3dInput_basic", + "ReflectionPad2dModule_Bottom", + "ReflectionPad2dModule_Left", + "ReflectionPad2dModule_Right", + "ReflectionPad2dModule_Top", + "ReflectionPad2dModule_basic", + "ReplicationPad2dModule_basic", + "ReplicationPad2dModule_bottom0", + "ReplicationPad2dModule_left0", + "ReplicationPad2dModule_right0", + "ReplicationPad2dModule_top0", + "RsubInt0d_NumToTensor_Module_basic", + "ScalarConstantTupleModule_basic", + "ScalarImplicitFloatModule_basic", + "ScaledDotProductAttentionDifferentModule_basic", + "ScatterReduceFloatMaxModule", + "ScatterReduceFloatMaxModuleIncludeSelf", + "ScatterReduceFloatMeanModule", + "ScatterReduceFloatMeanModuleIncludeSelf", + "ScatterReduceFloatMinModule", + "ScatterReduceFloatMinModuleIncludeSelf", + "ScatterReduceFloatProdModule", + "ScatterReduceFloatProdModuleIncludeSelf", + "ScatterReduceFloatSumModule", + "ScatterReduceFloatSumModuleIncludeSelf", + "ScatterReduceIntMaxModule", + "ScatterReduceIntMaxModuleIncludeSelf", + "ScatterReduceIntMeanModule", + "ScatterReduceIntMeanModuleIncludeSelf", + "ScatterReduceIntMinModule", + "ScatterReduceIntMinModuleIncludeSelf", + "ScatterReduceIntProdModule", + "ScatterReduceIntProdModuleIncludeSelf", + "ScatterReduceIntSumModule", + "ScatterReduceIntSumModuleIncludeSelf", + "ScatterSrcModule_basic", + "ScatterSrcStaticModule_basic", + "ScatterValueFloatModule_basic", + "ScatterValueIntModule_basic", + "SliceOutOfLowerBoundEndIndexModule_basic", + "SortIntListReverse_basic", + "SortIntList_basic", + "SortTensorDescending_basic", + "SortTensorInteger_basic", + "SortTensorNegativeDimension_basic", + "SortTensorSpecificDimension_basic", + "SortTensor_basic", + "SplitDimDynamicModule_basic", + "SplitDimStaticModule_basic", + "SqrtIntConstantModule_basic", + "SqrtIntModule_basic", + "SubFloatModule_basic", + "TModuleRank0_basic", + "TensorToBoolZeroRank_basic", + "TensorToBool_basic", + "TensorToFloatZeroRank_basic", + "TensorToFloat_basic", + "TensorToInt_basic", + "TestMultipleTensorAndPrimitiveTypesReturn_basic", + "Threshold1dFloatModule_basic", + "Threshold1dIntI32Module_basic", + "Threshold1dIntModule_basic", + "Threshold2dFloatModule_basic", + "Threshold2dIntModule_basic", + "Threshold3dFloatModule_basic", + "Threshold3dIntModule_basic", + "ThresholdBackward1dFloatModule_basic", + "ThresholdBackward1dIntModule_basic", + "ThresholdBackward1dMixedModule_basic", + "ThresholdBackward2dFloatModule_basic", + "ThresholdBackward2dIntModule_basic", + "ThresholdBackward2dMixedModule_basic", + "ThresholdBackward3dFloatModule_basic", + "ThresholdBackward3dIntModule_basic", + "ThresholdBackward3dMixedModule_basic", + "TorchPrimLoopForLikeModule_basic", + "TorchPrimLoopWhileLikeModule_basic", + "TraceModule_basic", + "TraceModule_empty", + "TraceModule_nonsquare", + "TraceSignedIntModule_basic", + "TraceUnsignedIntModule_basic", + "TraceUnsignedIntModule_empty", + "UnbindIntGetItem_Module_basic", + "UnbindIntListUnpack_Module_basic", + "UnsafeIndexPutHackedTwin1DFloatNonAccumulateModule_basic", + "UnsafeViewCollapseDynamicWithAtenSizeIntModule_basic", + "UpSampleNearest2dBackwardScalesNone_basic", + "UpSampleNearest2dBackward_basic", + "VarMeanBiasedModule_basic", + "VarMeanCorrectionNoneModule_basic", + "VarMeanUnbiasedModule_basic", + "ViewCollapseDynamicWithAtenSizeIntModule_basic", + "ViewSizeFromOtherTensor_basic", +} + +FX_IMPORTER_STABLEHLO_CRASHING_SET = { + "BatchNorm1DModule_basic", + "BatchNorm2DModule_basic", + "BatchNorm3DModule_basic", + "ResNet18Module_basic", + "ResNet18StaticModule_basic", + "MobilenetV3Module_basic", + "Conv2dBiasNoPaddingModule_basic", +} + STABLEHLO_PASS_SET = { "AdaptiveAvgPool1dNonUnitOutputSizeStaticModule_basic", "AdaptiveAvgPool1dUnitOutputSizeStaticModule_basic", diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/fx_importer_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/fx_importer_backend.py index 4d8ca8dc4968..0d75fe2ad3f0 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/fx_importer_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/fx_importer_backend.py @@ -82,9 +82,10 @@ def jit( class FxImporterTestConfig(TestConfig): """TestConfig that runs the torch.nn.Module with Fx Importer""" - def __init__(self, backend): + def __init__(self, backend, output_type="linalg-on-tensors"): super().__init__() - self.backend = backend + self._backend = backend + self._output_type = output_type def compile(self, program: torch.nn.Module) -> torch.nn.Module: return program @@ -95,9 +96,9 @@ def run(self, artifact: torch.nn.Module, trace: Trace) -> Trace: prog = torch.export.export(artifact, tuple(item.inputs)) module = jit(prog, func_name=artifact.__class__.__name__, - output_type="linalg-on-tensors") - module = self.backend.compile(module) - backend_module = self.backend.load(module) + output_type=self._output_type) + module = self._backend.compile(module) + backend_module = self._backend.load(module) params = { # **dict(artifact.named_parameters(remove_duplicate=False)), **dict(artifact.named_buffers(remove_duplicate=False)),