Introduced customized markers for backend errors

ci/default.yml

@@ -7,12 +7,9 @@ include:
     - nox -s test_model_stencils-3.10 -- --backend=$BACKEND --grid=$GRID
   parallel:
     matrix:
-    - BACKEND: [gtfn_cpu, gtfn_gpu, roundtrip]
+    - BACKEND: [embedded, gtfn_cpu, gtfn_gpu]
       GRID: [simple_grid, icon_grid]
   rules:
-    # exclude slow test configurations
-    - if: $BACKEND == "roundtrip" && $GRID == "icon_grid"
-      when: never
     - when: on_success
 # test_model_stencils_x86_64:
 #   extends: [.test_model_stencils, .test_template_x86_64]
@@ -31,11 +28,11 @@ test_tools_aarch64:
 .test_model_datatests:
   stage: test
   script:
-    - nox -s "test_model_datatest-3.10(subpackage='$COMPONENT')" -- --backend=$BACKEND 
+    - nox -s "test_model_datatest-3.10(subpackage='$COMPONENT')" -- --backend=$BACKEND
   parallel:
     matrix:
     - COMPONENT: [atmosphere/advection, atmosphere/diffusion, atmosphere/dycore, atmosphere/subgrid_scale_physics/microphysics, common, driver]
-      BACKEND: [gtfn_cpu, gtfn_gpu]
+      BACKEND: [embedded, gtfn_cpu, gtfn_gpu]
 # test_model_datatests_x86_64:
 #   extends: [.test_model_datatests, .test_template_x86_64]
 test_model_datatests_aarch64:

model/atmosphere/advection/tests/advection_stencil_tests/test_compute_ppm_all_face_values.py

@@ -21,6 +21,7 @@
 class TestComputePpmAllFaceValues(helpers.StencilTest):
     PROGRAM = compute_ppm_all_face_values
     OUTPUTS = ("p_face",)
+    MARKERS = (pytest.mark.requires_concat_where,)
 
     @staticmethod
     def reference(

model/atmosphere/advection/tests/advection_stencil_tests/test_reconstruct_cubic_coefficients_svd.py

@@ -33,6 +33,7 @@ class TestReconstructCubicCoefficientsSvd(helpers.StencilTest):
         "p_coeff_9_dsl",
         "p_coeff_10_dsl",
     )
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/advection/tests/advection_tests/test_advection.py

@@ -36,6 +36,7 @@
 # ------------------------------------
 
 
+@pytest.mark.embedded_remap_error
 @pytest.mark.datatest
 @pytest.mark.parametrize(
     "date, even_timestep, ntracer, horizontal_advection_type, horizontal_advection_limiter, vertical_advection_type, vertical_advection_limiter",

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_apply_diffusion_to_theta_and_exner.py

@@ -15,7 +15,6 @@
 from icon4py.model.common import dimension as dims
 from icon4py.model.common.grid import horizontal as h_grid
 from icon4py.model.common.utils.data_allocation import (
-    as_numpy,
     flatten_first_two_dims,
     random_field,
     random_mask,
@@ -35,6 +34,7 @@
 class TestApplyDiffusionToThetaAndExner(StencilTest):
     PROGRAM = apply_diffusion_to_theta_and_exner
     OUTPUTS = ("theta_v", "exner")
+    MARKERS = (pytest.mark.embedded_remap_error, pytest.mark.uses_as_offset)
 
     @staticmethod
     def reference(
@@ -82,10 +82,8 @@ def reference(
     def input_data(self, grid):
         # TODO [halungge]: understand why values do not verify intermittently
         # error message contained in truly_horizontal_diffusion_nabla_of_theta_over_steep_points_numpy
+        # TODO [nfarabullini]: set this as another marker
         pytest.xfail("fix start index issue")
-        if np.any(as_numpy(grid.connectivities[dims.C2E2CDim]) == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         kh_smag_e = random_field(grid, dims.EdgeDim, dims.KDim)
         inv_dual_edge_length = random_field(grid, dims.EdgeDim)
         theta_v_in = random_field(grid, dims.CellDim, dims.KDim)

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_apply_diffusion_to_w_and_compute_horizontal_gradients_for_turbulence.py

@@ -29,6 +29,7 @@
 class TestApplyDiffusionToWAndComputeHorizontalGradientsForTurbulence(StencilTest):
     PROGRAM = apply_diffusion_to_w_and_compute_horizontal_gradients_for_turbulence
     OUTPUTS = ("w", "dwdx", "dwdy")
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_apply_nabla2_to_w.py

@@ -36,6 +36,7 @@ def apply_nabla2_to_w_numpy(
 class TestMoApplyNabla2ToW(StencilTest):
     PROGRAM = apply_nabla2_to_w
     OUTPUTS = ("w",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_calculate_horizontal_gradients_for_turbulence.py

@@ -36,6 +36,7 @@ def calculate_horizontal_gradients_for_turbulence_numpy(
 class TestCalculateHorizontalGradientsForTurbulence(StencilTest):
     PROGRAM = calculate_horizontal_gradients_for_turbulence
     OUTPUTS = ("dwdx", "dwdy")
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_calculate_nabla2_and_smag_coefficients_for_vn.py

@@ -22,6 +22,7 @@
 class TestCalculateNabla2AndSmagCoefficientsForVn(StencilTest):
     PROGRAM = calculate_nabla2_and_smag_coefficients_for_vn
     OUTPUTS = ("kh_smag_e", "kh_smag_ec", "z_nabla2_e")
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -149,9 +150,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2C2VDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         u_vert = data_alloc.random_field(grid, dims.VertexDim, dims.KDim, dtype=vpfloat)
         v_vert = data_alloc.random_field(grid, dims.VertexDim, dims.KDim, dtype=vpfloat)
         smag_offset = vpfloat("9.0")

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_calculate_nabla2_for_w.py

@@ -31,6 +31,7 @@ def calculate_nabla2_for_w_numpy(
 class TestCalculateNabla2ForW(StencilTest):
     PROGRAM = calculate_nabla2_for_w
     OUTPUTS = ("z_nabla2_c",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_calculate_nabla4.py

@@ -66,6 +66,7 @@ def calculate_nabla4_numpy(
 class TestCalculateNabla4(StencilTest):
     PROGRAM = calculate_nabla4
     OUTPUTS = ("z_nabla4_e2",)
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -93,9 +94,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2C2VDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         u_vert = random_field(grid, dims.VertexDim, dims.KDim, dtype=vpfloat)
         v_vert = random_field(grid, dims.VertexDim, dims.KDim, dtype=vpfloat)
 

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_temporary_field_for_grid_point_cold_pools_enhancement.py

@@ -21,6 +21,7 @@
 class TestTemporaryFieldForGridPointColdPoolsEnhancement(StencilTest):
     PROGRAM = temporary_field_for_grid_point_cold_pools_enhancement
     OUTPUTS = ("enh_diffu_3d",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/diffusion/tests/diffusion_stencil_tests/test_truly_horizontal_diffusion_nabla_of_theta_over_steep_points.py

@@ -69,6 +69,7 @@ def truly_horizontal_diffusion_nabla_of_theta_over_steep_points_numpy(
 class TestTrulyHorizontalDiffusionNablaOfThetaOverSteepPoints(StencilTest):
     PROGRAM = truly_horizontal_diffusion_nabla_of_theta_over_steep_points
     OUTPUTS = ("z_temp",)
+    MARKERS = (pytest.mark.skip_value_error, pytest.mark.uses_as_offset)
 
     @staticmethod
     def reference(
@@ -98,9 +99,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.C2E2CDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         mask = random_mask(grid, dims.CellDim, dims.KDim)
 
         zd_vertoffset = zero_field(grid, dims.CellDim, dims.C2E2CDim, dims.KDim, dtype=gtx.int32)

model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py

@@ -13,7 +13,7 @@
 from icon4py.model.common.decomposition import definitions
 from icon4py.model.common.grid import vertical as v_grid
 from icon4py.model.common.utils import data_allocation as data_alloc
-from icon4py.model.testing import datatest_utils, parallel_helpers
+from icon4py.model.testing import datatest_utils, helpers, parallel_helpers
 
 from .. import utils
 
@@ -22,7 +22,7 @@
 @pytest.mark.parametrize("experiment", [datatest_utils.REGIONAL_EXPERIMENT])
 @pytest.mark.parametrize("ndyn_substeps", [2])
 @pytest.mark.parametrize("linit", [True, False])
-@pytest.mark.parametrize("orchestration", [True, False])
+@pytest.mark.parametrize("orchestration", [False, True])
 def test_parallel_diffusion(
     experiment,
     step_date_init,
@@ -44,8 +44,8 @@ def test_parallel_diffusion(
     backend,
     orchestration,
 ):
-    if orchestration and ("dace" not in backend.name.lower()):
-        raise pytest.skip("This test is only executed for `dace backends.")
+    if orchestration and not helpers.is_dace(backend):
+        raise pytest.skip("This test is only executed for `dace` backends.")
     caplog.set_level("INFO")
     parallel_helpers.check_comm_size(processor_props)
     print(
@@ -180,7 +180,7 @@ def test_parallel_diffusion_multiple_steps(
     caplog,
     backend,
 ):
-    if "dace" not in backend.name.lower():
+    if not helpers.is_dace(backend):
         raise pytest.skip("This test is only executed for `dace backends.")
     ######################################################################
     # Diffusion initialization

model/atmosphere/diffusion/tests/diffusion_tests/test_diffusion.py

@@ -404,7 +404,8 @@ def test_verify_diffusion_init_against_savepoint(
         (dt_utils.GLOBAL_EXPERIMENT, "2000-01-01T00:00:02.000", "2000-01-01T00:00:02.000"),
     ],
 )
-@pytest.mark.parametrize("ndyn_substeps, orchestration", [(2, [True, False])])
+@pytest.mark.parametrize("ndyn_substeps", [2])
+@pytest.mark.parametrize("orchestration", [False, True])
 def test_run_diffusion_single_step(
     savepoint_diffusion_init,
     savepoint_diffusion_exit,
@@ -419,8 +420,8 @@ def test_run_diffusion_single_step(
     backend,
     orchestration,
 ):
-    if orchestration and ("dace" not in backend.name.lower()):
-        pytest.skip(f"running backend = '{backend.name}': orchestration only on dace backends")
+    if not orchestration or not helpers.is_dace(backend):
+        pytest.skip("This test is only executed for orchestration on dace backends")
     grid = get_grid_for_experiment(experiment, backend)
     cell_geometry = get_cell_geometry_for_experiment(experiment, backend)
     edge_geometry = get_edge_geometry_for_experiment(experiment, backend)
@@ -522,8 +523,8 @@ def test_run_diffusion_multiple_steps(
     backend,
     icon_grid,
 ):
-    if "dace" not in backend.name.lower():
-        raise pytest.skip("This test is only executed for DaCe backends.")
+    if not helpers.is_dace(backend):
+        raise pytest.skip("This test is only executed for orchestration on dace backends")
     ######################################################################
     # Diffusion initialization
     ######################################################################
@@ -648,7 +649,8 @@ def test_run_diffusion_multiple_steps(
 
 @pytest.mark.datatest
 @pytest.mark.parametrize("experiment", [dt_utils.REGIONAL_EXPERIMENT])
-@pytest.mark.parametrize("linit, orchestration", [(True, [True, False])])
+@pytest.mark.parametrize("linit", [True])
+@pytest.mark.parametrize("orchestration", [False, True])
 def test_run_diffusion_initial_step(
     experiment,
     linit,
@@ -663,8 +665,8 @@ def test_run_diffusion_initial_step(
     backend,
     orchestration,
 ):
-    if orchestration and ("dace" not in backend.name.lower()):
-        pytest.skip(f"running backend = '{backend.name}': orchestration only on dace backends")
+    if not orchestration or not helpers.is_dace(backend):
+        pytest.skip("This test is only executed for orchestration only on dace backends")
     grid = get_grid_for_experiment(experiment, backend)
     cell_geometry = get_cell_geometry_for_experiment(experiment, backend)
     edge_geometry = get_edge_geometry_for_experiment(experiment, backend)

model/atmosphere/dycore/tests/dycore_stencil_tests/test_add_extra_diffusion_for_normal_wind_tendency_approaching_cfl.py

@@ -97,6 +97,7 @@ def add_extra_diffusion_for_normal_wind_tendency_approaching_cfl_numpy(
 class TestAddExtraDiffusionForNormalWindTendencyApproachingCfl(StencilTest):
     PROGRAM = add_extra_diffusion_for_normal_wind_tendency_approaching_cfl
     OUTPUTS = ("ddt_vn_apc",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @pytest.fixture
     def input_data(self, grid):

model/atmosphere/dycore/tests/dycore_stencil_tests/test_add_extra_diffusion_for_w_con_approaching_cfl.py

@@ -70,6 +70,7 @@ def add_extra_diffusion_for_w_con_approaching_cfl_numpy(
 class TestAddExtraDiffusionForWConApproachingCfl(StencilTest):
     PROGRAM = add_extra_diffusion_for_w_con_approaching_cfl
     OUTPUTS = ("ddt_w_adv",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/dycore/tests/dycore_stencil_tests/test_add_vertical_wind_derivative_to_divergence_damping.py

@@ -20,6 +20,7 @@
 class TestAddVerticalWindDerivativeToDivergenceDamping(helpers.StencilTest):
     PROGRAM = add_vertical_wind_derivative_to_divergence_damping
     OUTPUTS = ("z_graddiv_vn",)
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -46,9 +47,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(data_alloc.as_numpy(grid.connectivities[dims.E2CDim]) == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         hmask_dd3d = data_alloc.random_field(grid, dims.EdgeDim, dtype=ta.wpfloat)
         scalfac_dd3d = data_alloc.random_field(grid, dims.KDim, dtype=ta.wpfloat)
         inv_dual_edge_length = data_alloc.random_field(grid, dims.EdgeDim, dtype=ta.wpfloat)

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_advective_normal_wind_tendency.py

@@ -52,6 +52,7 @@ def compute_advective_normal_wind_tendency_numpy(
 class TestComputeAdvectiveNormalWindTendency(StencilTest):
     PROGRAM = compute_advective_normal_wind_tendency
     OUTPUTS = ("ddt_vn_apc",)
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -85,9 +86,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2CDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         z_kin_hor_e = data_alloc.random_field(grid, dims.EdgeDim, dims.KDim, dtype=vpfloat)
         coeff_gradekin = data_alloc.random_field(grid, dims.EdgeDim, dims.E2CDim, dtype=vpfloat)
         coeff_gradekin_new = data_alloc.as_1D_sparse_field(coeff_gradekin, dims.ECDim)

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_avg_vn.py

@@ -19,6 +19,7 @@
 class TestComputeAvgVn(StencilTest):
     PROGRAM = compute_avg_vn
     OUTPUTS = ("z_vn_avg",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_avg_vn_and_graddiv_vn_and_vt.py

@@ -21,6 +21,7 @@
 class TestComputeAvgVnAndGraddivVnAndVt(StencilTest):
     PROGRAM = compute_avg_vn_and_graddiv_vn_and_vt
     OUTPUTS = ("z_vn_avg", "z_graddiv_vn", "vt")
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_graddiv2_of_vn.py

@@ -19,6 +19,7 @@
 class TestComputeGraddiv2OfVn(StencilTest):
     PROGRAM = compute_graddiv2_of_vn
     OUTPUTS = ("z_graddiv2_vn",)
+    MARKERS = (pytest.mark.embedded_remap_error,)
 
     @staticmethod
     def reference(

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_horizontal_advection_of_rho_and_theta.py

@@ -113,6 +113,7 @@ def sten_16_numpy(
 class TestComputeBtraj(helpers.StencilTest):
     PROGRAM = compute_horizontal_advection_of_rho_and_theta
     OUTPUTS = ("z_rho_e", "z_theta_v_e")
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -175,9 +176,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2CDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         p_vn = random_field(grid, dims.EdgeDim, dims.KDim, dtype=wpfloat)
         p_vt = random_field(grid, dims.EdgeDim, dims.KDim, dtype=vpfloat)
         pos_on_tplane_e_1 = random_field(grid, dims.EdgeDim, dims.E2CDim, dtype=wpfloat)

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_horizontal_gradient_of_exner_pressure_for_flat_coordinates.py

@@ -21,6 +21,7 @@
 class TestComputeHorizontalGradientOfExnerPressureForFlatCoordinates(StencilTest):
     PROGRAM = compute_horizontal_gradient_of_exner_pressure_for_flat_coordinates
     OUTPUTS = ("z_gradh_exner",)
+    MARKERS = (pytest.mark.skip_value_error,)
 
     @staticmethod
     def reference(
@@ -39,9 +40,6 @@ def reference(
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2CDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         inv_dual_edge_length = random_field(grid, dims.EdgeDim, dtype=wpfloat)
         z_exner_ex_pr = random_field(grid, dims.CellDim, dims.KDim, dtype=vpfloat)
         z_gradh_exner = random_field(grid, dims.EdgeDim, dims.KDim, dtype=vpfloat)

model/atmosphere/dycore/tests/dycore_stencil_tests/test_compute_horizontal_gradient_of_exner_pressure_for_multiple_levels.py

@@ -25,6 +25,7 @@
 class TestComputeHorizontalGradientOfExnerPressureForMultipleLevels(StencilTest):
     PROGRAM = compute_horizontal_gradient_of_exner_pressure_for_multiple_levels
     OUTPUTS = ("z_gradh_exner",)
+    MARKERS = (pytest.mark.uses_as_offset, pytest.mark.skip_value_error)
 
     @staticmethod
     def reference(
@@ -71,9 +72,6 @@ def at_neighbor(i):
 
     @pytest.fixture
     def input_data(self, grid):
-        if np.any(grid.connectivities[dims.E2CDim] == -1):
-            pytest.xfail("Stencil does not support missing neighbors.")
-
         inv_dual_edge_length = random_field(grid, dims.EdgeDim, dtype=wpfloat)
         z_exner_ex_pr = random_field(grid, dims.CellDim, dims.KDim, dtype=vpfloat)
         zdiff_gradp = random_field(grid, dims.EdgeDim, dims.E2CDim, dims.KDim, dtype=vpfloat)