diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Error.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Error.tsx
new file mode 100644
index 00000000000..25bcd228c1d
--- /dev/null
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Error.tsx
@@ -0,0 +1,42 @@
+import { West } from "@mui/icons-material";
+import { Box, Button, Card, Stack, Typography } from "@mui/material";
+import React from "react";
+import ErrorIcon from "./ErrorIcon";
+
+export default function Error(props) {
+  const { onBack, error } = props;
+  console.log(">>>", { error });
+  return (
+    <Stack sx={{ height: "100%", p: 2 }} spacing={1}>
+      <Box>
+        <Button onClick={onBack} startIcon={<West />} color="secondary">
+          Back to Model Evaluation
+        </Button>
+      </Box>
+      <Card
+        sx={{
+          display: "flex",
+          justifyContent: "center",
+          alignItems: "center",
+          height: "100%",
+        }}
+      >
+        <Stack spacing={2} sx={{ alignItems: "center" }}>
+          <ErrorIcon sx={{ fontSize: 64 }} />
+          <Typography color="secondary">
+            Analyze and improve models collaboratively with your team
+          </Typography>
+          <Typography sx={{ fontWeight: 600 }}>
+            The Model Evaluation panel currently supports only classification,
+            detection, and segmentation evaluations
+          </Typography>
+        </Stack>
+      </Card>
+    </Stack>
+  );
+}
+
+type ErrorProps = {
+  onBack: () => void;
+  error: any;
+};
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/ErrorIcon.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/ErrorIcon.tsx
new file mode 100644
index 00000000000..8f595c416de
--- /dev/null
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/ErrorIcon.tsx
@@ -0,0 +1,33 @@
+import { SvgIcon, SvgIconProps } from "@mui/material";
+import React from "react";
+
+export default function ErrorIcon(props: SvgIconProps) {
+  return (
+    <SvgIcon width="54" height="61" viewBox="0 0 54 61" fill="none" {...props}>
+      <path
+        d="M27 3.16667C34.0667 3.16667 40.6667 3.9 45.5334 5.23333C48.4667 6.03333 50 6.83333 50.6667 7.3V53.7C50.0667 54.1667 48.5334 54.9667 45.5334 55.7667C40.6667 57.1 34.0667 57.8333 27 57.8333C19.9334 57.8333 13.3334 57.1 8.46669 55.7667C5.53335 54.9667 4.00002 54.1667 3.33335 53.7V7.3C3.93335 6.83333 5.46669 6.03333 8.46669 5.23333C13.3334 3.9 19.9334 3.16667 27 3.16667ZM27 0.5C19.7334 0.5 12.8667 1.23333 7.73335 2.63333C2.86669 3.96667 0.666687 5.56667 0.666687 6.56667V54.4333C0.666687 55.4333 2.86669 57.0333 7.73335 58.3667C12.8667 59.7 19.7334 60.5 27 60.5C34.2667 60.5 41.1334 59.7 46.2667 58.3667C51.1334 57.0333 53.3334 55.4333 53.3334 54.4333V6.56667C53.3334 5.56667 51.1334 3.96667 46.2667 2.63333C41.1334 1.23333 34.2667 0.5 27 0.5Z"
+        fill="#FFC59B"
+      />
+      <path
+        d="M18.2664 27.6333C14.3331 27.6333 11.0664 24.4333 11.0664 20.4333C11.0664 16.4333 14.3331 13.2999 18.2664 13.2999C22.1997 13.2999 25.3997 16.4999 25.3997 20.4333C25.3997 24.3666 22.1997 27.6333 18.2664 27.6333ZM18.2664 15.9666C15.7997 15.9666 13.7331 17.9666 13.7331 20.4999C13.7331 23.0333 15.7331 25.0333 18.2664 25.0333C20.7997 25.0333 22.7997 23.0333 22.7997 20.4999C22.7997 17.9666 20.7331 15.9666 18.2664 15.9666Z"
+        fill="#FFC59B"
+      />
+      <path
+        d="M35.7332 27.6333C31.7999 27.6333 28.5332 24.4333 28.5332 20.4333C28.5332 16.4333 31.7332 13.2333 35.7332 13.2333C39.7332 13.2333 42.9332 16.4999 42.9332 20.4333C42.9332 24.3666 39.6665 27.6333 35.7332 27.6333ZM35.7332 15.9666C33.2665 15.9666 31.1999 17.9666 31.1999 20.4999C31.1999 23.0333 33.1999 25.0333 35.7332 25.0333C38.2665 25.0333 40.2665 23.0333 40.2665 20.4999C40.2665 17.9666 38.1999 15.9666 35.7332 15.9666Z"
+        fill="#FFC59B"
+      />
+      <path
+        d="M18.2666 21.9C19.0766 21.9 19.7332 21.2434 19.7332 20.4333C19.7332 19.6233 19.0766 18.9667 18.2666 18.9667C17.4565 18.9667 16.7999 19.6233 16.7999 20.4333C16.7999 21.2434 17.4565 21.9 18.2666 21.9Z"
+        fill="#FFC59B"
+      />
+      <path
+        d="M35.7334 21.9C36.5434 21.9 37.2 21.2434 37.2 20.4333C37.2 19.6233 36.5434 18.9667 35.7334 18.9667C34.9233 18.9667 34.2667 19.6233 34.2667 20.4333C34.2667 21.2434 34.9233 21.9 35.7334 21.9Z"
+        fill="#FFC59B"
+      />
+      <path
+        d="M20.7333 41.2333C20.9333 40.9666 21.6 40.8333 21.9333 40.7666C22.5333 40.7 22.9333 40.9666 23.2 41.5C23.9333 43.0333 26.2666 41.7 25.5333 40.1666C24.2666 37.6333 19.9333 37.3666 18.4666 39.9C17.5333 41.4333 19.8666 42.7666 20.7333 41.2333Z"
+        fill="#FFC59B"
+      />
+    </SvgIcon>
+  );
+}
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
index 69aa0a4b395..9219ab56493 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
@@ -42,6 +42,7 @@ import {
 } from "@mui/material";
 import React, { useEffect, useMemo, useState } from "react";
 import { useRecoilState, useSetRecoilState } from "recoil";
+import Error from "./Error";
 import EvaluationNotes from "./EvaluationNotes";
 import EvaluationPlot from "./EvaluationPlot";
 import Status from "./Status";
@@ -90,6 +91,14 @@ export default function Evaluation(props: EvaluationProps) {
     const evaluation = data?.[`evaluation_${compareKey}`];
     return evaluation;
   }, [data]);
+  const evaluationError = useMemo(() => {
+    const evaluation = data?.[`evaluation_${name}_error`];
+    return evaluation;
+  }, [data]);
+  const compareEvaluationError = useMemo(() => {
+    const evaluation = data?.[`evaluation_${compareKey}_error`];
+    return evaluation;
+  }, [data]);
   const confusionMatrix = useMemo(() => {
     return getMatrix(evaluation?.confusion_matrices, confusionMatrixConfig);
   }, [evaluation, confusionMatrixConfig]);
@@ -145,6 +154,10 @@ export default function Evaluation(props: EvaluationProps) {
     setConfusionMatrixDialogConfig((state) => ({ ...state, open: false }));
   };
 
+  if (evaluationError) {
+    return <Error error={evaluationError} onBack={navigateBack} />;
+  }
+
   if (!evaluation) {
     return (
       <Box
@@ -516,7 +529,14 @@ export default function Evaluation(props: EvaluationProps) {
           </Stack>
         </Stack>
         <Stack sx={{ width: "50%" }} spacing={0.5}>
-          <Typography color="secondary">Compare against</Typography>
+          <Stack direction="row" spacing={1}>
+            <Typography color="secondary">Compare against</Typography>
+            {compareEvaluationError && (
+              <Typography sx={{ color: theme.palette.error.main }}>
+                Unsupported model evaluation type
+              </Typography>
+            )}
+          </Stack>
           {compareKeys.length === 0 ? (
             <Typography
               variant="body2"
diff --git a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
index e8a1aff3017..ef8d9b1c47d 100644
--- a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
+++ b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
@@ -32,6 +32,7 @@
     os.environ.get("FIFTYONE_DISABLE_EVALUATION_CACHING") not in TRUTHY_VALUES
 )
 CACHE_TTL = 30 * 24 * 60 * 60  # 30 days in seconds
+SUPPORTED_EVALUATION_TYPES = ["classification", "detection", "segmentation"]
 
 
 class EvaluationPanel(Panel):
@@ -299,6 +300,14 @@ def load_evaluation(self, ctx):
         )
         if evaluation_data is None:
             info = ctx.dataset.get_evaluation_info(computed_eval_key)
+            serialized_info = info.serialize()
+            evaluation_type = info.config.type
+            if evaluation_type not in SUPPORTED_EVALUATION_TYPES:
+                ctx.panel.set_data(
+                    f"evaluation_{computed_eval_key}_error",
+                    {"error": "unsupported", "info": serialized_info},
+                )
+                return
             results = ctx.dataset.load_evaluation_results(computed_eval_key)
             metrics = results.metrics()
             per_class_metrics = self.get_per_class_metrics(info, results)
@@ -311,7 +320,7 @@ def load_evaluation(self, ctx):
             metrics["mAP"] = self.get_map(results)
             evaluation_data = {
                 "metrics": metrics,
-                "info": info.serialize(),
+                "info": serialized_info,
                 "confusion_matrices": self.get_confusion_matrices(results),
                 "per_class_metrics": per_class_metrics,
             }