fix calls to untyped arbitrary expressions in generics (#24100)

fixes #24099

If an arbitrary expression without a proc type (in this case
`tyFromExpr`) is called, the compiler [passes it to overload
resolution](https://github.com/nim-lang/Nim/blob/793cee4de1934fd1f6271cf5fed46f01c5abb19b/compiler/semexprs.nim#L1223).
But overload resolution also can't handle arbitrary expressions and
treats them as not participating at all, matching with the state
`csEmpty`. The compiler checks for this and gives an "identifier
expected" error. Instead, now if we are in a generic context and an
arbitrary expression call matched with `csEmpty`, we now return
`csNoMatch` so that `semResolvedCall` can leave it untyped as
appropriate.

The expression flag `efNoDiagnostics` is replaced with this check. It's
not checked anywhere else and the only place that uses it is
`handleCaseStmtMacro`. Replacing it with `efNoUndeclared`, we just get
`csEmpty` instead of `csNoMatch`, which is handled the same way here. So
`efNoDiagnostics` is now removed and `efNoUndeclared` is used instead.

compiler/semcall.nim

@@ -558,7 +558,8 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
     if overloadsState == csEmpty and result.state == csEmpty:
       if efNoUndeclared notin flags: # for tests/pragmas/tcustom_pragma.nim
         result.state = csNoMatch
-        if efNoDiagnostics in flags:
+        if c.inGenericContext > 0 and nfExprCall in n.flags:
+          # untyped expression calls end up here, see #24099
           return
         # xxx adapt/use errorUndeclaredIdentifierHint(c, n, f.ident)
         localError(c.config, n.info, getMsgDiagnostic(c, flags, n, f))

compiler/semdata.nim

@@ -73,7 +73,6 @@ type
     efNoUndeclared, efIsDotCall, efCannotBeDotCall,
       # Use this if undeclared identifiers should not raise an error during
       # overload resolution.
-    efNoDiagnostics,
     efTypeAllowed # typeAllowed will be called after
     efWantNoDefaults
     efAllowSymChoice # symchoice node should not be resolved

compiler/semstmts.nim

@@ -1231,7 +1231,7 @@ proc handleCaseStmtMacro(c: PContext; n: PNode; flags: TExprFlags): PNode =
   toResolve.add n[0]
 
   var errors: CandidateErrors = @[]
-  var r = resolveOverloads(c, toResolve, toResolve, {skTemplate, skMacro}, {efNoDiagnostics},
+  var r = resolveOverloads(c, toResolve, toResolve, {skTemplate, skMacro}, {efNoUndeclared},
                            errors, false)
   if r.state == csMatch:
     var match = r.calleeSym
@@ -1245,8 +1245,6 @@ proc handleCaseStmtMacro(c: PContext; n: PNode; flags: TExprFlags): PNode =
     of skMacro: result = semMacroExpr(c, toExpand, toExpand, match, flags)
     of skTemplate: result = semTemplateExpr(c, toExpand, match, flags)
     else: result = errorNode(c, n[0])
-  elif r.state == csNoMatch:
-    result = errorNode(c, n[0])
   else:
     result = errorNode(c, n[0])
   if result.kind == nkEmpty:

tests/proc/tgenericdefaultparam.nim

@@ -20,3 +20,50 @@ block: # issue #20916
     tmp
   a()()
   doAssert val == 42
+
+import std/typetraits
+
+block: # issue #24099, original example
+  type
+    ColorRGBU = distinct array[3, uint8] ## RGB range 0..255
+    ColorRGBAU = distinct array[4, uint8] ## RGB range 0..255
+    ColorRGBUAny = ColorRGBU | ColorRGBAU
+  template componentType(t: typedesc[ColorRGBUAny]): typedesc =
+    ## Returns component type of a given color type.
+    arrayType distinctBase t
+  func `~=`[T: ColorRGBUAny](a, b: T, e = componentType(T)(1.0e-11)): bool =
+    ## Compares colors with given accuracy.
+    abs(a[0] - b[0]) < e and abs(a[1] - b[1]) < e and abs(a[2] - b[2]) < e
+
+block: # issue #24099, modified to actually work
+  type
+    ColorRGBU = distinct array[3, uint8] ## RGB range 0..255
+    ColorRGBAU = distinct array[4, uint8] ## RGB range 0..255
+    ColorRGBUAny = ColorRGBU | ColorRGBAU
+  template arrayType[I, T](t: typedesc[array[I, T]]): typedesc =
+    T
+  template `[]`(a: ColorRGBUAny, i: untyped): untyped = distinctBase(a)[i]
+  proc abs(a: uint8): uint8 = a
+  template componentType(t: typedesc[ColorRGBUAny]): typedesc =
+    ## Returns component type of a given color type.
+    arrayType distinctBase t
+  func `~=`[T: ColorRGBUAny](a, b: T, e = componentType(T)(1.0e-11)): bool =
+    ## Compares colors with given accuracy.
+    abs(a[0] - b[0]) <= e and abs(a[1] - b[1]) <= e and abs(a[2] - b[2]) <= e
+  doAssert ColorRGBU([1.uint8, 1, 1]) ~= ColorRGBU([1.uint8, 1, 1])
+
+block: # issue #24099, modified to work but using float32
+  type
+    ColorRGBU = distinct array[3, float32] ## RGB range 0..255
+    ColorRGBAU = distinct array[4, float32] ## RGB range 0..255
+    ColorRGBUAny = ColorRGBU | ColorRGBAU
+  template arrayType[I, T](t: typedesc[array[I, T]]): typedesc =
+    T
+  template `[]`(a: ColorRGBUAny, i: untyped): untyped = distinctBase(a)[i]
+  template componentType(t: typedesc[ColorRGBUAny]): typedesc =
+    ## Returns component type of a given color type.
+    arrayType distinctBase t
+  func `~=`[T: ColorRGBUAny](a, b: T, e = componentType(T)(1.0e-11)): bool =
+    ## Compares colors with given accuracy.
+    abs(a[0] - b[0]) < e and abs(a[1] - b[1]) < e and abs(a[2] - b[2]) < e
+  doAssert ColorRGBU([1.float32, 1, 1]) ~= ColorRGBU([1.float32, 1, 1])