sem: keep shape of array constructors in typed AST (#1340)

## Summary

Explicit indices in array constructors now stay in the AST after
typing. Since macros are able to observe this, this change is a
**breaking change**.

## Details

Keeping the explicit indices is also necessary for being able to
retype the AST after it was typed once (without the index, array
constructors for arrays not starting at zero would have a
different type afterwards).

Instead of dropping the explicit index together with the parent
`nkExprColonExpr` node, both are kept by `semArrayConstr`, and then
later discarded by `semfold` (during constant evaluation) and `transf`,
where they are no longer needed.

compiler/sem/semexprs.nim

@@ -772,32 +772,19 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     result.sons.setLen(n.len)
     for i, it in n.pairs:
       # first, analyse the index expression (if one exist)
-      let (idx, val) =
+      var (idx, val) =
         if i == 0: (first, firstIndex)
         else:      semArrayElementIndex(c, it, indexType)
 
-      # figure out the node that holds the element expression, and validate
-      # the index if one is provided
-      var e =
-        case idx.kind
-        of nkError:
-          let r = shallowCopy(it)
-          r[0] = idx
-          r[1] = it[1]
-          c.config.wrapError(r)
-        of nkEmpty:
-          it
-        else:
-          if val == lastIndex + 1:
-            it[1]
-          else:
-            # the specified index value doesn't match with the expected one
-            c.config.newError(it,
-                          PAstDiag(kind: adSemInvalidOrderInArrayConstructor))
+      if idx.kind notin {nkError, nkEmpty} and val != lastIndex + 1:
+        # the specified index value doesn't match with the expected one
+        idx = c.config.newError(idx,
+          PAstDiag(kind: adSemInvalidOrderInArrayConstructor))
 
-      if e.kind != nkError:
-        e = semExprWithType(c, e, {})
-        e = exprNotGenericRoutine(c, e)
+      # always analyze the expression, even when the index expression is
+      # erroneous
+      var e = semExprWithType(c, it.skipColon, {})
+      e = exprNotGenericRoutine(c, e)
 
       if typ.isNil:
         # must be the first item; initialize the common type:
@@ -816,7 +803,11 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
         # yet
         typ = commonType(c, typ, e.typ)
 
-      result[i] = e
+      if it.kind == nkExprColonExpr:
+        result[i] = newTreeI(nkExprColonExpr, it.info, [idx, e])
+      else:
+        result[i] = e
+
       inc lastIndex
 
     # watch out for ``sink T``!
@@ -831,8 +822,12 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     var hasError = false
     # fit all elements to be of the derived common type
     for it in result.sons.mitems:
-      it = fitNode(c, typ, it, it.info)
-      hasError = hasError or it.kind == nkError
+      if it.kind == nkExprColonExpr:
+        it[1] = fitNode(c, typ, it[1], it[1].info)
+        hasError = hasError or nkError in {it[0].kind, it[1].kind}
+      else:
+        it = fitNode(c, typ, it, it.info)
+        hasError = hasError or it.kind == nkError
 
     if hasError:
       result = c.config.wrapError(result)

compiler/sem/semfold.nim

@@ -759,7 +759,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
   of nkBracket, nkCurly:
     result = copyNode(n)
     for i, son in n.pairs:
-      let a = getConstExpr(m, son, idgen, g)
+      let a = getConstExpr(m, son.skipColon, idgen, g)
       if a == nil: return nil
       result.add a
     incl(result.flags, nfAllConst)
@@ -881,7 +881,7 @@ proc foldConstExprAux(m: PSym, n: PNode, idgen: IdGenerator, g: ModuleGraph): Fo
     # the last node is an expression
     result.add foldConstExprAux(m, n[^1], idgen, g)
   of nkExprColonExpr:
-    # comes here from tuple/object constructions
+    # comes here from array/tuple/object constructions
     result.add n[0]
     result.add foldConstExprAux(m, n[1], idgen, g)
     return

compiler/sem/transf.nim

@@ -1364,6 +1364,11 @@ proc transform(c: PTransf, n: PNode): PNode =
   of nkPragmaExpr:
     # not needed in transformed AST -> drop it
     result = transform(c, n.lastSon)
+  of nkBracket:
+    # replace elements where the index is specified with just the expression
+    result = shallowCopy(n)
+    for i, it in n.pairs:
+      result[i] = transform(c, it.skipColon)
   else:
     result = transformSons(c, n)
   when false:

tests/lang_exprs/ttyped_array_constr_expr.nim

@@ -0,0 +1,21 @@
+discard """
+  description: '''
+    Ensure that the shape of literal array construction expresions is
+    preserved in typed AST
+  '''
+  action: compile
+"""
+
+import std/macros
+
+macro m(x: typed) =
+  doAssert treeRepr(x) == """Bracket
+  ExprColonExpr
+    IntLit 1
+    StrLit "a"
+  StrLit "b"
+  ExprColonExpr
+    IntLit 3
+    StrLit "c""""
+
+m([1: "a", "b", 3: "c"])