codegen: better support for openArray

compiler/ast/types.nim

@@ -58,6 +58,11 @@ type
       # most useful, shows: symbol + resolved symbols if it differs, e.g.:
       # tuple[a: MyInt{int}, b: float]
 
+  BackendViewKind* = enum
+    bvcNone     ## no view
+    bvcSingle   ## single-location view
+    bvcSequence ## view of contiguous locations
+
 proc base*(t: PType): PType =
   result = t[0]
 
@@ -1586,3 +1591,23 @@ proc isCyclePossible*(typ: PType, g: ModuleGraph): bool =
   var marker = initIntSet() # keeps track of which object types we've already
                             # analysed
   result = check(marker, g, typ, typ, isInd=false)
+
+proc classifyBackendView*(t: PType): BackendViewKind =
+  ## For the *resolved* type `t`, computes and returns what kind of view
+  ## the type represents. This is meant to be used during the backend
+  ## phase.
+  case t.kind
+  of tyVar, tyLent:
+    if t.base.kind == tyOpenArray: bvcSequence
+    else:                          bvcSingle
+  of tyOpenArray, tyVarargs:
+    bvcSequence
+  of ConcreteTypes - {tyVar, tyLent, tyOpenArray}, tyNil, tyVoid, tyError,
+     tyUncheckedArray, tyTypeDesc:
+    bvcNone
+  of abstractInst - {tyTypeDesc}, tyUserTypeClasses, tyStatic:
+    classifyBackendView(t.lastSon)
+  of tyNone, tyEmpty, tyUntyped, tyTyped, tyGenericInvocation, tyGenericBody,
+     tyGenericParam, tyForward, tyBuiltInTypeClass, tyCompositeTypeClass,
+     tyAnd, tyOr, tyNot, tyAnything, tyFromExpr:
+    unreachable()

compiler/backend/ccgcalls.nim

@@ -216,10 +216,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
     case skipTypes(a.t, abstractVar+{tyStatic}).kind
     of tyOpenArray, tyVarargs:
       if reifiedOpenArray(n):
-        if a.t.kind in {tyVar, tyLent}:
-          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
-        else:
-          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
+        result = "$1.Field0, $1.Field1" % [rdLoc(a)]
       else:
         result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:

compiler/backend/ccgexprs.nim

@@ -172,39 +172,27 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc) =
 proc genAssignment(p: BProc, dest, src: TLoc) =
   # This function replaces all other methods for generating
   # the assignment operation in C.
-  if src.t != nil and src.t.kind == tyPtr:
-    # little HACK to support the new 'var T' as return type:
+  case mapType(p.config, dest.t, skVar)
+  of ctChar, ctBool, ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
+     ctFloat, ctFloat32, ctFloat64, ctFloat128,
+     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64,
+     ctStruct, ctPtrToArray, ctPtr, ctNimStr, ctNimSeq, ctProc,
+     ctCString:
     linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-    return
-  let ty = skipTypes(dest.t, abstractRange + tyUserTypeClasses + {tyStatic})
-  case ty.kind
-  of tyRef, tySequence, tyString, tyProc, tyTuple, tyObject:
-    linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-  of tyArray:
-      linefmt(p, cpsStmts,
-           "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
-           [rdLoc(dest), rdLoc(src), getTypeDesc(p.module, dest.t)])
-  of tyOpenArray, tyVarargs:
-    # open arrays are always on the stack - really? What if a sequence is
-    # passed to an open array?
+  of ctArray:
+    assert dest.t.skipTypes(irrelevantForBackend + abstractInst).kind != tyOpenArray
+    linefmt(p, cpsStmts, "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, $3);$n",
+            [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)])
+  of ctNimOpenArray:
+    # HACK: ``astgen`` elides to-openArray-conversion operations, so we
+    #       need to reconstruct that information here. Remove this case
+    #       once ``astgen`` no longer elides the operations
     if reifiedOpenArray(dest.lode):
       genOpenArrayConv(p, dest, src)
-    else:
-      linefmt(p, cpsStmts,
-           # bug #4799, keep the nimCopyMem for a while
-           #"#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
-           "$1 = $2;$n",
-           [rdLoc(dest), rdLoc(src)])
-  of tySet:
-    if mapSetType(p.config, ty) == ctArray:
-      linefmt(p, cpsStmts, "#nimCopyMem((void*)$1, (NIM_CONST void*)$2, $3);$n",
-              [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)])
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-  of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCstring,
-     tyInt..tyUInt64, tyRange, tyVar, tyLent, tyNil:
-    linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-  else: internalError(p.config, "genAssignment: " & $ty.kind)
+  of ctVoid:
+    unreachable("not a valid location type")
 
   if optMemTracker in p.options and dest.storage in {OnHeap, OnUnknown}:
     #writeStackTrace()

compiler/backend/ccgtypes.nim

@@ -135,7 +135,7 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind =
   of tySet: result = mapSetType(conf, typ)
   of tyOpenArray, tyVarargs:
     if kind == skParam: result = ctArray
-    else: result = ctStruct
+    else: result = ctNimOpenArray
   of tyArray, tyUncheckedArray: result = ctArray
   of tyObject, tyTuple: result = ctStruct
   of tyUserTypeClasses:
@@ -158,7 +158,10 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind =
   of tyPtr, tyVar, tyLent, tyRef:
     var base = skipTypes(typ.lastSon, typedescInst)
     case base.kind
-    of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctPtrToArray
+    of tyArray, tyUncheckedArray: result = ctPtrToArray
+    of tyOpenArray, tyVarargs:
+      if kind == skParam: result = ctPtrToArray
+      else:               result = ctNimOpenArray
     of tySet:
       if mapSetType(conf, base) == ctArray: result = ctPtrToArray
       else: result = ctPtr
@@ -593,6 +596,8 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
       addAbiCheck(m, t, result)
 
   result = getTypePre(m, t, sig)
+  # note: ``openArray`` types map to different C types depending on the
+  # context, so we always re-compute the C type for them
   if result != "" and t.kind != tyOpenArray:
     excl(check, t.id)
     return
@@ -601,7 +606,8 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     let star = "*"
     var et = origTyp.skipTypes(abstractInst).lastSon
     var etB = et.skipTypes(abstractInst)
-    if mapType(m.config, t, kind) == ctPtrToArray and (etB.kind != tyOpenArray or kind == skParam):
+    let origBase = etB
+    if mapType(m.config, t, kind) == ctPtrToArray:
       if etB.kind == tySet:
         et = getSysType(m.g.graph, unknownLineInfo, tyUInt8)
       else:
@@ -616,10 +622,19 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     of tySequence:
         result = getTypeDescWeak(m, et, check, kind) & star
         m.typeCache[sig] = result
+    of tyOpenArray:
+      result = getTypeDescAux(m, etB, check, kind)
     else:
       # else we have a strong dependency  :-(
       result = getTypeDescAux(m, et, check, kind) & star
       m.typeCache[sig] = result
+
+    # HACK: an openArray is mapped to different types depending on what context
+    #       we're in (`kind`). The context is not stored together with the cached
+    #       type, so we force the type to be computed again next time by deleting
+    #       the entry created above
+    if origBase.kind == tyOpenArray:
+      m.typeCache.del(sig)
   of tyOpenArray, tyVarargs:
     result = getOpenArrayDesc(m, t, check, kind)
   of tyEnum:

compiler/backend/cgen.nim

@@ -450,21 +450,29 @@ proc resetLoc(p: BProc, loc: var TLoc; doInitObj = true) =
         genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
 
 proc constructLoc(p: BProc, loc: var TLoc, isTemp = false; doInitObj = true) =
-  let typ = loc.t
-  if skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}:
+  let kind = mapTypeChooser(loc)
+  case mapType(p.config, loc.t, kind)
+  of ctChar, ctBool, ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
+     ctFloat, ctFloat32, ctFloat64, ctFloat128,
+     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64:
+    # numeric type
+    linefmt(p, cpsStmts, "$1 = 0;$n", [rdLoc(loc)])
+  of ctPtrToArray, ctPtr, ctCString, ctProc:
+    # a simple ptr-like type -> assign nil
+    linefmt(p, cpsStmts, "$1 = NIM_NIL;$n", [rdLoc(loc)])
+  of ctNimStr, ctNimSeq:
     linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)])
-  elif not isComplexValueType(typ):
-    linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc),
-      getTypeDesc(p.module, typ, mapTypeChooser(loc))])
-  else:
+  of ctArray, ctStruct, ctNimOpenArray:
     if not isTemp:
       # don't use nimZeroMem for temporary values for performance if we can
       # avoid it
       linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
-              [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))])
+              [addrLoc(p.config, loc), getTypeDesc(p.module, loc.t, kind)])
 
     if doInitObj:
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
+  of ctVoid:
+    unreachable()
 
 proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
   if sfNoInit notin v.flags:

compiler/backend/cgendata.nim

@@ -64,6 +64,7 @@ type
     ctFloat, ctFloat32, ctFloat64, ctFloat128,
     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64,
     ctArray, ctPtrToArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc,
+    ctNimOpenArray,
     ctCString
   TCFileSections* = array[TCFileSection, Rope] ## represents a generated C file
   TCProcSection* = enum       ## the sections a generated C proc consists of

compiler/backend/jsgen.nim

@@ -973,7 +973,7 @@ proc needsNoCopy(p: PProc; y: PNode): bool =
   return y.kind in nodeKindsNeedNoCopy or
         ((mapType(y.typ) != etyBaseIndex) and
           (skipTypes(y.typ, abstractInst).kind in
-            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc} + IntegralTypes))
+            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOpenArray} + IntegralTypes))
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   var a, b: TCompRes
@@ -1656,7 +1656,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       result = putToSeq("null", indirect)
   of tySequence, tyString:
     result = putToSeq("[]", indirect)
-  of tyCstring, tyProc:
+  of tyCstring, tyProc, tyOpenArray:
     result = putToSeq("null", indirect)
   of tyStatic:
     p.config.internalAssert(t.n != nil, "createVar: " & $t.kind)
@@ -1697,7 +1697,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     gen(p, n, a)
     case mapType(p, v.typ)
     of etyObject, etySeq:
-      if needsNoCopy(p, n):
+      if needsNoCopy(p, n) or classifyBackendView(v.typ) == bvcSequence:
         s = a.res
       else:
         useMagic(p, "nimCopy")

compiler/backend/jstypes.nim

@@ -134,7 +134,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
+  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet, tyOpenArray:
     var s =
       "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]

compiler/mir/astgen.nim

@@ -52,7 +52,6 @@ import
   ]
 
 from compiler/sem/semdata import makeVarType
-from compiler/sem/typeallowed import directViewType, noView
 
 # TODO: move the procedure somewhere common
 from compiler/vm/vmaux import findRecCase
@@ -534,7 +533,7 @@ proc canUseView(n: PNode): bool =
     of nkCallKinds:
       # if the call yields a view, use an lvalue reference (view) -- otherwise,
       # do not
-      return directViewType(n.typ) != noView
+      return classifyBackendView(n.typ) != bvcNone
     else:
       return false
 

compiler/mir/mirgen.nim

@@ -61,9 +61,6 @@ import
   compiler/front/[
     options
   ],
-  compiler/sem/[
-    typeallowed
-  ],
   compiler/utils/[
     containers,
     idioms
@@ -859,7 +856,7 @@ proc genCall(c: var TCtx, n: PNode): EValue =
       chain: genEmpty(c, n[i]) => arg(c)
     elif i == 1 and not fntyp[0].isEmptyType() and
          not isHandleLike(t) and
-         directViewType(fntyp[0]) != noView:
+         classifyBackendView(fntyp[0]) != bvcNone:
       # the procedure returns a view, but the first parameter is not something
       # that resembles a handle. We need to make sure that the first argument
       # (which the view could be created from), is passed by reference in that
@@ -1724,21 +1721,28 @@ proc genx(c: var TCtx, n: PNode, consume: bool): EValue =
     eval: genx(c, n[0]) => addrOp(c, n.typ)
   of nkHiddenAddr:
     let v = genx(c, n[0])
-    if directViewType(n.typ) != noView:
+    case classifyBackendView(n.typ)
+    of bvcSingle:
       # a view into the source operand is created
       eval: v => viewOp(c, n.typ)
-    else:
+    of bvcSequence:
+      # the addr operation is a no-op
+      v
+    of bvcNone:
       # a normal address-of operation
       eval: v => addrOp(c, n.typ)
 
   of nkDerefExpr:
     eval: genx(c, n[0]) => derefOp(c, n.typ)
   of nkHiddenDeref:
     let v = genx(c, n[0])
-    if directViewType(n[0].typ) != noView:
+    case classifyBackendView(n[0].typ)
+    of bvcSingle:
       # it's a deref of a view
       eval: v => derefViewOp(c, n.typ)
-    else:
+    of bvcSequence:
+      v
+    of bvcNone:
       # it's a ``ref`` or ``ptr`` deref
       eval: v => derefOp(c, n.typ)
 

compiler/sem/liftdestructors.nim

@@ -766,7 +766,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
   of tyNone, tyEmpty, tyVoid: discard
   of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring,
-      tyPtr, tyUncheckedArray, tyVar, tyLent:
+      tyPtr, tyUncheckedArray, tyVar, tyLent, tyVarargs, tyOpenArray:
     defaultOp(c, t, body, x, y)
   of tyRef:
     if c.g.config.selectedGC in {gcArc, gcOrc}:
@@ -817,12 +817,6 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       fillBody(c, t[0], body, x, y)
   of tyTuple:
     fillBodyTup(c, t, body, x, y)
-  of tyVarargs, tyOpenArray:
-    if c.kind == attachedDestructor and (tfHasAsgn in t.flags or useNoGc(c, t)):
-      forallElements(c, t, body, x, y)
-    else:
-      discard "cannot copy openArray"
-
   of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
      tyGenericParam, tyGenericBody, tyNil, tyUntyped, tyTyped,