Merge branch 'jtv/v2' into ee7/ci-add-tests-workflow

src/attrstore.nim

@@ -160,11 +160,14 @@ proc set*(ctx: RuntimeState, key: string, value: pointer, tid: TypeId,
 
   if curOpt.isSome():
     curInfo = curOpt.get()
+    newInfo.moduleLock = curInfo.moduleLock
 
     if newInfo.override and not override:
       return true # Pretend it was successful.
 
-    if curInfo.locked:
+    if curInfo.locked or
+       (curInfo.moduleLock != 0 and
+        curInfo.moduleLock != ctx.curModule.moduleId):
       if not override:
         if not call_eq(value, curInfo.contents, curInfo.tid):
           return false
@@ -179,6 +182,8 @@ proc set*(ctx: RuntimeState, key: string, value: pointer, tid: TypeId,
 
   if ctx.running:
     newInfo.lastset = addr ctx.curModule.instructions[ctx.ip]
+    if ctx.module_lock_stack.len() != 0:
+      newInfo.moduleLock = ctx.module_lock_stack[^1]
 
   # Don't trigger write lock if we're setting a default (i.e., internal is set).
   if (lock or wlock) and not internal:

src/codegen.nim

@@ -18,7 +18,7 @@
 # defer typing things at the top-level; this will be a subtlety we
 # deal with when we get to doing the REPL.
 
-import "."/irgen
+import "."/[irgen, compile]
 import ztypes/api
 
 proc findAndLoadModule(ctx: CompileCtx, location, fname, ext: string):
@@ -174,6 +174,8 @@ proc rawReprInstructions*(module: ZModuleInfo, ctx: ZObjectFile, fn = 0): Rope =
     of ZTCall:
       arg1 = text(hex(item.arg))
       lbl  = em(tCallReverseMap[item.arg])
+    of ZCallModule:
+      arg1 = text($(item.arg))
     of Z0Call:
       # Look up the symbol name by moduleId. It's always going to be a module,
       # so it's really about indexing into the right module in the
@@ -308,10 +310,11 @@ proc rawReprInstructions*(module: ZModuleInfo, ctx: ZObjectFile, fn = 0): Rope =
 
 proc disassembly*(ctx: ZObjectFile, skip_nops = true): Rope =
     for item in ctx.moduleContents:
-      if item.instructions.len() == 3 and skip_nops:
+      if item.instructions.len() == 4 and skip_nops:
         if item.instructions[0].op == ZNop and
-           item.instructions[1].op == ZModuleRet and
-           item.instructions[2].op == ZNop:
+           item.instructions[1].op == ZModuleEnter and
+           item.instructions[2].op == ZModuleRet and
+           item.instructions[3].op == ZNop:
           continue
       result = result + rawReprInstructions(item, ctx)
 
@@ -886,6 +889,7 @@ proc genExternCall(ctx: CodeGenState, cur: IrContents) =
   if tid != TVoid:
     ctx.emitInstruction(ZPushRes, tid = tid)
 
+
 proc genRunCallback(ctx: CodeGenState, cur: IrContents) =
   var
     i = cur.actuals.len()
@@ -1197,9 +1201,6 @@ proc oneIrNode(ctx: CodeGenState, n: IrNode) =
 
   ctx.curNode = saved
 
-proc addParameter(ctx: CodeGenState, symbol: SymbolInfo) =
-  discard
-
 proc stashSymbolInfo(scope: Scope, d: var Dict[int, string],
                      t: var seq[(int, TypeId)]) =
   for (name, sym) in scope.table.items(sort=true):
@@ -1233,6 +1234,74 @@ proc genFunction(ctx: CodeGenState, fn: FuncInfo) =
 
   ctx.mcur.objInfo.codesyms[zinfo.offset] = fullname
 
+proc stashParam(ctx: CodeGenState, m: Module, info: ParamInfo) =
+
+  var zparam = ZParamInfo(shortdoc: info.shortdoc,
+                          longdoc:  info.longdoc)
+
+  if info.sym.isAttr:
+    zparam.attr   = info.sym.name
+  else:
+    zparam.offset = info.sym.offset
+
+  if info.defaultIr.isSome():
+    # TODO: do I need to check if this is constant? Don't remember
+    zparam.haveDefault = true
+    zparam.default = info.defaultIr.get().value
+
+  zparam.tid = info.sym.tid
+
+  if info.validatorIr.isSome():
+    var
+      offset  = info.backpatchLoc
+      ### m.instructions[offset] is ZPushVmPtr
+      ### next one is ZRunCallback
+      litnode = info.validatorIr.get()
+      cb      = cast[ptr Callback](litnode.value)
+
+    if cb.impl.externInfo != nil:
+      # TODO Shouldn't the offset already be in cb.impl?
+      # Not sure we need to loop; just copying from above.
+      zparam.native = false
+      for i, item in ctx.zobj.ffiInfo:
+        let
+          p = cast[pointer](addr ctx.zobj.staticData[item.nameOffset])
+          s = $(cast[cstring](p))
+
+        if s == cb.impl.externName:
+          zparam.funcIx                           = int32(i)
+          m.objinfo.instructions[offset].arg      = zparam.funcIx
+          m.objinfo.instructions[offset].typeInfo = zparam.tid
+          break
+    else:
+      zparam.native = true
+      zparam.funcIx = int32(cb.impl.internalId)
+      m.objinfo.instructions[offset].arg = zparam.funcIx
+
+    offset += 1
+    m.objinfo.instructions[offset].typeInfo = cb.tid
+
+  else:
+    zparam.funcIx = -1
+
+
+  m.objInfo.parameters.add(zparam)
+
+proc placeholdParamValidation(ctx: CodeGenState, m: Module, i: int) =
+   m.params[i].backpatchLoc = m.objInfo.instructions.len()
+   ctx.emitInstruction(ZPushVmPtr)
+   ctx.emitInstruction(ZRunCallback, arg = 1)
+   ctx.emitInstruction(ZMoveSp, -1)
+   ctx.emitInstruction(ZPushRes)
+
+proc genParamChecks(ctx: CodeGenState, m: Module) =
+  for i, param in m.params:
+    if param.validatorIr.isNone():
+      continue
+    ctx.genLoadSymbol(param.sym)
+    ctx.placeholdParamValidation(m, i)
+    ctx.emitInstruction(ZParamCheck, arg = i)
+
 proc genModule(ctx: CodeGenState, m: Module) =
   let curMod = ctx.minfo[m.key]
   if curMod.processed:
@@ -1241,10 +1310,10 @@ proc genModule(ctx: CodeGenState, m: Module) =
   ctx.mcur = curMod
 
   ctx.genLabel("Module '" & m.modname & "' :")
+  ctx.emitInstruction(ZModuleEnter, arg = m.params.len())
 
-  for (_, sym) in m.moduleScope.table.items(sort=true):
-    if sym.pInfo != nil:
-      ctx.addParameter(sym)
+  if m.params.len() != 0:
+    ctx.genParamChecks(m)
 
   ctx.oneIrNode(m.ir)
   ctx.emitInstruction(ZModuleRet)
@@ -1258,6 +1327,9 @@ proc genModule(ctx: CodeGenState, m: Module) =
         continue
       ctx.genFunction(fn)
 
+  for param in m.params:
+      ctx.stashParam(m, param)
+
   curMod.processed = true
 
   let deps = ctx.minfo.items(sort = true)
@@ -1289,7 +1361,7 @@ proc applyArenaId(ctx: CodeGenState, scope: Scope, moduleId: int,
 
 proc applyArenasToGloballyVisibleFuncs(ctx: CodeGenState, scope: Scope,
                                        curFnId: var int) =
-  for (name, sym) in scope.table.items(sort=true):
+  for (name, sym) in scope.table.items(sort = true):
     for fn in sym.fimpls:
       if fn.externInfo != nil:
         continue
@@ -1299,7 +1371,7 @@ proc applyArenasToGloballyVisibleFuncs(ctx: CodeGenState, scope: Scope,
           mid = fn.defModule.objInfo.moduleId
           fi  = ZFnInfo(funcname: name, mid: int32(mid), tid: fn.tid,
                         offset: int32(fn.codeOffset))
-        ctx.zobj.funcInfo.adD(fi)
+        ctx.zobj.funcInfo.add(fi)
         fi.syms.initDict()
         fn.fnScope.stashSymbolInfo(fi.syms, fi.symTypes)
         curFnId += 1
@@ -1401,6 +1473,27 @@ proc extractSectionDocs(ctx: CodeGenState,
     let docObj = AttrDocs(shortdoc: node.shortdoc, longdoc: node.longdoc)
     d[sec] = docObj
 
+template setupOneModule() =
+    var mi = ZModuleInfo(moduleId: curArena, modname: module.modname,
+                         key: module.key, ext: module.ext, url: module.url,
+                         moduleVarSize: module.moduleScope.scopeSize)
+    module.objInfo = mi
+    ctx.applyArenaId(module.moduleScope, curArena, curFnId)
+    curArena              = curArena + 1
+    ctx.minfo[module.key] = module
+    ctx.zobj.moduleContents.add(mi)
+    mi.codesyms.initDict()
+    mi.datasyms.initDict()
+
+    # TODO: I have disabled eliding source code until incremental
+    # compilation is properly done.
+
+    #if not module.system:
+    mi.source = $(module.s.runes)
+
+    module.moduleScope.stashSymbolInfo(mi.datasyms, mi.symTypes)
+    mi.codesyms[0] = module.modname & ".__mod_run__()"
+
 proc setupModules(ctx: CodeGenState, d: Dict[string, AttrDocs]) =
   # This call applies a unique number to each module
   # and then applies a unique number to each function.
@@ -1417,32 +1510,29 @@ proc setupModules(ctx: CodeGenState, d: Dict[string, AttrDocs]) =
   ctx.cc.globalScope.stashSymbolInfo(ctx.zobj.globals, ctx.zobj.symTypes)
   ctx.zobj.globalScopeSz = ctx.cc.globalScope.scopeSize
 
-  for (_, module) in ctx.cc.modules.items(sort=true):
+  for (_, module) in ctx.cc.modules.items(sort = true):
     ctx.addFfiInfo(module)
     ctx.extractSectionDocs(module, d)
 
 
-  for (_, module) in ctx.cc.modules.items(sort=true):
-    var mi = ZModuleInfo(moduleId: curArena, modname: module.modname,
-                         moduleVarSize: module.moduleScope.scopeSize)
-    module.objInfo = mi
-    ctx.applyArenaId(module.moduleScope, curArena, curFnId)
-    curArena              = curArena + 1
-    ctx.minfo[module.key] = module
-    ctx.zobj.moduleContents.add(mi)
-    mi.codesyms.initDict()
-    mi.datasyms.initDict()
-
-    if not module.system:
-      mi.source = $(module.s.runes)
-
-    module.moduleScope.stashSymbolInfo(mi.datasyms, mi.symTypes)
-    mi.codesyms[0] = module.modname & ".__mod_run__()"
-
+  for (_, module) in ctx.cc.modules.items(sort = true):
+    setupOneModule()
   # Finally, apply module IDs to the functions that are globally visible
   # so we can look up their addresses.
   ctx.applyArenasToGloballyVisibleFuncs(ctx.cc.globalScope, curFnId)
 
+proc setupNewModules(ctx: CodeGenState, d: Dict[string, AttrDocs]) =
+  # The incremental version.
+  var
+    curArena = ctx.zobj.moduleContents[^1].moduleId + 1
+    curFnId  = ctx.zobj.funcInfo.len() + 1
+
+  for (_, module) in ctx.cc.modules.items(sort = true):
+    if module.objInfo != nil:
+      continue
+
+    setupOneModule()
+
 proc fillCallBackpatches(ctx: CodeGenState) =
   for (fn, dstmodule, patchloc) in ctx.callBackpatches:
     var cur = dstmodule.objInfo.instructions[patchloc]
@@ -1455,8 +1545,7 @@ proc cacheAllTypeInfo(ctx: CodeGenState) =
   # output, we go ahead and keep any type that does not forward.  This
   # is far less error prone than trying to make sure we always
   # remember to save off type info, but it might end up causing too
-  # much bloat. I doubt it, though! However, we definitely should
-  # collect metrics on this.
+  # much bloat. I doubt it, though!
 
   for (id, tObj) in typeStore.items(sort=true):
     if tObj.typeId != id:
@@ -1468,12 +1557,7 @@ proc cacheAllTypeInfo(ctx: CodeGenState) =
 
     discard ctx.zobj.tInfo.add(id, loc)
 
-  var typeCache: seq[string]
-  for (k, v) in ctx.zobj.tInfo.items(sort=true):
-    typeCache.add($(cast[int](k)))
-    typeCache.add($(cast[cstring](addr ctx.zobj.staticdata[v])))
-
-proc generateCode*(cc: CompileCtx, nextId = -1): RuntimeState =
+proc generateInitialCodeObject*(cc: CompileCtx, nextId = -1): RuntimeState =
   var ctx = CodeGenState()
 
   result     = RuntimeState()
@@ -1497,5 +1581,99 @@ proc generateCode*(cc: CompileCtx, nextId = -1): RuntimeState =
   result.obj.nextEntrypoint = int32(nextId)
   result.obj.spec           = ctx.cc.attrSpec
 
+when false:
+  proc setup_incremental_compile*(ctx: RuntimeState): CompileCtx =
+    result             = ctx.newCompileContext(ctx.obj.spec)
+    result.incremental = true
+
+    for (attr, contents) in ctx.attrs:
+      var sym = SymbolInfo(name: attr, tid: contents.tid,
+                           isAttr: true, inObj: true)
+      result.usedAttrs.table[attr] = sym
+
+    for (offset, name) in ctx.obj.globals.items():
+      var sym = SymbolInfo(name: name)
+
+      sym.tid          = x
+      sym.constValue   = x
+      sym.declaredType = x
+      sym.immutable    = x
+      sym.global       = true
+      sym.inObj        = true
+      sym.offset       = offset
+      sym.moduleId     = 0
+
+      result.globalScope.table[name] = sym
+
+    for zmod in ctx.obj.moduleContents:
+      var module = Module(loaded: true, system: true, modname: zmod.modname,
+                          where: zmod.location, ext: zmod.ext, key: zmod.key,
+                          exitNode: CfgNode(), processed: true,
+                          objinfo: zmod)
+      result.modules[module.key] = module
+
+    # global functions
+    # ffi functions
+    # module stubs
+else:
+  proc setup_incremental_compile*(ctx: RuntimeState): CompileCtx =
+    # Assume this all is going to just work...
+    result = CompileCtx()
+
+    result.globalScope = initScope()
+    result.usedAttrs   = initScope()
+    result.modules.initDict()
+
+    let entrypointix = ctx.obj.entrypoint
+    var entry: Module
+
+    var modlist: seq[Module]
+
+    for zmod in ctx.obj.moduleContents:
+      let m = newModuleObj(result, zmod.source, zmod.modname, zmod.location,
+                           zmod.ext, zmod.url, zmod.key)
+      if zmod.moduleId == entrypointix:
+        entry = m
+
+      m.objInfo = zmod
+      result.modules[zmod.key] = m
+
+      modlist.add(m)
+
+    for m in modlist:
+      m.system = true
+      result.loadModule(m)
+
+    discard result.build_program(entry)
+
+    for m in modlist:
+      m.processed = true
+
+proc generate_incremental_code*(rt: RuntimeState, cc: CompileCtx,
+                              newEntry: Module) =
+  # TODO -- Rebuild the string cache?
+  let start   = rt.obj.moduleContents.len()
+  var ctx     = CodeGenState()
+  ctx.cc      = cc
+  ctx.zobj    = rt.obj
+  ctx.memos   = Memos()
+  rt.obj.spec = cc.attrSpec
+  ctx.minfo   = cc.modules
+
+  ctx.strCache.initDict()
+  ctx.memos.map.initDict()
+  ctx.cacheAllTypeInfo()
+  ctx.setupNewModules(rt.sectionDocs)
+  ctx.genModule(newEntry)
+  ctx.fillCallBackpatches()
+
+  rt.obj.entrypoint = int32(newEntry.objInfo.moduleId)
+
+  for i, item in rt.obj.moduleContents:
+    if i < start:
+      continue
+    var moduleAllocation = newSeq[pointer](item.moduleVarSize * 2)
+    rt.moduleAllocations.add(moduleAllocation)
+
 # TODO -- pushTypeOf needs to handle attributes.
 # TODO -- varargs for func entry.