Hi.

I tried to re-write my "type register" to user SharedTable instead. I did not find a documentation page specifying the SharedTable API, so I read the code, but it's not all that obvious to me (in particular, the withXXX pattern). Mainly, atm, it seems my main problem is that SharedTable doesn't seem to have a "hasKey" or a "len" proc, and idk how to work around that. I tried to create a (very) simplified example of what I'm trying to do, but the compiler crashes on it, so I hope you at least get what I'm trying to do, even if this doesn't compile.

import locks
import typetraits
import sharedtables

type
  TypeRegister = object
    table: SharedTable[char, int]
    lock: Lock

proc register*(reg: ptr TypeRegister, T: typedesc): int =
  let tk = T.name[0] # In real code I have a "shared string" kind of thing as key
  if reg.table.hasKey(tk): # No "hasKey()"   :(
    result = reg.table.mget(tk) # I only need read access, but no "get(k)" or "[k]" AFAIK
  else:
    acquire(reg.lock)
    try:
      if reg.table.hasKey(tk):
        result = reg.table.mget(tk)
      else:
        let value: int = reg.table.len # No len() either. :(
        reg.table[tk] = value
        result = value
    finally:
      release(reg.lock)

when isMainModule:
  echo("Starting ...")
  let tr = createShared(TypeRegister)
  tr.table = initSharedTable[char, int]()
  initLock tr.lock
  echo("Registering bool ...")
  let id = register(tr, bool)
  echo("Done: " & $id)

On Windows 10 x64 I get a:

SIGSEGV: Illegal storage access. (Attempt to read from nil?)
when I try to compile this (but I don't get that in my real implementation).

Possible (ugly) work-around for missing:

  • hasKey(): just call mget(), and catch the exception.
  • len(): store the "theoretical" size of table in TypeRegister itself (size should only change in register() anyway).
2017-12-02 19:04:35

I have been trying to work around the missing hasKey() using mget() and a try/finally, but this fails to compile as well.

This is basically what I'm doing:

import sharedtables
import options
import typetraits

type
  TypeKey* = int
  TypeInfo*[M] = object
    myinfo: M
  TypeRegister[M] = object
    table: SharedTable[TypeKey, TypeInfo[M]]

proc createTypeRegister*(M: typedesc): TypeRegister[M] =
  result.table = initSharedTable[TypeKey, TypeInfo[M]](64)

proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
  ## Ugly work-around for missing SharedTable.hasKey()
  try:
    some(table.mget(k))
  except:
    none(B)

proc find*[M](reg: var TypeRegister[M], tk: TypeKey): Option[TypeInfo[M]] {.inline, noSideEffect.} =
  reg.table.find(tk)

proc `[]=`*[M](reg: var TypeRegister[M], tk: TypeKey, tv: TypeInfo[M]) {.inline.} =
  reg.table[tk] = tv

when isMainModule:
  echo("TESTING...")
  var t = createTypeRegister(bool)
  var tv: TypeInfo[bool]
  tv.myinfo = true
  t[TypeKey(4)] = tv
  let find2 = t.find(TypeKey(2))
  let find4 = t.find(TypeKey(4))
  assert(isNone(find2))
  assert(isSome(find4))
  echo("DONE")

This compiles and work fine. In my real code, the two "find()" procs are exactly the same as in this example. But calling "table.mget(k)" there results in this compilation error:

lib\pure\collections\sharedtables.nim(114, 10) Error: undeclared identifier: 'hasKey'

The code of SharedTable.mget() looks like this:

proc mget*[A, B](t: var SharedTable[A, B], key: A): var B =
  ## retrieves the value at ``t[key]``. The value can be modified.
  ## If `key` is not in `t`, the ``KeyError`` exception is raised.
  withLock t:
    var hc: Hash
    var index = rawGet(t, key, hc)
    let hasKey = index >= 0
    if hasKey: result = t.data[index].val
  if not hasKey:   # THIS IS LINE 114
    when compiles($key):
      raise newException(KeyError, "key not found: " & $key)
    else:
      raise newException(KeyError, "key not found")

I totally fail to see how this would produce a different result, if called by basically the same code.

This is the actual code, with one module (sharedarray) inserted "inline" for simplicity:

# A type-register maps types to IDs and back, at runtime.
# No attempt is made to return the same IDs on every run.

import locks
import typetraits
import hashes
import sharedtables
import options
import algorithm
import math
#import tables

#import sharedarray

#############################################################################
# This should be in imported module: sharedarray
#############################################################################
type
  # TODO Find a way to make the type restriction be recursive
  SharedArray*[T: not (ref|seq|string)] = ptr object
    ## Simple variable-length shared-heap array, with a pre-allocated capacity.
    ## Particularly useful because still compatible with open-array.
    ## It is *not* guarded by a lock, and therefore not thread-safe by default.
    size: int
      ## "first int" size, so we are compatible with openarray.
    mycapacity: int
      ## "second int" is required for seq-to-openarray compatibility, and used as "capacity" in our case.
    data: UncheckedArray[T]
      ## The data itself

proc len*[T](list: SharedArray[T]): int {.inline, noSideEffect.} =
  ## Returns the current length/size of the shared array. 0 if nil.
  if list != nil:
    list.size
  else:
    0

proc isNilOrEmpty*[T](list: SharedArray[T]): bool {.inline, noSideEffect.} =
  ## Returns true if the current length/size of the shared array is 0 or if it is nil.
  (list == nil) or (list.size == 0)

proc clear*[T](list: var SharedArray[T]): void =
  ## Sets the current length/size of the shared array to 0.
  if list != nil:
    list.size = 0

proc capacity*[T](list: SharedArray[T]): int {.inline, noSideEffect.} =
  ## Returns the fixed capacity of the shared array. 0 if nil.
  if list != nil:
    list.mycapacity
  else:
    0

iterator items*[T](it: var SharedArray[T]): T =
  ## Iterates over all array items. it cannot be nil.
  assert(it != nil)
  for i in 0..it.size-1:
    yield it.data[i]

proc initSharedArray*[T](capacity: int = 64): SharedArray[T] =
  ## Returns a new shared-heap array with a given maximum capacity, which must be power-of-two.
  assert(capacity >= 0)
  assert(isPowerOfTwo(capacity))
  result = cast[SharedArray[T]](allocShared0(2*sizeof(int)+capacity*sizeof(T)))
  result.mycapacity = capacity

proc deinitSharedArray*[T](list: var SharedArray[T]): void =
  ## Destroys shared-heap array, if not nil.
  if list != nil:
    deallocShared list
    list = nil

proc `[]`*[T](list: SharedArray[T]; i: int): var T {.inline, noSideEffect.} =
  ## Gets an array item.
  assert(list != nil)
  if (i < 0) or (i >= list.size):
    raise newException(Exception, "bad index: " & $i & " must be in [0, " & $list.size & "[")
  list.data[i]

proc `[]=`*[T](list: SharedArray[T]; i: int, x: T) {.inline.} =
  ## Sets an array item.
  assert(list != nil)
  if (i < 0) or (i >= list.size):
    raise newException(Exception, "bad index: " & $i & " must be in [0, " & $list.size & "[")
  list.data[i] = x

proc add*[T](list: var SharedArray[T]; x: T): void {.inline.} =
  ## Appends an array item.
  assert(list != nil)
  let size = list.size
  list.setLen(size+1)
  list.data[size] = x

proc toSharedArray*[T](src: openarray[T], capacity: int = -1): SharedArray[T] =
  ## Copies an open-array into a new SharedArray.
  ## capacity is used, if bigger than src.len
  let size = len(src)
  assert(size >= 0)
  result = initSharedArray[T](max(size, capacity))
  for i in 0..size-1:
    result.data[i] = src[i]
  result.size = size

proc toSeq*[T](list: SharedArray[T]): seq[T] =
  ## Copies a SharedArray into a Seq
  assert(list != nil)
  result = newSeq[T](list.size)
  for i in 0..list.size-1:
    result[i] = list.data[i]

template asOpenArray*[T](list: SharedArray[T]): openarray[T] =
  ## Casts a SharedArray to an open array.
  cast[seq[T]](list)

proc setLen*[T](list: var SharedArray[T], newLen: int): void =
  ## Sets the current length/size of the shared array. It cannot be nil.
  ## If newLen is greater than the capacity, the SharedArray is copied into
  ## a new SharedArray with the required capacity, list is set to the new
  ## SharedArray, and the old SharedArray is destroyed.
  assert(list != nil)
  if newLen < 0:
    raise newException(Exception, "newLen " & $newLen & " too small")
  if newLen > list.mycapacity:
    var old = list
    list = toSharedArray(asOpenArray(old), nextPowerOfTwo(newLen))
    deinitSharedArray(old)
  list.size = newLen

#############################################################################
# End of imported module: sharedarray
#############################################################################

type
  TypeKey* = object
    ## Represents a "key" for the type register.
    ## Types are automatically converted to it with a converter.
    myname: cstring
    myhash: Hash
  TypeInfo*[M] = object
    ## The info about a type: it's name, ID, it's size, and whatever meta-info the user also needed.
    myname: cstring
    myid: uint32
    mysize: uint32
    myinfo: M
  TypeRegister[M] = object
    ## A type register containing type T meta-info.
    table: SharedTable[TypeKey, TypeInfo[M]]
    list: SharedArray[TypeInfo[M]] ## Allows quickly accessing TypeInfo[M] by ID
    myfrozen: bool ## TypeRegister cannot be modified anymore.
    lock: Lock

template withLock(t, x: untyped) =
  acquire(t.lock)
  try:
    x
  finally:
    release(t.lock)


proc copyCString(s: cstring): cstring =
  ## Creates a copy of a (temporary) cstring into the shared-heap
  let len = s.len
  result = cast[cstring](allocShared(len+1))
  copyMem(cast[pointer](result), cast[pointer](s), len+1)

converter toTypeKey*(T: typedesc): TypeKey {.inline, noSideEffect.} =
  ## Automatically converts a type to a TypeKey. The type name is *not* cloned.
  result.myname = T.name
  result.myhash = hash(result.myname)

proc hash*(tk: TypeKey): Hash {.inline, noSideEffect.} =
  ## Returns the hash of the TypeKey
  result = tk.myhash

proc `$`*(tk: TypeKey): string {.inline.} =
  ## Returns the string representation of the TypeKey
  result = $tk.myname

proc `==`*(a, b: TypeKey): bool {.inline, noSideEffect.} =
  ## Compares TypeKeys
  (a.myhash == b.myhash) and (cmp(a.myname, b.myname) == 0)

proc `$`*[M](ti: TypeInfo[M]): string {.inline.} =
  ## Returns the string representation of the TypeKey
  result = $ti.myname

proc name*[M](ti: TypeInfo[M]): cstring {.inline, noSideEffect.} =
  ## Returns the id of the type
  result = ti.myname

proc id*[M](ti: TypeInfo[M]): uint32 {.inline, noSideEffect.} =
  ## Returns the id of the type
  result = ti.myid

proc size*[M](ti: TypeInfo[M]): uint32 {.inline, noSideEffect.} =
  ## Returns the size of the type
  result = ti.mysize

proc info*[M](ti: TypeInfo[M]): M {.inline, noSideEffect.} =
  ## Returns the meta-info of the type
  result = ti.myinfo

proc initTypeInfo*[M](name: cstring, id: uint32, size: uint32, info: M): TypeInfo[M] {.inline, noSideEffect.} =
  ## Initialises and return a TypeInfo[M]. name is *not* cloned.
  result.myname = name
  result.myid = id
  result.mysize = size
  result.myinfo = info

# NOT CURRENTLY THREAD-SAFE:
#iterator items*[M](reg: TypeRegister[M]): TypeInfo[M] =
#  ## iterates over any TypeInfo[M] in the TypeRegister `reg`.
#  reg.list.items

proc len*[M](reg: TypeRegister[M]): int {.inline, noSideEffect.} =
  ## Returns the current size of the TypeRegister.
  reg.list.len

proc createTypeRegister*(M: typedesc, capacity: int = 64): TypeRegister[M] =
  ## Creates a new type register.
  assert(capacity >= 0)
  assert(isPowerOfTwo(capacity))
  result.table = initSharedTable[TypeKey, TypeInfo[M]](capacity)
  result.list = initSharedArray[TypeInfo[M]](capacity)
  initLock result.lock

proc createTypeRegister*(M: typedesc, src: openArray[TypeInfo[M]]): TypeRegister[M] =
  ## Creates a new type register, and initialise it from src.
  ## Note: src will be *sorted* by ID during creation!
  # First, validate the hell out of the input.
  proc ti_id_cmp(a,b: TypeInfo[M]): int =
    int(a.myid) - int(b.myid)
  sort(src, ti_id_cmp)
  var nextID: uint32 = 0
  for ti in src:
    if ti.myname == nil:
      raise newException(Exception, "Type (ID: " & $ti.myid & "): name cannot be nil!")
    if ti.myname.len == 0:
      raise newException(Exception, "Type (ID: " & $ti.myid & "): name cannot be empty!")
    if ti.myname.split.len != 1:
      raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): name cannot contain whitespace!")
    if nextID != ti.myid:
      raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): ID should be " & $nextID)
    if ti.mysize == 0:
      raise newException(Exception, "Type (ID: " & $ti.myid & ", name: " & ti.myname & "): cannot have size 0")
    nextID.inc
  # XXX TODO!
  #[
  var keys = initTable[TypeKey, bool](nextPowerOfTwo(src.len))
  for ti in src:
    var tk : TypeKey
    tk.myname = ti.myname
    tk.myhash = hash(tk.myname)
    if keys.hasKeyOrPut(tk):
      raise newException(Exception, "Duplicate type name: " & $ti.myname)
  ]#
  # Input seems OK... so create it.
  result = createTypeRegister(M, src.len)
  result.withLock:
    for ti in src:
      var tk : TypeKey
      tk.myname = copyCString(ti.myname)
      tk.myhash = hash(tk.myname)
      let ti = initTypeInfo(tk.myname, ti.myid, ti.mysize, ti.myinfo)
      result.table[tk] = ti
      result.list.add(ti)

proc createTypeRegister*(M: typedesc, src: iterator : TypeInfo[M]): TypeRegister[M] =
  ## Creates a new type register, and initialise it from src.
  var infos = newSeq[TypeInfo[M]]
  for ti in src:
    infos.add(ti)
  createTypeRegister(M, infos)

proc destroyTypeRegister*[M](reg: var TypeRegister[M]): void =
  ## Destroys a new type register.
  reg.withLock:
    deinitSharedTable(reg.table)
    deinitSharedArray(reg.list)
  deinitLock(reg.lock)

proc freeze*[M](reg: var TypeRegister[M]): void =
  ## Freezes the type register, so that no new type can be registered.
  reg.withLock:
    reg.myfrozen = true

proc frozen*[M](reg: var TypeRegister[M]): bool =
  ## Returns true if no new type can be registered.
  reg.withLock:
    result = reg.myfrozen

proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
  ## Ugly work-around for missing SharedTable.hasKey()
  try:
    #############################################################################
    # Start of non-compiling code:
    #############################################################################
    some(table.mget(k))
    #############################################################################
    # End of non-compiling code.
    #############################################################################
  except:
    none(B)

proc find*[M](reg: var TypeRegister[M], tk: TypeKey): Option[TypeInfo[M]] {.inline, noSideEffect.} =
  ## Returns a type's type-info, if present.
  reg.table.find(tk)

proc register*[M](reg: var TypeRegister[M], T: typedesc, info: M): TypeInfo[M] =
  ## Register a new type, and returns it's type-info.
  let tk = toTypeKey(T)
  var opt = reg.find(tk)
  if isSome(opt):
    result = unsafeGet(opt)
  else:
    reg.withLock:
      var opt = reg.table.find(tk)
      if isSome(opt):
        result = unsafeGet(opt)
      else:
        let id = uint32(reg.len)
        if id == high(uint32):
          raise newException(Exception, "Too many types!")
        if reg.myfrozen:
          raise newException(Exception, "Type register is frozen!")
        var tk2 : TypeKey
        tk2.myname = copyCString(tk.myname)
        tk2.myhash = tk.myhash
        result = initTypeInfo(tk2.myname, id, uint32(sizeof(T)), info)
        reg.table[tk] = result
        reg.list.add(result)

proc get*[M](reg: var TypeRegister[M], T: typedesc): TypeInfo[M] {.inline.} =
  ## Returns a type's type-info, if present.
  ## Otherwise, register the type with a default meta-info!
  var m: M
  register(reg, T, m)

when isMainModule:
  import threadpool
  
  var reg = createTypeRegister(bool)
  
  proc typeIDTestHelper(): uint32 = reg.get(uint8).id
  
  proc testTypeID(): void =
    echo "STARTING testTypeID()"
    let t0 = reg.get(uint8).id
    let t1 = reg.get(bool).id
    let fv = spawn typeIDTestHelper()
    let t3 = ^fv
    assert(t0 != t1)
    assert(t0 == t3)
    echo "DONE testTypeID()"
  
  proc testTypeName(): void =
    echo "STARTING testTypeName()"
    let t0 = reg.get(uint8).name
    let t1 = reg.get(bool).name
    assert($t0 == "uint8")
    assert($t1 == "bool")
    echo "DONE testTypeName()"
  
  testTypeID()
  testTypeName()
  
  destroyTypeRegister(reg)

2017-12-03 19:42:38

You have your own withLock implementation that the instantiation of mget prefers which introduces a new scope and so hasKey is outside of the scope. Probably sharedtables should be changed so that withLock is not used as a mixin. In the meantime, name yours withL and it compiles if you also change your find to

proc find[A,B](table: var SharedTable[A,B], k: A): Option[B] {.inline, noSideEffect.} =
  ## Ugly work-around for missing SharedTable.hasKey()
  result = try:
      some(table.mget(k))
    except:
      none(B)

That len and hasKey are missing is no oversight btw, it's almost impossible to use these without introducing subtle races. Yes, yes, I know, you only insert at program startup and then only read from it. But still. Why not use the variant of withValue that takes 2 pieces of code.

2017-12-03 21:26:26

@Araq Thanks! This seems to work.

it's almost impossible to use these without introducing subtle races

I assume you mean races in the user code, rahter than the SharedTable code, I guess.

But I must still ask: I've seen the "withLock" pattern used im multiple place (OK, mine was slightly changed). As long as it's not "public", I thought they could not interfere with each other? How is that possible? Is it because it's a template?

2017-12-03 22:48:36