I have this code that works on Windows, but not on Ubuntu MATE on the Raspberry Pi 3:

#
# nim_module.nim
#

type
  PySizeT* = clong
  
  # typedef int (*traverseproc)(PyObject *, visitproc, void *);
  TraverseProc* = proc (ob: PyObjectPtr,
                        prc: VisitProc, p: pointer): cint{.cdecl.}
  
  # typedef int (*inquiry)(PyObject *);
  Inquiry* = proc (ob: PyObjectPtr): cint{.cdecl.}
  
  # typedef void (*freefunc)(void *);
  FreeFunc* = proc (p: pointer){.cdecl.}
  
  # typedef int (*visitproc)(PyObject *, void *);
  VisitProc* = proc (ob: PyObjectPtr, p: pointer): cint{.cdecl.}
  
  # typedef struct _typeobject PyTypeObject; /* opaque */
  PyTypeObjectPtr* = ptr PyTypeObject
  PyTypeObject* {.final.} = object # Defined in "Include/object.h"
  
  # typedef struct _object {
  #     _PyObject_HEAD_EXTRA
  #     Py_ssize_t ob_refcnt;
  #     struct _typeobject *ob_type;
  # } PyObject;
  #
  # #define _PyObject_HEAD_EXTRA
  PyObjectPtr* = ptr PyObject
  PyObject* {.final.} = object # Defined in "Include/object.h"
    obRefcnt*: PySizeT
    obType*: PyTypeObjectPtr
  
  # struct PyMethodDef {
  #     const char  *ml_name;   /* The name of the built-in function/method */
  #     PyCFunction ml_meth;    /* The C function that implements it */
  #     int         ml_flags;   /* Combination of METH_xxx flags, which mostly
  #                                describe the args expected by the C func */
  #     const char  *ml_doc;    /* The __doc__ attribute, or NULL */
  # };
  # typedef struct PyMethodDef PyMethodDef;
  PyMethodDefPtr* = ptr PyMethodDef
  PyMethodDef* {.final.} = object  # Defined in "Include/methodobject.h"
    mlName*: cstring
    mlMeth*: PyCFunction
    mlFlags*: cint
    mlDoc*: cstring
  
  # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
  PyCFunction* = proc (self, args: PyObjectPtr): PyObjectPtr{.cdecl.}
  
  # typedef struct PyModuleDef{
  #   PyModuleDef_Base m_base;
  #   const char* m_name;
  #   const char* m_doc;
  #   Py_ssize_t m_size;
  #   PyMethodDef *m_methods;
  #   struct PyModuleDef_Slot* m_slots;
  #   traverseproc m_traverse;
  #   inquiry m_clear;
  #   freefunc m_free;
  # }PyModuleDef;
  PyModuleDefPtr* = ptr PyModuleDef
  PyModuleDef* {.final.}  = object
    m_base*: PyModuleDefBase
    m_name*: cstring
    m_doc*: cstring
    m_size*: PySizeT
    m_methods*: PyMethodDefPtr
    m_slots*: PyModuleDefSlotPtr
    m_traverse*: TraverseProc
    m_clear*: Inquiry
    m_free*: FreeFunc
  
  # typedef struct PyModuleDef_Base {
  #   PyObject_HEAD
  #   PyObject* (*m_init)(void);
  #   Py_ssize_t m_index;
  #   PyObject* m_copy;
  # } PyModuleDef_Base;
  # #define PyObject_HEAD PyObject ob_base;
  PyModuleDefBasePtr* = ptr PyModuleDefBase
  PyModuleDefBase* {.final.}  = object
    ob_base*: PyObject
    m_init*: proc (): PyObjectPtr {.cdecl.}
    m_index*: PySizeT
    m_copy*: PyObjectPtr
  
  # typedef struct PyModuleDef_Slot{
  #     int slot;
  #     void *value;
  # } PyModuleDef_Slot;
  PyModuleDefSlotPtr* = ptr PyModuleDefSlot
  PyModuleDefSlot* {.final.} = object
    slot*: cint
    value*: pointer

# PyObject* PyModule_Create2(struct PyModuleDef* module, int module_api_version)
proc moduleCreate2*(module: PyModuleDefPtr;
                    module_api_version: cint): PyObjectPtr {.cdecl,
                    importc: "PyModule_Create2" dynlib: "libpython3.5m.so.1".}
proc moduleCreate*(module: PyModuleDefPtr): PyObjectPtr {.cdecl.} =
  result = moduleCreate2(module, 3)

var
    nim_method_array: array[1, PyMethodDef] = [
        PyMethodDef(mlName:nil, mlMeth:nil, mlFlags:0, mlDoc:nil)
    ]
    nim_methods* {.exportc.}: PyMethodDefPtr = cast[PyMethodDefPtr](
        addr(nim_method_array)
    )

var nim_lexers* {.exportc.}: PyModuleDef = PyModuleDef(
    m_base: PyModuleDefBase(
        ob_base:PyObject(obRefcnt:1, obType:nil),
        m_init:nil,
        m_index:0,
        m_copy:nil),
    m_name: "nim_lexers",
    m_doc: "",
    m_size: -1,
    m_methods: nim_methods,
)

# Hack needed on windows
{.emit: """N_CDECL(void, NimMain)(void);""".}

proc PyInit_nim_lexers(): PyObjectPtr {.exportc, cdecl.} =
    {.emit: """NimMain();""".}
    result = moduleCreate(addr(nim_lexers))

It compiles without an error, but when trying to import the module from Python3 on Ubuntu MATE (RPi3) it throws:
Traceback (most recent call last)
nim_module.nim(140)    PyInit_nim_module
nim_module.nim(115)    moduleCreate
SIGSEGV: Illegal storage access. (Attempt to read from nil?)

I suspect it's an error in one of the types since I got the same errors when making the python3 wrapper, but I've been looking at the code for so long and cannot find anything. I copied all of the types from the python3 wrapper and added the C declarations on top of every type and proc definition for clearity, but maybe I missed something. It's

If anyone can see any mistakes or knows more about Python3, please help!

Thanks, Matic

2017-04-22 20:10:13
nim_methods is not NULL terminated / misses the sentinel.
2017-04-22 21:15:49

Hey Araq,

Doesn't nim_methods point to the array with only the sentinel, or am I missing something?

2017-04-22 21:40:09
Oh my bad.
2017-04-22 21:50:51

Ok, I corrected the PyObject object (it has only the obRefcnt and obType fields) but now I'm getting the following error:

*** Error in 'python3': free(): invalid pointer: 0x76508050 ***
Traceback (most recent call last)
nim_module.nim(133)    PyInit_nim_module
nim_module.nim(106)    moduleCreate
SIGABRT: Abnormal termination.
I checked that the Python3 and C compilers that are used for compiling are the same, which they are. I'm guessing there is either a mistake in one of the object declarations or I'm missing something different between Windows/Linux. I checked the Nim generated C files and they are very similar except for a few naming differences.

I have no more good ideas. If anyone finds anything, please let me know!

Matic

2017-04-28 13:12:16
Any way you can run it through Valgrind to see where the invalid free is coming from?
2017-04-29 11:12:45

Hey @JohnS,

Installed Valgrind and as a test I ran it with:

valgrind python3 -c "print('TEST')"
but I get an illegal instruction error. Also tried adding the python suppression file but the error still persists.

Any ideas?

2017-04-29 20:24:59
I now tried the following. Python3 initializes a module by using the C function PyInit_{MODULE_NAME}, inside which I only initialize the Python3 stuff using the C Py_Initialize function like so:
# The imported Python3 initialization function
proc initialize*(){.cdecl, importc: "Py_Initialize" dynlib: "libpython3.5m.so.1".}

# The module's init function
proc PyInit_nim_module() {.cdecl, exportc.} =
  {.emit: """NimMain();""".}
  initialize() # <--- ERROR
and already the error appears! I have no idea why? Is it maybe something that the NimMain() call does?
2017-04-30 17:16:48

Maybe you have the newer python3.6 on Ubuntu or the .so file has a different link name ? For me the second example compiles with python3.6 on openSuse Tumbleweed.

# The imported Python3 initialization function
proc initialize*(){.cdecl, importc: "Py_Initialize" dynlib: "libpython3.6m.so.1.0".}

# The module's init function
proc PyInit_nim_module() {.cdecl, exportc.} =
  {.emit: """NimMain();""".}
  initialize() #

Your first example does not compile , some error in nim_lexers.

2017-05-01 00:06:57

Hey @qqtop,

I fixed the first example above, could you try it again? Did you try importing the second compiled module from python3? It compiles on my machine too, but when I import the module from python3 it throws the error.

Thanks, Matic

2017-05-01 07:00:56
<<<••12••>>>