makeStructs.py

# Copyright 2004-2010 Grant T. Olson.
# See license.txt for terms.

import re, sys, glob, os, time

#These files are in the 'include' directory, but are modules.
#They won't get built properly
skipFiles = ('datetime.h','py_curses.h','structseq.h','symtable.h')


def emitFileHeader():
    print """/* Copyright 2004-2006 Grant T. Olson. See license.txt for terms.*/
#include <Python.h>

/* file autogenerated by pyasm's makeStructs script on """ + time.ctime() + """ %/

/* Preprocessor abuse at it's finest.

   We could probably do all of this in a straight python file, but then
   it wouldn't be in sync with a particular build.  This insures we have
   the right offsets for our structs in a way we can't in pure python
*/

#define OFFSET_STRING(f) #f
#define OFFSET(m,s,f) \
    offset = PyInt_FromLong((long)&(((s*)0)->f)); \
    Py_INCREF(offset); \
    PyModule_AddObject(m, OFFSET_STRING(f), offset);

/* Py_DEBUG implies Py_TRACE_REFS. */
#if defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
#define Py_TRACE_REFS
#endif

/* Py_TRACE_REFS implies Py_REF_DEBUG. */
#if defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
#define Py_REF_DEBUG
#endif


static PyObject *StructsError;
static PyObject *offset;

static PyMethodDef StructsMethods[] = {

    {NULL, NULL, 0, NULL}        /* Sentinel */
};

""" 

def emitFileBody():
    print """
static void
load_PyObject(PyObject* module)
{
#ifdef Py_TRACE_REFS
    OFFSET(module,PyObject,_ob_next);
    OFFSET(module,PyObject,_ob_prev);
#endif
    OFFSET(module,PyObject,ob_refcnt);
    OFFSET(module,PyObject,ob_type);
}

static void
load_PyVarObject(PyObject* module)
{
    load_PyObject(module);
    OFFSET(module,PyVarObject,ob_size);
}


"""

def emitFileFooter(modules):
    print """

PyMODINIT_FUNC
initstructs(void)
{
    
    PyObject *m, *n, *o;
    /*PyObject *offset;*/

    m = Py_InitModule("structs", StructsMethods);
    n = Py_InitModule("PyObject", StructsMethods);
    o = Py_InitModule("PyVarObject", StructsMethods);
    
    StructsError = PyErr_NewException("structs.StructsError", NULL, NULL);
    Py_INCREF(StructsError);
    PyModule_AddObject(m, "StructsError", StructsError);

    load_PyObject(n);
    Py_INCREF(n);
    PyModule_AddObject(m, "PyObject", n);

    load_PyVarObject(o);
    Py_INCREF(o);
    PyModule_AddObject(m, "PyVarObject", o);
    
    %s
}""" % ''.join(["    load_%s(m);\n" % x for x in modules])


def emitModuleHeader(moduleName):
    print """static void
load_%(funcname)s(PyObject *structs)
{
    PyObject *sm = Py_InitModule("%(funcname)s",StructsMethods);

""" % {'funcname':moduleName}

def emitModuleFooter(moduleName):
    print """

    Py_INCREF(sm);
    PyModule_AddObject(structs,"%(funcname)s",sm);
}""" % {'funcname':moduleName}

structsRe = re.compile("typedef\s+struct\s*\w*\s*{(.*?)}\s*(\w+)",re.DOTALL)
typeofRe = re.compile(r"(?P<type>\w+)\s*(?P<rest>[^;]+);")
variablesRe = re.compile(r"(\(|\)|\*\*|\*|\[|\]|\w+)[,\s]*")
names = []

def emitComment(commentText):
    print "/* %s */" % commentText

def emitRaw(rawText):
    print rawText

def emitOffset(name,val):
    print "    OFFSET(sm,%s,%s);" % (name,val)

def parse_filetext(filetext):
    global names
    for struct in structsRe.findall(filetext):
        body,name = struct
        if name in ('PyObject','PyVarObject', 'PyFrameObject'):
            emitComment("Skipping object %s" % name)
            continue
        print >> sys.stderr, "NAME", name

        startComment = body.find("/*")
        while startComment >= 0: #strip multiline comments
          endComment = body.find("*/",startComment) + 2
          body = body[:startComment] + body[endComment:]
          startComment = body.find("/*")

        lines = body.split("\n")
        isPyObject = False
        for line in lines:
            
            line = line.strip()
            if not line:
                continue
            print >> sys.stderr, "LINE:" , line
            if line.startswith("#"):
                print >> sys.stderr, "PREPROCESSOR DIRECTIVE"
                emitRaw(line)
            elif line == 'PyObject_HEAD':
                print >> sys.stderr, "HEADER" , line
                
                isPyObject = True
                emitModuleHeader(name)
                names.append(name)
                emitRaw("    load_PyObject(sm);")
            elif line == 'PyObject_VAR_HEAD':
                print >> sys.stderr, "HEADER" , line
                
                isPyObject = True
                emitModuleHeader(name)
                names.append(name)
                emitRaw("    load_PyVarObject(sm);")
            elif line:
                if isPyObject == False:
                    print >> sys.stderr, "NOT A PyObject: SKIPPING" , name
                    emitComment("Skipping struct %s, not a PyObject based struct" % name)
                    break
                typeof,rest = typeofRe.match(line).groups()
                print >> sys.stderr, "TYPE", typeof
                vars = variablesRe.findall(rest)
                vars.reverse()

                if typeof == "struct": # skip struct def
                    print >> sys.stderr, "STRUCT", vars
                    vars.pop()
                
                while vars:
                    var = vars.pop()
                    if var in ('*', '**'):
                        var = vars.pop()
                        if var == "(":
                            #function pointer
                            print >> sys.stderr, "FUNCTION POINTER", vars
                            var = vars.pop()
                            if var != "*":
                                print >> sys.stderr, var, vars
                                raise RuntimeError("Invalid Function Pointer "
                                               "format: %s.  Expected '*' got %s from %s" % (line,var,vars))
                            var = vars.pop()
                            emitOffset(name, var)
                            vars = None
                        else:
                            print >> sys.stderr, "POINTER", var
                            emitOffset(name, var)
                    elif var == '(':
                        print >> sys.stderr, "FUNCTION POINTER", vars
                        var = vars.pop()
                        print >> sys.stderr, "NAME VAR" , name, var
                        if var != "*":
                            print >> sys.stderr, var, vars
                            raise RuntimeError("Invalid Function Pointer "
                                               "format: %s.  Expected '*' got %s from %s" % (line,var,vars))
                        var = vars.pop()
                        
                        emitOffset(name, var)
                        vars = None
                    elif var == "[":

                        print >> sys.stderr, "SKIPPING ARRAY STUB" , vars
                        var = vars.pop()
                        var = vars.pop()
                    else:
                        print >> sys.stderr, "normal", var
                        emitOffset(name,var)

        if isPyObject == True:
            emitModuleFooter(name)
                        
def parse_headers():
    headerDir = os.path.join(sys.exec_prefix, "include")
    headerFiles = glob.glob(os.path.join(headerDir,"*.h"))
    headerFiles = [x for x in headerFiles if os.path.split(x)[1] not in skipFiles]
    for filename in headerFiles:
        
        print >> sys.stderr, "PROCESSING FILE", filename
        print "\n\n/* Generated from file %s */\n\n" % filename
        f = file(filename) 
        filetext = f.read()
        f.close()
        parse_filetext(filetext)

def make_struct_c():
    emitFileHeader()
    emitFileBody()
    parse_headers()
    emitFileFooter(names)

make_struct_c()