parsing with libclang : getting CXX_BASE_SPECIFIER cursors when base types unknown

c++, clang, parsing, python

Solution

I'm going to answer this one myself, because the code I came up with can be useful to future googlers.

In the end, I have coded the both methods that are supposed to work to retreive the list of base classes in the inheritance list on a class declaration line.

one using the AST cursor and one fully manual, coping as much as it can with C++ complexity.

here is the whole result:

#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on 2013/12/09

@author: voddou
'''

import sys
import re
import clang.cindex
import os
import string

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    CYAN = '\033[96m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    MAGENTA = '\033[95m'
    GREY = '\033[90m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''
        self.CYAN = ''
        self.MAGENTA = ''
        self.GREY = ''

from contextlib import contextmanager

@contextmanager
def scopedColorizer(color):
    sys.stdout.write(color)
    yield
    sys.stdout.write(bcolors.ENDC)

#clang.cindex.Config.set_library_file("C:/python27/DLLs/libclang.dll")

src_filepath = sys.argv[1]
src_basename = os.path.basename(src_filepath)

parseeLines = file(src_filepath).readlines()

def trim_all(astring):
    return "".join(astring.split()) 

def has_token(line, token):
    trimed = trim_all(line)
    pos = string.find(trimed, token)
    return pos != -1

def has_any_token(line, token_list):
    results = [has_token(line, t) for t in token_list]
    return any(results)

def is_any(astring, some_strings):
    return any([x == astring for x in some_strings])

def comment_out(line):
    return "//" + line

# alter the original file to remove #inlude directives and protective ifdef blocks
for i, l in enumerate(parseeLines):
    if has_token(l, "#include"):
        parseeLines[i] = comment_out(l)
    elif has_any_token(l, ["#ifdef", "#ifdefined", "#ifndef", "#if!defined", "#endif", "#elif", "#else"]):
        parseeLines[i] = comment_out(l)

index = clang.cindex.Index.create()
tu = index.parse(src_basename,
                 args=["-std=c++98"],
                 unsaved_files=[(src_basename, "".join(parseeLines))],
                 options=clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)

print 'Translation unit:', tu.spelling, "\n"

def gather_until(strlist, ifrom, endtokens):
    """make one string out of a list of strings, starting from a given index, until one token in endtokens is found.
    ex: gather_until(["foo", "toto", "bar", "kaz"], 1, ["r", "z"])
        will yield "totoba"
    """
    result = strlist[ifrom]
    nextline = ifrom + 1
    while not any([string.find(result, token) != -1 for token in endtokens]):
        result = result + strlist[nextline]
        nextline = nextline + 1
    nearest = result
    for t in endtokens:
        nearest = nearest.partition(t)[0]
    return nearest

def strip_templates_parameters(declline):
    """remove any content between < >
    """
    res = ""
    nested = 0
    for c in declline:
        if c == '>':
            nested = nested - 1
        if nested == 0:
            res = res + c
        if c == '<':
            nested = nested + 1
    return res

# thanks Markus Jarderot from Stackoverflow.com
def comment_remover(text):
    def replacer(match):
        s = match.group(0)
        if s.startswith('/'):
            return ""
        else:
            return s
    pattern = re.compile(
        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
        re.DOTALL | re.MULTILINE
    )
    return re.sub(pattern, replacer, text)

def replace_any_of(haystack, list_of_candidates, by_what):
    for cand in list_of_candidates:
        haystack = string.replace(haystack, cand, by_what)
    return haystack

cxx_keywords = ["class", "struct", "public", "private", "protected"]

def clean_name(displayname):
    """remove namespace and type tags
    """
    r = displayname.rpartition("::")[2]
    r = replace_any_of(r, cxx_keywords, "")
    return r

def find_parents_using_clang(node):
    l = []
    for c in node.get_children():
        if c.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
            l.append(clean_name(c.displayname))
    return None if len(l) == 0 else l

# syntax based custom parsing
def find_parents_list(node):
    ideclline = node.location.line - 1
    declline = parseeLines[ideclline]
    with scopedColorizer(bcolors.WARNING):
        print "class decl line:", declline.strip()
    fulldecl = gather_until(parseeLines, ideclline, ["{", ";"])
    fulldecl = clean_name(fulldecl)
    fulldecl = trim_all(fulldecl)
    if string.find(fulldecl, ":") != -1:    # if inheritance exists on the declaration line
        baselist = fulldecl.partition(":")[2]
        res = strip_templates_parameters(baselist)  # because they are separated by commas, they would break the split(",")
        res = comment_remover(res)
        res = res.split(",")
        return res
    return None

# documentation generator
def make_htll_visitor(node):
    if (node.kind == clang.cindex.CursorKind.CLASS_DECL
        or node.kind == clang.cindex.CursorKind.STRUCT_DECL
        or node.kind == clang.cindex.CursorKind.CLASS_TEMPLATE):

        bases2 = find_parents_list(node)
        bases = find_parents_using_clang(node)
        if bases is not None:
            with scopedColorizer(bcolors.CYAN):
                print "class clang list of bases:", str(bases)

        if bases2 is not None:
            with scopedColorizer(bcolors.MAGENTA):
                print "class manual list of bases:", str(bases2)


def visit(node, func):
    func(node)
    for c in node.get_children():
        visit(c, func)

visit(tu.cursor, make_htll_visitor)

with scopedColorizer(bcolors.OKGREEN):
    print "all over"

this code has allowed me to accept incomplete C++ translation units, correctly parsing declarations such as this:

struct ComplexBuffer
    : IAnimatable
    , Bugger,

        Mozafoka
{
};

coping also with these:

struct AnimHandler : NonCopyable, IHandlerPrivateGetter< AnimHandler, AafHandler > // CRTP
{
...
};

giving me this output:

class manual list of bases: ['NonCopyable', 'IHandlerPrivateGetter<>']

which is nice, the `clang` function version didn't return a single class in the base list. Now it is forseeable to merge the result of both these functions using a `set` to be on the safe side in case the manual parser would miss something. However I'm thinking this could cause subtle duplications because of the difference between `displayname` and my own parser.

But there you go, googlers, a nice clang python documentation generator template that doesn't need full correctness of the build options and is pretty fast because it totally ignores the `include`s statements.

nice day to all.

Problem

Original source