gr-osmosdr/docs/doxygen/update_pydoc.py

#
# Copyright 2010-2012 Free Software Foundation, Inc.
#
# This file was generated by gr_modtool, a tool from the GNU Radio framework
# This file is a part of gnuradio
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
#
"""
Updates the *pydoc_h files for a module
Execute using: python update_pydoc.py xml_path outputfilename

The file instructs Pybind11 to transfer the doxygen comments into the
python docstrings.

"""

import os, sys, time, glob, re, json
from argparse import ArgumentParser

from doxyxml import DoxyIndex, DoxyClass, DoxyFriend, DoxyFunction, DoxyFile
from doxyxml import DoxyOther, base

def py_name(name):
    bits = name.split('_')
    return '_'.join(bits[1:])

def make_name(name):
    bits = name.split('_')
    return bits[0] + '_make_' + '_'.join(bits[1:])


class Block(object):
    """
    Checks if doxyxml produced objects correspond to a gnuradio block.
    """

    @classmethod
    def includes(cls, item):
        if not isinstance(item, DoxyClass):
            return False
        # Check for a parsing error.
        if item.error():
            return False
        friendname = make_name(item.name())
        is_a_block = item.has_member(friendname, DoxyFriend)
        # But now sometimes the make function isn't a friend so check again.
        if not is_a_block:
            is_a_block = di.has_member(friendname, DoxyFunction)
        return is_a_block

class Block2(object):
    """
    Checks if doxyxml produced objects correspond to a new style
    gnuradio block.
    """

    @classmethod
    def includes(cls, item):
        if not isinstance(item, DoxyClass):
            return False
        # Check for a parsing error.
        if item.error():
            return False
        is_a_block2 = item.has_member('make', DoxyFunction) and item.has_member('sptr', DoxyOther)
        return is_a_block2


def utoascii(text):
    """
    Convert unicode text into ascii and escape quotes and backslashes.
    """
    if text is None:
        return ''
    out = text.encode('ascii', 'replace')
    # swig will require us to replace blackslash with 4 backslashes
    # TODO: evaluate what this should be for pybind11
    out = out.replace(b'\\', b'\\\\\\\\')
    out = out.replace(b'"', b'\\"').decode('ascii')
    return str(out)


def combine_descriptions(obj):
    """
    Combines the brief and detailed descriptions of an object together.
    """
    description = []
    bd = obj.brief_description.strip()
    dd = obj.detailed_description.strip()
    if bd:
        description.append(bd)
    if dd:
        description.append(dd)
    return utoascii('\n\n'.join(description)).strip()

def format_params(parameteritems):
    output = ['Args:']
    template = '    {0} : {1}'
    for pi in parameteritems:
        output.append(template.format(pi.name, pi.description))
    return '\n'.join(output)

entry_templ = '%feature("docstring") {name} "{docstring}"'
def make_entry(obj, name=None, templ="{description}", description=None, params=[]):
    """
    Create a docstring key/value pair, where the key is the object name.

    obj - a doxyxml object from which documentation will be extracted.
    name - the name of the C object (defaults to obj.name())
    templ - an optional template for the docstring containing only one
            variable named 'description'.
    description - if this optional variable is set then it's value is
            used as the description instead of extracting it from obj.
    """
    if name is None:
        name=obj.name()
        if hasattr(obj,'_parse_data') and hasattr(obj._parse_data,'definition'):
            name=obj._parse_data.definition.split(' ')[-1]
    if "operator " in name:
        return ''
    if description is None:
        description = combine_descriptions(obj)
    if params:
        description += '\n\n'
        description += utoascii(format_params(params))
    docstring = templ.format(description=description)

    return {name: docstring}


def make_class_entry(klass, description=None, ignored_methods=[], params=None):
    """
    Create a class docstring key/value pair.
    """
    if params is None:
        params = klass.params
    output = {}
    output.update(make_entry(klass, description=description, params=params))
    for func in klass.in_category(DoxyFunction):
        if func.name() not in ignored_methods:
            name = klass.name() + '::' + func.name()
            output.update(make_entry(func, name=name))
    return output


def make_block_entry(di, block):
    """
    Create class and function docstrings of a gnuradio block
    """
    descriptions = []
    # Get the documentation associated with the class.
    class_desc = combine_descriptions(block)
    if class_desc:
        descriptions.append(class_desc)
    # Get the documentation associated with the make function
    make_func = di.get_member(make_name(block.name()), DoxyFunction)
    make_func_desc = combine_descriptions(make_func)
    if make_func_desc:
        descriptions.append(make_func_desc)
    # Get the documentation associated with the file
    try:
        block_file = di.get_member(block.name() + ".h", DoxyFile)
        file_desc = combine_descriptions(block_file)
        if file_desc:
            descriptions.append(file_desc)
    except base.Base.NoSuchMember:
        # Don't worry if we can't find a matching file.
        pass
    # And join them all together to make a super duper description.
    super_description = "\n\n".join(descriptions)
    # Associate the combined description with the class and
    # the make function.
    output = {}
    output.update(make_class_entry(block, description=super_description))
    output.update(make_entry(make_func, description=super_description,
                                  params=block.params))
    return output

def make_block2_entry(di, block):
    """
    Create class and function docstrings of a new style gnuradio block
    """
    # For new style blocks all the relevant documentation should be
    # associated with the 'make' method.
    class_description = combine_descriptions(block)
    make_func = block.get_member('make', DoxyFunction)
    make_description = combine_descriptions(make_func)
    description = class_description + "\n\nConstructor Specific Documentation:\n\n" + make_description
    # Associate the combined description with the class and
    # the make function.
    output = {}
    output.update(make_class_entry(
            block, description=description,
            ignored_methods=['make'], params=make_func.params))
    makename = block.name() + '::make'
    output.update(make_entry(
            make_func, name=makename, description=description,
            params=make_func.params))
    return output

def get_docstrings_dict(di, custom_output=None):

    output = {}
    if custom_output:
        output.update(custom_output)

    # Create docstrings for the blocks.
    blocks = di.in_category(Block)
    blocks2 = di.in_category(Block2)

    make_funcs = set([])
    for block in blocks:
        try:
            make_func = di.get_member(make_name(block.name()), DoxyFunction)
            # Don't want to risk writing to output twice.
            if make_func.name() not in make_funcs:
                make_funcs.add(make_func.name())
                output.update(make_block_entry(di, block))
        except block.ParsingError:
            sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
            raise

    for block in blocks2:
        try:
            make_func = block.get_member('make', DoxyFunction)
            make_func_name = block.name() +'::make'
            # Don't want to risk writing to output twice.
            if make_func_name not in make_funcs:
                make_funcs.add(make_func_name)
                output.update(make_block2_entry(di, block))
        except block.ParsingError:
            sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
            raise

    # Create docstrings for functions
    # Don't include the make functions since they have already been dealt with.
    funcs = [f for f in di.in_category(DoxyFunction)
             if f.name() not in make_funcs and not f.name().startswith('std::')]
    for f in funcs:
        try:
            output.update(make_entry(f))
        except f.ParsingError:
            sys.stderr.write('Parsing error for function {0}\n'.format(f.name()))

    # Create docstrings for classes
    block_names = [block.name() for block in blocks]
    block_names += [block.name() for block in blocks2]
    klasses = [k for k in di.in_category(DoxyClass)
               if k.name() not in block_names and not k.name().startswith('std::')]
    for k in klasses:
        try:
            output.update(make_class_entry(k))
        except k.ParsingError:
            sys.stderr.write('Parsing error for class {0}\n'.format(k.name()))

    # Docstrings are not created for anything that is not a function or a class.
    # If this excludes anything important please add it here.

    return output

def sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, output_dir, filter_str=None):
    if filter_str:
        docstrings_dict = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str)}

    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
    
        for pydoc_file in pydoc_files:
            if filter_str:
                filter_str2 = "::".join((filter_str,os.path.split(pydoc_file)[-1].split('_pydoc_template.h')[0]))
                docstrings_dict2 = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str2)}
            else:
                docstrings_dict2 = docstrings_dict



            file_in = open(pydoc_file,'r').read()
            for key, value in docstrings_dict2.items(): 
                file_in_tmp = file_in 
                try:
                    doc_key = key.split("::")
                    # if 'gr' in doc_key:
                    #     doc_key.remove('gr') 
                    doc_key = '_'.join(doc_key)
                    regexp = r'(__doc_{} =\sR\"doc\()[^)]*(\)doc\")'.format(doc_key)
                    regexp = re.compile(regexp, re.MULTILINE)
                
                    (file_in, nsubs) = regexp.subn(r'\1'+value+r'\2', file_in, count=1)
                    if nsubs == 1:
                        status_file.write("PASS: " + pydoc_file + "\n")
                except KeyboardInterrupt:
                    raise KeyboardInterrupt
                except: # be permissive, TODO log, but just leave the docstring blank
                    status_file.write("FAIL: " + pydoc_file + "\n")
                    file_in = file_in_tmp

            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
            # FIXME: Remove this debug print
            print('output docstrings to {}'.format(output_pathname))
            with open(output_pathname,'w') as file_out:
                file_out.write(file_in)

def copy_docstring_templates(pydoc_files, output_dir):
    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
        for pydoc_file in pydoc_files:
            file_in = open(pydoc_file,'r').read()
            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
            # FIXME: Remove this debug print
            print('copy docstrings to {}'.format(output_pathname))
            with open(output_pathname,'w') as file_out:
                file_out.write(file_in)
        status_file.write("DONE")

def argParse():
    """Parses commandline args."""
    desc='Scrape the doxygen generated xml for docstrings to insert into python bindings'
    parser = ArgumentParser(description=desc)
    
    parser.add_argument("function", help="Operation to perform on docstrings", choices=["scrape","sub","copy"])

    parser.add_argument("--xml_path")
    parser.add_argument("--bindings_dir")
    parser.add_argument("--output_dir")
    parser.add_argument("--json_path")
    parser.add_argument("--filter", default=None)

    return parser.parse_args()

if __name__ == "__main__":
    # Parse command line options and set up doxyxml.
    args = argParse()
    if args.function.lower() == 'scrape':
        di = DoxyIndex(args.xml_path)
        docstrings_dict = get_docstrings_dict(di)
        with open(args.json_path, 'w') as fp:
            json.dump(docstrings_dict, fp)
    elif args.function.lower() == 'sub':
        with open(args.json_path, 'r') as fp:
            docstrings_dict = json.load(fp)
        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
        sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, args.output_dir, args.filter)
    elif args.function.lower() == 'copy':
        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
        copy_docstring_templates(pydoc_files, args.output_dir)