#! /usr/bin/python3
# -*- coding: UTF-8 -*-

from typing import Any, Optional, Union, List, Dict, Callable, Iterable
import sys
import re
import subprocess
import warnings

errors = 0
def warn(msg: str, error: Any = None) -> None:
    global errors
    errors += 1
    if error is None:
        warnings.warn("-- "+str(errors)+" --\n  "+msg, RuntimeWarning, 2)
    else:
        warnings.warn("-- "+str(errors)+" --\n  "+msg+
                      "\n  error was "+str(error), RuntimeWarning, 2)
#fu

# beware, stupid python interprets backslashes in repl only partially!
def s(string: str, pattern: str, repl: Union[str, Callable[[re.Match], str]], count: int =0) -> str:
    return re.sub(pattern, repl, string, count)
def m(string: str, pattern: str) -> Optional[re.Match]:
    return re.match(pattern, string)
def sorted_keys(dict: Dict[str, Any]) -> List[str]:
    keys = list(dict.keys())
    keys.sort()
    return keys

# we make up a few formatter routines to help in the processing:
def html2docbook(text: str) -> str:
    """ the C comment may contain html markup - simulate with docbook tags """
    return (
        s(s(s(s(s(s(s(s(s(s(s(text,
                              r"<br\s*/?>",""),
                            r"(</?)em>",r"\1emphasis>"),
                          r"<code>","<userinput>"),
                        r"</code>","</userinput>"),
                      r"<link>","<function>"),
                    r"</link>","</function>"),
                  r"(?s)\s*</screen>","</screen>"),
#               r"<ul>","</para><itemizedlist>"),
#             r"</ul>","</itemizedlist><para>"),
#           r"<li>","<listitem><para>"),
#         r"</li>","</para></listitem>\n"),
                r"<ul>","</para><programlisting>\n"),
              r"</ul>","</programlisting><para>"),
            r"<li>",""),
          r"</li>",""))
def paramdef2html(text: str) -> str:
    return s(s(s(s(s(text,
                     r"\s+<paramdef>", r"\n<nobr>"),
                   r"<paramdef>",r"<nobr>"),
                 r"</paramdef>",r"</nobr>"),
               r"<parameters>",r"\n <code>"),
             r"</parameters>",r"</code>\n")
def section2html(text: str) -> str:
    mapping = { "<screen>" : "<pre>", "</screen>" : "</pre>",
                "<para>" : "<p>", "</para>" : "</p>" ,
                "<function>" : "<link>", "</function>" : "</link>" }
    for m in mapping:
        text = text.replace(m, mapping[m])
    return text
def cdata1(text: str) -> str:
    return text.replace("&",  "&amp;")
def cdata31(text: str) -> str:
    text = text.replace("<","&lt;")
    return text.replace(">","&gt;")
def cdata3(text: str) -> str:
    return cdata31(cdata1(text))
def cdata43(text: str) -> str:
    return text.replace("\"", "&quot;")
def cdata41(text: str) -> str:
    return cdata43(cdata31(text))
def cdata4(text: str) -> str:
    return cdata43(cdata3(text))
def markup_as_screen41 (text: str) -> str:
    """ used for non-star lines in comment blocks """
    return " <screen> " + s(cdata41(text), r"(?m)^", r" ") +" </screen> "

def file_comment2section(text: str) -> str:
    """ convert a C comment into a series of <para> and <screen> parts """
    return ("<para>\n"+
            s(s(s(s(s(s(s(text,
                          r"(?s){<([\w\.\-]+\@[\w\.\-]+\w\w)>",
                          r"&lt;\1&gt;"),
                        r"(?mx) ^\s?\s?\s? ([^\*\s]+ .*) $",
                        lambda x : markup_as_screen41 (x.group(1))),
                      r"(?mx) ^\s*[*]\s* $", r" \n</para><para>\n"),
                    r"(?mx) ^\s?\s?\s?\* (.*) $", r" \1 "),
                  r"(?sx) </screen>(\s*)<screen> ", r"\1"),
                r"(?sx) <([^<>\;]+\@[^<>\;]+)> ", r"<email>\1</email>"),
              r"(?sx) \&lt\;([^<>\&\;]+\@[^<>\&\;]+)\&gt\; ",
              r"<email>\1</email>") + "\n</para>")
def func_comment2section(text: str) -> str:
    """ convert a C comment into a series of <para> and <screen> parts
        and sanitize a few markups already present in the comment text
    """
    return ("<para>\n"+
            s(s(s(s(s(s(s(s(s(s(s(text,
                                  r"<c>",r"<code>"),   r"</c>", r"</code>"),
                              r"(?mx) ^\s?\s?\s? ([^\*\s]+.*)",
                              lambda x: markup_as_screen41 (x.group(1))),
                            r"(?mx) ^\s?\s?\s?\* (.*) $", r" <br /> \1"),
                          r"(?mx) ^\s*<br\s*\/>\s* $", r"\n</para><para>\n"),
                        r"<<",r"&lt;"),   r">>",r"&gt;"),
                    r"(?sx) (</?para>\s*)<br\s*\/?>",r"\1"),
                  r"(?sx) (</?para>\s*)<br\s*\/?>",r"\1"),
                r"(?sx) (<br\s*\/?>\s*)<br\s*\/?>",r"\1"),
              r"(?sx) <\/screen>(\s*)<screen>",r"\1") + "\n</para>")
def markup_link_syntax(text: str) -> str:
    """ markup the link-syntax ` => somewhere ` in the text block """
    return (
        s(s(s(s(text,
                r"(?mx) (^|\s)\=\>\"([^\"]*)\"", r"\1<link>\2</link>"),
              r"(?mx) (^|\s)\=\>\'([^\"]*)\'", r"\1<link>\2</link>"),
            r"(?mx) (^|\s)\=\>\s(\w[\w.]*\w)\b", r"\1<link>\2</link>"),
          r"(?mx) (^|\s)\=\>\s([^\s\,\.\!\?\:\;\<\>\&\'\=\-]+)",
          r"\1<link>\2</link>"))
def this_function_link(text: str, name: str) -> str:
    return s(text, r"(?sx) (T|t)his \s (function|procedure) ", lambda x
             : "<function>"+x.group(1)+"he "+name+" "+x.group(2)+"</function>")

# -----------------------------------------------------------------------
class Options:
    verbose: int
    version:str
    package: str
    suffix: str
    mainheader: str
    def __init__(self) -> None:
        self.verbose = 0
        self.version = ""
        self.package = ""
        self.suffix = ""
        self.mainheader = ""

o = Options()
o.version = s( subprocess.check_output(
    """ grep -i "^version *:" *.spec 2>/dev/null |
        sed -e "s/[Vv]ersion *: *//" """).decode("utf-8"),  r"\s*",r"")
o.package = s(subprocess.check_output(
    """ grep -i "^name *:" *.spec 2>/dev/null |
        sed -e "s/[Nn]ame *: *//" """).decode("utf-8"),     r"\s*",r"")

if not len(o.version):
    o.version = subprocess.check_output(""" date +%Y.%m.%d """).decode("utf-8")
if not len(o.package):
    o.package = "_project"

o.suffix = "-doc3"
o.mainheader = o.package+".h"

class File:
    name: str
    mainheader: str
    authors: str
    copyright: str
    include: Optional[str]
    comment: Optional[str]
    section: Optional[str]
    def __init__(self, filename: str) -> None:
        self.name = filename
        self.mainheader = o.mainheader
        self.authors = ""
        self.copyright = ""
        self.include = None
        self.comment = None
        self.section = None
    def set_author(self, text: str) -> str:
        if self.authors:
            self.authors += "\n"
        self.authors += text
        return text
    def set_copyright(self, text: str) -> str:
        self.copyright = text
        return text

class FuncDeclaration:
    file: Optional[File]
    comment: str # note that both decl.comment and
    prototype: str # decl.prototype are in cdata1 format
    id: int
    def filename(self) -> str:
        if self.file:
            return self.file.name
        return "#"+str(self.id)
class InputFiles:
    """ for each set of input files we can create an object
        it does correspond with a single html-output page and
        a single docbook <reference> master page to be output
    """
    def __init__(self) -> None:
        # the id will tell us in which order
        # we did meet each function definition
        self.id = 1000
        self.files: List[File] = [] # file_list
        self.funcs: List[FuncDeclaration]  = [] # func_list: of hidden class FuncDeclaration
        self.file: Optional[File] = None # current file
    def new_File(self, name: str) -> File:
        self.file = File(name)
        self.files.append(self.file)
        return self.file
    def next_id(self) -> int:
        id = self.id ; self.id += 1
        return id
    def add_function_declaration(self, comment: str, prototype: str) -> str:
        func = FuncDeclaration()
        func.file = self.file
        func.comment = s(comment, # need to take out email-style markups
                         r"<([\w\.\-]+\@[\w\.\-]+\w\w)>", r"&lt;\1&gt;")
        func.prototype = prototype
        func.id = all.next_id()
        self.funcs.append(func)
        # print id
        return prototype

def scan_options (options: Iterable[str], list: InputFiles) -> None:
    def encode(text: str) -> str:
        return s(s(text, r"¬",  r"&#AC;"), r"\*/",r"¬")
    def decode(text: str) -> str:
        return s(text, r"¬", r"*/")

    for name in options:
        found = m(name, r"^(\w+)=(.*)")
        if found:
            o.var[found.group(1)] = found.group(2)
            continue
        #else
        try:
            input = open(name, "r")
        except IOError as error:
            warn(#...... (scan_options) ...............
                "can not open input file: "+name, error)
            continue
        text = input.read() ; input.close()
        text = encode (cdata1 (text))

        file = list.new_File(name)
        
        # cut per-function comment block
        text = s(text, r"(?x) [/][*][*](?=\s) ([^¬]+) ¬ ([^\{\}\;\#]+) [\{\;]",
                 lambda x : list.add_function_declaration(
            decode(x.group(1)), decode(x.group(2))))

        # cut per-file comment block
        found = m(text, r"(?sx)  [/][*]+(?=\s) ([^¬]+) ¬ "
                  r"(?:\s*\#define\s*\S+)*"
                  r"(\s*\#include\s*<[^<>]*>(?:\s*//[^\n]*)?)")
        if found:
            file.comment = decode(found.group(1))
            file.include = cdata31(found.group(2))
        else:
            file.comment = None
            file.include = None
            found = m(text, r"(?sx)  ^ [/][*]+(?=\s) ([^¬]+) ¬ ")
            if found:
                file.comment = decode(found.group(1))
        #fi
        # throw away the rest - further processing on memorized strings only

    return None

all = InputFiles()
scan_options (sys.argv[1:], all)

if not o.docbookfile:
    o.docbookfile = o.package+o.suffix+".docbook"
if not o.libhtmlfile:
    o.libhtmlfile = o.package+o.suffix+".html"
if not o.dumpdocfile:
    o.dumpdocfile = o.package+o.suffix+".dxml"

# ...........................................................................
# check out information in the file.comment section

def all_files_comment2section(list: List[File]) -> None:
    for file in list:
        if file.comment is None: continue
        file.section = file_comment2section(file.comment)
    
        file.section = s(
            file.section, r"(?sx) \b[Aa]uthor\s*:(.*</email>) ", lambda x
            : "<author>" + file.set_author(x.group(1)) + "</author>")
        file.section = s(
            file.section, r"(?sx) \b[Cc]opyright\s*:([^<>]*)</para> ",lambda x
            : "<copyright>" + file.set_copyright(x.group(1)) + "</copyright>")
        # if "file" in file.name: print >> sys.stderr, file.comment # 2.3
    #od
all_files_comment2section(all.files)

# -----------------------------------------------------------------------

class Function:
    " <prespec>void* </><namespec>hello</><namespec> (int) const</callspec> "
    src:  FuncDeclaration
    name: str
    prespec: str
    namespec: str
    callspec: str
    body: str
    head: str
    into: Optional[str]
    seealso: Optional[str]
    def __init__(self, src: FuncDeclaration, name: str = "") -> None:
        self.src = src
        self.name = name
        self.prespec  = ""
        self.namespec = ""
        self.callspec = ""
        self.body = ""
        self.head = ""
#    def set(self, **defines):
#        name = defines.keys()[0]
#        self.__dict__[name] = defines[name]
#        return defines[name]
#    def cut(self, **defines):
#        name = defines.keys()[0]
#        self.__dict__[name] += defines[name]
#        return ""
    def dict(self) -> Dict[str, Any]:
        return self.__dict__
    def dict_sorted_keys(self) -> List[str]:
        keys = list(self.__dict__.keys())
        keys.sort()
        return keys
    def parse(self, prototype: str) -> Optional[str]:
        found = m(prototype, r"(?sx) ^(.*[^.]) \b(\w[\w.]*\w)\b (\s*\(.*) $ ")
        if found:
            self.prespec = found.group(1).lstrip()
            self.namespec = found.group(2)
            self.callspec = found.group(3).lstrip()
            self.name = self.namespec.strip()
            return self.name
        return None

# pass 1 of per-func strings ...............................................
# (a) cut prototype into prespec/namespec/callspec
# (b) cut out first line of comment as headline information
# (c) sanitize rest of comment block into proper docbook formatted .body
# 
# do this while copying strings from all.funcs to function_list
# and remember the original order in name_list

def markup_callspec(text: str) -> str:
    return (
        s(s(s(s(s(text,
                  r"(?sx) ^([^\(\)]*)\(", r"\1<parameters>(<paramdef>",1),
                r"(?sx) \)([^\(\)]*)$", r"</paramdef>)</parameters>\1",1),
              r"(?sx) , ", r"</paramdef>,<paramdef>"),
            r"(?sx) <paramdef>(\s+) ", r"\1<paramdef>"),
          r"(?sx) (\s+)</paramdef>", r"</paramdef>\1"))

def parse_all_functions(func_list: List[FuncDeclaration]) -> List[Function]: # list of FunctionDeclarations
    """ parse all FunctionDeclarations and create a list of Functions """
    list: List[Function] = []
    for func in func_list:
        function = Function(func)
        if not function.parse (func.prototype): continue

        list.append(function)

        function.body = markup_link_syntax(func.comment)
        if "\n" not in function.body: # single-line comment is the head
            function.head = function.body
            function.body = ""
        else: # cut comment in first-line and only keep the rest as descr body
            function.head = s(function.body,  r"(?sx) ^([^\n]*\n).*",r"\1",1)
            function.body = s(function.body,  r"(?sx)  ^[^\n]*\n",   r"",  1)
        #fi
        if m(function.head, r"(?sx) ^\s*$ "): # empty head line, autofill here
            function.head = s("("+func.filename()+")", r"[.][.][/]", r"")

        function.body = func_comment2section(function.body)

        # add extra docbook markups to callspec in $fn-hash
        function.callspec = markup_callspec (function.callspec)
    #od
    return list
function_list = parse_all_functions(all.funcs)

def examine_head_anchors(func_list: List[Function]) -> None:
    """ .into tells later steps which func-name is the leader of a man 
        page and that this func should add its descriptions over there. """
    for function in func_list:
        function.into = None
        function.seealso = None
        
        if function.head:
            found = m(function.head, r"(?sx) ^ \s* <link>(\w[\w.]*\w)<\/link>")
            # if found and found.group(1) in func_list.names:
            if found and found.group(1):
                function.into = found.group(1)

        def set_seealso(f: Function, value: str) -> str:
            f.seealso = value
            return value
        if function.head:
            function.head = s(function.head, r"(.*)also:(.*)", lambda x
                          : set_seealso(function, x.group(2)) and x.group(1))
        if function.seealso and None:
            print("function[" + function.name + "].seealso=" + function.seealso)
examine_head_anchors(function_list)

# =============================================================== HTML =====

def find_by_name(func_list: List[Function], name: str) -> Optional[Function]:
    for func in func_list:
        if func.name == name:
            return func
    #od
    return None
#fu

class HtmlFunction:
    src: FuncDeclaration
    into: Optional[str]
    name: str
    synopsis: str
    anchor: str
    section: str
    def __init__(self, func: Function):
        self.src = func.src
        self.into = func.into
        self.name = func.name
        self.toc_line = paramdef2html(
            "  <td valign=\"top\"><code>"+func.prespec+"</code></td>\n"+
            "  <td valign=\"top\">&nbsp;&nbsp;</td>\n"+
            "  <td valign=\"top\"><a href=\"#"+func.name+"\">\n"+
            "                       <code>"+func.namespec+"</code>"+
            "  </a></td>\n"+
            "  <td valign=\"top\">&nbsp;&nbsp;</td>\n"+
            "  <td valign=\"top\">"+func.callspec+"</td>\n")
        self.synopsis = paramdef2html(
            "  <code>"+func.prespec+"</code>\n"+
            "  <br /><b><code>"+func.namespec+"</code></b>\n"+
            "   &nbsp; <code>"+func.callspec+"</code>\n")
        self.anchor = "<a name=\""+func.name+"\" />"
        self.section = "<para><em> &nbsp;"+func.head+"\n"+ \
                       "\n</em></para>"+section2html(func.body)
#class

class HtmlFunctionFamily(HtmlFunction):
    toc_line_list: List[str]
    synopsis_list: List[str]
    achor_list: List[str]
    section_list: List[str]
    def __init__(self, func: Function) -> None:
        HtmlFunction.__init__(self, func)
        self.toc_line_list = [ self.toc_line ]
        # self.html_txt     = self.synopsis
        self.synopsis_list = [ self.synopsis ]
        self.anchor_list   = [ self.anchor ]
        self.section_list  = [ this_function_link(self.section, func.name) ]

def ensure_name(text: str, name: str) -> str:
    adds = "<small><code>"+name+"</code></small> -"
    match = r"(?sx) .*>[^<>]*\b" + name + r"\b[^<>]*<.*"
    found = m(text, match)
    if found: return text
    found = m(text, r".*<p(ara)?>.*")
    if found: return s(text, r"(<p(ara)?>)", r"\1"+adds, 1)
    return adds+text

def combined_html_pages(func_list: List[Function]) -> List[HtmlFunctionFamily]:
    """ and now add descriptions of non-leader entries (html-mode) """
    combined: Dict[str, HtmlFunctionFamily] = {}
    
    for func in func_list: # assemble leader pages
        if func.into is not None: continue
        combined[func.name] =  HtmlFunctionFamily(func)

    for func in func_list: 
        if func.into is None: continue
        if func.into not in combined :
            warn(#......... (combine_html_pages) ..............
                "function '"+func.name+"'s into => '"+func.into+
                "\n: no such target function: "+func.into)
            combined[func.name] = HtmlFunctionFamily(func)
            continue
        #fi
        page = HtmlFunction(func)
        into = combined[func.into]
        into.toc_line_list.append( page.toc_line )
        into.anchor_list.append( page.anchor )
        into.synopsis_list.append( page.synopsis )
        into.section_list.append(
            s(ensure_name(this_function_link(section2html( func.body),
                                             func.name), func.name),
              r"(?sx) (</?para>\s*) <br\s*\/>", r"\1"))
    return list(combined.values())
html_pages = combined_html_pages(function_list)

def html_resolve_links_on_page(text: str, list: List[Function]) -> str:
    """ link ref-names of a page with its endpoint on the same html page"""
    def html_link (name: str, extra: str) -> str:
        """ make <link>s to <href> of correct target or make it <code> """
        if find_by_name(list, name) is None:
            return "<code>"+name+extra+"</code>"
        else:
            return "<a href=\"#"+name+"\"><code>"+name+extra+"</code></a>"
    #fu html_link
    return s(s(text, r"(?sx) <link>(\w+)([^<>]*)<\/link> ",
               lambda x : html_link(x.group(1),x.group(2))),
             r"(?sx) \-\> ", r"<small>-&gt;</small>") # just sanitize..
#fu html_resolve_links

class HtmlPage:
    toc: str
    txt: str
    package: str
    version: str
    def __init__(self) -> None:
        self.toc = ""
        self.txt = ""
        self.package = o.package
        self.version = o.version
    def page_text(self) -> str:
        """ render .toc and .txt parts into proper <html> page """
        T = ""
        T += "<html><head>"
        T += "<title>"+self.package+"autodoc documentation </title>"
        T += "</head>\n<body>\n"
        T += "\n<h1>"+self.package+" <small><small><i>- "+self.version
        T += "</i></small></small></h1>"
        T += "\n<table border=0 cellspacing=2 cellpadding=0>"
        T +=  self.toc
        T += "\n</table>"
        T += "\n<h3>Documentation</h3>\n\n<dl>"
        T += html_resolve_links_on_page(self.txt, function_list)
        T += "\n</dl>\n</body></html>\n"
        return T
    def add_page_map(self, family: Dict[str, HtmlFunctionFamily]) -> None:
        """ generate the index-block at the start of the onepage-html file """
        keys = list(family.keys())
        keys.sort()
        for name in keys:
            self.toc += "<tr valign=\"top\">\n"+ \
                        "\n</tr><tr valign=\"top\">\n".join(
                family[name].toc_line_list)+"</tr>\n"
            self.txt += "\n<dt>"+" ".join(family[name].anchor_list)
            self.txt += "\n"+"\n<br />".join(family[name].synopsis_list)+"<dt>"
            self.txt += "\n<dd>\n"+"\n".join(family[name].section_list)
            self.txt += ("\n<p align=\"right\">"+
                         "<small>("+family[name].src.filename()+")</small>"+
                         "</p></dd>")
    def add_page_list(self, functions: List[HtmlFunctionFamily]) -> None:
        """ generate the index-block at the start of the onepage-html file """
        mapp: Dict[str, HtmlFunctionFamily] = {}
        for func in functions:
            mapp[func.name] = func
        #od
        self.add_page_map(mapp)
#end

html = HtmlPage()
# html.add_function_dict(Fn)
# html.add_function_list(Fn.sort.values())
html.add_page_list(html_pages)

# and finally print the html-formatted output
try:
    F = open(o.libhtmlfile, "w")
except IOError as error:
    warn(# ............. open(o.libhtmlfile, "w") ..............
        "can not open html output file: "+o.libhtmlfile, error)
else:
    print(html.page_text(), file=F)
    F.close()
#fi

# ========================================================== DOCBOOK =====
# let's go for the pure docbook, a reference type master for all man pages

class RefPage:
    refhint: str
    refentry: Optional[str]
    refentry_date: str
    refentry_productname: str
    refentry_title: Optional[str]
    refentryinfo: Optional[str]
    manvolnum: str
    refentrytitle: Optional[str]
    refmeta: Optional[str]
    refpurpose: Optional[str]
    refname: Optional[str]
    refname_list: List[str]
    refnamediv: Optional[str]
    mainheader: str
    includes: Optional[str]
    funcsynopsisinfo: str
    funcsynopsis: str
    funcsynopsis_list: List[str]
    description: Optional[str]
    description_list: List[str]
    authors_list: List[str]
    authors: Optional[str]
    copyright_list: List[str]
    copyright: Optional[str]
    seealso_list: List[str]
    seealso: Optional[str]
    func: Function
    file_authors: Optional[str]
    file_copyright: Optional[str]
    def __init__(self, func: Function) -> None:
        """ initialize the fields needed for a man page entry - the fields are
           named after the docbook-markup that encloses (!!) the text we store
           the entries like X.refhint = "hello" will be printed therefore as
           <refhint>hello</refhint>. Names with underscores are only used as
           temporaries but they are memorized, perhaps for later usage. """
        self.refhint = "\n<!--========= "+func.name+" (3) ===========-->\n"
        self.refentry = None
        self.refentry_date = o.version.strip()        # //refentryinfo/date
        self.refentry_productname = o.package.strip() # //refentryinfo/prod*
        self.refentry_title = None                    # //refentryinfo/title
        self.refentryinfo = None                      # override
        self.manvolnum = "3"                         # //refmeta/manvolnum
        self.refentrytitle = None                    # //refmeta/refentrytitle
        self.refmeta = None                          # override
        self.refpurpose = None                       # //refnamediv/refpurpose
        self.refname = None                          # //refnamediv/refname
        self.refname_list = []
        self.refnamediv = None                       # override
        self.mainheader = func.src.file.mainheader if func.src.file else ""
        self.includes = func.src.file.include if func.src.file else None
        self.funcsynopsisinfo = ""       # //funcsynopsisdiv/funcsynopsisinfo
        self.funcsynopsis = ""         # //funcsynopsisdiv/funcsynopsis
        self.funcsynopsis_list = []
        self.description = None
        self.description_list = []
        # optional sections
        self.authors_list = []           # //sect1[authors]/listitem
        self.authors = None              # override
        self.copyright = None
        self.copyright_list = []
        self.seealso = None
        self.seealso_list = []
        if  func.seealso:
            self.seealso_list.append(func.seealso)
        # func.func references
        self.func = func
        self.file_authors = None
        self.file_copyright = None
        if func.src.file:
            if  func.src.file.authors:
                self.file_authors = func.src.file.authors
            if  func.src.file.copyright:
                self.file_copyright = func.src.file.copyright
    #fu
    def refentryinfo_text(self) -> str:
        """ the manvol formatter wants to render a footer line and header line
            on each manpage and such info is set in <refentryinfo> """
        if self.refentryinfo:
            return self.refentryinfo
        if self.refentry_date and \
           self.refentry_productname and \
           self.refentry_title: return (
            "\n <date>"+self.refentry_date+"</date>"+ 
            "\n <productname>"+self.refentry_productname+"</productname>"+
            "\n <title>"+self.refentry_title+"</title>")
        if self.refentry_date and \
           self.refentry_productname: return (
            "\n <date>"+self.refentry_date+"</date>"+ 
            "\n <productname>"+self.refentry_productname+"</productname>")
        return ""
    def refmeta_text(self) -> str:
        """ the manvol formatter needs to know the filename of the manpage to
            be made up and these parts are set in <refmeta> actually """
        if self.refmeta:
            return self.refmeta
        if self.manvolnum and self.refentrytitle:
            return (
                "\n <refentrytitle>"+self.refentrytitle+"</refentrytitle>"+
                "\n <manvolnum>"+self.manvolnum+"</manvolnum>")
        if self.manvolnum and self.func.name:
            return (
                "\n <refentrytitle>"+self.func.name+"</refentrytitle>"+
                "\n <manvolnum>"+self.manvolnum+"</manvolnum>")
        return ""
    def refnamediv_text(self) -> str:
        """ the manvol formatter prints a header line with a <refpurpose> line
            and <refname>'d functions that are described later. For each of
            the <refname>s listed here, a mangpage is generated, and for each
            of the <refname>!=<refentrytitle> then a symlink is created """
        if self.refnamediv:
            return self.refnamediv
        if self.refpurpose and self.refname:
            return ("\n <refname>"+self.refname+'</refname>'+
                    "\n <refpurpose>"+self.refpurpose+" </refpurpose>")
        if self.refpurpose and self.refname_list:
            T = ""
            for refname in self.refname_list:
                T += "\n <refname>"+refname+'</refname>'
            T += "\n <refpurpose>"+self.refpurpose+" </refpurpose>"
            return T
        return ""
    def funcsynopsisdiv_text(self) -> str:
        """ refsynopsisdiv shall be between the manvol mangemaent information
            and the reference page description blocks """
        T=""
        if self.funcsynopsis:
            T += "\n<funcsynopsis>"
            if self.funcsynopsisinfo:
                T += "\n<funcsynopsisinfo>"+    self.funcsynopsisinfo + \
                     "\n</funcsynopsisinfo>\n"
            T += self.funcsynopsis + \
                 "\n</funcsynopsis>\n"
        if self.funcsynopsis_list:
            T += "\n<funcsynopsis>"
            if self.funcsynopsisinfo:
                T += "\n<funcsynopsisinfo>"+    self.funcsynopsisinfo + \
                     "\n</funcsynopsisinfo>\n"
            for funcsynopsis in self.funcsynopsis_list:
                T += funcsynopsis
            T += "\n</funcsynopsis>\n"
        #fi
        return T
    def description_text(self) -> str:
        """ the description section on a manpage is the main part. Here
            it is generated from the per-function comment area. """
        if self.description:
            return self.description
        if self.description_list:
            T = ""
            for description in self.description_list:
                if not description: continue
                T += description
            if T: return T
        return ""
    def authors_text(self) -> str:
        """ part of the footer sections on a manpage and a description of
            original authors. We prever an itimizedlist to let the manvol
            show a nice vertical aligment of authors of this ref item """
        if self.authors:
            return self.authors
        if self.authors_list:
            T = "<itemizedlist>"
            previous=""
            for authors in self.authors_list:
                if not authors: continue
                if previous == authors: continue
                T += "\n <listitem><para>"+authors+"</para></listitem>"
                previous = authors
            T += "</itemizedlist>"
            return T
        if self.authors:
            return self.authors
        return ""
    def copyright_text(self) -> str:
        """ the copyright section is almost last on a manpage and purely
            optional. We list the part of the per-file copyright info """
        if self.copyright:
            return self.copyright
        """ we only return the first valid instead of merging them """
        if self.copyright_list:
            T = ""
            for copyright in self.copyright_list:
                if not copyright: continue
                return copyright # !!!
        return ""
    def seealso_text(self) -> str:
        """ the last section on a manpage is called 'SEE ALSO' usually and
            contains a comma-separated list of references. Some manpage
            viewers can parse these and convert them into hyperlinks """
        if self.seealso:
            return self.seealso
        if self.seealso_list:
            T = ""
            for seealso in self.seealso_list:
                if not seealso: continue
                if T: T += ", "
                T += seealso
            if T: return T
        return ""
    def refentry_text(self, ref: Optional[str]=None) -> str:
        """ combine fields into a proper docbook refentry """
        if ref is None:
            ref = self.refentry
        if ref:
            T = '<refentry id="'+ref+'">'
        else:
            T = '<refentry>' # this is an error
           
        if self.refentryinfo_text():
            T += "\n<refentryinfo>"+       self.refentryinfo_text()+ \
                 "\n</refentryinfo>\n"
        if self.refmeta_text():
            T += "\n<refmeta>"+            self.refmeta_text() + \
                 "\n</refmeta>\n" 
        if self.refnamediv_text():
            T += "\n<refnamediv>"+         self.refnamediv_text() + \
                 "\n</refnamediv>\n"
        if self.funcsynopsisdiv_text():     
            T += "\n<refsynopsisdiv>\n"+   self.funcsynopsisdiv_text()+ \
                 "\n</refsynopsisdiv>\n"
        if self.description_text():
            T += "\n<refsect1><title>Description</title> " + \
                 self.description_text() + "\n</refsect1>"
        if self.authors_text():
            T += "\n<refsect1><title>Author</title> " + \
                 self.authors_text() + "\n</refsect1>"
        if self.copyright_text():
            T += "\n<refsect1><title>Copyright</title> " + \
                 self.copyright_text() + "\n</refsect1>\n"
        if self.seealso_text():
            T += "\n<refsect1><title>See Also</title><para> " + \
                 self.seealso_text() + "\n</para></refsect1>\n"

        T +=  "\n</refentry>\n"
        return T
    #fu
#end

# -----------------------------------------------------------------------
class FunctionRefPage(RefPage):
    def reinit(self) -> RefPage:
        """ here we parse the input function for its values """
        if self.func.into:
            self.refhint = "\n              <!-- see "+self.func.into+" -->\n"
        #fi
        self.refentry = self.func.name               # //refentry@id
        self.refentry_title = self.func.name.strip() # //refentryinfo/title
        self.refentrytitle = self.func.name          # //refmeta/refentrytitle
        if self.includes:
            self.funcsynopsisinfo += "\n"+self.includes
        if not self.funcsynopsisinfo:
            self.funcsynopsisinfo="\n"+' #include &lt;'+self.mainheader+'&gt;'
        self.refpurpose = self.func.head
        self.refname = self.func.name

        def funcsynopsis_of(func: Function) -> str:
            return (
                "\n <funcprototype>\n <funcdef>"+func.prespec+
                " <function>"+func.name+"</function></funcdef>"+
                "\n"+s(s(s(func.callspec,
                           r"<parameters>\s*\(",r" "),
                         r"\)\s*</parameters>",r" "),
                       r"</paramdef>\s*,\s*",r"</paramdef>\n ")+
                " </funcprototype>")
        self.funcsynopsis = funcsynopsis_of(self.func)

        self.description = (
            html2docbook(this_function_link(self.func.body, self.func.name)))

        if self.file_authors:
            def add_authors(page: RefPage, ename: str, email: str) -> str:
                page.authors_list.append( ename+' '+email )
                return ename+email
            s(self.file_authors,
              r"(?sx) \s* ([^<>]*) (<email>[^<>]*</email>) ", lambda x
              : add_authors(self, x.group(1), x.group(2)))
        #fi

        if self.file_copyright:
            self.copyright = "<screen>\n"+self.file_copyright+"</screen>\n"
        #fi
        return self
    def __init__(self,func: Function) -> None:
        RefPage.__init__(self, func)
        FunctionRefPage.reinit(self)
    
def refpage_list_from_function_list(funclist: List[Function]) -> List[FunctionRefPage]:
    list: List[FunctionRefPage] = []
    mapp: Dict[str, Function] = {}
    for func in funclist:
        mapp[func.name] = func
    #od
    for func in funclist:
        page = FunctionRefPage(func)
        if func.into and func.into not in mapp:
            warn (# ............ (refpage_list_from_function_list) .......
                "page '"+page.func.name+"' has no target => "+
                "'"+(page.func.into or "?")+"'"
                "\n: going to reset .into of Function '"+page.func.name+"'")
            func.into = None
        #fi
        list.append(FunctionRefPage(func))
    return list
#fu
    
# ordered list of pages
refpage_list = refpage_list_from_function_list(function_list)

class FunctionFamilyRefPage(RefPage):
    seealso_list: List[str]
    refhint_list: List[str]
    def __init__(self, page: RefPage)-> None:
        RefPage.__init__(self, page.func)
        self.seealso_list = [] # reset
        self.refhint_list = []
    def refhint_list_text(self) -> str:
        T = ""
        for hint in self.refhint_list:
            T += hint
        return T
    def refentry_text(self, ref: Optional[str]=None) -> str:
        return self.refhint_list_text() + "\n" + \
               RefPage.refentry_text(self)
    pass

def docbook_pages_recombine(pagelist: Iterable[FunctionRefPage]) -> List[FunctionFamilyRefPage]:
    """ take a list of RefPages and create a new list where sections are
        recombined in a way that their description is listed on the same
        page and the manvol formatter creates symlinks to the combined
        function description page - use the attribute 'into' to guide the
        processing here as each of these will be removed from the output
        list. If no into-pages are there then the returned list should
        render to the very same output text like the input list would do """

    list: List[FunctionFamilyRefPage] = []
    combined = {}
    for orig in pagelist:
        if orig.func.into: continue
        page = FunctionFamilyRefPage(orig)
        combined[orig.func.name] = page ; list.append(page)

        page.refentry = orig.refentry              # //refentry@id
        page.refentry_title = orig.refentrytitle   # //refentryinfo/title
        page.refentrytitle = orig.refentrytitle    # //refmeta/refentrytitle
        page.includes = orig.includes
        page.funcsynopsisinfo = orig.funcsynopsisinfo
        page.refpurpose = orig.refpurpose
        if orig.refhint:
            page.refhint_list.append( orig.refhint )
        if orig.refname:
            page.refname_list.append( orig.refname )
        elif orig.refname_list:
            page.refname_list.extend( orig.refname_list )
        if orig.funcsynopsis:
            page.funcsynopsis_list.append( orig.funcsynopsis )
        elif orig.refname_list:
            page.funcsynopsis_list.extend( orig.funcsynopsis_list )
        if orig.description:
            page.description_list.append( orig.description )
        elif orig.refname_list:
            page.description_list.extend( orig.description_list )
        if orig.seealso:
            page.seealso_list.append( orig.seealso )
        elif orig.seealso_list:
            page.seealso_list.extend( orig.seealso_list )
        if orig.authors:
            page.authors_list.append( orig.authors )
        elif orig.authors_list:
            page.authors_list.extend( orig.authors_list )
        if orig.copyright:
            page.copyright_list.append( orig.copyright )
        elif orig.refname_list:
            page.copyright_list.extend( orig.copyright_list )
    #od
    for orig in pagelist:
        if not orig.func.into: continue
        if orig.func.into not in combined:
            warn("page for '"+orig.func.name+
                 "' has no target => '"+orig.func.into+"'")
            page = FunctionFamilyRefPage(orig)
        else:
            page = combined[orig.func.into]

        if orig.refname:
            page.refname_list.append( orig.refname )
        elif orig.refname_list:
            page.refname_list.extend( orig.refname_list )
        if orig.funcsynopsis:
            page.funcsynopsis_list.append( orig.funcsynopsis )
        elif orig.refname_list:
            page.funcsynopsis_list.extend( orig.funcsynopsis_list )
        if orig.description:
            page.description_list.append( orig.description )
        elif orig.refname_list:
            page.description_list.extend( orig.description_list )
        if orig.seealso:
            page.seealso_list.append( orig.seealso )
        elif orig.seealso_list:
            page.seealso_list.extend( orig.seealso_list )
        if orig.authors:
            page.authors_list.append( orig.authors )
        elif orig.authors_list:
            page.authors_list.extend( orig.authors_list )
        if orig.copyright:
            page.copyright_list.append( orig.copyright )
        elif orig.refname_list:
            page.copyright_list.extend( orig.copyright_list )
    #od
    return list
#fu

combined_pages = docbook_pages_recombine(pagelist = refpage_list)

# -----------------------------------------------------------------------

class HeaderRefPage(RefPage):
    pass

def docbook_refpages_perheader(page_list: Iterable[FunctionRefPage]) -> Dict[str, HeaderRefPage]: # headerlist
    " creating the per-header manpage - a combination of function man pages "
    header: Dict[str, HeaderRefPage] = {}
    for page in page_list:
        assert not page.func.into
        main = page.func.src.file.mainheader if page.func.src.file else ""
        if not main:
            warn("no mainheader found %s", page.func.src.filename())
            continue
        if main not in header:
            filerefname = s(main, r"[^\w\.]","-")
            header[main] = HeaderRefPage(page.func)
            header[main].refentry = filerefname
            header[main].refentryinfo = None
            header[main].refentry_date = page.refentry_date
            header[main].refentry_productname = (
                "the library "+page.refentry_productname)
            header[main].manvolnum = page.manvolnum
            header[main].refentrytitle = main
            header[main].funcsynopsis = ""
        if 1: # or += or if not header[file].refnamediv:
            header[main].refpurpose = " library "
            header[main].refname = filerefname

        if not header[main].funcsynopsisinfo and page.funcsynopsisinfo:
            header[main].funcsynopsisinfo  = page.funcsynopsisinfo
        if page.funcsynopsis:
            header[main].funcsynopsis  += "\n"+page.funcsynopsis
        if not header[main].copyright and page.copyright:
            header[main].copyright = page.copyright
        if not header[main].authors and page.authors:
            header[main].authors = page.authors
        if not header[main].authors and page.authors_list:
            header[main].authors_list = page.authors_list
        if not header[main].description:
            found = m(subprocess.check_output("cat "+o.package+".spec").decode("utf-8"),
                      r"(?s)\%description\b([^\%]*)\%")
            if found:
                header[main].description = found.group(1)
            elif not header[main].description:
                header[main].description = "<para>" + (
                    page.refentry_productname + " library") + "</para>";
            #fi
        #fi
    #od
    return header#list
#fu

def leaders(pagelist: List[FunctionRefPage]) -> List[FunctionRefPage]:
    list = []
    for page in pagelist:
        if page.func.into : continue
        list.append(page)
    return list
header_refpages = docbook_refpages_perheader(leaders(refpage_list))

# -----------------------------------------------------------------------
# printing the docbook file is a two-phase process - we spit out the
# leader pages first - later we add more pages with _refstart pointing
# to the leader page, so that xmlto will add the functions there. Only the
# leader page contains some extra info needed for troff page processing.

doctype = '<!DOCTYPE reference PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"'
doctype += "\n     "
doctype += '"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd">'+"\n"

try:
    F = open(o.docbookfile,"w")
except IOError as error:
    warn("can not open docbook output file: "+o.docbookfile, error)
else:
    print(doctype + '<reference><title>Manual Pages</title>', file=F)

    for page in combined_pages:
        print(page.refentry_text(), file=F)
    #od

    for refpage in header_refpages.values():
        if not refpage.refentry: continue
        print("\n<!-- _______ "+(refpage.refname or "?")+" _______ -->" + refpage.refentry_text(), file=F)
    #od

    print("\n</reference>\n", file=F)
    F.close()
#fi

# _____________________________________________________________________
try:
    F = open( o.dumpdocfile, "w")
except IOError as error:
    warn ("can not open"+o.dumpdocfile,error)
else:
    for func in function_list:
        name = func.name
        print("<fn id=\""+name+"\">"+"<!-- FOR \""+name+"\" -->\n", file=F)
        for H in sorted_keys(func.dict()):
            print("<"+H+" name=\""+name+"\">" + str(func.dict()[H]) + "</"+H+">", file=F)
        #od
        print("</fn><!-- END \""+name+"\" -->\n\n", file=F)
    #od
    F.close();
#fi

if errors: sys.exit(errors)
