#!/usr/bin/env python # -*- mode:Python; tab-width: 3 -*- """ This is a helper script which reads the .c files of the examples directory, extracts the comments from the source, and updates allegro._tx's appropriate section with the comments after hyperlinking functions and examples. It is not meant to be run by the end user, only by the maintainer packaging Allegro. To run the script go to the Allegro root dir and type "python misc/genexamp.py". If everything goes ok, you will end up with a patch in stdout you can apply manually after having verified it is ok. Usage example: python misc/genexamp.py | less ...review... python misc/genexamp.py | patch -p0 In order to work, this script requires Python (tested with 2.3.3) and the diff binary in your path. Written by Grzegorz Adam Hankiewicz, gradha@users.sourceforge.net. Notify me of any broken behaviour, like uncaught exceptions. This script falls under Allegro's giftware license. """ import glob import os import os.path import popen2 import re import string import sys path_to_documentation = apply(os.path.join, ["docs", "src", "allegro._tx"]) path_to_example_dir = apply(os.path.join, ["examples"]) # number of max erefs, below 1 desactivates limit limit_example_erefs = 10 limit_example_reference = "Available Allegro examples" # text tags used to detect where the examples should be put START_MARK = "start genexamp.py chunk" END_MARK = "end genexamp.py chunk" XREF_WIDTH = 70 regular_expression_for_tx_identifiers = r"@[@\\][^@]+@(?P\w+)" comment_line = re.compile(r"Example pr|Modified by") # the order of the examples is aimed at the newbie, going from easy # to difficult examples_order = ["exhello", "exmem", "expal", "expat", "exflame", "exdbuf", "exflip", "exfixed", "exfont", "exmouse", "extimer", "exkeys", "exjoy", "exsample", "exmidi", "exgui", "excustom", "exunicod", "exbitmap", "exscale", "exconfig", "exdata", "exsprite", "exexedat", "extrans", "extruec", "excolmap", "exrgbhsv", "exshade", "exblend", "exxfade", "exalpha", "exlights", "ex3d", "excamera", "exquat", "exstars", "exscn3d", "exzbuf", "exscroll", "ex3buf", "ex12bit", "exaccel", "exspline", "exsyscur", "exupdate", "exswitch", "exstream", "expackf"] # Holds examples' short descriptions. Loaded from disk at runtime. short_descriptions = {} def detect_all_available_examples(path): """func(path_to_directory) Checks if the maintainer is doing his work and didn't miss some example in the global list, printing any missing examples to stderr. """ examples = filter(lambda x: x not in examples_order, map(lambda x: os.path.splitext(os.path.basename(x))[0], glob.glob(os.path.join(path, "*.c")))) if examples: sys.stderr.write("Warning! There are examples not listed in the default\n" "examples_order variable. You should correct this:\n") for example in examples: sys.stderr.write("\t%s\n" % example) examples_order.extend(examples) def load_example_short_descriptions(path): """func(path_to_directory) Loads the file examples.txt from the specified directory and extracts the short example descriptions which are in "exblah - text" format. The short descriptions are stored in the global dictionary short_descriptions. """ file_input = file(os.path.join(path, "examples.txt"), "rt") try: regex = re.compile(r"(ex\w+)[.]c\s+-\s(.*)") line = file_input.readline() while line: m = regex.match(line) if m: short_descriptions[m.group(1)] = m.group(2) line = file_input.readline() finally: file_input.close() def retrieve_documentation_identifiers(file_name): """func(path_to_documentation._tx) -> {documentation_identifiers} Used on a text file with the format of Allegro's documentation, extracts all the valid identifiers and returns them as the keys of a dictionary, whose values are all 1 and can be ignored. """ dic = {} exp = re.compile(regular_expression_for_tx_identifiers) file = open(file_name, "rt") for line in file.readlines(): match = exp.match(line) # avoid the identifiers of existant examples if match and match.group("name") not in examples_order: dic[match.group("name")] = 1 file.close() return dic def retrieve_source_data(file_name): """func(path_to_example.c) -> ([comment_lines], {detected_identifiers}) Used on the source code of an example, extracts the first comment after deleting the initial lines which credit authors. The comment will be used as the text for the documentation. Also returns a dictionary with all the keys being hypotetical identifiers used in the code, that is, every single word susceptible of being used as an allegro function, like return, printf, allegro_init, SCREEN_W, etc. """ dic = {} exp = re.compile(r"\w+") file = open(file_name, "rt") getting_comment = 1 comment = [] for line in file.readlines(): if getting_comment: # stop reading after first comment end marker if string.find(line, "*/") >= 0: getting_comment = 0 else: # ok, retrieve all lines removing first 3 characters comment.append(string.strip(line[2:])) else: # retrieve all possible keywords for match in re.findall(exp, line): dic[match] = 1 file.close() # post-process comment, removing initial credits while len(comment): if string.strip(comment[0]) == "": comment.pop(0) elif comment_line.search(comment[0]): while len(comment) and string.strip(comment[0]) != "": comment.pop(0) else: break return comment, dic def get_valid_example_ids(example_ids, global_ids, incremental = None): """func({example_ids}, {global_ids}, bool) -> [valid_identifiers] Compares the dictionary of a source example with all its possible identifiers against the dictionary with all the available documentation identifiers and returns a filtered list of the example identifiers which are also in the global dictionary. If the incremental parameter is true, the identifiers of global_ids that are also contained in example_ids will be erased. This is useful if you are building example after example and you don't want new lists to include the identifiers returned in previous calls to this function. """ example_ids = example_ids.keys() example_ids.sort() valid_ids = [] if incremental: for id in example_ids: try: del global_ids[id] valid_ids.append(id) except KeyError: pass else: for id in example_ids: if global_ids.has_key(id): valid_ids.append(id) return valid_ids def build_formatted_lines(word_list): """func([word1, word2, ...]) -> [line1, line2, ...] Consumes the given list of words and returns a list of text lines. Each line will contain several words joined with the string ", ". All lines will have a width <= XREF_WIDTH. """ new_list = [] new_line = [] length = 0 while len(word_list) > 0: new_word = word_list.pop(0) if length + len(new_word) + 2 < XREF_WIDTH: length = length + len(new_word) + 2 new_line.append(new_word) else: new_list.append(string.join(new_line, ", ")) new_line = [new_word] length = len(new_word) + 2 new_list.append(string.join(new_line, ", ")) return new_list def build_xref_block(xref_list, xref_tag = None): """func([word1, word2, ...], string) -> [line1, line2, ...] Behaves like build_formatted_lines, adding to the beginning of each returned line the string given as second parameter. If the string is None, "@xref" will be prepended. """ lines = [] if not xref_tag: xref_tag = "@xref" for line in build_formatted_lines(xref_list): lines.append("%s %s\n" % (xref_tag, line)) return lines def build_example_output(example_name, comment_lines, xref_list): """func(example_name, [comment], [xrefs]) -> [formatted_lines] Given an example name (the part of the file_name without extension), its source commentary, and the references it makes, returns a list of text lines formatted like this: @@Example @example_name @@xrefs @shortdesc comment """ lines = ["@@Example @%s\n" % example_name] lines.extend(build_xref_block(xref_list)) if short_descriptions.has_key(example_name): lines.append("@shortdesc %s\n" % short_descriptions[example_name]) lines.extend(map(lambda x: " %s\n" % x, comment_lines)) lines.append("\n") return lines def generate_example_chapter(path_to_documentation, example_files, incremental = None): """func(path._tx, [paths_to_examples], bool) -> ([doc], {ids_to_examples}) First retrieves fom path_to_documentation the valid identifiers. Then goes through the list of paths to examples and extracts from each of them the used identifiers. Returns a list of lines which contains the new generated documentation. Also returns a dictionary whose keys are identifiers found in the documentation file, with the values being a list of the examples which refer to the key. If incremental is false, each documented example will reference all the possible identifiers found in the documentation. But if it's true, each example will create references only to identifiers which still have not been referenced before. """ reverse_docs = {} doc_ids = retrieve_documentation_identifiers(path_to_documentation) lines = [] for example in example_files: comment, ex_ids = retrieve_source_data(example) name = os.path.splitext(os.path.basename(example))[0] valid_ids = get_valid_example_ids(ex_ids, doc_ids, incremental) # build reverse lookup for id in valid_ids: try: reverse_docs[id].append(name) except KeyError: reverse_docs[id] = [name] # finally add the text chunk lines.extend(build_example_output(name, comment, valid_ids)) return lines, reverse_docs def replace_example_chapter(path_to_documentation, chapter_lines): """func(path_to_doc._tx, [new_chapter]) -> [regenerated_documentation] Opens the documentation and searches for the text section separated through the global marks START_MARK/END_MARK. Returns the opened file with that section replaced by chapter_lines. """ lines = [] file = open(path_to_documentation, "rt") state = 0 for line in file.readlines(): if state == 0: lines.append(line) if string.find(line, START_MARK) > 0: state = state + 1 elif state == 1: if string.find(line, END_MARK) > 0: lines.extend(chapter_lines) lines.append(line) state = state + 1 else: lines.append(line) return lines def find_differences(file_name, lines): """func(path_to_doc._tx, [text_lines]) -> [diff_lines] This function runs diff over the given document file and text lines. To make the comparison a temporary file with extension .tmp is build and later removed. The function returns the output of the diff commant as a list of lines. """ temp_name = string.replace(path_to_documentation, "._tx", ".tmp") file = open(temp_name, "wt") file.writelines(lines) file.close() # call external tool and read it's stdout stdout, stdin = popen2.popen2 (["diff", "-u", file_name, temp_name]) stdin.close() diff_lines = stdout.readlines() stdout.close() os.unlink(temp_name) return diff_lines def replace_example_references(documentation, ids_to_examples): """func([lines], {id: [words]}) -> [new_lines] Goes through the documentation in memory searching for the identifiers. When one is found, it looks it up in the provided dictionary. If it's not found, nothing happens. Otherwise whatches closely the following text lines for @erefs and updates them accordingly. If no @erefs are found, they are created. Finally a new list is returned with the updated documentation. There is a special hardcoded case: if limit_example_erefs is greater than 0, this value will be used as the limit of example references generated per identifier. When the number of example references is greater than this limit, a single reference is generated to the value stored in the string limit_example_reference, which should point to the correct Allegro documentation chapter. """ new_lines = [] short_desc = [] found_id = "" exp = re.compile(regular_expression_for_tx_identifiers) for line in documentation: if found_id: if line[0] == '@': # Don't append erefs, which will be regenerated. if line[1:5] == "xref" or line[1] == "@" or line[1:3] == "\\ ": new_lines.append(line) # Append shortdesc, but after @erefs as a special case. if line[1:10] == "shortdesc": short_desc = [line] else: # create the eref block and append before normal text eref_list = ids_to_examples[found_id] eref_list.sort() if len(eref_list) > limit_example_erefs: new_lines.append("@eref %s\n" % limit_example_reference) else: new_lines.extend(build_xref_block(eref_list, "@eref")) if short_desc: new_lines.extend(short_desc) short_desc = [] new_lines.append(line) found_id = "" else: if short_desc: new_lines.extend(short_desc) short_desc = [] new_lines.append(line) match = exp.match(line) if match and ids_to_examples.has_key(match.group("name")): found_id = match.group("name") return new_lines def main(path_to_documentation, path_to_example_dir, incremental): detect_all_available_examples(path_to_example_dir) load_example_short_descriptions(path_to_example_dir) chapter, ids_to_examples = generate_example_chapter(path_to_documentation, map(lambda x: os.path.join(path_to_example_dir, "%s.c" % x), examples_order), incremental) documentation = replace_example_chapter(path_to_documentation, chapter) documentation = replace_example_references(documentation, ids_to_examples) differences = find_differences(path_to_documentation, documentation) sys.stdout.writelines(differences) if __name__ == "__main__": main(path_to_documentation, path_to_example_dir, None)