diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 628d6d5b2f..17a43636b5 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -34,25 +34,29 @@ @author: Ward Poelmans (Ghent University) """ import copy +import inspect +import os from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.tools.filetools import read_file from easybuild.tools.ordereddict import OrderedDict -from easybuild.tools.utilities import quote_str +from easybuild.tools.utilities import import_available_modules, quote_str FORMAT_RST = 'rst' FORMAT_TXT = 'txt' +def det_col_width(entries, title): + """Determine column width based on column title and list of entries.""" + return max(map(len, entries + [title])) + + def avail_easyconfig_params_rst(title, grouped_params): """ Compose overview of available easyconfig parameters, in RST format. """ - def det_col_width(entries, title): - """Determine column width based on column title and list of entries.""" - return max(map(len, entries + [title])) - # main title lines = [ title, @@ -65,33 +69,19 @@ def det_col_width(entries, title): lines.append("%s parameters" % grpname) lines.extend(['-' * len(lines[-1]), '']) - name_title = "**Parameter name**" - descr_title = "**Description**" - dflt_title = "**Default value**" - - # figure out column widths - nw = det_col_width(grouped_params[grpname].keys(), name_title) + 4 # +4 for raw format ("``foo``") - dw = det_col_width([x[0] for x in grouped_params[grpname].values()], descr_title) - dfw = det_col_width([str(quote_str(x[1])) for x in grouped_params[grpname].values()], dflt_title) + titles = ["**Parameter name**", "**Description**", "**Default value**"] + values = [ + ['``' + name + '``' for name in grouped_params[grpname].keys()], # parameter name + [x[0] for x in grouped_params[grpname].values()], # description + [str(quote_str(x[1])) for x in grouped_params[grpname].values()] # default value + ] - # 3 columns (name, description, default value), left-aligned, {c} is fill char - line_tmpl = "{0:{c}<%s} {1:{c}<%s} {2:{c}<%s}" % (nw, dw, dfw) - table_line = line_tmpl.format('', '', '', c='=', nw=nw, dw=dw, dfw=dfw) - - # table header - lines.append(table_line) - lines.append(line_tmpl.format(name_title, descr_title, dflt_title, c=' ')) - lines.append(line_tmpl.format('', '', '', c='-')) - - # table rows by parameter - for name, (descr, dflt) in sorted(grouped_params[grpname].items()): - rawname = '``%s``' % name - lines.append(line_tmpl.format(rawname, descr, str(quote_str(dflt)), c=' ')) - lines.append(table_line) + lines.extend(mk_rst_table(titles, values)) lines.append('') return '\n'.join(lines) + def avail_easyconfig_params_txt(title, grouped_params): """ Compose overview of available easyconfig parameters, in plain text format. @@ -117,6 +107,7 @@ def avail_easyconfig_params_txt(title, grouped_params): return '\n'.join(lines) + def avail_easyconfig_params(easyblock, output_format): """ Compose overview of available easyconfig parameters, in specified format. @@ -160,3 +151,156 @@ def avail_easyconfig_params(easyblock, output_format): FORMAT_TXT: avail_easyconfig_params_txt, } return avail_easyconfig_params_functions[output_format](title, grouped_params) + + +def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): + """ + Compose overview of all easyblocks in the given package in rst format + """ + modules = import_available_modules(package_name) + docs = [] + all_blocks = [] + + # get all blocks + for mod in modules: + for name,obj in inspect.getmembers(mod, inspect.isclass): + eb_class = getattr(mod, name) + # skip imported classes that are not easyblocks + if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks: + all_blocks.append(eb_class) + + for eb_class in sorted(all_blocks, key=lambda c: c.__name__): + docs.append(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks)) + + title = 'Overview of generic easyblocks' + + heading = [ + '*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*', + '', + '=' * len(title), + title, + '=' * len(title), + '', + ] + + contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)] + toc = ' - '.join(contents) + heading.append(toc) + heading.append('') + + return heading + docs + + +def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks): + """ + Compose overview of one easyblock given class object of the easyblock in rst format + """ + classname = eb_class.__name__ + + lines = [ + '.. _' + classname + ':', + '', + '``' + classname + '``', + '=' * (len(classname)+4), + '', + ] + + bases = [] + for b in eb_class.__bases__: + base = ':ref:`' + b.__name__ +'`' if b in all_blocks else b.__name__ + bases.append(base) + + derived = '(derives from ' + ', '.join(bases) + ')' + lines.extend([derived, '']) + + # Description (docstring) + lines.extend([eb_class.__doc__.strip(), '']) + + # Add extra options, if any + if eb_class.extra_options(): + extra_parameters = 'Extra easyconfig parameters specific to ``' + classname + '`` easyblock' + lines.extend([extra_parameters, '-' * len(extra_parameters), '']) + ex_opt = eb_class.extra_options() + + titles = ['easyconfig parameter', 'description', 'default value'] + values = [ + ['``' + key + '``' for key in ex_opt], # parameter name + [val[1] for val in ex_opt.values()], # description + ['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()] # default value + ] + + lines.extend(mk_rst_table(titles, values)) + + # Add commonly used parameters + if classname in common_params: + commonly_used = 'Commonly used easyconfig parameters with ``' + classname + '`` easyblock' + lines.extend([commonly_used, '-' * len(commonly_used)]) + + titles = ['easyconfig parameter', 'description'] + values = [ + [opt for opt in common_params[classname]], + [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]], + ] + + lines.extend(mk_rst_table(titles, values)) + + lines.append('') + + # Add docstring for custom steps + custom = [] + inh = '' + f = None + for func in doc_functions: + if func in eb_class.__dict__: + f = eb_class.__dict__[func] + + if f.__doc__: + custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh) + + if custom: + title = 'Customised steps in ``' + classname + '`` easyblock' + lines.extend([title, '-' * len(title)] + custom) + lines.append('') + + # Add example if available + if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)): + title = 'Example for ``' + classname + '`` easyblock' + lines.extend(['', title, '-' * len(title), '', '::', '']) + for line in read_file(os.path.join(path_to_examples, classname+'.eb')).split('\n'): + lines.append(' ' + line.strip()) + lines.append('') # empty line after literal block + + return '\n'.join(lines) + + +def mk_rst_table(titles, values): + """ + Returns an rst table with given titles and values (a nested list of string values for each column) + """ + num_col = len(titles) + table = [] + col_widths = [] + tmpl = [] + line= [] + + # figure out column widths + for i in range(0, num_col): + col_widths.append(det_col_width(values[i], titles[i])) + + # make line template + tmpl.append('{' + str(i) + ':{c}<' + str(col_widths[i]) + '}') + line.append('') # needed for table line + + line_tmpl = ' '.join(tmpl) + table_line = line_tmpl.format(*line, c="=") + + table.append(table_line) + table.append(line_tmpl.format(*titles, c=' ')) + table.append(table_line) + + for i in range(0, len(values[0])): + table.append(line_tmpl.format(*[v[i] for v in values], c=' ')) + + table.extend([table_line, '']) + + return table diff --git a/test/framework/docs.py b/test/framework/docs.py new file mode 100644 index 0000000000..50c8e1c3bb --- /dev/null +++ b/test/framework/docs.py @@ -0,0 +1,122 @@ +# # +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for docs.py. +""" + +import os +import re +import sys +import inspect + +from easybuild.tools.docs import gen_easyblocks_overview_rst, mk_rst_table +from easybuild.tools.utilities import import_available_modules +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader, main + +class DocsTest(EnhancedTestCase): + + def test_rst_table(self): + """ Test mk_rst_table function """ + entries = [['one', 'two', 'three']] + t = 'This title is longer than the entries in the column' + titles = [t] + + # small table + table = '\n'.join(mk_rst_table(titles, entries)) + check = '\n'.join([ + '=' * len(t), + t, + '=' * len(t), + 'one' + ' ' * (len(t) - 3), + 'two' + ' ' * (len(t) -3), + 'three' + ' ' * (len(t) - 5), + '=' * len(t), + '', + ]) + + self.assertEqual(table, check) + + def test_gen_easyblocks(self): + """ Test gen_easyblocks_overview_rst function """ + module = 'easybuild.easyblocks.generic' + modules = import_available_modules(module) + common_params = { + 'ConfigureMake' : ['configopts', 'buildopts', 'installopts'], + } + doc_functions = ['build_step', 'configure_step', 'test_step'] + + eb_overview = gen_easyblocks_overview_rst(module, 'easyconfigs', common_params, doc_functions) + ebdoc = '\n'.join(eb_overview) + + # extensive check for ConfigureMake easyblock + check_configuremake = '\n'.join([ + ".. _ConfigureMake:", + '', + "``ConfigureMake``", + "=================", + '', + "(derives from EasyBlock)", + '', + "Dummy support for building and installing applications with configure/make/make install.", + '', + "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", + "--------------------------------------------------------------------", + "==================== ================================================================", + "easyconfig parameter description ", + "==================== ================================================================", + "configopts Extra options passed to configure (default already has --prefix)", + "buildopts Extra options passed to make step (default already has -j X) ", + "installopts Extra options for installation ", + "==================== ================================================================", + ]) + + self.assertTrue(check_configuremake in ebdoc) + names = [] + + for mod in modules: + for name, obj in inspect.getmembers(mod, inspect.isclass): + eb_class = getattr(mod, name) + # skip imported classes that are not easyblocks + if eb_class.__module__.startswith(module): + self.assertTrue(name in ebdoc) + names.append(name) + + toc = [":ref:`" + n + "`" for n in sorted(names)] + pattern = " - ".join(toc) + + regex = re.compile(pattern) + self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc)) + + +def suite(): + """ returns all test cases in this module """ + return TestLoader().loadTestsFromTestCase(DocsTest) + +if __name__ == '__main__': + # also check the setUp for debug + # logToScreen(enable=True) + # setLogLevelDebug() + main() diff --git a/test/framework/suite.py b/test/framework/suite.py index 9affffc9ae..34caa3ec8f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -62,6 +62,7 @@ import test.framework.easyconfigformat as ef import test.framework.ebconfigobj as ebco import test.framework.easyconfigversion as ev +import test.framework.docs as d import test.framework.filetools as f import test.framework.format_convert as f_c import test.framework.general as gen @@ -103,7 +104,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, - p, i, pkg] + p, i, pkg, d] SUITE = unittest.TestSuite([x.suite() for x in tests])