diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst index 4a7d863f8..4c4e7a69e 100644 --- a/doc/advanced_configuration.rst +++ b/doc/advanced_configuration.rst @@ -18,6 +18,7 @@ file: - ``examples_dirs`` and ``gallery_dirs`` (:ref:`multiple_galleries_config`) - ``filename_pattern`` (:ref:`build_pattern`) - ``subsection_order`` (:ref:`sub_gallery_order`) +- ``within_subsection_order`` (:ref:`within_gallery_order`) - ``reference_url`` (:ref:`link_to_documentation`) - ``backreferences_dir`` and ``doc_module`` (:ref:`references_to_examples`) - ``default_thumb_file`` (:ref:`custom_default_thumb`) @@ -142,6 +143,33 @@ If you so desire you can implement your own sorting key. It will be provided the relative paths to `conf.py` of each sub gallery folder. +.. _within_gallery_order: + +Sorting gallery examples +======================== + +Within a given gallery (sub)section, the example files are ordered by +using the standard :func:`sorted` function with the ``key`` argument by default +set to +:class:`NumberOfCodeLinesSortKey(src_dir) `, +which sorts the files based on the number of code lines:: + + from sphinx_gallery.sorting import NumberOfCodeLinesSortKey + sphinx_gallery_conf = { + ... + 'within_subsection_order': NumberOfCodeLinesSortKey, + } + +In addition, multiple convenience classes are provided for use with +``within_subsection_order``: + +- :class:`sphinx_gallery.sorting.NumberOfCodeLinesSortKey` (default) to sort by + the number of code lines. +- :class:`sphinx_gallery.sorting.FileSizeSortKey` to sort by file size. +- :class:`sphinx_gallery.sorting.FileNameSortKey` to sort by file name. +- :class:`sphinx_gallery.sorting.ExampleTitleSortKey` to sort by example title. + + .. _link_to_documentation: Linking to documentation diff --git a/doc/conf.py b/doc/conf.py index eaa5dff23..63cae10b4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -306,7 +306,7 @@ def setup(app): 'mayavi': ('http://docs.enthought.com/mayavi/mayavi', None), } -from sphinx_gallery.sorting import ExplicitOrder +from sphinx_gallery.sorting import ExplicitOrder, NumberOfCodeLinesSortKey examples_dirs = ['../examples', '../tutorials'] gallery_dirs = ['auto_examples', 'tutorials'] @@ -336,6 +336,7 @@ def setup(app): 'subsection_order': ExplicitOrder(['../examples/sin_func', '../examples/no_output', '../tutorials/seaborn']), + 'within_subsection_order': NumberOfCodeLinesSortKey, 'find_mayavi_figures': find_mayavi_figures, 'expected_failing_examples': ['../examples/no_output/plot_raise.py', '../examples/no_output/plot_syntaxerror.py'] diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py index 0951e4bd0..077c0ae49 100644 --- a/sphinx_gallery/gen_gallery.py +++ b/sphinx_gallery/gen_gallery.py @@ -21,6 +21,7 @@ from .gen_rst import generate_dir_rst, SPHX_GLR_SIG from .docs_resolv import embed_code_links from .downloads import generate_zipfiles +from .sorting import NumberOfCodeLinesSortKey try: FileNotFoundError @@ -32,6 +33,7 @@ 'filename_pattern': re.escape(os.sep) + 'plot', 'examples_dirs': os.path.join('..', 'examples'), 'subsection_order': None, + 'within_subsection_order': NumberOfCodeLinesSortKey, 'gallery_dirs': 'auto_examples', 'backreferences_dir': None, 'doc_module': (), diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py index aa6d06f95..283c4b4b6 100644 --- a/sphinx_gallery/gen_rst.py +++ b/sphinx_gallery/gen_rst.py @@ -153,22 +153,30 @@ def codestr2rst(codestr, lang='python', lineno=None): return code_directive + indented_block -def extract_intro(filename, docstring): +def extract_intro_and_title(filename, docstring): """ Extract the first paragraph of module-level docstring. max:95 char""" # lstrip is just in case docstring has a '\n\n' at the beginning paragraphs = docstring.lstrip().split('\n\n') - if len(paragraphs) > 1: - first_paragraph = re.sub('\n', ' ', paragraphs[1]) - first_paragraph = (first_paragraph[:95] + '...' - if len(first_paragraph) > 95 else first_paragraph) - else: + # remove comments and other syntax like `.. _link:` + paragraphs = [p for p in paragraphs if not p.startswith('.. ')] + if len(paragraphs) <= 1: raise ValueError( "Example docstring should have a header for the example title " "and at least a paragraph explaining what the example is about. " "Please check the example file:\n {}\n".format(filename)) + # Title is the first paragraph with any ReSTructuredText title chars + # removed, i.e. lines that consist of (all the same) 7-bit non-ASCII chars. + # This conditional is not perfect but should hopefully be good enough. + title = paragraphs[0].strip().split('\n') + title = ' '.join(t for t in title if len(t) > 0 and + (ord(t[0]) >= 128 or t[0].isalnum())) + # Concatenate all lines of the first paragraph and truncate at 95 chars + first_paragraph = re.sub('\n', ' ', paragraphs[1]) + first_paragraph = (first_paragraph[:95] + '...' + if len(first_paragraph) > 95 else first_paragraph) - return first_paragraph + return first_paragraph, title def get_md5sum(src_file): @@ -375,8 +383,10 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs): if not os.path.exists(target_dir): os.makedirs(target_dir) - sorted_listdir = [fname for fname in sorted(os.listdir(src_dir)) - if fname.endswith('.py')] + listdir = [fname for fname in os.listdir(src_dir) + if fname.endswith('.py')] + sorted_listdir = sorted( + listdir, key=gallery_conf['within_subsection_order'](src_dir)) entries_text = [] computation_times = [] build_target_dir = os.path.relpath(target_dir, gallery_conf['src_dir']) @@ -385,29 +395,25 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs): 'Generating gallery for %s ' % build_target_dir, length=len(sorted_listdir)) for fname in iterator: - intro, amount_of_code, time_elapsed = generate_file_rst( + intro, time_elapsed = generate_file_rst( fname, target_dir, src_dir, gallery_conf) computation_times.append((time_elapsed, fname)) - new_fname = os.path.join(src_dir, fname) this_entry = _thumbnail_div(build_target_dir, fname, intro) + """ .. toctree:: :hidden: /%s\n""" % os.path.join(build_target_dir, fname[:-3]).replace(os.sep, '/') - entries_text.append((amount_of_code, this_entry)) + entries_text.append(this_entry) if gallery_conf['backreferences_dir']: write_backreferences(seen_backrefs, gallery_conf, target_dir, fname, intro) - # sort to have the smallest entries in the beginning - entries_text.sort() - - for _, entry_text in entries_text: + for entry_text in entries_text: fhindex += entry_text # clear at the end of the section @@ -531,8 +537,6 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): ------- intro: str The introduction of the example - amount_of_code : int - character count of the corresponding python script in file time_elapsed : float seconds required to run the script """ @@ -541,13 +545,10 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): example_file = os.path.join(target_dir, fname) shutil.copyfile(src_file, example_file) file_conf, script_blocks = split_code_and_text_blocks(src_file) - amount_of_code = sum([len(bcontent) - for blabel, bcontent, lineno in script_blocks - if blabel == 'code']) - intro = extract_intro(fname, script_blocks[0][1]) + intro, title = extract_intro_and_title(fname, script_blocks[0][1]) if md5sum_is_current(example_file): - return intro, amount_of_code, 0 + return intro, 0 image_dir = os.path.join(target_dir, 'images') if not os.path.exists(image_dir): @@ -646,4 +647,4 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): if block_vars['execute_script']: logger.debug("%s ran in : %.2g seconds", src_file, time_elapsed) - return intro, amount_of_code, time_elapsed + return intro, time_elapsed diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 98b7dbe0d..d889e1c94 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- r""" -Sorters for Sphinx-Gallery subsections -====================================== +Sorters for Sphinx-Gallery (sub)sections +======================================== -Sorting key functions for gallery subsection folders +Sorting key functions for gallery subsection folders and section files. """ # Created: Sun May 21 20:38:59 2017 # Author: Óscar Nájera @@ -13,21 +13,24 @@ import os import types +from .gen_rst import extract_intro_and_title +from .py_source_parser import split_code_and_text_blocks + class ExplicitOrder(object): - """Sorting key for all galleries subsections + """Sorting key for all gallery subsections. - This requires all folders to be listed otherwise an exception is raised + This requires all folders to be listed otherwise an exception is raised. Parameters ---------- ordered_list : list, tuple, types.GeneratorType - Hold the paths of each galleries' subsections + Hold the paths of each galleries' subsections. Raises ------ ValueError - Wrong input type or Subgallery path missing + Wrong input type or Subgallery path missing. """ def __init__(self, ordered_list): @@ -46,3 +49,71 @@ def __call__(self, item): raise ValueError('If you use an explicit folder ordering, you ' 'must specify all folders. Explicit order not ' 'found for {}'.format(item)) + + +class _SortKey(object): + """Base class for section order key classes.""" + + def __init__(self, src_dir): + self.src_dir = src_dir + + +class NumberOfCodeLinesSortKey(_SortKey): + """Sort examples in src_dir by the number of code lines. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) + file_conf, script_blocks = split_code_and_text_blocks(src_file) + amount_of_code = sum([len(bcontent) + for blabel, bcontent, lineno in script_blocks + if blabel == 'code']) + return amount_of_code + + +class FileSizeSortKey(_SortKey): + """Sort examples in src_dir by file size. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) + return os.stat(src_file).st_size + + +class FileNameSortKey(_SortKey): + """Sort examples in src_dir by file size. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + return filename + + +class ExampleTitleSortKey(_SortKey): + """Sort examples in src_dir by example title. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) + _, script_blocks = split_code_and_text_blocks(src_file) + _, title = extract_intro_and_title(src_file, script_blocks[0][1]) + return title diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py index 79e0b49b0..09c11e880 100644 --- a/sphinx_gallery/tests/test_gen_gallery.py +++ b/sphinx_gallery/tests/test_gen_gallery.py @@ -5,15 +5,17 @@ Test Sphinx-Gallery """ -from __future__ import division, absolute_import, print_function, unicode_literals +from __future__ import (division, absolute_import, print_function, + unicode_literals) +import codecs import os -import tempfile +import re import shutil +import tempfile import pytest from sphinx.application import Sphinx from sphinx.errors import ExtensionError from sphinx_gallery.gen_rst import MixedEncodingStringIO -from sphinx_gallery.gen_gallery import DEFAULT_GALLERY_CONF from sphinx_gallery import sphinx_compatibility @@ -162,3 +164,69 @@ def test_config_backreferences(config_app): 'gen_modules', 'backreferences') build_warn = config_app._warning.getvalue() assert build_warn == "" + + +def _check_order(config_app, key): + index_fname = os.path.join(config_app.outdir, '..', 'ex', 'index.rst') + order = list() + regex = '.*:%s=(.):.*' % key + with codecs.open(index_fname, 'r', 'utf-8') as fid: + for line in fid: + if 'sphx-glr-thumbcontainer' in line: + order.append(int(re.match(regex, line).group(1))) + assert len(order) == 3 + assert order == [1, 2, 3] + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', +}""") +def test_example_sorting_default(config_app): + """Test sorting of examples by default key (number of code lines).""" + _check_order(config_app, 'lines') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import FileSizeSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': FileSizeSortKey, +}""") +def test_example_sorting_filesize(config_app): + """Test sorting of examples by filesize.""" + _check_order(config_app, 'filesize') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import FileNameSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': FileNameSortKey, +}""") +def test_example_sorting_filename(config_app): + """Test sorting of examples by filename.""" + _check_order(config_app, 'filename') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import ExampleTitleSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': ExampleTitleSortKey, +}""") +def test_example_sorting_title(config_app): + """Test sorting of examples by title.""" + _check_order(config_app, 'title') diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py index 1047f375b..75d967d96 100644 --- a/sphinx_gallery/tests/test_gen_rst.py +++ b/sphinx_gallery/tests/test_gen_rst.py @@ -25,7 +25,8 @@ import matplotlib.pyplot as plt CONTENT = [ - '"""' + '"""', + '================', 'Docstring header', '================', '', @@ -109,11 +110,13 @@ def test_codestr2rst(): assert reference == output -def test_extract_intro(): - result = sg.extract_intro('', '\n'.join(CONTENT[1:9])) - assert 'Docstring' not in result - assert result == 'This is the description of the example which goes on and on, Óscar' # noqa - assert 'second paragraph' not in result +def test_extract_intro_and_title(): + intro, title = sg.extract_intro_and_title('', + '\n'.join(CONTENT[1:10])) + assert title == 'Docstring header' + assert 'Docstring' not in intro + assert intro == 'This is the description of the example which goes on and on, Óscar' # noqa + assert 'second paragraph' not in intro def test_md5sums(): diff --git a/sphinx_gallery/tests/test_sorting.py b/sphinx_gallery/tests/test_sorting.py index 036e4648c..d6a866d3a 100644 --- a/sphinx_gallery/tests/test_sorting.py +++ b/sphinx_gallery/tests/test_sorting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- r""" -Tests for sorting keys on gallery sections -========================================== +Tests for sorting keys on gallery (sub)sections +=============================================== """ # Author: Óscar Nájera diff --git a/sphinx_gallery/tests/testconfs/src/plot_1.py b/sphinx_gallery/tests/testconfs/src/plot_1.py new file mode 100644 index 000000000..721bfc10a --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_1.py @@ -0,0 +1,11 @@ +""" +====== +B test +====== + +:filename=1:title=2:lines=3:filesize=2: +""" + +print('foo') +print('bar') +print('again') diff --git a/sphinx_gallery/tests/testconfs/src/plot_2.py b/sphinx_gallery/tests/testconfs/src/plot_2.py new file mode 100644 index 000000000..d468d799c --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_2.py @@ -0,0 +1,9 @@ +""" +====== +C test +====== + +:filename=2:title=3:lines=1:filesize=1: +""" + +print('foo') diff --git a/sphinx_gallery/tests/testconfs/src/plot_3.py b/sphinx_gallery/tests/testconfs/src/plot_3.py new file mode 100644 index 000000000..ceea4c0a3 --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_3.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" +.. _extra_ref: + +=========================================================== +A test with a really long title to make the filesize larger +=========================================================== + +:filename=3:title=1:lines=2:filesize=3: +""" + +print('foo') +print('bar')