From 30030bf9a05c6bb7581f686355be1a17199eeddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20N=C3=A1jera?= Date: Mon, 31 Jul 2017 18:44:01 +0200 Subject: [PATCH 1/8] Expose examples sortkey --- doc/conf.py | 3 ++- sphinx_gallery/gen_gallery.py | 1 + sphinx_gallery/gen_rst.py | 25 +++++++++---------------- sphinx_gallery/sorting.py | 22 ++++++++++++++++++++++ 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index eaa5dff23..d0666e9e4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -306,7 +306,7 @@ def setup(app): 'mayavi': ('http://docs.enthought.com/mayavi/mayavi', None), } -from sphinx_gallery.sorting import ExplicitOrder +from sphinx_gallery.sorting import ExplicitOrder, amount_of_code examples_dirs = ['../examples', '../tutorials'] gallery_dirs = ['auto_examples', 'tutorials'] @@ -336,6 +336,7 @@ def setup(app): 'subsection_order': ExplicitOrder(['../examples/sin_func', '../examples/no_output', '../tutorials/seaborn']), + 'within_subsection_order': amount_of_code, 'find_mayavi_figures': find_mayavi_figures, 'expected_failing_examples': ['../examples/no_output/plot_raise.py', '../examples/no_output/plot_syntaxerror.py'] diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py index 0951e4bd0..0f1e0f72d 100644 --- a/sphinx_gallery/gen_gallery.py +++ b/sphinx_gallery/gen_gallery.py @@ -32,6 +32,7 @@ 'filename_pattern': re.escape(os.sep) + 'plot', 'examples_dirs': os.path.join('..', 'examples'), 'subsection_order': None, + 'within_subsection_order': None, 'gallery_dirs': 'auto_examples', 'backreferences_dir': None, 'doc_module': (), diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py index aa6d06f95..fe299c8fe 100644 --- a/sphinx_gallery/gen_rst.py +++ b/sphinx_gallery/gen_rst.py @@ -375,8 +375,10 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs): if not os.path.exists(target_dir): os.makedirs(target_dir) - sorted_listdir = [fname for fname in sorted(os.listdir(src_dir)) - if fname.endswith('.py')] + listdir = [fname for fname in os.listdir(src_dir) + if fname.endswith('.py')] + sorted_listdir = sorted( + listdir, key=gallery_conf['within_subsection_order'](src_dir)) entries_text = [] computation_times = [] build_target_dir = os.path.relpath(target_dir, gallery_conf['src_dir']) @@ -385,29 +387,25 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs): 'Generating gallery for %s ' % build_target_dir, length=len(sorted_listdir)) for fname in iterator: - intro, amount_of_code, time_elapsed = generate_file_rst( + intro, time_elapsed = generate_file_rst( fname, target_dir, src_dir, gallery_conf) computation_times.append((time_elapsed, fname)) - new_fname = os.path.join(src_dir, fname) this_entry = _thumbnail_div(build_target_dir, fname, intro) + """ .. toctree:: :hidden: /%s\n""" % os.path.join(build_target_dir, fname[:-3]).replace(os.sep, '/') - entries_text.append((amount_of_code, this_entry)) + entries_text.append(this_entry) if gallery_conf['backreferences_dir']: write_backreferences(seen_backrefs, gallery_conf, target_dir, fname, intro) - # sort to have the smallest entries in the beginning - entries_text.sort() - - for _, entry_text in entries_text: + for entry_text in entries_text: fhindex += entry_text # clear at the end of the section @@ -531,8 +529,6 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): ------- intro: str The introduction of the example - amount_of_code : int - character count of the corresponding python script in file time_elapsed : float seconds required to run the script """ @@ -541,13 +537,10 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): example_file = os.path.join(target_dir, fname) shutil.copyfile(src_file, example_file) file_conf, script_blocks = split_code_and_text_blocks(src_file) - amount_of_code = sum([len(bcontent) - for blabel, bcontent, lineno in script_blocks - if blabel == 'code']) intro = extract_intro(fname, script_blocks[0][1]) if md5sum_is_current(example_file): - return intro, amount_of_code, 0 + return intro, 0 image_dir = os.path.join(target_dir, 'images') if not os.path.exists(image_dir): @@ -646,4 +639,4 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): if block_vars['execute_script']: logger.debug("%s ran in : %.2g seconds", src_file, time_elapsed) - return intro, amount_of_code, time_elapsed + return intro, time_elapsed diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 98b7dbe0d..27e0c0ea5 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -13,6 +13,8 @@ import os import types +from .py_source_parser import split_code_and_text_blocks + class ExplicitOrder(object): """Sorting key for all galleries subsections @@ -46,3 +48,23 @@ def __call__(self, item): raise ValueError('If you use an explicit folder ordering, you ' 'must specify all folders. Explicit order not ' 'found for {}'.format(item)) + + +def amount_of_code(src_dir): + """Sort examples in src_dir by amount of code """ + def sort_files(filename): + src_file = os.path.normpath(os.path.join(src_dir, filename)) + file_conf, script_blocks = split_code_and_text_blocks(src_file) + amount_of_code = sum([len(bcontent) + for blabel, bcontent, lineno in script_blocks + if blabel == 'code']) + return amount_of_code + return sort_files + + +def file_size(src_dir): + """Sort examples in src_dir by file size""" + def sort_files(filename): + src_file = os.path.normpath(os.path.join(src_dir, filename)) + return os.stat(src_file).st_size + return sort_files From af055fca7175b6d160aba2d2e841d17c975f9747 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 1 Aug 2017 22:19:06 -0600 Subject: [PATCH 2/8] FIX: Add tests --- doc/conf.py | 4 +- sphinx_gallery/gen_gallery.py | 3 +- sphinx_gallery/sorting.py | 66 +++++++++++++++----- sphinx_gallery/tests/test_gen_gallery.py | 60 +++++++++++++++++- sphinx_gallery/tests/test_sorting.py | 4 +- sphinx_gallery/tests/testconfs/src/plot_1.py | 11 ++++ sphinx_gallery/tests/testconfs/src/plot_2.py | 9 +++ sphinx_gallery/tests/testconfs/src/plot_3.py | 10 +++ 8 files changed, 142 insertions(+), 25 deletions(-) create mode 100644 sphinx_gallery/tests/testconfs/src/plot_1.py create mode 100644 sphinx_gallery/tests/testconfs/src/plot_2.py create mode 100644 sphinx_gallery/tests/testconfs/src/plot_3.py diff --git a/doc/conf.py b/doc/conf.py index d0666e9e4..43fc3e9f1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -306,7 +306,7 @@ def setup(app): 'mayavi': ('http://docs.enthought.com/mayavi/mayavi', None), } -from sphinx_gallery.sorting import ExplicitOrder, amount_of_code +from sphinx_gallery.sorting import ExplicitOrder, AmountOfCodeSortKey examples_dirs = ['../examples', '../tutorials'] gallery_dirs = ['auto_examples', 'tutorials'] @@ -336,7 +336,7 @@ def setup(app): 'subsection_order': ExplicitOrder(['../examples/sin_func', '../examples/no_output', '../tutorials/seaborn']), - 'within_subsection_order': amount_of_code, + 'within_subsection_order': AmountOfCodeSortKey, 'find_mayavi_figures': find_mayavi_figures, 'expected_failing_examples': ['../examples/no_output/plot_raise.py', '../examples/no_output/plot_syntaxerror.py'] diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py index 0f1e0f72d..0a96dfb35 100644 --- a/sphinx_gallery/gen_gallery.py +++ b/sphinx_gallery/gen_gallery.py @@ -21,6 +21,7 @@ from .gen_rst import generate_dir_rst, SPHX_GLR_SIG from .docs_resolv import embed_code_links from .downloads import generate_zipfiles +from .sorting import AmountOfCodeSortKey try: FileNotFoundError @@ -32,7 +33,7 @@ 'filename_pattern': re.escape(os.sep) + 'plot', 'examples_dirs': os.path.join('..', 'examples'), 'subsection_order': None, - 'within_subsection_order': None, + 'within_subsection_order': AmountOfCodeSortKey, 'gallery_dirs': 'auto_examples', 'backreferences_dir': None, 'doc_module': (), diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 27e0c0ea5..f5833733e 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- r""" -Sorters for Sphinx-Gallery subsections -====================================== +Sorters for Sphinx-Gallery (sub)sections +======================================== -Sorting key functions for gallery subsection folders +Sorting key functions for gallery subsection folders and section files. """ # Created: Sun May 21 20:38:59 2017 # Author: Óscar Nájera @@ -17,19 +17,19 @@ class ExplicitOrder(object): - """Sorting key for all galleries subsections + """Sorting key for all gallery subsections. - This requires all folders to be listed otherwise an exception is raised + This requires all folders to be listed otherwise an exception is raised. Parameters ---------- ordered_list : list, tuple, types.GeneratorType - Hold the paths of each galleries' subsections + Hold the paths of each galleries' subsections. Raises ------ ValueError - Wrong input type or Subgallery path missing + Wrong input type or Subgallery path missing. """ def __init__(self, ordered_list): @@ -50,21 +50,53 @@ def __call__(self, item): 'found for {}'.format(item)) -def amount_of_code(src_dir): - """Sort examples in src_dir by amount of code """ - def sort_files(filename): - src_file = os.path.normpath(os.path.join(src_dir, filename)) +class _SortKey(object): + """Base class for section order key classes.""" + + def __init__(self, src_dir): + self.src_dir = src_dir + + +class AmountOfCodeSortKey(_SortKey): + """Sort examples in src_dir by amount of code. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) file_conf, script_blocks = split_code_and_text_blocks(src_file) amount_of_code = sum([len(bcontent) for blabel, bcontent, lineno in script_blocks if blabel == 'code']) return amount_of_code - return sort_files -def file_size(src_dir): - """Sort examples in src_dir by file size""" - def sort_files(filename): - src_file = os.path.normpath(os.path.join(src_dir, filename)) +class FileSizeSortKey(_SortKey): + """Sort examples in src_dir by file size. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) return os.stat(src_file).st_size - return sort_files + + +class FileNameSortKey(_SortKey): + """Sort examples in src_dir by file size. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + return filename diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py index 79e0b49b0..fa9f64833 100644 --- a/sphinx_gallery/tests/test_gen_gallery.py +++ b/sphinx_gallery/tests/test_gen_gallery.py @@ -5,15 +5,17 @@ Test Sphinx-Gallery """ -from __future__ import division, absolute_import, print_function, unicode_literals +from __future__ import (division, absolute_import, print_function, + unicode_literals) +import codecs import os -import tempfile +import re import shutil +import tempfile import pytest from sphinx.application import Sphinx from sphinx.errors import ExtensionError from sphinx_gallery.gen_rst import MixedEncodingStringIO -from sphinx_gallery.gen_gallery import DEFAULT_GALLERY_CONF from sphinx_gallery import sphinx_compatibility @@ -162,3 +164,55 @@ def test_config_backreferences(config_app): 'gen_modules', 'backreferences') build_warn = config_app._warning.getvalue() assert build_warn == "" + + +def _check_order(config_app, key): + index_fname = os.path.join(config_app.outdir, '..', 'ex', 'index.rst') + order = list() + regex = '.*:%s=(.):.*' % key + with codecs.open(index_fname, 'r', 'utf-8') as fid: + for line in fid: + if 'sphx-glr-thumbcontainer' in line: + order.append(int(re.match(regex, line).group(1))) + assert len(order) == 3 + assert order == [1, 2, 3] + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', +}""") +def test_example_sorting_default(config_app): + """Test sorting of examples by default key (number of code lines).""" + _check_order(config_app, 'lines') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import FileSizeSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': FileSizeSortKey, +}""") +def test_example_sorting_filesize(config_app): + """Test sorting of examples by default key (number of code lines).""" + _check_order(config_app, 'filesize') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import FileNameSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': FileNameSortKey, +}""") +def test_example_sorting_filename(config_app): + """Test sorting of examples by default key (number of code lines).""" + _check_order(config_app, 'filename') diff --git a/sphinx_gallery/tests/test_sorting.py b/sphinx_gallery/tests/test_sorting.py index 036e4648c..d6a866d3a 100644 --- a/sphinx_gallery/tests/test_sorting.py +++ b/sphinx_gallery/tests/test_sorting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- r""" -Tests for sorting keys on gallery sections -========================================== +Tests for sorting keys on gallery (sub)sections +=============================================== """ # Author: Óscar Nájera diff --git a/sphinx_gallery/tests/testconfs/src/plot_1.py b/sphinx_gallery/tests/testconfs/src/plot_1.py new file mode 100644 index 000000000..721bfc10a --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_1.py @@ -0,0 +1,11 @@ +""" +====== +B test +====== + +:filename=1:title=2:lines=3:filesize=2: +""" + +print('foo') +print('bar') +print('again') diff --git a/sphinx_gallery/tests/testconfs/src/plot_2.py b/sphinx_gallery/tests/testconfs/src/plot_2.py new file mode 100644 index 000000000..d468d799c --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_2.py @@ -0,0 +1,9 @@ +""" +====== +C test +====== + +:filename=2:title=3:lines=1:filesize=1: +""" + +print('foo') diff --git a/sphinx_gallery/tests/testconfs/src/plot_3.py b/sphinx_gallery/tests/testconfs/src/plot_3.py new file mode 100644 index 000000000..42b4c33b9 --- /dev/null +++ b/sphinx_gallery/tests/testconfs/src/plot_3.py @@ -0,0 +1,10 @@ +""" +=========================================================== +A test with a really long title to make the filesize larger +=========================================================== + +:filename=3:title=1:lines=2:filesize=3: +""" + +print('foo') +print('bar') From 5305f9f78571d2c2806d5f3a44a32ea381a0af07 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 1 Aug 2017 22:38:06 -0600 Subject: [PATCH 3/8] ENH: Add ExampleTitleSortKey --- sphinx_gallery/sorting.py | 16 ++++++++++++++++ sphinx_gallery/tests/test_gen_gallery.py | 18 ++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index f5833733e..5be8b5823 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -100,3 +100,19 @@ class FileNameSortKey(_SortKey): def __call__(self, filename): return filename + + +class ExampleTitleSortKey(_SortKey): + """Sort examples in src_dir by example title. + + Parameters + ---------- + src_dir : str + The source directory. + """ + + def __call__(self, filename): + src_file = os.path.normpath(os.path.join(self.src_dir, filename)) + _, script_blocks = split_code_and_text_blocks(src_file) + # title should be the second line of the first script_blocks entry + return script_blocks[0][1].strip().split('\n')[1] diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py index fa9f64833..09c11e880 100644 --- a/sphinx_gallery/tests/test_gen_gallery.py +++ b/sphinx_gallery/tests/test_gen_gallery.py @@ -200,7 +200,7 @@ def test_example_sorting_default(config_app): 'within_subsection_order': FileSizeSortKey, }""") def test_example_sorting_filesize(config_app): - """Test sorting of examples by default key (number of code lines).""" + """Test sorting of examples by filesize.""" _check_order(config_app, 'filesize') @@ -214,5 +214,19 @@ def test_example_sorting_filesize(config_app): 'within_subsection_order': FileNameSortKey, }""") def test_example_sorting_filename(config_app): - """Test sorting of examples by default key (number of code lines).""" + """Test sorting of examples by filename.""" _check_order(config_app, 'filename') + + +@pytest.mark.conf_file(content=""" +import sphinx_gallery +from sphinx_gallery.sorting import ExampleTitleSortKey +extensions = ['sphinx_gallery.gen_gallery'] +sphinx_gallery_conf = { + 'examples_dirs': 'src', + 'gallery_dirs': 'ex', + 'within_subsection_order': ExampleTitleSortKey, +}""") +def test_example_sorting_title(config_app): + """Test sorting of examples by title.""" + _check_order(config_app, 'title') From c51c93e710d92310f476424684807f5a20e7f9d0 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 3 Aug 2017 22:59:34 -0600 Subject: [PATCH 4/8] DOC: Add docs --- doc/advanced_configuration.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst index 4a7d863f8..ffa425bf0 100644 --- a/doc/advanced_configuration.rst +++ b/doc/advanced_configuration.rst @@ -18,6 +18,7 @@ file: - ``examples_dirs`` and ``gallery_dirs`` (:ref:`multiple_galleries_config`) - ``filename_pattern`` (:ref:`build_pattern`) - ``subsection_order`` (:ref:`sub_gallery_order`) +- ``within_subsection_order`` (:ref:`within_gallery_order`) - ``reference_url`` (:ref:`link_to_documentation`) - ``backreferences_dir`` and ``doc_module`` (:ref:`references_to_examples`) - ``default_thumb_file`` (:ref:`custom_default_thumb`) @@ -142,6 +143,29 @@ If you so desire you can implement your own sorting key. It will be provided the relative paths to `conf.py` of each sub gallery folder. +.. _within_gallery_order: + +Sorting gallery examples +======================== + +Within a given gallery (sub)section, the example files by default are ordered +by the amount of code, i.e. :class:`sphinx_gallery.sorting.AmountOfCodeSortKey` +as:: + + from sphinx_gallery.sorting import AmountOfCodeSortKey + sphinx_gallery_conf = { + ... + 'within_subsection_order': AmountOfCodeSortKey, + } + +However, other options exist that can be instantiated for use with +``within_subsection_order``: + +- :class:`sphinx_gallery.sorting.FileSizeSortKey` to sort by file size. +- :class:`sphinx_gallery.sorting.FileNameSortKey` to sort by file name. +- :class:`sphinx_gallery.sorting.ExampleTitleSortKey` to sort by example title. + + .. _link_to_documentation: Linking to documentation From 7dba0b7e20ab258da3783b430966aa7967bda359 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 4 Aug 2017 19:43:02 -0600 Subject: [PATCH 5/8] FIX: Rename key --- doc/advanced_configuration.rst | 16 ++++++++++------ doc/conf.py | 4 ++-- sphinx_gallery/gen_gallery.py | 4 ++-- sphinx_gallery/sorting.py | 7 ++++--- sphinx_gallery/tests/testconfs/src/plot_1.py | 8 +++++--- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst index ffa425bf0..da4cb31fb 100644 --- a/doc/advanced_configuration.rst +++ b/doc/advanced_configuration.rst @@ -148,19 +148,23 @@ provided the relative paths to `conf.py` of each sub gallery folder. Sorting gallery examples ======================== -Within a given gallery (sub)section, the example files by default are ordered -by the amount of code, i.e. :class:`sphinx_gallery.sorting.AmountOfCodeSortKey` -as:: +Within a given gallery (sub)section, the example files are ordered by +using the standard :func:`sorted` function with the ``key`` argument by default +set to +:class:`NumberOfCodeLinesSortKey(src_dir) `, +which sorts the files based on the number of code lines:: - from sphinx_gallery.sorting import AmountOfCodeSortKey + from sphinx_gallery.sorting import NumberOfCodeLinesSortKey sphinx_gallery_conf = { ... - 'within_subsection_order': AmountOfCodeSortKey, + 'within_subsection_order': NumberOfCodeLinesSortKey, } -However, other options exist that can be instantiated for use with +However, multiple convenience classes are provided for use with ``within_subsection_order``: +- :class:`sphinx_gallery.sorting.NumberOfCodeLinesSortKey` (default) to sort by + the number of code lines. - :class:`sphinx_gallery.sorting.FileSizeSortKey` to sort by file size. - :class:`sphinx_gallery.sorting.FileNameSortKey` to sort by file name. - :class:`sphinx_gallery.sorting.ExampleTitleSortKey` to sort by example title. diff --git a/doc/conf.py b/doc/conf.py index 43fc3e9f1..63cae10b4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -306,7 +306,7 @@ def setup(app): 'mayavi': ('http://docs.enthought.com/mayavi/mayavi', None), } -from sphinx_gallery.sorting import ExplicitOrder, AmountOfCodeSortKey +from sphinx_gallery.sorting import ExplicitOrder, NumberOfCodeLinesSortKey examples_dirs = ['../examples', '../tutorials'] gallery_dirs = ['auto_examples', 'tutorials'] @@ -336,7 +336,7 @@ def setup(app): 'subsection_order': ExplicitOrder(['../examples/sin_func', '../examples/no_output', '../tutorials/seaborn']), - 'within_subsection_order': AmountOfCodeSortKey, + 'within_subsection_order': NumberOfCodeLinesSortKey, 'find_mayavi_figures': find_mayavi_figures, 'expected_failing_examples': ['../examples/no_output/plot_raise.py', '../examples/no_output/plot_syntaxerror.py'] diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py index 0a96dfb35..077c0ae49 100644 --- a/sphinx_gallery/gen_gallery.py +++ b/sphinx_gallery/gen_gallery.py @@ -21,7 +21,7 @@ from .gen_rst import generate_dir_rst, SPHX_GLR_SIG from .docs_resolv import embed_code_links from .downloads import generate_zipfiles -from .sorting import AmountOfCodeSortKey +from .sorting import NumberOfCodeLinesSortKey try: FileNotFoundError @@ -33,7 +33,7 @@ 'filename_pattern': re.escape(os.sep) + 'plot', 'examples_dirs': os.path.join('..', 'examples'), 'subsection_order': None, - 'within_subsection_order': AmountOfCodeSortKey, + 'within_subsection_order': NumberOfCodeLinesSortKey, 'gallery_dirs': 'auto_examples', 'backreferences_dir': None, 'doc_module': (), diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 5be8b5823..d8a5c9d26 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -57,8 +57,8 @@ def __init__(self, src_dir): self.src_dir = src_dir -class AmountOfCodeSortKey(_SortKey): - """Sort examples in src_dir by amount of code. +class NumberOfCodeLinesSortKey(_SortKey): + """Sort examples in src_dir by the number of code lines. Parameters ---------- @@ -115,4 +115,5 @@ def __call__(self, filename): src_file = os.path.normpath(os.path.join(self.src_dir, filename)) _, script_blocks = split_code_and_text_blocks(src_file) # title should be the second line of the first script_blocks entry - return script_blocks[0][1].strip().split('\n')[1] + title = script_blocks[0][1].strip().split('\n')[1] + return title diff --git a/sphinx_gallery/tests/testconfs/src/plot_1.py b/sphinx_gallery/tests/testconfs/src/plot_1.py index 721bfc10a..bcf9a0f5f 100644 --- a/sphinx_gallery/tests/testconfs/src/plot_1.py +++ b/sphinx_gallery/tests/testconfs/src/plot_1.py @@ -1,7 +1,9 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- """ -====== -B test -====== +====================================== +B test, with two junk lines at the top +====================================== :filename=1:title=2:lines=3:filesize=2: """ From 933d51c7237cfd9e64b44e6cd9511531341c3dc9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 5 Aug 2017 13:22:31 -0600 Subject: [PATCH 6/8] FIX: Extract title and intro properly --- doc/advanced_configuration.rst | 2 +- sphinx_gallery/gen_rst.py | 9 ++++++--- sphinx_gallery/tests/test_gen_gallery.py | 1 + sphinx_gallery/tests/test_gen_rst.py | 13 +++++++------ sphinx_gallery/tests/testconfs/src/plot_1.py | 8 +++----- sphinx_gallery/tests/testconfs/src/plot_3.py | 4 ++++ 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst index da4cb31fb..4c4e7a69e 100644 --- a/doc/advanced_configuration.rst +++ b/doc/advanced_configuration.rst @@ -160,7 +160,7 @@ which sorts the files based on the number of code lines:: 'within_subsection_order': NumberOfCodeLinesSortKey, } -However, multiple convenience classes are provided for use with +In addition, multiple convenience classes are provided for use with ``within_subsection_order``: - :class:`sphinx_gallery.sorting.NumberOfCodeLinesSortKey` (default) to sort by diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py index fe299c8fe..14a718a47 100644 --- a/sphinx_gallery/gen_rst.py +++ b/sphinx_gallery/gen_rst.py @@ -153,12 +153,15 @@ def codestr2rst(codestr, lang='python', lineno=None): return code_directive + indented_block -def extract_intro(filename, docstring): +def extract_intro_title(filename, docstring): """ Extract the first paragraph of module-level docstring. max:95 char""" # lstrip is just in case docstring has a '\n\n' at the beginning paragraphs = docstring.lstrip().split('\n\n') + paragraphs = [p for p in paragraphs if not p.startswith('..')] if len(paragraphs) > 1: + title = paragraphs[0].split('\n') + title = ' '.join(t for t in title if t[0] not in ['=', '-', '^', '~']) first_paragraph = re.sub('\n', ' ', paragraphs[1]) first_paragraph = (first_paragraph[:95] + '...' if len(first_paragraph) > 95 else first_paragraph) @@ -168,7 +171,7 @@ def extract_intro(filename, docstring): "and at least a paragraph explaining what the example is about. " "Please check the example file:\n {}\n".format(filename)) - return first_paragraph + return first_paragraph, title def get_md5sum(src_file): @@ -537,7 +540,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): example_file = os.path.join(target_dir, fname) shutil.copyfile(src_file, example_file) file_conf, script_blocks = split_code_and_text_blocks(src_file) - intro = extract_intro(fname, script_blocks[0][1]) + intro, title = extract_intro_title(fname, script_blocks[0][1]) if md5sum_is_current(example_file): return intro, 0 diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py index 09c11e880..f346230fa 100644 --- a/sphinx_gallery/tests/test_gen_gallery.py +++ b/sphinx_gallery/tests/test_gen_gallery.py @@ -172,6 +172,7 @@ def _check_order(config_app, key): regex = '.*:%s=(.):.*' % key with codecs.open(index_fname, 'r', 'utf-8') as fid: for line in fid: + print(line) if 'sphx-glr-thumbcontainer' in line: order.append(int(re.match(regex, line).group(1))) assert len(order) == 3 diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py index 1047f375b..cd6cc4a03 100644 --- a/sphinx_gallery/tests/test_gen_rst.py +++ b/sphinx_gallery/tests/test_gen_rst.py @@ -25,7 +25,7 @@ import matplotlib.pyplot as plt CONTENT = [ - '"""' + '"""', 'Docstring header', '================', '', @@ -109,11 +109,12 @@ def test_codestr2rst(): assert reference == output -def test_extract_intro(): - result = sg.extract_intro('', '\n'.join(CONTENT[1:9])) - assert 'Docstring' not in result - assert result == 'This is the description of the example which goes on and on, Óscar' # noqa - assert 'second paragraph' not in result +def test_extract_intro_title(): + intro, title = sg.extract_intro_title('', '\n'.join(CONTENT[1:9])) + assert title == 'Docstring header' + assert 'Docstring' not in intro + assert intro == 'This is the description of the example which goes on and on, Óscar' # noqa + assert 'second paragraph' not in intro def test_md5sums(): diff --git a/sphinx_gallery/tests/testconfs/src/plot_1.py b/sphinx_gallery/tests/testconfs/src/plot_1.py index bcf9a0f5f..721bfc10a 100644 --- a/sphinx_gallery/tests/testconfs/src/plot_1.py +++ b/sphinx_gallery/tests/testconfs/src/plot_1.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- """ -====================================== -B test, with two junk lines at the top -====================================== +====== +B test +====== :filename=1:title=2:lines=3:filesize=2: """ diff --git a/sphinx_gallery/tests/testconfs/src/plot_3.py b/sphinx_gallery/tests/testconfs/src/plot_3.py index 42b4c33b9..ceea4c0a3 100644 --- a/sphinx_gallery/tests/testconfs/src/plot_3.py +++ b/sphinx_gallery/tests/testconfs/src/plot_3.py @@ -1,4 +1,8 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- """ +.. _extra_ref: + =========================================================== A test with a really long title to make the filesize larger =========================================================== From 2f97da79f32649a292300dd32b0b8e93ed36f35f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 5 Aug 2017 13:26:24 -0600 Subject: [PATCH 7/8] FIX: Cleaner extraction --- sphinx_gallery/sorting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index d8a5c9d26..659b4041b 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -13,6 +13,7 @@ import os import types +from .gen_rst import extract_intro_title from .py_source_parser import split_code_and_text_blocks @@ -114,6 +115,5 @@ class ExampleTitleSortKey(_SortKey): def __call__(self, filename): src_file = os.path.normpath(os.path.join(self.src_dir, filename)) _, script_blocks = split_code_and_text_blocks(src_file) - # title should be the second line of the first script_blocks entry - title = script_blocks[0][1].strip().split('\n')[1] + _, title = extract_intro_title(src_file, script_blocks[0][1]) return title From e28140927eb609f6581eb1fbbb57d06963944ec4 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 5 Aug 2017 15:08:48 -0600 Subject: [PATCH 8/8] FIX: Fix naming --- sphinx_gallery/gen_rst.py | 25 ++++++++++++++---------- sphinx_gallery/sorting.py | 4 ++-- sphinx_gallery/tests/test_gen_gallery.py | 1 - sphinx_gallery/tests/test_gen_rst.py | 6 ++++-- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py index 14a718a47..283c4b4b6 100644 --- a/sphinx_gallery/gen_rst.py +++ b/sphinx_gallery/gen_rst.py @@ -153,23 +153,28 @@ def codestr2rst(codestr, lang='python', lineno=None): return code_directive + indented_block -def extract_intro_title(filename, docstring): +def extract_intro_and_title(filename, docstring): """ Extract the first paragraph of module-level docstring. max:95 char""" # lstrip is just in case docstring has a '\n\n' at the beginning paragraphs = docstring.lstrip().split('\n\n') - paragraphs = [p for p in paragraphs if not p.startswith('..')] - if len(paragraphs) > 1: - title = paragraphs[0].split('\n') - title = ' '.join(t for t in title if t[0] not in ['=', '-', '^', '~']) - first_paragraph = re.sub('\n', ' ', paragraphs[1]) - first_paragraph = (first_paragraph[:95] + '...' - if len(first_paragraph) > 95 else first_paragraph) - else: + # remove comments and other syntax like `.. _link:` + paragraphs = [p for p in paragraphs if not p.startswith('.. ')] + if len(paragraphs) <= 1: raise ValueError( "Example docstring should have a header for the example title " "and at least a paragraph explaining what the example is about. " "Please check the example file:\n {}\n".format(filename)) + # Title is the first paragraph with any ReSTructuredText title chars + # removed, i.e. lines that consist of (all the same) 7-bit non-ASCII chars. + # This conditional is not perfect but should hopefully be good enough. + title = paragraphs[0].strip().split('\n') + title = ' '.join(t for t in title if len(t) > 0 and + (ord(t[0]) >= 128 or t[0].isalnum())) + # Concatenate all lines of the first paragraph and truncate at 95 chars + first_paragraph = re.sub('\n', ' ', paragraphs[1]) + first_paragraph = (first_paragraph[:95] + '...' + if len(first_paragraph) > 95 else first_paragraph) return first_paragraph, title @@ -540,7 +545,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): example_file = os.path.join(target_dir, fname) shutil.copyfile(src_file, example_file) file_conf, script_blocks = split_code_and_text_blocks(src_file) - intro, title = extract_intro_title(fname, script_blocks[0][1]) + intro, title = extract_intro_and_title(fname, script_blocks[0][1]) if md5sum_is_current(example_file): return intro, 0 diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 659b4041b..d889e1c94 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -13,7 +13,7 @@ import os import types -from .gen_rst import extract_intro_title +from .gen_rst import extract_intro_and_title from .py_source_parser import split_code_and_text_blocks @@ -115,5 +115,5 @@ class ExampleTitleSortKey(_SortKey): def __call__(self, filename): src_file = os.path.normpath(os.path.join(self.src_dir, filename)) _, script_blocks = split_code_and_text_blocks(src_file) - _, title = extract_intro_title(src_file, script_blocks[0][1]) + _, title = extract_intro_and_title(src_file, script_blocks[0][1]) return title diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py index f346230fa..09c11e880 100644 --- a/sphinx_gallery/tests/test_gen_gallery.py +++ b/sphinx_gallery/tests/test_gen_gallery.py @@ -172,7 +172,6 @@ def _check_order(config_app, key): regex = '.*:%s=(.):.*' % key with codecs.open(index_fname, 'r', 'utf-8') as fid: for line in fid: - print(line) if 'sphx-glr-thumbcontainer' in line: order.append(int(re.match(regex, line).group(1))) assert len(order) == 3 diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py index cd6cc4a03..75d967d96 100644 --- a/sphinx_gallery/tests/test_gen_rst.py +++ b/sphinx_gallery/tests/test_gen_rst.py @@ -26,6 +26,7 @@ CONTENT = [ '"""', + '================', 'Docstring header', '================', '', @@ -109,8 +110,9 @@ def test_codestr2rst(): assert reference == output -def test_extract_intro_title(): - intro, title = sg.extract_intro_title('', '\n'.join(CONTENT[1:9])) +def test_extract_intro_and_title(): + intro, title = sg.extract_intro_and_title('', + '\n'.join(CONTENT[1:10])) assert title == 'Docstring header' assert 'Docstring' not in intro assert intro == 'This is the description of the example which goes on and on, Óscar' # noqa