From 0e0e71c75eaf0c9af5b8f64a8045abc0c3ca08d5 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 10 Jan 2025 02:06:26 -0500 Subject: [PATCH 1/4] Remove unused pixfmt_pre_type --- src/_image_resample.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/_image_resample.h b/src/_image_resample.h index 19dc05b32e2a..7e6c32c6bf64 100644 --- a/src/_image_resample.h +++ b/src/_image_resample.h @@ -519,17 +519,6 @@ struct type_mapping agg::pixfmt_alpha_blend_gray, agg::pixfmt_alpha_blend_rgba >; - using pixfmt_pre_type = std::conditional_t< - is_grayscale_v, - pixfmt_type, - agg::pixfmt_alpha_blend_rgba< - std::conditional_t< - std::is_same_v, - fixed_blender_rgba_pre, - agg::blender_rgba_pre - >, - agg::rendering_buffer> - >; template using span_gen_affine_type = std::conditional_t< is_grayscale_v, agg::span_image_resample_gray_affine, From 2aca299ff929b7562b0faf10cef710ec87e19f8c Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 10 Jan 2025 04:33:33 -0500 Subject: [PATCH 2/4] Remove unused fixed_blender_rgba_pre --- src/agg_workaround.h | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/src/agg_workaround.h b/src/agg_workaround.h index 476219519280..a167be97e171 100644 --- a/src/agg_workaround.h +++ b/src/agg_workaround.h @@ -8,46 +8,6 @@ blending of RGBA32 pixels does not preserve enough precision */ -template -struct fixed_blender_rgba_pre : agg::conv_rgba_pre -{ - typedef ColorT color_type; - typedef Order order_type; - typedef typename color_type::value_type value_type; - typedef typename color_type::calc_type calc_type; - typedef typename color_type::long_type long_type; - enum base_scale_e - { - base_shift = color_type::base_shift, - base_mask = color_type::base_mask - }; - - //-------------------------------------------------------------------- - static AGG_INLINE void blend_pix(value_type* p, - value_type cr, value_type cg, value_type cb, - value_type alpha, agg::cover_type cover) - { - blend_pix(p, - color_type::mult_cover(cr, cover), - color_type::mult_cover(cg, cover), - color_type::mult_cover(cb, cover), - color_type::mult_cover(alpha, cover)); - } - - //-------------------------------------------------------------------- - static AGG_INLINE void blend_pix(value_type* p, - value_type cr, value_type cg, value_type cb, - value_type alpha) - { - alpha = base_mask - alpha; - p[Order::R] = (value_type)(((p[Order::R] * alpha) >> base_shift) + cr); - p[Order::G] = (value_type)(((p[Order::G] * alpha) >> base_shift) + cg); - p[Order::B] = (value_type)(((p[Order::B] * alpha) >> base_shift) + cb); - p[Order::A] = (value_type)(base_mask - ((alpha * (base_mask - p[Order::A])) >> base_shift)); - } -}; - - template struct fixed_blender_rgba_plain : agg::conv_rgba_plain { From 2b73e7e33fd337de95869b6b5b37fd628a2441d8 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 10 Jan 2025 04:47:58 -0500 Subject: [PATCH 3/4] Resample RGB images in C++ Agg already has RGB resampling with output to RGBA builtin, so we just need to correctly wire up the corresponding templates. With this RGB resampling mode, we save the extra copy from RGB to RGBA in NumPy land that was required for the previous always-RGBA resampling. --- lib/matplotlib/image.py | 30 ++++-------- lib/matplotlib/tests/test_image.py | 4 +- src/_image_resample.h | 77 ++++++++++++++++++++---------- src/_image_wrapper.cpp | 54 +++++++++++++-------- 4 files changed, 98 insertions(+), 67 deletions(-) diff --git a/lib/matplotlib/image.py b/lib/matplotlib/image.py index f71d49db8ad8..28e6acbb8c81 100644 --- a/lib/matplotlib/image.py +++ b/lib/matplotlib/image.py @@ -160,8 +160,7 @@ def flush_images(): flush_images() -def _resample( - image_obj, data, out_shape, transform, *, resample=None, alpha=1): +def _resample(image_obj, data, out_shape, transform, *, resample=None, alpha=1): """ Convenience wrapper around `._image.resample` to resample *data* to *out_shape* (with a third dimension if *data* is RGBA) that takes care of @@ -204,7 +203,10 @@ def _resample( interpolation = 'nearest' else: interpolation = 'hanning' - out = np.zeros(out_shape + data.shape[2:], data.dtype) # 2D->2D, 3D->3D. + if len(data.shape) == 3: + # Always output RGBA. + out_shape += (4, ) + out = np.zeros(out_shape, data.dtype) if resample is None: resample = image_obj.get_resample() _image.resample(data, out, transform, @@ -216,20 +218,6 @@ def _resample( return out -def _rgb_to_rgba(A): - """ - Convert an RGB image to RGBA, as required by the image resample C++ - extension. - """ - rgba = np.zeros((A.shape[0], A.shape[1], 4), dtype=A.dtype) - rgba[:, :, :3] = A - if rgba.dtype == np.uint8: - rgba[:, :, 3] = 255 - else: - rgba[:, :, 3] = 1.0 - return rgba - - class _ImageBase(mcolorizer.ColorizingArtist): """ Base class for images. @@ -508,10 +496,10 @@ def _make_image(self, A, in_bbox, out_bbox, clip_bbox, magnification=1.0, # alpha channel below. output_alpha = (255 * alpha) if A.dtype == np.uint8 else alpha else: - output_alpha = _resample( # resample alpha channel - self, A[..., 3], out_shape, t, alpha=alpha) - output = _resample( # resample rgb channels - self, _rgb_to_rgba(A[..., :3]), out_shape, t, alpha=alpha) + # resample alpha channel + output_alpha = _resample(self, A[..., 3], out_shape, t, alpha=alpha) + # resample rgb channels + output = _resample(self, A[..., :3], out_shape, t, alpha=alpha) output[..., 3] = output_alpha # recombine rgb and alpha # output is now either a 2D array of normed (int or float) data diff --git a/lib/matplotlib/tests/test_image.py b/lib/matplotlib/tests/test_image.py index 24a0ab929bbf..e1d0e3e700d8 100644 --- a/lib/matplotlib/tests/test_image.py +++ b/lib/matplotlib/tests/test_image.py @@ -1577,8 +1577,8 @@ def test__resample_valid_output(): resample(np.zeros((9, 9)), np.zeros((9, 9, 4))) with pytest.raises(ValueError, match="different dimensionalities"): resample(np.zeros((9, 9, 4)), np.zeros((9, 9))) - with pytest.raises(ValueError, match="3D input array must be RGBA"): - resample(np.zeros((9, 9, 3)), np.zeros((9, 9, 4))) + with pytest.raises(ValueError, match="3D input array must be RGB"): + resample(np.zeros((9, 9, 2)), np.zeros((9, 9, 4))) with pytest.raises(ValueError, match="3D output array must be RGBA"): resample(np.zeros((9, 9, 4)), np.zeros((9, 9, 3))) with pytest.raises(ValueError, match="mismatched types"): diff --git a/src/_image_resample.h b/src/_image_resample.h index 7e6c32c6bf64..5af48c8f5e57 100644 --- a/src/_image_resample.h +++ b/src/_image_resample.h @@ -8,6 +8,7 @@ #include "agg_image_accessors.h" #include "agg_path_storage.h" #include "agg_pixfmt_gray.h" +#include "agg_pixfmt_rgb.h" #include "agg_pixfmt_rgba.h" #include "agg_renderer_base.h" #include "agg_renderer_scanline.h" @@ -16,6 +17,7 @@ #include "agg_span_allocator.h" #include "agg_span_converter.h" #include "agg_span_image_filter_gray.h" +#include "agg_span_image_filter_rgb.h" #include "agg_span_image_filter_rgba.h" #include "agg_span_interpolator_adaptor.h" #include "agg_span_interpolator_linear.h" @@ -496,16 +498,38 @@ typedef enum { } interpolation_e; -// T is rgba if and only if it has an T::r field. +// T is rgb(a) if and only if it has an T::r field. template struct is_grayscale : std::true_type {}; template struct is_grayscale> : std::false_type {}; template constexpr bool is_grayscale_v = is_grayscale::value; -template +template struct type_mapping { - using blender_type = std::conditional_t< + using input_blender_type = std::conditional_t< + is_grayscale_v, + agg::blender_gray, + std::conditional_t< + input_has_alpha, + std::conditional_t< + std::is_same_v, + fixed_blender_rgba_plain, + agg::blender_rgba_plain + >, + agg::blender_rgb + > + >; + using input_pixfmt_type = std::conditional_t< + is_grayscale_v, + agg::pixfmt_alpha_blend_gray, + std::conditional_t< + input_has_alpha, + agg::pixfmt_alpha_blend_rgba, + agg::pixfmt_alpha_blend_rgb + > + >; + using output_blender_type = std::conditional_t< is_grayscale_v, agg::blender_gray, std::conditional_t< @@ -514,25 +538,37 @@ struct type_mapping agg::blender_rgba_plain > >; - using pixfmt_type = std::conditional_t< + using output_pixfmt_type = std::conditional_t< is_grayscale_v, - agg::pixfmt_alpha_blend_gray, - agg::pixfmt_alpha_blend_rgba + agg::pixfmt_alpha_blend_gray, + agg::pixfmt_alpha_blend_rgba >; template using span_gen_affine_type = std::conditional_t< is_grayscale_v, agg::span_image_resample_gray_affine , - agg::span_image_resample_rgba_affine + std::conditional_t< + input_has_alpha, + agg::span_image_resample_rgba_affine , + agg::span_image_resample_rgb_affine + > >; template using span_gen_filter_type = std::conditional_t< is_grayscale_v, agg::span_image_filter_gray, - agg::span_image_filter_rgba + std::conditional_t< + input_has_alpha, + agg::span_image_filter_rgba, + agg::span_image_filter_rgb + > >; template using span_gen_nn_type = std::conditional_t< is_grayscale_v, agg::span_image_filter_gray_nn, - agg::span_image_filter_rgba_nn + std::conditional_t< + input_has_alpha, + agg::span_image_filter_rgba_nn, + agg::span_image_filter_rgb_nn + > >; }; @@ -686,16 +722,16 @@ static void get_filter(const resample_params_t ¶ms, } -template +template void resample( - const void *input, int in_width, int in_height, - void *output, int out_width, int out_height, + const void *input, int in_width, int in_height, int in_stride, + void *output, int out_width, int out_height, int out_stride, resample_params_t ¶ms) { - using type_mapping_t = type_mapping; + using type_mapping_t = type_mapping; - using input_pixfmt_t = typename type_mapping_t::pixfmt_type; - using output_pixfmt_t = typename type_mapping_t::pixfmt_type; + using input_pixfmt_t = typename type_mapping_t::input_pixfmt_type; + using output_pixfmt_t = typename type_mapping_t::output_pixfmt_type; using renderer_t = agg::renderer_base; using rasterizer_t = agg::rasterizer_scanline_aa; @@ -711,11 +747,6 @@ void resample( using arbitrary_interpolator_t = agg::span_interpolator_adaptor, lookup_distortion>; - size_t itemsize = sizeof(color_type); - if (is_grayscale::value) { - itemsize /= 2; // agg::grayXX includes an alpha channel which we don't have. - } - if (params.interpolation != NEAREST && params.is_affine && fabs(params.affine.sx) == 1.0 && @@ -732,14 +763,12 @@ void resample( span_conv_alpha_t conv_alpha(params.alpha); agg::rendering_buffer input_buffer; - input_buffer.attach( - (unsigned char *)input, in_width, in_height, in_width * itemsize); + input_buffer.attach((unsigned char *)input, in_width, in_height, in_stride); input_pixfmt_t input_pixfmt(input_buffer); image_accessor_t input_accessor(input_pixfmt); agg::rendering_buffer output_buffer; - output_buffer.attach( - (unsigned char *)output, out_width, out_height, out_width * itemsize); + output_buffer.attach((unsigned char *)output, out_width, out_height, out_stride); output_pixfmt_t output_pixfmt(output_buffer); renderer_t renderer(output_pixfmt); diff --git a/src/_image_wrapper.cpp b/src/_image_wrapper.cpp index 0f7b0da88de8..c791899c7f88 100644 --- a/src/_image_wrapper.cpp +++ b/src/_image_wrapper.cpp @@ -106,10 +106,14 @@ image_resample(py::array input_array, throw std::invalid_argument("Input array must be a 2D or 3D array"); } - if (ndim == 3 && input_array.shape(2) != 4) { - throw std::invalid_argument( - "3D input array must be RGBA with shape (M, N, 4), has trailing dimension of {}"_s.format( - input_array.shape(2))); + py::ssize_t ncomponents = 0; + if (ndim == 3) { + ncomponents = input_array.shape(2); + if (ncomponents != 3 && ncomponents != 4) { + throw std::invalid_argument( + "3D input array must be RGB with shape (M, N, 3) or RGBA with shape (M, N, 4), " + "has trailing dimension of {}"_s.format(ncomponents)); + } } // Ensure input array is contiguous, regardless of dtype @@ -173,25 +177,35 @@ image_resample(py::array input_array, if (auto resampler = (ndim == 2) ? ( - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : nullptr) : ( - // ndim == 3 - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - (dtype.equal(py::dtype::of())) ? resample : - nullptr)) { + // ndim == 3 + (ncomponents == 4) ? ( + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + nullptr + ) : ( + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + nullptr))) + { Py_BEGIN_ALLOW_THREADS resampler( - input_array.data(), input_array.shape(1), input_array.shape(0), - output_array.mutable_data(), output_array.shape(1), output_array.shape(0), + input_array.data(), input_array.shape(1), input_array.shape(0), input_array.strides(0), + output_array.mutable_data(), output_array.shape(1), output_array.shape(0), output_array.strides(0), params); Py_END_ALLOW_THREADS } else { From 769f7a49403a8f520f0a0283d8744a98b7ed761d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 17 Jan 2025 03:49:07 -0500 Subject: [PATCH 4/4] Avoid another copy when RGBA is resampled as RGB In the case of RGBA, the RGB and A channels are resampled separately, but they are created as a view on the original to pass to the C++ code. The C++ code then copies it to a contiguous buffer, but Agg's RGB resampler supports manually stepping the RGB input by a custom stride. As this step is a template parameter, we can't handle any arbitraray array, but can special case steps of 3 or 4 units, which should cover the common cases of RGB or RGBA-viewed-as-RGB input. --- src/_image_resample.h | 10 ++++++---- src/_image_wrapper.cpp | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/_image_resample.h b/src/_image_resample.h index 5af48c8f5e57..66577ea4ae96 100644 --- a/src/_image_resample.h +++ b/src/_image_resample.h @@ -504,7 +504,8 @@ template struct is_grayscale> : std:: template constexpr bool is_grayscale_v = is_grayscale::value; -template +// rgb_step is only used if input_has_alpha=false. +template struct type_mapping { using input_blender_type = std::conditional_t< @@ -526,7 +527,7 @@ struct type_mapping std::conditional_t< input_has_alpha, agg::pixfmt_alpha_blend_rgba, - agg::pixfmt_alpha_blend_rgb + agg::pixfmt_alpha_blend_rgb > >; using output_blender_type = std::conditional_t< @@ -722,13 +723,14 @@ static void get_filter(const resample_params_t ¶ms, } -template +// rgb_step is only used if input_has_alpha=false. +template void resample( const void *input, int in_width, int in_height, int in_stride, void *output, int out_width, int out_height, int out_stride, resample_params_t ¶ms) { - using type_mapping_t = type_mapping; + using type_mapping_t = type_mapping; using input_pixfmt_t = typename type_mapping_t::input_pixfmt_type; using output_pixfmt_t = typename type_mapping_t::output_pixfmt_type; diff --git a/src/_image_wrapper.cpp b/src/_image_wrapper.cpp index c791899c7f88..5b30282b140a 100644 --- a/src/_image_wrapper.cpp +++ b/src/_image_wrapper.cpp @@ -107,17 +107,29 @@ image_resample(py::array input_array, } py::ssize_t ncomponents = 0; + int rgb_step = 0; if (ndim == 3) { ncomponents = input_array.shape(2); - if (ncomponents != 3 && ncomponents != 4) { + if (ncomponents == 3) { + // We special-case a few options in order to avoid copying in the common case. + auto rgb_stride = input_array.strides(1); + auto item_stride = input_array.strides(2); + if (rgb_stride == 3 * item_stride) { + rgb_step = 3; + } else if (rgb_stride == 4 * item_stride) { + rgb_step = 4; + } + } else if (ncomponents != 4) { throw std::invalid_argument( "3D input array must be RGB with shape (M, N, 3) or RGBA with shape (M, N, 4), " "has trailing dimension of {}"_s.format(ncomponents)); } } - // Ensure input array is contiguous, regardless of dtype - input_array = py::array::ensure(input_array, py::array::c_style); + if (rgb_step == 0) { + // Ensure input array is contiguous, regardless of dtype + input_array = py::array::ensure(input_array, py::array::c_style); + } // Validate output array auto out_ndim = output_array.ndim(); @@ -194,13 +206,22 @@ image_resample(py::array input_array, dtype.equal(py::dtype::of()) ? resample : nullptr ) : ( - dtype.equal(py::dtype::of()) ? resample : - dtype.equal(py::dtype::of()) ? resample : - dtype.equal(py::dtype::of()) ? resample : - dtype.equal(py::dtype::of()) ? resample : - dtype.equal(py::dtype::of()) ? resample : - dtype.equal(py::dtype::of()) ? resample : - nullptr))) + (rgb_step == 4) ? ( + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + nullptr + ) : ( + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + dtype.equal(py::dtype::of()) ? resample : + nullptr)))) { Py_BEGIN_ALLOW_THREADS resampler(